Total coverage: 140196 (8%)of 1837355 




















   14 











   14 









   14 



    6 


    1 






    6 



















    6 
    4 
    2 












    6 






























    3 








    5 
    5 







    1 


    3 


    1 
    1 






    3 


















































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 *        IPV6 GSO/GRO offload support
 *        Linux INET6 implementation
 *
 *      UDPv6 GSO support
 */
#include <linux/skbuff.h>
#include <linux/netdevice.h>
#include <linux/indirect_call_wrapper.h>
#include <net/protocol.h>
#include <net/ipv6.h>
#include <net/udp.h>
#include <net/ip6_checksum.h>
#include "ip6_offload.h"
#include <net/gro.h>
#include <net/gso.h>

static struct sk_buff *udp6_ufo_fragment(struct sk_buff *skb,
                                         netdev_features_t features)
{
        struct sk_buff *segs = ERR_PTR(-EINVAL);
        unsigned int mss;
        unsigned int unfrag_ip6hlen, unfrag_len;
        struct frag_hdr *fptr;
        u8 *packet_start, *prevhdr;
        u8 nexthdr;
        u8 frag_hdr_sz = sizeof(struct frag_hdr);
        __wsum csum;
        int tnl_hlen;
        int err;

        if (skb->encapsulation && skb_shinfo(skb)->gso_type &
            (SKB_GSO_UDP_TUNNEL|SKB_GSO_UDP_TUNNEL_CSUM))
                segs = skb_udp_tunnel_segment(skb, features, true);
        else {
                const struct ipv6hdr *ipv6h;
                struct udphdr *uh;

                if (!(skb_shinfo(skb)->gso_type & (SKB_GSO_UDP | SKB_GSO_UDP_L4)))
                        goto out;

                if (!pskb_may_pull(skb, sizeof(struct udphdr)))
                        goto out;

                if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4)
                        return __udp_gso_segment(skb, features, true);

                mss = skb_shinfo(skb)->gso_size;
                if (unlikely(skb->len <= mss))
                        goto out;

                /* Do software UFO. Complete and fill in the UDP checksum as HW cannot
                 * do checksum of UDP packets sent as multiple IP fragments.
                 */

                uh = udp_hdr(skb);
                ipv6h = ipv6_hdr(skb);

                uh->check = 0;
                csum = skb_checksum(skb, 0, skb->len, 0);
                uh->check = udp_v6_check(skb->len, &ipv6h->saddr,
                                          &ipv6h->daddr, csum);
                if (uh->check == 0)
                        uh->check = CSUM_MANGLED_0;

                skb->ip_summed = CHECKSUM_UNNECESSARY;

                /* If there is no outer header we can fake a checksum offload
                 * due to the fact that we have already done the checksum in
                 * software prior to segmenting the frame.
                 */
                if (!skb->encap_hdr_csum)
                        features |= NETIF_F_HW_CSUM;

                /* Check if there is enough headroom to insert fragment header. */
                tnl_hlen = skb_tnl_header_len(skb);
                if (skb->mac_header < (tnl_hlen + frag_hdr_sz)) {
                        if (gso_pskb_expand_head(skb, tnl_hlen + frag_hdr_sz))
                                goto out;
                }

                /* Find the unfragmentable header and shift it left by frag_hdr_sz
                 * bytes to insert fragment header.
                 */
                err = ip6_find_1stfragopt(skb, &prevhdr);
                if (err < 0)
                        return ERR_PTR(err);
                unfrag_ip6hlen = err;
                nexthdr = *prevhdr;
                *prevhdr = NEXTHDR_FRAGMENT;
                unfrag_len = (skb_network_header(skb) - skb_mac_header(skb)) +
                             unfrag_ip6hlen + tnl_hlen;
                packet_start = (u8 *) skb->head + SKB_GSO_CB(skb)->mac_offset;
                memmove(packet_start-frag_hdr_sz, packet_start, unfrag_len);

                SKB_GSO_CB(skb)->mac_offset -= frag_hdr_sz;
                skb->mac_header -= frag_hdr_sz;
                skb->network_header -= frag_hdr_sz;

                fptr = (struct frag_hdr *)(skb_network_header(skb) + unfrag_ip6hlen);
                fptr->nexthdr = nexthdr;
                fptr->reserved = 0;
                fptr->identification = ipv6_proxy_select_ident(dev_net(skb->dev), skb);

                /* Fragment the skb. ipv6 header and the remaining fields of the
                 * fragment header are updated in ipv6_gso_segment()
                 */
                segs = skb_segment(skb, features);
        }

out:
        return segs;
}

static struct sock *udp6_gro_lookup_skb(struct sk_buff *skb, __be16 sport,
                                        __be16 dport)
{
        const struct ipv6hdr *iph = skb_gro_network_header(skb);
        struct net *net = dev_net(skb->dev);
        int iif, sdif;

        inet6_get_iif_sdif(skb, &iif, &sdif);

        return __udp6_lib_lookup(net, &iph->saddr, sport,
                                 &iph->daddr, dport, iif,
                                 sdif, net->ipv4.udp_table, NULL);
}

INDIRECT_CALLABLE_SCOPE
struct sk_buff *udp6_gro_receive(struct list_head *head, struct sk_buff *skb)
{
        struct udphdr *uh = udp_gro_udphdr(skb);
        struct sock *sk = NULL;
        struct sk_buff *pp;

        if (unlikely(!uh))
                goto flush;

        /* Don't bother verifying checksum if we're going to flush anyway. */
        if (NAPI_GRO_CB(skb)->flush)
                goto skip;

        if (skb_gro_checksum_validate_zero_check(skb, IPPROTO_UDP, uh->check,
                                                 ip6_gro_compute_pseudo))
                goto flush;
        else if (uh->check)
                skb_gro_checksum_try_convert(skb, IPPROTO_UDP,
                                             ip6_gro_compute_pseudo);

skip:
        NAPI_GRO_CB(skb)->is_ipv6 = 1;

        if (static_branch_unlikely(&udpv6_encap_needed_key))
                sk = udp6_gro_lookup_skb(skb, uh->source, uh->dest);

        pp = udp_gro_receive(head, skb, uh, sk);
        return pp;

flush:
        NAPI_GRO_CB(skb)->flush = 1;
        return NULL;
}

INDIRECT_CALLABLE_SCOPE int udp6_gro_complete(struct sk_buff *skb, int nhoff)
{
        const u16 offset = NAPI_GRO_CB(skb)->network_offsets[skb->encapsulation];
        const struct ipv6hdr *ipv6h = (struct ipv6hdr *)(skb->data + offset);
        struct udphdr *uh = (struct udphdr *)(skb->data + nhoff);

        /* do fraglist only if there is no outer UDP encap (or we already processed it) */
        if (NAPI_GRO_CB(skb)->is_flist && !NAPI_GRO_CB(skb)->encap_mark) {
                uh->len = htons(skb->len - nhoff);

                skb_shinfo(skb)->gso_type |= (SKB_GSO_FRAGLIST|SKB_GSO_UDP_L4);
                skb_shinfo(skb)->gso_segs = NAPI_GRO_CB(skb)->count;

                __skb_incr_checksum_unnecessary(skb);

                return 0;
        }

        if (uh->check)
                uh->check = ~udp_v6_check(skb->len - nhoff, &ipv6h->saddr,
                                          &ipv6h->daddr, 0);

        return udp_gro_complete(skb, nhoff, udp6_lib_lookup_skb);
}

int __init udpv6_offload_init(void)
{
        net_hotdata.udpv6_offload = (struct net_offload) {
                .callbacks = {
                        .gso_segment        =        udp6_ufo_fragment,
                        .gro_receive        =        udp6_gro_receive,
                        .gro_complete        =        udp6_gro_complete,
                },
        };
        return inet6_add_offload(&net_hotdata.udpv6_offload, IPPROTO_UDP);
}

int udpv6_offload_exit(void)
{
        return inet6_del_offload(&net_hotdata.udpv6_offload, IPPROTO_UDP);
}















    8 



















   13 








    2 


























































   11 

   12 







   10 
















































    7 

    3 




















1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
// SPDX-License-Identifier: GPL-2.0 OR MIT
/*
 * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
 *
 * This is based in part on Andrew Moon's poly1305-donna, which is in the
 * public domain.
 */

#include <linux/kernel.h>
#include <linux/unaligned.h>
#include <crypto/internal/poly1305.h>

void poly1305_core_setkey(struct poly1305_core_key *key,
                          const u8 raw_key[POLY1305_BLOCK_SIZE])
{
        u64 t0, t1;

        /* r &= 0xffffffc0ffffffc0ffffffc0fffffff */
        t0 = get_unaligned_le64(&raw_key[0]);
        t1 = get_unaligned_le64(&raw_key[8]);

        key->key.r64[0] = t0 & 0xffc0fffffffULL;
        key->key.r64[1] = ((t0 >> 44) | (t1 << 20)) & 0xfffffc0ffffULL;
        key->key.r64[2] = ((t1 >> 24)) & 0x00ffffffc0fULL;

        /* s = 20*r */
        key->precomputed_s.r64[0] = key->key.r64[1] * 20;
        key->precomputed_s.r64[1] = key->key.r64[2] * 20;
}
EXPORT_SYMBOL(poly1305_core_setkey);

void poly1305_core_blocks(struct poly1305_state *state,
                          const struct poly1305_core_key *key, const void *src,
                          unsigned int nblocks, u32 hibit)
{
        const u8 *input = src;
        u64 hibit64;
        u64 r0, r1, r2;
        u64 s1, s2;
        u64 h0, h1, h2;
        u64 c;
        u128 d0, d1, d2, d;

        if (!nblocks)
                return;

        hibit64 = ((u64)hibit) << 40;

        r0 = key->key.r64[0];
        r1 = key->key.r64[1];
        r2 = key->key.r64[2];

        h0 = state->h64[0];
        h1 = state->h64[1];
        h2 = state->h64[2];

        s1 = key->precomputed_s.r64[0];
        s2 = key->precomputed_s.r64[1];

        do {
                u64 t0, t1;

                /* h += m[i] */
                t0 = get_unaligned_le64(&input[0]);
                t1 = get_unaligned_le64(&input[8]);

                h0 += t0 & 0xfffffffffffULL;
                h1 += ((t0 >> 44) | (t1 << 20)) & 0xfffffffffffULL;
                h2 += (((t1 >> 24)) & 0x3ffffffffffULL) | hibit64;

                /* h *= r */
                d0 = (u128)h0 * r0;
                d = (u128)h1 * s2;
                d0 += d;
                d = (u128)h2 * s1;
                d0 += d;
                d1 = (u128)h0 * r1;
                d = (u128)h1 * r0;
                d1 += d;
                d = (u128)h2 * s2;
                d1 += d;
                d2 = (u128)h0 * r2;
                d = (u128)h1 * r1;
                d2 += d;
                d = (u128)h2 * r0;
                d2 += d;

                /* (partial) h %= p */
                c = (u64)(d0 >> 44);
                h0 = (u64)d0 & 0xfffffffffffULL;
                d1 += c;
                c = (u64)(d1 >> 44);
                h1 = (u64)d1 & 0xfffffffffffULL;
                d2 += c;
                c = (u64)(d2 >> 42);
                h2 = (u64)d2 & 0x3ffffffffffULL;
                h0 += c * 5;
                c = h0 >> 44;
                h0 = h0 & 0xfffffffffffULL;
                h1 += c;

                input += POLY1305_BLOCK_SIZE;
        } while (--nblocks);

        state->h64[0] = h0;
        state->h64[1] = h1;
        state->h64[2] = h2;
}
EXPORT_SYMBOL(poly1305_core_blocks);

void poly1305_core_emit(const struct poly1305_state *state, const u32 nonce[4],
                        void *dst)
{
        u8 *mac = dst;
        u64 h0, h1, h2, c;
        u64 g0, g1, g2;
        u64 t0, t1;

        /* fully carry h */
        h0 = state->h64[0];
        h1 = state->h64[1];
        h2 = state->h64[2];

        c = h1 >> 44;
        h1 &= 0xfffffffffffULL;
        h2 += c;
        c = h2 >> 42;
        h2 &= 0x3ffffffffffULL;
        h0 += c * 5;
        c = h0 >> 44;
        h0 &= 0xfffffffffffULL;
        h1 += c;
        c = h1 >> 44;
        h1 &= 0xfffffffffffULL;
        h2 += c;
        c = h2 >> 42;
        h2 &= 0x3ffffffffffULL;
        h0 += c * 5;
        c = h0 >> 44;
        h0 &= 0xfffffffffffULL;
        h1 += c;

        /* compute h + -p */
        g0 = h0 + 5;
        c  = g0 >> 44;
        g0 &= 0xfffffffffffULL;
        g1 = h1 + c;
        c  = g1 >> 44;
        g1 &= 0xfffffffffffULL;
        g2 = h2 + c - (1ULL << 42);

        /* select h if h < p, or h + -p if h >= p */
        c = (g2 >> ((sizeof(u64) * 8) - 1)) - 1;
        g0 &= c;
        g1 &= c;
        g2 &= c;
        c  = ~c;
        h0 = (h0 & c) | g0;
        h1 = (h1 & c) | g1;
        h2 = (h2 & c) | g2;

        if (likely(nonce)) {
                /* h = (h + nonce) */
                t0 = ((u64)nonce[1] << 32) | nonce[0];
                t1 = ((u64)nonce[3] << 32) | nonce[2];

                h0 += t0 & 0xfffffffffffULL;
                c = h0 >> 44;
                h0 &= 0xfffffffffffULL;
                h1 += (((t0 >> 44) | (t1 << 20)) & 0xfffffffffffULL) + c;
                c = h1 >> 44;
                h1 &= 0xfffffffffffULL;
                h2 += (((t1 >> 24)) & 0x3ffffffffffULL) + c;
                h2 &= 0x3ffffffffffULL;
        }

        /* mac = h % (2^128) */
        h0 = h0 | (h1 << 44);
        h1 = (h1 >> 20) | (h2 << 24);

        put_unaligned_le64(h0, &mac[0]);
        put_unaligned_le64(h1, &mac[8]);
}
EXPORT_SYMBOL(poly1305_core_emit);































































    5 
    5 

























































    3 
    3 














































































    5 

    3 



































    5 





















































































































































































































































































  777 













































































































































































































































































































































































































    3 





    3 
    3 



    3 














    3 

































    3 












    3 









    3 


    3 

    3 







    3 


    3 








    3 
























    3 




























    3 




    3 

























    3 






















    3 


    3 






























































































    5 

    5 































































































    4 












    5 










    5 












    5 




    5 


    5 

    3 
    2 




    3 

















    5 
    5 

    5 

    5 

    5 

    5 





















    5 


    5 
    5 


    3 





























































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































    3 












    3 














    3 







    3 








    3 


























































































































































































    3 








    3 






    3 
    3 


































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965
2966
2967
2968
2969
2970
2971
2972
2973
2974
2975
2976
2977
2978
2979
2980
2981
2982
2983
2984
2985
2986
2987
2988
2989
2990
2991
2992
2993
2994
2995
2996
2997
2998
2999
3000
3001
3002
3003
3004
3005
3006
3007
3008
3009
3010
3011
3012
3013
3014
3015
3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029
3030
3031
3032
3033
3034
3035
3036
3037
3038
3039
3040
3041
3042
3043
3044
3045
3046
3047
3048
3049
3050
3051
3052
3053
3054
3055
3056
3057
3058
3059
3060
3061
3062
3063
3064
3065
3066
3067
3068
3069
3070
3071
3072
3073
3074
3075
3076
3077
3078
3079
3080
3081
3082
3083
3084
3085
3086
3087
3088
3089
3090
3091
3092
3093
3094
3095
3096
3097
3098
3099
3100
3101
3102
3103
3104
3105
3106
3107
3108
3109
3110
3111
3112
3113
3114
3115
3116
3117
3118
3119
3120
3121
3122
3123
3124
3125
3126
3127
3128
3129
3130
3131
3132
3133
3134
3135
3136
3137
3138
3139
3140
3141
3142
3143
3144
3145
3146
3147
3148
3149
3150
3151
3152
3153
3154
3155
3156
3157
3158
// SPDX-License-Identifier: GPL-2.0-only
/*
 *  linux/fs/buffer.c
 *
 *  Copyright (C) 1991, 1992, 2002  Linus Torvalds
 */

/*
 * Start bdflush() with kernel_thread not syscall - Paul Gortmaker, 12/95
 *
 * Removed a lot of unnecessary code and simplified things now that
 * the buffer cache isn't our primary cache - Andrew Tridgell 12/96
 *
 * Speed up hash, lru, and free list operations.  Use gfp() for allocating
 * hash table, use SLAB cache for buffer heads. SMP threading.  -DaveM
 *
 * Added 32k buffer block sizes - these are required older ARM systems. - RMK
 *
 * async buffer flushing, 1999 Andrea Arcangeli <andrea@suse.de>
 */

#include <linux/kernel.h>
#include <linux/sched/signal.h>
#include <linux/syscalls.h>
#include <linux/fs.h>
#include <linux/iomap.h>
#include <linux/mm.h>
#include <linux/percpu.h>
#include <linux/slab.h>
#include <linux/capability.h>
#include <linux/blkdev.h>
#include <linux/file.h>
#include <linux/quotaops.h>
#include <linux/highmem.h>
#include <linux/export.h>
#include <linux/backing-dev.h>
#include <linux/writeback.h>
#include <linux/hash.h>
#include <linux/suspend.h>
#include <linux/buffer_head.h>
#include <linux/task_io_accounting_ops.h>
#include <linux/bio.h>
#include <linux/cpu.h>
#include <linux/bitops.h>
#include <linux/mpage.h>
#include <linux/bit_spinlock.h>
#include <linux/pagevec.h>
#include <linux/sched/mm.h>
#include <trace/events/block.h>
#include <linux/fscrypt.h>
#include <linux/fsverity.h>
#include <linux/sched/isolation.h>

#include "internal.h"

static int fsync_buffers_list(spinlock_t *lock, struct list_head *list);
static void submit_bh_wbc(blk_opf_t opf, struct buffer_head *bh,
                          enum rw_hint hint, struct writeback_control *wbc);

#define BH_ENTRY(list) list_entry((list), struct buffer_head, b_assoc_buffers)

inline void touch_buffer(struct buffer_head *bh)
{
        trace_block_touch_buffer(bh);
        folio_mark_accessed(bh->b_folio);
}
EXPORT_SYMBOL(touch_buffer);

void __lock_buffer(struct buffer_head *bh)
{
        wait_on_bit_lock_io(&bh->b_state, BH_Lock, TASK_UNINTERRUPTIBLE);
}
EXPORT_SYMBOL(__lock_buffer);

void unlock_buffer(struct buffer_head *bh)
{
        clear_bit_unlock(BH_Lock, &bh->b_state);
        smp_mb__after_atomic();
        wake_up_bit(&bh->b_state, BH_Lock);
}
EXPORT_SYMBOL(unlock_buffer);

/*
 * Returns if the folio has dirty or writeback buffers. If all the buffers
 * are unlocked and clean then the folio_test_dirty information is stale. If
 * any of the buffers are locked, it is assumed they are locked for IO.
 */
void buffer_check_dirty_writeback(struct folio *folio,
                                     bool *dirty, bool *writeback)
{
        struct buffer_head *head, *bh;
        *dirty = false;
        *writeback = false;

        BUG_ON(!folio_test_locked(folio));

        head = folio_buffers(folio);
        if (!head)
                return;

        if (folio_test_writeback(folio))
                *writeback = true;

        bh = head;
        do {
                if (buffer_locked(bh))
                        *writeback = true;

                if (buffer_dirty(bh))
                        *dirty = true;

                bh = bh->b_this_page;
        } while (bh != head);
}

/*
 * Block until a buffer comes unlocked.  This doesn't stop it
 * from becoming locked again - you have to lock it yourself
 * if you want to preserve its state.
 */
void __wait_on_buffer(struct buffer_head * bh)
{
        wait_on_bit_io(&bh->b_state, BH_Lock, TASK_UNINTERRUPTIBLE);
}
EXPORT_SYMBOL(__wait_on_buffer);

static void buffer_io_error(struct buffer_head *bh, char *msg)
{
        if (!test_bit(BH_Quiet, &bh->b_state))
                printk_ratelimited(KERN_ERR
                        "Buffer I/O error on dev %pg, logical block %llu%s\n",
                        bh->b_bdev, (unsigned long long)bh->b_blocknr, msg);
}

/*
 * End-of-IO handler helper function which does not touch the bh after
 * unlocking it.
 * Note: unlock_buffer() sort-of does touch the bh after unlocking it, but
 * a race there is benign: unlock_buffer() only use the bh's address for
 * hashing after unlocking the buffer, so it doesn't actually touch the bh
 * itself.
 */
static void __end_buffer_read_notouch(struct buffer_head *bh, int uptodate)
{
        if (uptodate) {
                set_buffer_uptodate(bh);
        } else {
                /* This happens, due to failed read-ahead attempts. */
                clear_buffer_uptodate(bh);
        }
        unlock_buffer(bh);
}

/*
 * Default synchronous end-of-IO handler..  Just mark it up-to-date and
 * unlock the buffer.
 */
void end_buffer_read_sync(struct buffer_head *bh, int uptodate)
{
        __end_buffer_read_notouch(bh, uptodate);
        put_bh(bh);
}
EXPORT_SYMBOL(end_buffer_read_sync);

void end_buffer_write_sync(struct buffer_head *bh, int uptodate)
{
        if (uptodate) {
                set_buffer_uptodate(bh);
        } else {
                buffer_io_error(bh, ", lost sync page write");
                mark_buffer_write_io_error(bh);
                clear_buffer_uptodate(bh);
        }
        unlock_buffer(bh);
        put_bh(bh);
}
EXPORT_SYMBOL(end_buffer_write_sync);

/*
 * Various filesystems appear to want __find_get_block to be non-blocking.
 * But it's the page lock which protects the buffers.  To get around this,
 * we get exclusion from try_to_free_buffers with the blockdev mapping's
 * i_private_lock.
 *
 * Hack idea: for the blockdev mapping, i_private_lock contention
 * may be quite high.  This code could TryLock the page, and if that
 * succeeds, there is no need to take i_private_lock.
 */
static struct buffer_head *
__find_get_block_slow(struct block_device *bdev, sector_t block)
{
        struct address_space *bd_mapping = bdev->bd_mapping;
        const int blkbits = bd_mapping->host->i_blkbits;
        struct buffer_head *ret = NULL;
        pgoff_t index;
        struct buffer_head *bh;
        struct buffer_head *head;
        struct folio *folio;
        int all_mapped = 1;
        static DEFINE_RATELIMIT_STATE(last_warned, HZ, 1);

        index = ((loff_t)block << blkbits) / PAGE_SIZE;
        folio = __filemap_get_folio(bd_mapping, index, FGP_ACCESSED, 0);
        if (IS_ERR(folio))
                goto out;

        spin_lock(&bd_mapping->i_private_lock);
        head = folio_buffers(folio);
        if (!head)
                goto out_unlock;
        bh = head;
        do {
                if (!buffer_mapped(bh))
                        all_mapped = 0;
                else if (bh->b_blocknr == block) {
                        ret = bh;
                        get_bh(bh);
                        goto out_unlock;
                }
                bh = bh->b_this_page;
        } while (bh != head);

        /* we might be here because some of the buffers on this page are
         * not mapped.  This is due to various races between
         * file io on the block device and getblk.  It gets dealt with
         * elsewhere, don't buffer_error if we had some unmapped buffers
         */
        ratelimit_set_flags(&last_warned, RATELIMIT_MSG_ON_RELEASE);
        if (all_mapped && __ratelimit(&last_warned)) {
                printk("__find_get_block_slow() failed. block=%llu, "
                       "b_blocknr=%llu, b_state=0x%08lx, b_size=%zu, "
                       "device %pg blocksize: %d\n",
                       (unsigned long long)block,
                       (unsigned long long)bh->b_blocknr,
                       bh->b_state, bh->b_size, bdev,
                       1 << blkbits);
        }
out_unlock:
        spin_unlock(&bd_mapping->i_private_lock);
        folio_put(folio);
out:
        return ret;
}

static void end_buffer_async_read(struct buffer_head *bh, int uptodate)
{
        unsigned long flags;
        struct buffer_head *first;
        struct buffer_head *tmp;
        struct folio *folio;
        int folio_uptodate = 1;

        BUG_ON(!buffer_async_read(bh));

        folio = bh->b_folio;
        if (uptodate) {
                set_buffer_uptodate(bh);
        } else {
                clear_buffer_uptodate(bh);
                buffer_io_error(bh, ", async page read");
        }

        /*
         * Be _very_ careful from here on. Bad things can happen if
         * two buffer heads end IO at almost the same time and both
         * decide that the page is now completely done.
         */
        first = folio_buffers(folio);
        spin_lock_irqsave(&first->b_uptodate_lock, flags);
        clear_buffer_async_read(bh);
        unlock_buffer(bh);
        tmp = bh;
        do {
                if (!buffer_uptodate(tmp))
                        folio_uptodate = 0;
                if (buffer_async_read(tmp)) {
                        BUG_ON(!buffer_locked(tmp));
                        goto still_busy;
                }
                tmp = tmp->b_this_page;
        } while (tmp != bh);
        spin_unlock_irqrestore(&first->b_uptodate_lock, flags);

        folio_end_read(folio, folio_uptodate);
        return;

still_busy:
        spin_unlock_irqrestore(&first->b_uptodate_lock, flags);
        return;
}

struct postprocess_bh_ctx {
        struct work_struct work;
        struct buffer_head *bh;
};

static void verify_bh(struct work_struct *work)
{
        struct postprocess_bh_ctx *ctx =
                container_of(work, struct postprocess_bh_ctx, work);
        struct buffer_head *bh = ctx->bh;
        bool valid;

        valid = fsverity_verify_blocks(bh->b_folio, bh->b_size, bh_offset(bh));
        end_buffer_async_read(bh, valid);
        kfree(ctx);
}

static bool need_fsverity(struct buffer_head *bh)
{
        struct folio *folio = bh->b_folio;
        struct inode *inode = folio->mapping->host;

        return fsverity_active(inode) &&
                /* needed by ext4 */
                folio->index < DIV_ROUND_UP(inode->i_size, PAGE_SIZE);
}

static void decrypt_bh(struct work_struct *work)
{
        struct postprocess_bh_ctx *ctx =
                container_of(work, struct postprocess_bh_ctx, work);
        struct buffer_head *bh = ctx->bh;
        int err;

        err = fscrypt_decrypt_pagecache_blocks(bh->b_folio, bh->b_size,
                                               bh_offset(bh));
        if (err == 0 && need_fsverity(bh)) {
                /*
                 * We use different work queues for decryption and for verity
                 * because verity may require reading metadata pages that need
                 * decryption, and we shouldn't recurse to the same workqueue.
                 */
                INIT_WORK(&ctx->work, verify_bh);
                fsverity_enqueue_verify_work(&ctx->work);
                return;
        }
        end_buffer_async_read(bh, err == 0);
        kfree(ctx);
}

/*
 * I/O completion handler for block_read_full_folio() - pages
 * which come unlocked at the end of I/O.
 */
static void end_buffer_async_read_io(struct buffer_head *bh, int uptodate)
{
        struct inode *inode = bh->b_folio->mapping->host;
        bool decrypt = fscrypt_inode_uses_fs_layer_crypto(inode);
        bool verify = need_fsverity(bh);

        /* Decrypt (with fscrypt) and/or verify (with fsverity) if needed. */
        if (uptodate && (decrypt || verify)) {
                struct postprocess_bh_ctx *ctx =
                        kmalloc(sizeof(*ctx), GFP_ATOMIC);

                if (ctx) {
                        ctx->bh = bh;
                        if (decrypt) {
                                INIT_WORK(&ctx->work, decrypt_bh);
                                fscrypt_enqueue_decrypt_work(&ctx->work);
                        } else {
                                INIT_WORK(&ctx->work, verify_bh);
                                fsverity_enqueue_verify_work(&ctx->work);
                        }
                        return;
                }
                uptodate = 0;
        }
        end_buffer_async_read(bh, uptodate);
}

/*
 * Completion handler for block_write_full_folio() - folios which are unlocked
 * during I/O, and which have the writeback flag cleared upon I/O completion.
 */
static void end_buffer_async_write(struct buffer_head *bh, int uptodate)
{
        unsigned long flags;
        struct buffer_head *first;
        struct buffer_head *tmp;
        struct folio *folio;

        BUG_ON(!buffer_async_write(bh));

        folio = bh->b_folio;
        if (uptodate) {
                set_buffer_uptodate(bh);
        } else {
                buffer_io_error(bh, ", lost async page write");
                mark_buffer_write_io_error(bh);
                clear_buffer_uptodate(bh);
        }

        first = folio_buffers(folio);
        spin_lock_irqsave(&first->b_uptodate_lock, flags);

        clear_buffer_async_write(bh);
        unlock_buffer(bh);
        tmp = bh->b_this_page;
        while (tmp != bh) {
                if (buffer_async_write(tmp)) {
                        BUG_ON(!buffer_locked(tmp));
                        goto still_busy;
                }
                tmp = tmp->b_this_page;
        }
        spin_unlock_irqrestore(&first->b_uptodate_lock, flags);
        folio_end_writeback(folio);
        return;

still_busy:
        spin_unlock_irqrestore(&first->b_uptodate_lock, flags);
        return;
}

/*
 * If a page's buffers are under async readin (end_buffer_async_read
 * completion) then there is a possibility that another thread of
 * control could lock one of the buffers after it has completed
 * but while some of the other buffers have not completed.  This
 * locked buffer would confuse end_buffer_async_read() into not unlocking
 * the page.  So the absence of BH_Async_Read tells end_buffer_async_read()
 * that this buffer is not under async I/O.
 *
 * The page comes unlocked when it has no locked buffer_async buffers
 * left.
 *
 * PageLocked prevents anyone starting new async I/O reads any of
 * the buffers.
 *
 * PageWriteback is used to prevent simultaneous writeout of the same
 * page.
 *
 * PageLocked prevents anyone from starting writeback of a page which is
 * under read I/O (PageWriteback is only ever set against a locked page).
 */
static void mark_buffer_async_read(struct buffer_head *bh)
{
        bh->b_end_io = end_buffer_async_read_io;
        set_buffer_async_read(bh);
}

static void mark_buffer_async_write_endio(struct buffer_head *bh,
                                          bh_end_io_t *handler)
{
        bh->b_end_io = handler;
        set_buffer_async_write(bh);
}

void mark_buffer_async_write(struct buffer_head *bh)
{
        mark_buffer_async_write_endio(bh, end_buffer_async_write);
}
EXPORT_SYMBOL(mark_buffer_async_write);


/*
 * fs/buffer.c contains helper functions for buffer-backed address space's
 * fsync functions.  A common requirement for buffer-based filesystems is
 * that certain data from the backing blockdev needs to be written out for
 * a successful fsync().  For example, ext2 indirect blocks need to be
 * written back and waited upon before fsync() returns.
 *
 * The functions mark_buffer_dirty_inode(), fsync_inode_buffers(),
 * inode_has_buffers() and invalidate_inode_buffers() are provided for the
 * management of a list of dependent buffers at ->i_mapping->i_private_list.
 *
 * Locking is a little subtle: try_to_free_buffers() will remove buffers
 * from their controlling inode's queue when they are being freed.  But
 * try_to_free_buffers() will be operating against the *blockdev* mapping
 * at the time, not against the S_ISREG file which depends on those buffers.
 * So the locking for i_private_list is via the i_private_lock in the address_space
 * which backs the buffers.  Which is different from the address_space 
 * against which the buffers are listed.  So for a particular address_space,
 * mapping->i_private_lock does *not* protect mapping->i_private_list!  In fact,
 * mapping->i_private_list will always be protected by the backing blockdev's
 * ->i_private_lock.
 *
 * Which introduces a requirement: all buffers on an address_space's
 * ->i_private_list must be from the same address_space: the blockdev's.
 *
 * address_spaces which do not place buffers at ->i_private_list via these
 * utility functions are free to use i_private_lock and i_private_list for
 * whatever they want.  The only requirement is that list_empty(i_private_list)
 * be true at clear_inode() time.
 *
 * FIXME: clear_inode should not call invalidate_inode_buffers().  The
 * filesystems should do that.  invalidate_inode_buffers() should just go
 * BUG_ON(!list_empty).
 *
 * FIXME: mark_buffer_dirty_inode() is a data-plane operation.  It should
 * take an address_space, not an inode.  And it should be called
 * mark_buffer_dirty_fsync() to clearly define why those buffers are being
 * queued up.
 *
 * FIXME: mark_buffer_dirty_inode() doesn't need to add the buffer to the
 * list if it is already on a list.  Because if the buffer is on a list,
 * it *must* already be on the right one.  If not, the filesystem is being
 * silly.  This will save a ton of locking.  But first we have to ensure
 * that buffers are taken *off* the old inode's list when they are freed
 * (presumably in truncate).  That requires careful auditing of all
 * filesystems (do it inside bforget()).  It could also be done by bringing
 * b_inode back.
 */

/*
 * The buffer's backing address_space's i_private_lock must be held
 */
static void __remove_assoc_queue(struct buffer_head *bh)
{
        list_del_init(&bh->b_assoc_buffers);
        WARN_ON(!bh->b_assoc_map);
        bh->b_assoc_map = NULL;
}

int inode_has_buffers(struct inode *inode)
{
        return !list_empty(&inode->i_data.i_private_list);
}

/*
 * osync is designed to support O_SYNC io.  It waits synchronously for
 * all already-submitted IO to complete, but does not queue any new
 * writes to the disk.
 *
 * To do O_SYNC writes, just queue the buffer writes with write_dirty_buffer
 * as you dirty the buffers, and then use osync_inode_buffers to wait for
 * completion.  Any other dirty buffers which are not yet queued for
 * write will not be flushed to disk by the osync.
 */
static int osync_buffers_list(spinlock_t *lock, struct list_head *list)
{
        struct buffer_head *bh;
        struct list_head *p;
        int err = 0;

        spin_lock(lock);
repeat:
        list_for_each_prev(p, list) {
                bh = BH_ENTRY(p);
                if (buffer_locked(bh)) {
                        get_bh(bh);
                        spin_unlock(lock);
                        wait_on_buffer(bh);
                        if (!buffer_uptodate(bh))
                                err = -EIO;
                        brelse(bh);
                        spin_lock(lock);
                        goto repeat;
                }
        }
        spin_unlock(lock);
        return err;
}

/**
 * sync_mapping_buffers - write out & wait upon a mapping's "associated" buffers
 * @mapping: the mapping which wants those buffers written
 *
 * Starts I/O against the buffers at mapping->i_private_list, and waits upon
 * that I/O.
 *
 * Basically, this is a convenience function for fsync().
 * @mapping is a file or directory which needs those buffers to be written for
 * a successful fsync().
 */
int sync_mapping_buffers(struct address_space *mapping)
{
        struct address_space *buffer_mapping = mapping->i_private_data;

        if (buffer_mapping == NULL || list_empty(&mapping->i_private_list))
                return 0;

        return fsync_buffers_list(&buffer_mapping->i_private_lock,
                                        &mapping->i_private_list);
}
EXPORT_SYMBOL(sync_mapping_buffers);

/**
 * generic_buffers_fsync_noflush - generic buffer fsync implementation
 * for simple filesystems with no inode lock
 *
 * @file:        file to synchronize
 * @start:        start offset in bytes
 * @end:        end offset in bytes (inclusive)
 * @datasync:        only synchronize essential metadata if true
 *
 * This is a generic implementation of the fsync method for simple
 * filesystems which track all non-inode metadata in the buffers list
 * hanging off the address_space structure.
 */
int generic_buffers_fsync_noflush(struct file *file, loff_t start, loff_t end,
                                  bool datasync)
{
        struct inode *inode = file->f_mapping->host;
        int err;
        int ret;

        err = file_write_and_wait_range(file, start, end);
        if (err)
                return err;

        ret = sync_mapping_buffers(inode->i_mapping);
        if (!(inode->i_state & I_DIRTY_ALL))
                goto out;
        if (datasync && !(inode->i_state & I_DIRTY_DATASYNC))
                goto out;

        err = sync_inode_metadata(inode, 1);
        if (ret == 0)
                ret = err;

out:
        /* check and advance again to catch errors after syncing out buffers */
        err = file_check_and_advance_wb_err(file);
        if (ret == 0)
                ret = err;
        return ret;
}
EXPORT_SYMBOL(generic_buffers_fsync_noflush);

/**
 * generic_buffers_fsync - generic buffer fsync implementation
 * for simple filesystems with no inode lock
 *
 * @file:        file to synchronize
 * @start:        start offset in bytes
 * @end:        end offset in bytes (inclusive)
 * @datasync:        only synchronize essential metadata if true
 *
 * This is a generic implementation of the fsync method for simple
 * filesystems which track all non-inode metadata in the buffers list
 * hanging off the address_space structure. This also makes sure that
 * a device cache flush operation is called at the end.
 */
int generic_buffers_fsync(struct file *file, loff_t start, loff_t end,
                          bool datasync)
{
        struct inode *inode = file->f_mapping->host;
        int ret;

        ret = generic_buffers_fsync_noflush(file, start, end, datasync);
        if (!ret)
                ret = blkdev_issue_flush(inode->i_sb->s_bdev);
        return ret;
}
EXPORT_SYMBOL(generic_buffers_fsync);

/*
 * Called when we've recently written block `bblock', and it is known that
 * `bblock' was for a buffer_boundary() buffer.  This means that the block at
 * `bblock + 1' is probably a dirty indirect block.  Hunt it down and, if it's
 * dirty, schedule it for IO.  So that indirects merge nicely with their data.
 */
void write_boundary_block(struct block_device *bdev,
                        sector_t bblock, unsigned blocksize)
{
        struct buffer_head *bh = __find_get_block(bdev, bblock + 1, blocksize);
        if (bh) {
                if (buffer_dirty(bh))
                        write_dirty_buffer(bh, 0);
                put_bh(bh);
        }
}

void mark_buffer_dirty_inode(struct buffer_head *bh, struct inode *inode)
{
        struct address_space *mapping = inode->i_mapping;
        struct address_space *buffer_mapping = bh->b_folio->mapping;

        mark_buffer_dirty(bh);
        if (!mapping->i_private_data) {
                mapping->i_private_data = buffer_mapping;
        } else {
                BUG_ON(mapping->i_private_data != buffer_mapping);
        }
        if (!bh->b_assoc_map) {
                spin_lock(&buffer_mapping->i_private_lock);
                list_move_tail(&bh->b_assoc_buffers,
                                &mapping->i_private_list);
                bh->b_assoc_map = mapping;
                spin_unlock(&buffer_mapping->i_private_lock);
        }
}
EXPORT_SYMBOL(mark_buffer_dirty_inode);

/**
 * block_dirty_folio - Mark a folio as dirty.
 * @mapping: The address space containing this folio.
 * @folio: The folio to mark dirty.
 *
 * Filesystems which use buffer_heads can use this function as their
 * ->dirty_folio implementation.  Some filesystems need to do a little
 * work before calling this function.  Filesystems which do not use
 * buffer_heads should call filemap_dirty_folio() instead.
 *
 * If the folio has buffers, the uptodate buffers are set dirty, to
 * preserve dirty-state coherency between the folio and the buffers.
 * Buffers added to a dirty folio are created dirty.
 *
 * The buffers are dirtied before the folio is dirtied.  There's a small
 * race window in which writeback may see the folio cleanness but not the
 * buffer dirtiness.  That's fine.  If this code were to set the folio
 * dirty before the buffers, writeback could clear the folio dirty flag,
 * see a bunch of clean buffers and we'd end up with dirty buffers/clean
 * folio on the dirty folio list.
 *
 * We use i_private_lock to lock against try_to_free_buffers() while
 * using the folio's buffer list.  This also prevents clean buffers
 * being added to the folio after it was set dirty.
 *
 * Context: May only be called from process context.  Does not sleep.
 * Caller must ensure that @folio cannot be truncated during this call,
 * typically by holding the folio lock or having a page in the folio
 * mapped and holding the page table lock.
 *
 * Return: True if the folio was dirtied; false if it was already dirtied.
 */
bool block_dirty_folio(struct address_space *mapping, struct folio *folio)
{
        struct buffer_head *head;
        bool newly_dirty;

        spin_lock(&mapping->i_private_lock);
        head = folio_buffers(folio);
        if (head) {
                struct buffer_head *bh = head;

                do {
                        set_buffer_dirty(bh);
                        bh = bh->b_this_page;
                } while (bh != head);
        }
        /*
         * Lock out page's memcg migration to keep PageDirty
         * synchronized with per-memcg dirty page counters.
         */
        folio_memcg_lock(folio);
        newly_dirty = !folio_test_set_dirty(folio);
        spin_unlock(&mapping->i_private_lock);

        if (newly_dirty)
                __folio_mark_dirty(folio, mapping, 1);

        folio_memcg_unlock(folio);

        if (newly_dirty)
                __mark_inode_dirty(mapping->host, I_DIRTY_PAGES);

        return newly_dirty;
}
EXPORT_SYMBOL(block_dirty_folio);

/*
 * Write out and wait upon a list of buffers.
 *
 * We have conflicting pressures: we want to make sure that all
 * initially dirty buffers get waited on, but that any subsequently
 * dirtied buffers don't.  After all, we don't want fsync to last
 * forever if somebody is actively writing to the file.
 *
 * Do this in two main stages: first we copy dirty buffers to a
 * temporary inode list, queueing the writes as we go.  Then we clean
 * up, waiting for those writes to complete.
 * 
 * During this second stage, any subsequent updates to the file may end
 * up refiling the buffer on the original inode's dirty list again, so
 * there is a chance we will end up with a buffer queued for write but
 * not yet completed on that list.  So, as a final cleanup we go through
 * the osync code to catch these locked, dirty buffers without requeuing
 * any newly dirty buffers for write.
 */
static int fsync_buffers_list(spinlock_t *lock, struct list_head *list)
{
        struct buffer_head *bh;
        struct address_space *mapping;
        int err = 0, err2;
        struct blk_plug plug;
        LIST_HEAD(tmp);

        blk_start_plug(&plug);

        spin_lock(lock);
        while (!list_empty(list)) {
                bh = BH_ENTRY(list->next);
                mapping = bh->b_assoc_map;
                __remove_assoc_queue(bh);
                /* Avoid race with mark_buffer_dirty_inode() which does
                 * a lockless check and we rely on seeing the dirty bit */
                smp_mb();
                if (buffer_dirty(bh) || buffer_locked(bh)) {
                        list_add(&bh->b_assoc_buffers, &tmp);
                        bh->b_assoc_map = mapping;
                        if (buffer_dirty(bh)) {
                                get_bh(bh);
                                spin_unlock(lock);
                                /*
                                 * Ensure any pending I/O completes so that
                                 * write_dirty_buffer() actually writes the
                                 * current contents - it is a noop if I/O is
                                 * still in flight on potentially older
                                 * contents.
                                 */
                                write_dirty_buffer(bh, REQ_SYNC);

                                /*
                                 * Kick off IO for the previous mapping. Note
                                 * that we will not run the very last mapping,
                                 * wait_on_buffer() will do that for us
                                 * through sync_buffer().
                                 */
                                brelse(bh);
                                spin_lock(lock);
                        }
                }
        }

        spin_unlock(lock);
        blk_finish_plug(&plug);
        spin_lock(lock);

        while (!list_empty(&tmp)) {
                bh = BH_ENTRY(tmp.prev);
                get_bh(bh);
                mapping = bh->b_assoc_map;
                __remove_assoc_queue(bh);
                /* Avoid race with mark_buffer_dirty_inode() which does
                 * a lockless check and we rely on seeing the dirty bit */
                smp_mb();
                if (buffer_dirty(bh)) {
                        list_add(&bh->b_assoc_buffers,
                                 &mapping->i_private_list);
                        bh->b_assoc_map = mapping;
                }
                spin_unlock(lock);
                wait_on_buffer(bh);
                if (!buffer_uptodate(bh))
                        err = -EIO;
                brelse(bh);
                spin_lock(lock);
        }
        
        spin_unlock(lock);
        err2 = osync_buffers_list(lock, list);
        if (err)
                return err;
        else
                return err2;
}

/*
 * Invalidate any and all dirty buffers on a given inode.  We are
 * probably unmounting the fs, but that doesn't mean we have already
 * done a sync().  Just drop the buffers from the inode list.
 *
 * NOTE: we take the inode's blockdev's mapping's i_private_lock.  Which
 * assumes that all the buffers are against the blockdev.  Not true
 * for reiserfs.
 */
void invalidate_inode_buffers(struct inode *inode)
{
        if (inode_has_buffers(inode)) {
                struct address_space *mapping = &inode->i_data;
                struct list_head *list = &mapping->i_private_list;
                struct address_space *buffer_mapping = mapping->i_private_data;

                spin_lock(&buffer_mapping->i_private_lock);
                while (!list_empty(list))
                        __remove_assoc_queue(BH_ENTRY(list->next));
                spin_unlock(&buffer_mapping->i_private_lock);
        }
}
EXPORT_SYMBOL(invalidate_inode_buffers);

/*
 * Remove any clean buffers from the inode's buffer list.  This is called
 * when we're trying to free the inode itself.  Those buffers can pin it.
 *
 * Returns true if all buffers were removed.
 */
int remove_inode_buffers(struct inode *inode)
{
        int ret = 1;

        if (inode_has_buffers(inode)) {
                struct address_space *mapping = &inode->i_data;
                struct list_head *list = &mapping->i_private_list;
                struct address_space *buffer_mapping = mapping->i_private_data;

                spin_lock(&buffer_mapping->i_private_lock);
                while (!list_empty(list)) {
                        struct buffer_head *bh = BH_ENTRY(list->next);
                        if (buffer_dirty(bh)) {
                                ret = 0;
                                break;
                        }
                        __remove_assoc_queue(bh);
                }
                spin_unlock(&buffer_mapping->i_private_lock);
        }
        return ret;
}

/*
 * Create the appropriate buffers when given a folio for data area and
 * the size of each buffer.. Use the bh->b_this_page linked list to
 * follow the buffers created.  Return NULL if unable to create more
 * buffers.
 *
 * The retry flag is used to differentiate async IO (paging, swapping)
 * which may not fail from ordinary buffer allocations.
 */
struct buffer_head *folio_alloc_buffers(struct folio *folio, unsigned long size,
                                        gfp_t gfp)
{
        struct buffer_head *bh, *head;
        long offset;
        struct mem_cgroup *memcg, *old_memcg;

        /* The folio lock pins the memcg */
        memcg = folio_memcg(folio);
        old_memcg = set_active_memcg(memcg);

        head = NULL;
        offset = folio_size(folio);
        while ((offset -= size) >= 0) {
                bh = alloc_buffer_head(gfp);
                if (!bh)
                        goto no_grow;

                bh->b_this_page = head;
                bh->b_blocknr = -1;
                head = bh;

                bh->b_size = size;

                /* Link the buffer to its folio */
                folio_set_bh(bh, folio, offset);
        }
out:
        set_active_memcg(old_memcg);
        return head;
/*
 * In case anything failed, we just free everything we got.
 */
no_grow:
        if (head) {
                do {
                        bh = head;
                        head = head->b_this_page;
                        free_buffer_head(bh);
                } while (head);
        }

        goto out;
}
EXPORT_SYMBOL_GPL(folio_alloc_buffers);

struct buffer_head *alloc_page_buffers(struct page *page, unsigned long size)
{
        gfp_t gfp = GFP_NOFS | __GFP_ACCOUNT;

        return folio_alloc_buffers(page_folio(page), size, gfp);
}
EXPORT_SYMBOL_GPL(alloc_page_buffers);

static inline void link_dev_buffers(struct folio *folio,
                struct buffer_head *head)
{
        struct buffer_head *bh, *tail;

        bh = head;
        do {
                tail = bh;
                bh = bh->b_this_page;
        } while (bh);
        tail->b_this_page = head;
        folio_attach_private(folio, head);
}

static sector_t blkdev_max_block(struct block_device *bdev, unsigned int size)
{
        sector_t retval = ~((sector_t)0);
        loff_t sz = bdev_nr_bytes(bdev);

        if (sz) {
                unsigned int sizebits = blksize_bits(size);
                retval = (sz >> sizebits);
        }
        return retval;
}

/*
 * Initialise the state of a blockdev folio's buffers.
 */ 
static sector_t folio_init_buffers(struct folio *folio,
                struct block_device *bdev, unsigned size)
{
        struct buffer_head *head = folio_buffers(folio);
        struct buffer_head *bh = head;
        bool uptodate = folio_test_uptodate(folio);
        sector_t block = div_u64(folio_pos(folio), size);
        sector_t end_block = blkdev_max_block(bdev, size);

        do {
                if (!buffer_mapped(bh)) {
                        bh->b_end_io = NULL;
                        bh->b_private = NULL;
                        bh->b_bdev = bdev;
                        bh->b_blocknr = block;
                        if (uptodate)
                                set_buffer_uptodate(bh);
                        if (block < end_block)
                                set_buffer_mapped(bh);
                }
                block++;
                bh = bh->b_this_page;
        } while (bh != head);

        /*
         * Caller needs to validate requested block against end of device.
         */
        return end_block;
}

/*
 * Create the page-cache folio that contains the requested block.
 *
 * This is used purely for blockdev mappings.
 *
 * Returns false if we have a failure which cannot be cured by retrying
 * without sleeping.  Returns true if we succeeded, or the caller should retry.
 */
static bool grow_dev_folio(struct block_device *bdev, sector_t block,
                pgoff_t index, unsigned size, gfp_t gfp)
{
        struct address_space *mapping = bdev->bd_mapping;
        struct folio *folio;
        struct buffer_head *bh;
        sector_t end_block = 0;

        folio = __filemap_get_folio(mapping, index,
                        FGP_LOCK | FGP_ACCESSED | FGP_CREAT, gfp);
        if (IS_ERR(folio))
                return false;

        bh = folio_buffers(folio);
        if (bh) {
                if (bh->b_size == size) {
                        end_block = folio_init_buffers(folio, bdev, size);
                        goto unlock;
                }

                /*
                 * Retrying may succeed; for example the folio may finish
                 * writeback, or buffers may be cleaned.  This should not
                 * happen very often; maybe we have old buffers attached to
                 * this blockdev's page cache and we're trying to change
                 * the block size?
                 */
                if (!try_to_free_buffers(folio)) {
                        end_block = ~0ULL;
                        goto unlock;
                }
        }

        bh = folio_alloc_buffers(folio, size, gfp | __GFP_ACCOUNT);
        if (!bh)
                goto unlock;

        /*
         * Link the folio to the buffers and initialise them.  Take the
         * lock to be atomic wrt __find_get_block(), which does not
         * run under the folio lock.
         */
        spin_lock(&mapping->i_private_lock);
        link_dev_buffers(folio, bh);
        end_block = folio_init_buffers(folio, bdev, size);
        spin_unlock(&mapping->i_private_lock);
unlock:
        folio_unlock(folio);
        folio_put(folio);
        return block < end_block;
}

/*
 * Create buffers for the specified block device block's folio.  If
 * that folio was dirty, the buffers are set dirty also.  Returns false
 * if we've hit a permanent error.
 */
static bool grow_buffers(struct block_device *bdev, sector_t block,
                unsigned size, gfp_t gfp)
{
        loff_t pos;

        /*
         * Check for a block which lies outside our maximum possible
         * pagecache index.
         */
        if (check_mul_overflow(block, (sector_t)size, &pos) || pos > MAX_LFS_FILESIZE) {
                printk(KERN_ERR "%s: requested out-of-range block %llu for device %pg\n",
                        __func__, (unsigned long long)block,
                        bdev);
                return false;
        }

        /* Create a folio with the proper size buffers */
        return grow_dev_folio(bdev, block, pos / PAGE_SIZE, size, gfp);
}

static struct buffer_head *
__getblk_slow(struct block_device *bdev, sector_t block,
             unsigned size, gfp_t gfp)
{
        /* Size must be multiple of hard sectorsize */
        if (unlikely(size & (bdev_logical_block_size(bdev)-1) ||
                        (size < 512 || size > PAGE_SIZE))) {
                printk(KERN_ERR "getblk(): invalid block size %d requested\n",
                                        size);
                printk(KERN_ERR "logical block size: %d\n",
                                        bdev_logical_block_size(bdev));

                dump_stack();
                return NULL;
        }

        for (;;) {
                struct buffer_head *bh;

                bh = __find_get_block(bdev, block, size);
                if (bh)
                        return bh;

                if (!grow_buffers(bdev, block, size, gfp))
                        return NULL;
        }
}

/*
 * The relationship between dirty buffers and dirty pages:
 *
 * Whenever a page has any dirty buffers, the page's dirty bit is set, and
 * the page is tagged dirty in the page cache.
 *
 * At all times, the dirtiness of the buffers represents the dirtiness of
 * subsections of the page.  If the page has buffers, the page dirty bit is
 * merely a hint about the true dirty state.
 *
 * When a page is set dirty in its entirety, all its buffers are marked dirty
 * (if the page has buffers).
 *
 * When a buffer is marked dirty, its page is dirtied, but the page's other
 * buffers are not.
 *
 * Also.  When blockdev buffers are explicitly read with bread(), they
 * individually become uptodate.  But their backing page remains not
 * uptodate - even if all of its buffers are uptodate.  A subsequent
 * block_read_full_folio() against that folio will discover all the uptodate
 * buffers, will set the folio uptodate and will perform no I/O.
 */

/**
 * mark_buffer_dirty - mark a buffer_head as needing writeout
 * @bh: the buffer_head to mark dirty
 *
 * mark_buffer_dirty() will set the dirty bit against the buffer, then set
 * its backing page dirty, then tag the page as dirty in the page cache
 * and then attach the address_space's inode to its superblock's dirty
 * inode list.
 *
 * mark_buffer_dirty() is atomic.  It takes bh->b_folio->mapping->i_private_lock,
 * i_pages lock and mapping->host->i_lock.
 */
void mark_buffer_dirty(struct buffer_head *bh)
{
        WARN_ON_ONCE(!buffer_uptodate(bh));

        trace_block_dirty_buffer(bh);

        /*
         * Very *carefully* optimize the it-is-already-dirty case.
         *
         * Don't let the final "is it dirty" escape to before we
         * perhaps modified the buffer.
         */
        if (buffer_dirty(bh)) {
                smp_mb();
                if (buffer_dirty(bh))
                        return;
        }

        if (!test_set_buffer_dirty(bh)) {
                struct folio *folio = bh->b_folio;
                struct address_space *mapping = NULL;

                folio_memcg_lock(folio);
                if (!folio_test_set_dirty(folio)) {
                        mapping = folio->mapping;
                        if (mapping)
                                __folio_mark_dirty(folio, mapping, 0);
                }
                folio_memcg_unlock(folio);
                if (mapping)
                        __mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
        }
}
EXPORT_SYMBOL(mark_buffer_dirty);

void mark_buffer_write_io_error(struct buffer_head *bh)
{
        set_buffer_write_io_error(bh);
        /* FIXME: do we need to set this in both places? */
        if (bh->b_folio && bh->b_folio->mapping)
                mapping_set_error(bh->b_folio->mapping, -EIO);
        if (bh->b_assoc_map) {
                mapping_set_error(bh->b_assoc_map, -EIO);
                errseq_set(&bh->b_assoc_map->host->i_sb->s_wb_err, -EIO);
        }
}
EXPORT_SYMBOL(mark_buffer_write_io_error);

/**
 * __brelse - Release a buffer.
 * @bh: The buffer to release.
 *
 * This variant of brelse() can be called if @bh is guaranteed to not be NULL.
 */
void __brelse(struct buffer_head *bh)
{
        if (atomic_read(&bh->b_count)) {
                put_bh(bh);
                return;
        }
        WARN(1, KERN_ERR "VFS: brelse: Trying to free free buffer\n");
}
EXPORT_SYMBOL(__brelse);

/**
 * __bforget - Discard any dirty data in a buffer.
 * @bh: The buffer to forget.
 *
 * This variant of bforget() can be called if @bh is guaranteed to not
 * be NULL.
 */
void __bforget(struct buffer_head *bh)
{
        clear_buffer_dirty(bh);
        if (bh->b_assoc_map) {
                struct address_space *buffer_mapping = bh->b_folio->mapping;

                spin_lock(&buffer_mapping->i_private_lock);
                list_del_init(&bh->b_assoc_buffers);
                bh->b_assoc_map = NULL;
                spin_unlock(&buffer_mapping->i_private_lock);
        }
        __brelse(bh);
}
EXPORT_SYMBOL(__bforget);

static struct buffer_head *__bread_slow(struct buffer_head *bh)
{
        lock_buffer(bh);
        if (buffer_uptodate(bh)) {
                unlock_buffer(bh);
                return bh;
        } else {
                get_bh(bh);
                bh->b_end_io = end_buffer_read_sync;
                submit_bh(REQ_OP_READ, bh);
                wait_on_buffer(bh);
                if (buffer_uptodate(bh))
                        return bh;
        }
        brelse(bh);
        return NULL;
}

/*
 * Per-cpu buffer LRU implementation.  To reduce the cost of __find_get_block().
 * The bhs[] array is sorted - newest buffer is at bhs[0].  Buffers have their
 * refcount elevated by one when they're in an LRU.  A buffer can only appear
 * once in a particular CPU's LRU.  A single buffer can be present in multiple
 * CPU's LRUs at the same time.
 *
 * This is a transparent caching front-end to sb_bread(), sb_getblk() and
 * sb_find_get_block().
 *
 * The LRUs themselves only need locking against invalidate_bh_lrus.  We use
 * a local interrupt disable for that.
 */

#define BH_LRU_SIZE        16

struct bh_lru {
        struct buffer_head *bhs[BH_LRU_SIZE];
};

static DEFINE_PER_CPU(struct bh_lru, bh_lrus) = {{ NULL }};

#ifdef CONFIG_SMP
#define bh_lru_lock()        local_irq_disable()
#define bh_lru_unlock()        local_irq_enable()
#else
#define bh_lru_lock()        preempt_disable()
#define bh_lru_unlock()        preempt_enable()
#endif

static inline void check_irqs_on(void)
{
#ifdef irqs_disabled
        BUG_ON(irqs_disabled());
#endif
}

/*
 * Install a buffer_head into this cpu's LRU.  If not already in the LRU, it is
 * inserted at the front, and the buffer_head at the back if any is evicted.
 * Or, if already in the LRU it is moved to the front.
 */
static void bh_lru_install(struct buffer_head *bh)
{
        struct buffer_head *evictee = bh;
        struct bh_lru *b;
        int i;

        check_irqs_on();
        bh_lru_lock();

        /*
         * the refcount of buffer_head in bh_lru prevents dropping the
         * attached page(i.e., try_to_free_buffers) so it could cause
         * failing page migration.
         * Skip putting upcoming bh into bh_lru until migration is done.
         */
        if (lru_cache_disabled() || cpu_is_isolated(smp_processor_id())) {
                bh_lru_unlock();
                return;
        }

        b = this_cpu_ptr(&bh_lrus);
        for (i = 0; i < BH_LRU_SIZE; i++) {
                swap(evictee, b->bhs[i]);
                if (evictee == bh) {
                        bh_lru_unlock();
                        return;
                }
        }

        get_bh(bh);
        bh_lru_unlock();
        brelse(evictee);
}

/*
 * Look up the bh in this cpu's LRU.  If it's there, move it to the head.
 */
static struct buffer_head *
lookup_bh_lru(struct block_device *bdev, sector_t block, unsigned size)
{
        struct buffer_head *ret = NULL;
        unsigned int i;

        check_irqs_on();
        bh_lru_lock();
        if (cpu_is_isolated(smp_processor_id())) {
                bh_lru_unlock();
                return NULL;
        }
        for (i = 0; i < BH_LRU_SIZE; i++) {
                struct buffer_head *bh = __this_cpu_read(bh_lrus.bhs[i]);

                if (bh && bh->b_blocknr == block && bh->b_bdev == bdev &&
                    bh->b_size == size) {
                        if (i) {
                                while (i) {
                                        __this_cpu_write(bh_lrus.bhs[i],
                                                __this_cpu_read(bh_lrus.bhs[i - 1]));
                                        i--;
                                }
                                __this_cpu_write(bh_lrus.bhs[0], bh);
                        }
                        get_bh(bh);
                        ret = bh;
                        break;
                }
        }
        bh_lru_unlock();
        return ret;
}

/*
 * Perform a pagecache lookup for the matching buffer.  If it's there, refresh
 * it in the LRU and mark it as accessed.  If it is not present then return
 * NULL
 */
struct buffer_head *
__find_get_block(struct block_device *bdev, sector_t block, unsigned size)
{
        struct buffer_head *bh = lookup_bh_lru(bdev, block, size);

        if (bh == NULL) {
                /* __find_get_block_slow will mark the page accessed */
                bh = __find_get_block_slow(bdev, block);
                if (bh)
                        bh_lru_install(bh);
        } else
                touch_buffer(bh);

        return bh;
}
EXPORT_SYMBOL(__find_get_block);

/**
 * bdev_getblk - Get a buffer_head in a block device's buffer cache.
 * @bdev: The block device.
 * @block: The block number.
 * @size: The size of buffer_heads for this @bdev.
 * @gfp: The memory allocation flags to use.
 *
 * The returned buffer head has its reference count incremented, but is
 * not locked.  The caller should call brelse() when it has finished
 * with the buffer.  The buffer may not be uptodate.  If needed, the
 * caller can bring it uptodate either by reading it or overwriting it.
 *
 * Return: The buffer head, or NULL if memory could not be allocated.
 */
struct buffer_head *bdev_getblk(struct block_device *bdev, sector_t block,
                unsigned size, gfp_t gfp)
{
        struct buffer_head *bh = __find_get_block(bdev, block, size);

        might_alloc(gfp);
        if (bh)
                return bh;

        return __getblk_slow(bdev, block, size, gfp);
}
EXPORT_SYMBOL(bdev_getblk);

/*
 * Do async read-ahead on a buffer..
 */
void __breadahead(struct block_device *bdev, sector_t block, unsigned size)
{
        struct buffer_head *bh = bdev_getblk(bdev, block, size,
                        GFP_NOWAIT | __GFP_MOVABLE);

        if (likely(bh)) {
                bh_readahead(bh, REQ_RAHEAD);
                brelse(bh);
        }
}
EXPORT_SYMBOL(__breadahead);

/**
 * __bread_gfp() - Read a block.
 * @bdev: The block device to read from.
 * @block: Block number in units of block size.
 * @size: The block size of this device in bytes.
 * @gfp: Not page allocation flags; see below.
 *
 * You are not expected to call this function.  You should use one of
 * sb_bread(), sb_bread_unmovable() or __bread().
 *
 * Read a specified block, and return the buffer head that refers to it.
 * If @gfp is 0, the memory will be allocated using the block device's
 * default GFP flags.  If @gfp is __GFP_MOVABLE, the memory may be
 * allocated from a movable area.  Do not pass in a complete set of
 * GFP flags.
 *
 * The returned buffer head has its refcount increased.  The caller should
 * call brelse() when it has finished with the buffer.
 *
 * Context: May sleep waiting for I/O.
 * Return: NULL if the block was unreadable.
 */
struct buffer_head *__bread_gfp(struct block_device *bdev, sector_t block,
                unsigned size, gfp_t gfp)
{
        struct buffer_head *bh;

        gfp |= mapping_gfp_constraint(bdev->bd_mapping, ~__GFP_FS);

        /*
         * Prefer looping in the allocator rather than here, at least that
         * code knows what it's doing.
         */
        gfp |= __GFP_NOFAIL;

        bh = bdev_getblk(bdev, block, size, gfp);

        if (likely(bh) && !buffer_uptodate(bh))
                bh = __bread_slow(bh);
        return bh;
}
EXPORT_SYMBOL(__bread_gfp);

static void __invalidate_bh_lrus(struct bh_lru *b)
{
        int i;

        for (i = 0; i < BH_LRU_SIZE; i++) {
                brelse(b->bhs[i]);
                b->bhs[i] = NULL;
        }
}
/*
 * invalidate_bh_lrus() is called rarely - but not only at unmount.
 * This doesn't race because it runs in each cpu either in irq
 * or with preempt disabled.
 */
static void invalidate_bh_lru(void *arg)
{
        struct bh_lru *b = &get_cpu_var(bh_lrus);

        __invalidate_bh_lrus(b);
        put_cpu_var(bh_lrus);
}

bool has_bh_in_lru(int cpu, void *dummy)
{
        struct bh_lru *b = per_cpu_ptr(&bh_lrus, cpu);
        int i;
        
        for (i = 0; i < BH_LRU_SIZE; i++) {
                if (b->bhs[i])
                        return true;
        }

        return false;
}

void invalidate_bh_lrus(void)
{
        on_each_cpu_cond(has_bh_in_lru, invalidate_bh_lru, NULL, 1);
}
EXPORT_SYMBOL_GPL(invalidate_bh_lrus);

/*
 * It's called from workqueue context so we need a bh_lru_lock to close
 * the race with preemption/irq.
 */
void invalidate_bh_lrus_cpu(void)
{
        struct bh_lru *b;

        bh_lru_lock();
        b = this_cpu_ptr(&bh_lrus);
        __invalidate_bh_lrus(b);
        bh_lru_unlock();
}

void folio_set_bh(struct buffer_head *bh, struct folio *folio,
                  unsigned long offset)
{
        bh->b_folio = folio;
        BUG_ON(offset >= folio_size(folio));
        if (folio_test_highmem(folio))
                /*
                 * This catches illegal uses and preserves the offset:
                 */
                bh->b_data = (char *)(0 + offset);
        else
                bh->b_data = folio_address(folio) + offset;
}
EXPORT_SYMBOL(folio_set_bh);

/*
 * Called when truncating a buffer on a page completely.
 */

/* Bits that are cleared during an invalidate */
#define BUFFER_FLAGS_DISCARD \
        (1 << BH_Mapped | 1 << BH_New | 1 << BH_Req | \
         1 << BH_Delay | 1 << BH_Unwritten)

static void discard_buffer(struct buffer_head * bh)
{
        unsigned long b_state;

        lock_buffer(bh);
        clear_buffer_dirty(bh);
        bh->b_bdev = NULL;
        b_state = READ_ONCE(bh->b_state);
        do {
        } while (!try_cmpxchg(&bh->b_state, &b_state,
                              b_state & ~BUFFER_FLAGS_DISCARD));
        unlock_buffer(bh);
}

/**
 * block_invalidate_folio - Invalidate part or all of a buffer-backed folio.
 * @folio: The folio which is affected.
 * @offset: start of the range to invalidate
 * @length: length of the range to invalidate
 *
 * block_invalidate_folio() is called when all or part of the folio has been
 * invalidated by a truncate operation.
 *
 * block_invalidate_folio() does not have to release all buffers, but it must
 * ensure that no dirty buffer is left outside @offset and that no I/O
 * is underway against any of the blocks which are outside the truncation
 * point.  Because the caller is about to free (and possibly reuse) those
 * blocks on-disk.
 */
void block_invalidate_folio(struct folio *folio, size_t offset, size_t length)
{
        struct buffer_head *head, *bh, *next;
        size_t curr_off = 0;
        size_t stop = length + offset;

        BUG_ON(!folio_test_locked(folio));

        /*
         * Check for overflow
         */
        BUG_ON(stop > folio_size(folio) || stop < length);

        head = folio_buffers(folio);
        if (!head)
                return;

        bh = head;
        do {
                size_t next_off = curr_off + bh->b_size;
                next = bh->b_this_page;

                /*
                 * Are we still fully in range ?
                 */
                if (next_off > stop)
                        goto out;

                /*
                 * is this block fully invalidated?
                 */
                if (offset <= curr_off)
                        discard_buffer(bh);
                curr_off = next_off;
                bh = next;
        } while (bh != head);

        /*
         * We release buffers only if the entire folio is being invalidated.
         * The get_block cached value has been unconditionally invalidated,
         * so real IO is not possible anymore.
         */
        if (length == folio_size(folio))
                filemap_release_folio(folio, 0);
out:
        folio_clear_mappedtodisk(folio);
        return;
}
EXPORT_SYMBOL(block_invalidate_folio);

/*
 * We attach and possibly dirty the buffers atomically wrt
 * block_dirty_folio() via i_private_lock.  try_to_free_buffers
 * is already excluded via the folio lock.
 */
struct buffer_head *create_empty_buffers(struct folio *folio,
                unsigned long blocksize, unsigned long b_state)
{
        struct buffer_head *bh, *head, *tail;
        gfp_t gfp = GFP_NOFS | __GFP_ACCOUNT | __GFP_NOFAIL;

        head = folio_alloc_buffers(folio, blocksize, gfp);
        bh = head;
        do {
                bh->b_state |= b_state;
                tail = bh;
                bh = bh->b_this_page;
        } while (bh);
        tail->b_this_page = head;

        spin_lock(&folio->mapping->i_private_lock);
        if (folio_test_uptodate(folio) || folio_test_dirty(folio)) {
                bh = head;
                do {
                        if (folio_test_dirty(folio))
                                set_buffer_dirty(bh);
                        if (folio_test_uptodate(folio))
                                set_buffer_uptodate(bh);
                        bh = bh->b_this_page;
                } while (bh != head);
        }
        folio_attach_private(folio, head);
        spin_unlock(&folio->mapping->i_private_lock);

        return head;
}
EXPORT_SYMBOL(create_empty_buffers);

/**
 * clean_bdev_aliases: clean a range of buffers in block device
 * @bdev: Block device to clean buffers in
 * @block: Start of a range of blocks to clean
 * @len: Number of blocks to clean
 *
 * We are taking a range of blocks for data and we don't want writeback of any
 * buffer-cache aliases starting from return from this function and until the
 * moment when something will explicitly mark the buffer dirty (hopefully that
 * will not happen until we will free that block ;-) We don't even need to mark
 * it not-uptodate - nobody can expect anything from a newly allocated buffer
 * anyway. We used to use unmap_buffer() for such invalidation, but that was
 * wrong. We definitely don't want to mark the alias unmapped, for example - it
 * would confuse anyone who might pick it with bread() afterwards...
 *
 * Also..  Note that bforget() doesn't lock the buffer.  So there can be
 * writeout I/O going on against recently-freed buffers.  We don't wait on that
 * I/O in bforget() - it's more efficient to wait on the I/O only if we really
 * need to.  That happens here.
 */
void clean_bdev_aliases(struct block_device *bdev, sector_t block, sector_t len)
{
        struct address_space *bd_mapping = bdev->bd_mapping;
        const int blkbits = bd_mapping->host->i_blkbits;
        struct folio_batch fbatch;
        pgoff_t index = ((loff_t)block << blkbits) / PAGE_SIZE;
        pgoff_t end;
        int i, count;
        struct buffer_head *bh;
        struct buffer_head *head;

        end = ((loff_t)(block + len - 1) << blkbits) / PAGE_SIZE;
        folio_batch_init(&fbatch);
        while (filemap_get_folios(bd_mapping, &index, end, &fbatch)) {
                count = folio_batch_count(&fbatch);
                for (i = 0; i < count; i++) {
                        struct folio *folio = fbatch.folios[i];

                        if (!folio_buffers(folio))
                                continue;
                        /*
                         * We use folio lock instead of bd_mapping->i_private_lock
                         * to pin buffers here since we can afford to sleep and
                         * it scales better than a global spinlock lock.
                         */
                        folio_lock(folio);
                        /* Recheck when the folio is locked which pins bhs */
                        head = folio_buffers(folio);
                        if (!head)
                                goto unlock_page;
                        bh = head;
                        do {
                                if (!buffer_mapped(bh) || (bh->b_blocknr < block))
                                        goto next;
                                if (bh->b_blocknr >= block + len)
                                        break;
                                clear_buffer_dirty(bh);
                                wait_on_buffer(bh);
                                clear_buffer_req(bh);
next:
                                bh = bh->b_this_page;
                        } while (bh != head);
unlock_page:
                        folio_unlock(folio);
                }
                folio_batch_release(&fbatch);
                cond_resched();
                /* End of range already reached? */
                if (index > end || !index)
                        break;
        }
}
EXPORT_SYMBOL(clean_bdev_aliases);

static struct buffer_head *folio_create_buffers(struct folio *folio,
                                                struct inode *inode,
                                                unsigned int b_state)
{
        struct buffer_head *bh;

        BUG_ON(!folio_test_locked(folio));

        bh = folio_buffers(folio);
        if (!bh)
                bh = create_empty_buffers(folio,
                                1 << READ_ONCE(inode->i_blkbits), b_state);
        return bh;
}

/*
 * NOTE! All mapped/uptodate combinations are valid:
 *
 *        Mapped        Uptodate        Meaning
 *
 *        No        No                "unknown" - must do get_block()
 *        No        Yes                "hole" - zero-filled
 *        Yes        No                "allocated" - allocated on disk, not read in
 *        Yes        Yes                "valid" - allocated and up-to-date in memory.
 *
 * "Dirty" is valid only with the last case (mapped+uptodate).
 */

/*
 * While block_write_full_folio is writing back the dirty buffers under
 * the page lock, whoever dirtied the buffers may decide to clean them
 * again at any time.  We handle that by only looking at the buffer
 * state inside lock_buffer().
 *
 * If block_write_full_folio() is called for regular writeback
 * (wbc->sync_mode == WB_SYNC_NONE) then it will redirty a page which has a
 * locked buffer.   This only can happen if someone has written the buffer
 * directly, with submit_bh().  At the address_space level PageWriteback
 * prevents this contention from occurring.
 *
 * If block_write_full_folio() is called with wbc->sync_mode ==
 * WB_SYNC_ALL, the writes are posted using REQ_SYNC; this
 * causes the writes to be flagged as synchronous writes.
 */
int __block_write_full_folio(struct inode *inode, struct folio *folio,
                        get_block_t *get_block, struct writeback_control *wbc)
{
        int err;
        sector_t block;
        sector_t last_block;
        struct buffer_head *bh, *head;
        size_t blocksize;
        int nr_underway = 0;
        blk_opf_t write_flags = wbc_to_write_flags(wbc);

        head = folio_create_buffers(folio, inode,
                                    (1 << BH_Dirty) | (1 << BH_Uptodate));

        /*
         * Be very careful.  We have no exclusion from block_dirty_folio
         * here, and the (potentially unmapped) buffers may become dirty at
         * any time.  If a buffer becomes dirty here after we've inspected it
         * then we just miss that fact, and the folio stays dirty.
         *
         * Buffers outside i_size may be dirtied by block_dirty_folio;
         * handle that here by just cleaning them.
         */

        bh = head;
        blocksize = bh->b_size;

        block = div_u64(folio_pos(folio), blocksize);
        last_block = div_u64(i_size_read(inode) - 1, blocksize);

        /*
         * Get all the dirty buffers mapped to disk addresses and
         * handle any aliases from the underlying blockdev's mapping.
         */
        do {
                if (block > last_block) {
                        /*
                         * mapped buffers outside i_size will occur, because
                         * this folio can be outside i_size when there is a
                         * truncate in progress.
                         */
                        /*
                         * The buffer was zeroed by block_write_full_folio()
                         */
                        clear_buffer_dirty(bh);
                        set_buffer_uptodate(bh);
                } else if ((!buffer_mapped(bh) || buffer_delay(bh)) &&
                           buffer_dirty(bh)) {
                        WARN_ON(bh->b_size != blocksize);
                        err = get_block(inode, block, bh, 1);
                        if (err)
                                goto recover;
                        clear_buffer_delay(bh);
                        if (buffer_new(bh)) {
                                /* blockdev mappings never come here */
                                clear_buffer_new(bh);
                                clean_bdev_bh_alias(bh);
                        }
                }
                bh = bh->b_this_page;
                block++;
        } while (bh != head);

        do {
                if (!buffer_mapped(bh))
                        continue;
                /*
                 * If it's a fully non-blocking write attempt and we cannot
                 * lock the buffer then redirty the folio.  Note that this can
                 * potentially cause a busy-wait loop from writeback threads
                 * and kswapd activity, but those code paths have their own
                 * higher-level throttling.
                 */
                if (wbc->sync_mode != WB_SYNC_NONE) {
                        lock_buffer(bh);
                } else if (!trylock_buffer(bh)) {
                        folio_redirty_for_writepage(wbc, folio);
                        continue;
                }
                if (test_clear_buffer_dirty(bh)) {
                        mark_buffer_async_write_endio(bh,
                                end_buffer_async_write);
                } else {
                        unlock_buffer(bh);
                }
        } while ((bh = bh->b_this_page) != head);

        /*
         * The folio and its buffers are protected by the writeback flag,
         * so we can drop the bh refcounts early.
         */
        BUG_ON(folio_test_writeback(folio));
        folio_start_writeback(folio);

        do {
                struct buffer_head *next = bh->b_this_page;
                if (buffer_async_write(bh)) {
                        submit_bh_wbc(REQ_OP_WRITE | write_flags, bh,
                                      inode->i_write_hint, wbc);
                        nr_underway++;
                }
                bh = next;
        } while (bh != head);
        folio_unlock(folio);

        err = 0;
done:
        if (nr_underway == 0) {
                /*
                 * The folio was marked dirty, but the buffers were
                 * clean.  Someone wrote them back by hand with
                 * write_dirty_buffer/submit_bh.  A rare case.
                 */
                folio_end_writeback(folio);

                /*
                 * The folio and buffer_heads can be released at any time from
                 * here on.
                 */
        }
        return err;

recover:
        /*
         * ENOSPC, or some other error.  We may already have added some
         * blocks to the file, so we need to write these out to avoid
         * exposing stale data.
         * The folio is currently locked and not marked for writeback
         */
        bh = head;
        /* Recovery: lock and submit the mapped buffers */
        do {
                if (buffer_mapped(bh) && buffer_dirty(bh) &&
                    !buffer_delay(bh)) {
                        lock_buffer(bh);
                        mark_buffer_async_write_endio(bh,
                                end_buffer_async_write);
                } else {
                        /*
                         * The buffer may have been set dirty during
                         * attachment to a dirty folio.
                         */
                        clear_buffer_dirty(bh);
                }
        } while ((bh = bh->b_this_page) != head);
        BUG_ON(folio_test_writeback(folio));
        mapping_set_error(folio->mapping, err);
        folio_start_writeback(folio);
        do {
                struct buffer_head *next = bh->b_this_page;
                if (buffer_async_write(bh)) {
                        clear_buffer_dirty(bh);
                        submit_bh_wbc(REQ_OP_WRITE | write_flags, bh,
                                      inode->i_write_hint, wbc);
                        nr_underway++;
                }
                bh = next;
        } while (bh != head);
        folio_unlock(folio);
        goto done;
}
EXPORT_SYMBOL(__block_write_full_folio);

/*
 * If a folio has any new buffers, zero them out here, and mark them uptodate
 * and dirty so they'll be written out (in order to prevent uninitialised
 * block data from leaking). And clear the new bit.
 */
void folio_zero_new_buffers(struct folio *folio, size_t from, size_t to)
{
        size_t block_start, block_end;
        struct buffer_head *head, *bh;

        BUG_ON(!folio_test_locked(folio));
        head = folio_buffers(folio);
        if (!head)
                return;

        bh = head;
        block_start = 0;
        do {
                block_end = block_start + bh->b_size;

                if (buffer_new(bh)) {
                        if (block_end > from && block_start < to) {
                                if (!folio_test_uptodate(folio)) {
                                        size_t start, xend;

                                        start = max(from, block_start);
                                        xend = min(to, block_end);

                                        folio_zero_segment(folio, start, xend);
                                        set_buffer_uptodate(bh);
                                }

                                clear_buffer_new(bh);
                                mark_buffer_dirty(bh);
                        }
                }

                block_start = block_end;
                bh = bh->b_this_page;
        } while (bh != head);
}
EXPORT_SYMBOL(folio_zero_new_buffers);

static int
iomap_to_bh(struct inode *inode, sector_t block, struct buffer_head *bh,
                const struct iomap *iomap)
{
        loff_t offset = (loff_t)block << inode->i_blkbits;

        bh->b_bdev = iomap->bdev;

        /*
         * Block points to offset in file we need to map, iomap contains
         * the offset at which the map starts. If the map ends before the
         * current block, then do not map the buffer and let the caller
         * handle it.
         */
        if (offset >= iomap->offset + iomap->length)
                return -EIO;

        switch (iomap->type) {
        case IOMAP_HOLE:
                /*
                 * If the buffer is not up to date or beyond the current EOF,
                 * we need to mark it as new to ensure sub-block zeroing is
                 * executed if necessary.
                 */
                if (!buffer_uptodate(bh) ||
                    (offset >= i_size_read(inode)))
                        set_buffer_new(bh);
                return 0;
        case IOMAP_DELALLOC:
                if (!buffer_uptodate(bh) ||
                    (offset >= i_size_read(inode)))
                        set_buffer_new(bh);
                set_buffer_uptodate(bh);
                set_buffer_mapped(bh);
                set_buffer_delay(bh);
                return 0;
        case IOMAP_UNWRITTEN:
                /*
                 * For unwritten regions, we always need to ensure that regions
                 * in the block we are not writing to are zeroed. Mark the
                 * buffer as new to ensure this.
                 */
                set_buffer_new(bh);
                set_buffer_unwritten(bh);
                fallthrough;
        case IOMAP_MAPPED:
                if ((iomap->flags & IOMAP_F_NEW) ||
                    offset >= i_size_read(inode)) {
                        /*
                         * This can happen if truncating the block device races
                         * with the check in the caller as i_size updates on
                         * block devices aren't synchronized by i_rwsem for
                         * block devices.
                         */
                        if (S_ISBLK(inode->i_mode))
                                return -EIO;
                        set_buffer_new(bh);
                }
                bh->b_blocknr = (iomap->addr + offset - iomap->offset) >>
                                inode->i_blkbits;
                set_buffer_mapped(bh);
                return 0;
        default:
                WARN_ON_ONCE(1);
                return -EIO;
        }
}

int __block_write_begin_int(struct folio *folio, loff_t pos, unsigned len,
                get_block_t *get_block, const struct iomap *iomap)
{
        size_t from = offset_in_folio(folio, pos);
        size_t to = from + len;
        struct inode *inode = folio->mapping->host;
        size_t block_start, block_end;
        sector_t block;
        int err = 0;
        size_t blocksize;
        struct buffer_head *bh, *head, *wait[2], **wait_bh=wait;

        BUG_ON(!folio_test_locked(folio));
        BUG_ON(to > folio_size(folio));
        BUG_ON(from > to);

        head = folio_create_buffers(folio, inode, 0);
        blocksize = head->b_size;
        block = div_u64(folio_pos(folio), blocksize);

        for (bh = head, block_start = 0; bh != head || !block_start;
            block++, block_start=block_end, bh = bh->b_this_page) {
                block_end = block_start + blocksize;
                if (block_end <= from || block_start >= to) {
                        if (folio_test_uptodate(folio)) {
                                if (!buffer_uptodate(bh))
                                        set_buffer_uptodate(bh);
                        }
                        continue;
                }
                if (buffer_new(bh))
                        clear_buffer_new(bh);
                if (!buffer_mapped(bh)) {
                        WARN_ON(bh->b_size != blocksize);
                        if (get_block)
                                err = get_block(inode, block, bh, 1);
                        else
                                err = iomap_to_bh(inode, block, bh, iomap);
                        if (err)
                                break;

                        if (buffer_new(bh)) {
                                clean_bdev_bh_alias(bh);
                                if (folio_test_uptodate(folio)) {
                                        clear_buffer_new(bh);
                                        set_buffer_uptodate(bh);
                                        mark_buffer_dirty(bh);
                                        continue;
                                }
                                if (block_end > to || block_start < from)
                                        folio_zero_segments(folio,
                                                to, block_end,
                                                block_start, from);
                                continue;
                        }
                }
                if (folio_test_uptodate(folio)) {
                        if (!buffer_uptodate(bh))
                                set_buffer_uptodate(bh);
                        continue; 
                }
                if (!buffer_uptodate(bh) && !buffer_delay(bh) &&
                    !buffer_unwritten(bh) &&
                     (block_start < from || block_end > to)) {
                        bh_read_nowait(bh, 0);
                        *wait_bh++=bh;
                }
        }
        /*
         * If we issued read requests - let them complete.
         */
        while(wait_bh > wait) {
                wait_on_buffer(*--wait_bh);
                if (!buffer_uptodate(*wait_bh))
                        err = -EIO;
        }
        if (unlikely(err))
                folio_zero_new_buffers(folio, from, to);
        return err;
}

int __block_write_begin(struct folio *folio, loff_t pos, unsigned len,
                get_block_t *get_block)
{
        return __block_write_begin_int(folio, pos, len, get_block, NULL);
}
EXPORT_SYMBOL(__block_write_begin);

static void __block_commit_write(struct folio *folio, size_t from, size_t to)
{
        size_t block_start, block_end;
        bool partial = false;
        unsigned blocksize;
        struct buffer_head *bh, *head;

        bh = head = folio_buffers(folio);
        if (!bh)
                return;
        blocksize = bh->b_size;

        block_start = 0;
        do {
                block_end = block_start + blocksize;
                if (block_end <= from || block_start >= to) {
                        if (!buffer_uptodate(bh))
                                partial = true;
                } else {
                        set_buffer_uptodate(bh);
                        mark_buffer_dirty(bh);
                }
                if (buffer_new(bh))
                        clear_buffer_new(bh);

                block_start = block_end;
                bh = bh->b_this_page;
        } while (bh != head);

        /*
         * If this is a partial write which happened to make all buffers
         * uptodate then we can optimize away a bogus read_folio() for
         * the next read(). Here we 'discover' whether the folio went
         * uptodate as a result of this (potentially partial) write.
         */
        if (!partial)
                folio_mark_uptodate(folio);
}

/*
 * block_write_begin takes care of the basic task of block allocation and
 * bringing partial write blocks uptodate first.
 *
 * The filesystem needs to handle block truncation upon failure.
 */
int block_write_begin(struct address_space *mapping, loff_t pos, unsigned len,
                struct folio **foliop, get_block_t *get_block)
{
        pgoff_t index = pos >> PAGE_SHIFT;
        struct folio *folio;
        int status;

        folio = __filemap_get_folio(mapping, index, FGP_WRITEBEGIN,
                        mapping_gfp_mask(mapping));
        if (IS_ERR(folio))
                return PTR_ERR(folio);

        status = __block_write_begin_int(folio, pos, len, get_block, NULL);
        if (unlikely(status)) {
                folio_unlock(folio);
                folio_put(folio);
                folio = NULL;
        }

        *foliop = folio;
        return status;
}
EXPORT_SYMBOL(block_write_begin);

int block_write_end(struct file *file, struct address_space *mapping,
                        loff_t pos, unsigned len, unsigned copied,
                        struct folio *folio, void *fsdata)
{
        size_t start = pos - folio_pos(folio);

        if (unlikely(copied < len)) {
                /*
                 * The buffers that were written will now be uptodate, so
                 * we don't have to worry about a read_folio reading them
                 * and overwriting a partial write. However if we have
                 * encountered a short write and only partially written
                 * into a buffer, it will not be marked uptodate, so a
                 * read_folio might come in and destroy our partial write.
                 *
                 * Do the simplest thing, and just treat any short write to a
                 * non uptodate folio as a zero-length write, and force the
                 * caller to redo the whole thing.
                 */
                if (!folio_test_uptodate(folio))
                        copied = 0;

                folio_zero_new_buffers(folio, start+copied, start+len);
        }
        flush_dcache_folio(folio);

        /* This could be a short (even 0-length) commit */
        __block_commit_write(folio, start, start + copied);

        return copied;
}
EXPORT_SYMBOL(block_write_end);

int generic_write_end(struct file *file, struct address_space *mapping,
                        loff_t pos, unsigned len, unsigned copied,
                        struct folio *folio, void *fsdata)
{
        struct inode *inode = mapping->host;
        loff_t old_size = inode->i_size;
        bool i_size_changed = false;

        copied = block_write_end(file, mapping, pos, len, copied, folio, fsdata);

        /*
         * No need to use i_size_read() here, the i_size cannot change under us
         * because we hold i_rwsem.
         *
         * But it's important to update i_size while still holding folio lock:
         * page writeout could otherwise come in and zero beyond i_size.
         */
        if (pos + copied > inode->i_size) {
                i_size_write(inode, pos + copied);
                i_size_changed = true;
        }

        folio_unlock(folio);
        folio_put(folio);

        if (old_size < pos)
                pagecache_isize_extended(inode, old_size, pos);
        /*
         * Don't mark the inode dirty under page lock. First, it unnecessarily
         * makes the holding time of page lock longer. Second, it forces lock
         * ordering of page lock and transaction start for journaling
         * filesystems.
         */
        if (i_size_changed)
                mark_inode_dirty(inode);
        return copied;
}
EXPORT_SYMBOL(generic_write_end);

/*
 * block_is_partially_uptodate checks whether buffers within a folio are
 * uptodate or not.
 *
 * Returns true if all buffers which correspond to the specified part
 * of the folio are uptodate.
 */
bool block_is_partially_uptodate(struct folio *folio, size_t from, size_t count)
{
        unsigned block_start, block_end, blocksize;
        unsigned to;
        struct buffer_head *bh, *head;
        bool ret = true;

        head = folio_buffers(folio);
        if (!head)
                return false;
        blocksize = head->b_size;
        to = min_t(unsigned, folio_size(folio) - from, count);
        to = from + to;
        if (from < blocksize && to > folio_size(folio) - blocksize)
                return false;

        bh = head;
        block_start = 0;
        do {
                block_end = block_start + blocksize;
                if (block_end > from && block_start < to) {
                        if (!buffer_uptodate(bh)) {
                                ret = false;
                                break;
                        }
                        if (block_end >= to)
                                break;
                }
                block_start = block_end;
                bh = bh->b_this_page;
        } while (bh != head);

        return ret;
}
EXPORT_SYMBOL(block_is_partially_uptodate);

/*
 * Generic "read_folio" function for block devices that have the normal
 * get_block functionality. This is most of the block device filesystems.
 * Reads the folio asynchronously --- the unlock_buffer() and
 * set/clear_buffer_uptodate() functions propagate buffer state into the
 * folio once IO has completed.
 */
int block_read_full_folio(struct folio *folio, get_block_t *get_block)
{
        struct inode *inode = folio->mapping->host;
        sector_t iblock, lblock;
        struct buffer_head *bh, *head, *arr[MAX_BUF_PER_PAGE];
        size_t blocksize;
        int nr, i;
        int fully_mapped = 1;
        bool page_error = false;
        loff_t limit = i_size_read(inode);

        /* This is needed for ext4. */
        if (IS_ENABLED(CONFIG_FS_VERITY) && IS_VERITY(inode))
                limit = inode->i_sb->s_maxbytes;

        VM_BUG_ON_FOLIO(folio_test_large(folio), folio);

        head = folio_create_buffers(folio, inode, 0);
        blocksize = head->b_size;

        iblock = div_u64(folio_pos(folio), blocksize);
        lblock = div_u64(limit + blocksize - 1, blocksize);
        bh = head;
        nr = 0;
        i = 0;

        do {
                if (buffer_uptodate(bh))
                        continue;

                if (!buffer_mapped(bh)) {
                        int err = 0;

                        fully_mapped = 0;
                        if (iblock < lblock) {
                                WARN_ON(bh->b_size != blocksize);
                                err = get_block(inode, iblock, bh, 0);
                                if (err)
                                        page_error = true;
                        }
                        if (!buffer_mapped(bh)) {
                                folio_zero_range(folio, i * blocksize,
                                                blocksize);
                                if (!err)
                                        set_buffer_uptodate(bh);
                                continue;
                        }
                        /*
                         * get_block() might have updated the buffer
                         * synchronously
                         */
                        if (buffer_uptodate(bh))
                                continue;
                }
                arr[nr++] = bh;
        } while (i++, iblock++, (bh = bh->b_this_page) != head);

        if (fully_mapped)
                folio_set_mappedtodisk(folio);

        if (!nr) {
                /*
                 * All buffers are uptodate or get_block() returned an
                 * error when trying to map them - we can finish the read.
                 */
                folio_end_read(folio, !page_error);
                return 0;
        }

        /* Stage two: lock the buffers */
        for (i = 0; i < nr; i++) {
                bh = arr[i];
                lock_buffer(bh);
                mark_buffer_async_read(bh);
        }

        /*
         * Stage 3: start the IO.  Check for uptodateness
         * inside the buffer lock in case another process reading
         * the underlying blockdev brought it uptodate (the sct fix).
         */
        for (i = 0; i < nr; i++) {
                bh = arr[i];
                if (buffer_uptodate(bh))
                        end_buffer_async_read(bh, 1);
                else
                        submit_bh(REQ_OP_READ, bh);
        }
        return 0;
}
EXPORT_SYMBOL(block_read_full_folio);

/* utility function for filesystems that need to do work on expanding
 * truncates.  Uses filesystem pagecache writes to allow the filesystem to
 * deal with the hole.  
 */
int generic_cont_expand_simple(struct inode *inode, loff_t size)
{
        struct address_space *mapping = inode->i_mapping;
        const struct address_space_operations *aops = mapping->a_ops;
        struct folio *folio;
        void *fsdata = NULL;
        int err;

        err = inode_newsize_ok(inode, size);
        if (err)
                goto out;

        err = aops->write_begin(NULL, mapping, size, 0, &folio, &fsdata);
        if (err)
                goto out;

        err = aops->write_end(NULL, mapping, size, 0, 0, folio, fsdata);
        BUG_ON(err > 0);

out:
        return err;
}
EXPORT_SYMBOL(generic_cont_expand_simple);

static int cont_expand_zero(struct file *file, struct address_space *mapping,
                            loff_t pos, loff_t *bytes)
{
        struct inode *inode = mapping->host;
        const struct address_space_operations *aops = mapping->a_ops;
        unsigned int blocksize = i_blocksize(inode);
        struct folio *folio;
        void *fsdata = NULL;
        pgoff_t index, curidx;
        loff_t curpos;
        unsigned zerofrom, offset, len;
        int err = 0;

        index = pos >> PAGE_SHIFT;
        offset = pos & ~PAGE_MASK;

        while (index > (curidx = (curpos = *bytes)>>PAGE_SHIFT)) {
                zerofrom = curpos & ~PAGE_MASK;
                if (zerofrom & (blocksize-1)) {
                        *bytes |= (blocksize-1);
                        (*bytes)++;
                }
                len = PAGE_SIZE - zerofrom;

                err = aops->write_begin(file, mapping, curpos, len,
                                            &folio, &fsdata);
                if (err)
                        goto out;
                folio_zero_range(folio, offset_in_folio(folio, curpos), len);
                err = aops->write_end(file, mapping, curpos, len, len,
                                                folio, fsdata);
                if (err < 0)
                        goto out;
                BUG_ON(err != len);
                err = 0;

                balance_dirty_pages_ratelimited(mapping);

                if (fatal_signal_pending(current)) {
                        err = -EINTR;
                        goto out;
                }
        }

        /* page covers the boundary, find the boundary offset */
        if (index == curidx) {
                zerofrom = curpos & ~PAGE_MASK;
                /* if we will expand the thing last block will be filled */
                if (offset <= zerofrom) {
                        goto out;
                }
                if (zerofrom & (blocksize-1)) {
                        *bytes |= (blocksize-1);
                        (*bytes)++;
                }
                len = offset - zerofrom;

                err = aops->write_begin(file, mapping, curpos, len,
                                            &folio, &fsdata);
                if (err)
                        goto out;
                folio_zero_range(folio, offset_in_folio(folio, curpos), len);
                err = aops->write_end(file, mapping, curpos, len, len,
                                                folio, fsdata);
                if (err < 0)
                        goto out;
                BUG_ON(err != len);
                err = 0;
        }
out:
        return err;
}

/*
 * For moronic filesystems that do not allow holes in file.
 * We may have to extend the file.
 */
int cont_write_begin(struct file *file, struct address_space *mapping,
                        loff_t pos, unsigned len,
                        struct folio **foliop, void **fsdata,
                        get_block_t *get_block, loff_t *bytes)
{
        struct inode *inode = mapping->host;
        unsigned int blocksize = i_blocksize(inode);
        unsigned int zerofrom;
        int err;

        err = cont_expand_zero(file, mapping, pos, bytes);
        if (err)
                return err;

        zerofrom = *bytes & ~PAGE_MASK;
        if (pos+len > *bytes && zerofrom & (blocksize-1)) {
                *bytes |= (blocksize-1);
                (*bytes)++;
        }

        return block_write_begin(mapping, pos, len, foliop, get_block);
}
EXPORT_SYMBOL(cont_write_begin);

void block_commit_write(struct page *page, unsigned from, unsigned to)
{
        struct folio *folio = page_folio(page);
        __block_commit_write(folio, from, to);
}
EXPORT_SYMBOL(block_commit_write);

/*
 * block_page_mkwrite() is not allowed to change the file size as it gets
 * called from a page fault handler when a page is first dirtied. Hence we must
 * be careful to check for EOF conditions here. We set the page up correctly
 * for a written page which means we get ENOSPC checking when writing into
 * holes and correct delalloc and unwritten extent mapping on filesystems that
 * support these features.
 *
 * We are not allowed to take the i_mutex here so we have to play games to
 * protect against truncate races as the page could now be beyond EOF.  Because
 * truncate writes the inode size before removing pages, once we have the
 * page lock we can determine safely if the page is beyond EOF. If it is not
 * beyond EOF, then the page is guaranteed safe against truncation until we
 * unlock the page.
 *
 * Direct callers of this function should protect against filesystem freezing
 * using sb_start_pagefault() - sb_end_pagefault() functions.
 */
int block_page_mkwrite(struct vm_area_struct *vma, struct vm_fault *vmf,
                         get_block_t get_block)
{
        struct folio *folio = page_folio(vmf->page);
        struct inode *inode = file_inode(vma->vm_file);
        unsigned long end;
        loff_t size;
        int ret;

        folio_lock(folio);
        size = i_size_read(inode);
        if ((folio->mapping != inode->i_mapping) ||
            (folio_pos(folio) >= size)) {
                /* We overload EFAULT to mean page got truncated */
                ret = -EFAULT;
                goto out_unlock;
        }

        end = folio_size(folio);
        /* folio is wholly or partially inside EOF */
        if (folio_pos(folio) + end > size)
                end = size - folio_pos(folio);

        ret = __block_write_begin_int(folio, 0, end, get_block, NULL);
        if (unlikely(ret))
                goto out_unlock;

        __block_commit_write(folio, 0, end);

        folio_mark_dirty(folio);
        folio_wait_stable(folio);
        return 0;
out_unlock:
        folio_unlock(folio);
        return ret;
}
EXPORT_SYMBOL(block_page_mkwrite);

int block_truncate_page(struct address_space *mapping,
                        loff_t from, get_block_t *get_block)
{
        pgoff_t index = from >> PAGE_SHIFT;
        unsigned blocksize;
        sector_t iblock;
        size_t offset, length, pos;
        struct inode *inode = mapping->host;
        struct folio *folio;
        struct buffer_head *bh;
        int err = 0;

        blocksize = i_blocksize(inode);
        length = from & (blocksize - 1);

        /* Block boundary? Nothing to do */
        if (!length)
                return 0;

        length = blocksize - length;
        iblock = ((loff_t)index * PAGE_SIZE) >> inode->i_blkbits;

        folio = filemap_grab_folio(mapping, index);
        if (IS_ERR(folio))
                return PTR_ERR(folio);

        bh = folio_buffers(folio);
        if (!bh)
                bh = create_empty_buffers(folio, blocksize, 0);

        /* Find the buffer that contains "offset" */
        offset = offset_in_folio(folio, from);
        pos = blocksize;
        while (offset >= pos) {
                bh = bh->b_this_page;
                iblock++;
                pos += blocksize;
        }

        if (!buffer_mapped(bh)) {
                WARN_ON(bh->b_size != blocksize);
                err = get_block(inode, iblock, bh, 0);
                if (err)
                        goto unlock;
                /* unmapped? It's a hole - nothing to do */
                if (!buffer_mapped(bh))
                        goto unlock;
        }

        /* Ok, it's mapped. Make sure it's up-to-date */
        if (folio_test_uptodate(folio))
                set_buffer_uptodate(bh);

        if (!buffer_uptodate(bh) && !buffer_delay(bh) && !buffer_unwritten(bh)) {
                err = bh_read(bh, 0);
                /* Uhhuh. Read error. Complain and punt. */
                if (err < 0)
                        goto unlock;
        }

        folio_zero_range(folio, offset, length);
        mark_buffer_dirty(bh);

unlock:
        folio_unlock(folio);
        folio_put(folio);

        return err;
}
EXPORT_SYMBOL(block_truncate_page);

/*
 * The generic ->writepage function for buffer-backed address_spaces
 */
int block_write_full_folio(struct folio *folio, struct writeback_control *wbc,
                void *get_block)
{
        struct inode * const inode = folio->mapping->host;
        loff_t i_size = i_size_read(inode);

        /* Is the folio fully inside i_size? */
        if (folio_pos(folio) + folio_size(folio) <= i_size)
                return __block_write_full_folio(inode, folio, get_block, wbc);

        /* Is the folio fully outside i_size? (truncate in progress) */
        if (folio_pos(folio) >= i_size) {
                folio_unlock(folio);
                return 0; /* don't care */
        }

        /*
         * The folio straddles i_size.  It must be zeroed out on each and every
         * writepage invocation because it may be mmapped.  "A file is mapped
         * in multiples of the page size.  For a file that is not a multiple of
         * the page size, the remaining memory is zeroed when mapped, and
         * writes to that region are not written out to the file."
         */
        folio_zero_segment(folio, offset_in_folio(folio, i_size),
                        folio_size(folio));
        return __block_write_full_folio(inode, folio, get_block, wbc);
}

sector_t generic_block_bmap(struct address_space *mapping, sector_t block,
                            get_block_t *get_block)
{
        struct inode *inode = mapping->host;
        struct buffer_head tmp = {
                .b_size = i_blocksize(inode),
        };

        get_block(inode, block, &tmp, 0);
        return tmp.b_blocknr;
}
EXPORT_SYMBOL(generic_block_bmap);

static void end_bio_bh_io_sync(struct bio *bio)
{
        struct buffer_head *bh = bio->bi_private;

        if (unlikely(bio_flagged(bio, BIO_QUIET)))
                set_bit(BH_Quiet, &bh->b_state);

        bh->b_end_io(bh, !bio->bi_status);
        bio_put(bio);
}

static void submit_bh_wbc(blk_opf_t opf, struct buffer_head *bh,
                          enum rw_hint write_hint,
                          struct writeback_control *wbc)
{
        const enum req_op op = opf & REQ_OP_MASK;
        struct bio *bio;

        BUG_ON(!buffer_locked(bh));
        BUG_ON(!buffer_mapped(bh));
        BUG_ON(!bh->b_end_io);
        BUG_ON(buffer_delay(bh));
        BUG_ON(buffer_unwritten(bh));

        /*
         * Only clear out a write error when rewriting
         */
        if (test_set_buffer_req(bh) && (op == REQ_OP_WRITE))
                clear_buffer_write_io_error(bh);

        if (buffer_meta(bh))
                opf |= REQ_META;
        if (buffer_prio(bh))
                opf |= REQ_PRIO;

        bio = bio_alloc(bh->b_bdev, 1, opf, GFP_NOIO);

        fscrypt_set_bio_crypt_ctx_bh(bio, bh, GFP_NOIO);

        bio->bi_iter.bi_sector = bh->b_blocknr * (bh->b_size >> 9);
        bio->bi_write_hint = write_hint;

        bio_add_folio_nofail(bio, bh->b_folio, bh->b_size, bh_offset(bh));

        bio->bi_end_io = end_bio_bh_io_sync;
        bio->bi_private = bh;

        /* Take care of bh's that straddle the end of the device */
        guard_bio_eod(bio);

        if (wbc) {
                wbc_init_bio(wbc, bio);
                wbc_account_cgroup_owner(wbc, bh->b_folio, bh->b_size);
        }

        submit_bio(bio);
}

void submit_bh(blk_opf_t opf, struct buffer_head *bh)
{
        submit_bh_wbc(opf, bh, WRITE_LIFE_NOT_SET, NULL);
}
EXPORT_SYMBOL(submit_bh);

void write_dirty_buffer(struct buffer_head *bh, blk_opf_t op_flags)
{
        lock_buffer(bh);
        if (!test_clear_buffer_dirty(bh)) {
                unlock_buffer(bh);
                return;
        }
        bh->b_end_io = end_buffer_write_sync;
        get_bh(bh);
        submit_bh(REQ_OP_WRITE | op_flags, bh);
}
EXPORT_SYMBOL(write_dirty_buffer);

/*
 * For a data-integrity writeout, we need to wait upon any in-progress I/O
 * and then start new I/O and then wait upon it.  The caller must have a ref on
 * the buffer_head.
 */
int __sync_dirty_buffer(struct buffer_head *bh, blk_opf_t op_flags)
{
        WARN_ON(atomic_read(&bh->b_count) < 1);
        lock_buffer(bh);
        if (test_clear_buffer_dirty(bh)) {
                /*
                 * The bh should be mapped, but it might not be if the
                 * device was hot-removed. Not much we can do but fail the I/O.
                 */
                if (!buffer_mapped(bh)) {
                        unlock_buffer(bh);
                        return -EIO;
                }

                get_bh(bh);
                bh->b_end_io = end_buffer_write_sync;
                submit_bh(REQ_OP_WRITE | op_flags, bh);
                wait_on_buffer(bh);
                if (!buffer_uptodate(bh))
                        return -EIO;
        } else {
                unlock_buffer(bh);
        }
        return 0;
}
EXPORT_SYMBOL(__sync_dirty_buffer);

int sync_dirty_buffer(struct buffer_head *bh)
{
        return __sync_dirty_buffer(bh, REQ_SYNC);
}
EXPORT_SYMBOL(sync_dirty_buffer);

static inline int buffer_busy(struct buffer_head *bh)
{
        return atomic_read(&bh->b_count) |
                (bh->b_state & ((1 << BH_Dirty) | (1 << BH_Lock)));
}

static bool
drop_buffers(struct folio *folio, struct buffer_head **buffers_to_free)
{
        struct buffer_head *head = folio_buffers(folio);
        struct buffer_head *bh;

        bh = head;
        do {
                if (buffer_busy(bh))
                        goto failed;
                bh = bh->b_this_page;
        } while (bh != head);

        do {
                struct buffer_head *next = bh->b_this_page;

                if (bh->b_assoc_map)
                        __remove_assoc_queue(bh);
                bh = next;
        } while (bh != head);
        *buffers_to_free = head;
        folio_detach_private(folio);
        return true;
failed:
        return false;
}

/**
 * try_to_free_buffers - Release buffers attached to this folio.
 * @folio: The folio.
 *
 * If any buffers are in use (dirty, under writeback, elevated refcount),
 * no buffers will be freed.
 *
 * If the folio is dirty but all the buffers are clean then we need to
 * be sure to mark the folio clean as well.  This is because the folio
 * may be against a block device, and a later reattachment of buffers
 * to a dirty folio will set *all* buffers dirty.  Which would corrupt
 * filesystem data on the same device.
 *
 * The same applies to regular filesystem folios: if all the buffers are
 * clean then we set the folio clean and proceed.  To do that, we require
 * total exclusion from block_dirty_folio().  That is obtained with
 * i_private_lock.
 *
 * Exclusion against try_to_free_buffers may be obtained by either
 * locking the folio or by holding its mapping's i_private_lock.
 *
 * Context: Process context.  @folio must be locked.  Will not sleep.
 * Return: true if all buffers attached to this folio were freed.
 */
bool try_to_free_buffers(struct folio *folio)
{
        struct address_space * const mapping = folio->mapping;
        struct buffer_head *buffers_to_free = NULL;
        bool ret = 0;

        BUG_ON(!folio_test_locked(folio));
        if (folio_test_writeback(folio))
                return false;

        if (mapping == NULL) {                /* can this still happen? */
                ret = drop_buffers(folio, &buffers_to_free);
                goto out;
        }

        spin_lock(&mapping->i_private_lock);
        ret = drop_buffers(folio, &buffers_to_free);

        /*
         * If the filesystem writes its buffers by hand (eg ext3)
         * then we can have clean buffers against a dirty folio.  We
         * clean the folio here; otherwise the VM will never notice
         * that the filesystem did any IO at all.
         *
         * Also, during truncate, discard_buffer will have marked all
         * the folio's buffers clean.  We discover that here and clean
         * the folio also.
         *
         * i_private_lock must be held over this entire operation in order
         * to synchronise against block_dirty_folio and prevent the
         * dirty bit from being lost.
         */
        if (ret)
                folio_cancel_dirty(folio);
        spin_unlock(&mapping->i_private_lock);
out:
        if (buffers_to_free) {
                struct buffer_head *bh = buffers_to_free;

                do {
                        struct buffer_head *next = bh->b_this_page;
                        free_buffer_head(bh);
                        bh = next;
                } while (bh != buffers_to_free);
        }
        return ret;
}
EXPORT_SYMBOL(try_to_free_buffers);

/*
 * Buffer-head allocation
 */
static struct kmem_cache *bh_cachep __ro_after_init;

/*
 * Once the number of bh's in the machine exceeds this level, we start
 * stripping them in writeback.
 */
static unsigned long max_buffer_heads __ro_after_init;

int buffer_heads_over_limit;

struct bh_accounting {
        int nr;                        /* Number of live bh's */
        int ratelimit;                /* Limit cacheline bouncing */
};

static DEFINE_PER_CPU(struct bh_accounting, bh_accounting) = {0, 0};

static void recalc_bh_state(void)
{
        int i;
        int tot = 0;

        if (__this_cpu_inc_return(bh_accounting.ratelimit) - 1 < 4096)
                return;
        __this_cpu_write(bh_accounting.ratelimit, 0);
        for_each_online_cpu(i)
                tot += per_cpu(bh_accounting, i).nr;
        buffer_heads_over_limit = (tot > max_buffer_heads);
}

struct buffer_head *alloc_buffer_head(gfp_t gfp_flags)
{
        struct buffer_head *ret = kmem_cache_zalloc(bh_cachep, gfp_flags);
        if (ret) {
                INIT_LIST_HEAD(&ret->b_assoc_buffers);
                spin_lock_init(&ret->b_uptodate_lock);
                preempt_disable();
                __this_cpu_inc(bh_accounting.nr);
                recalc_bh_state();
                preempt_enable();
        }
        return ret;
}
EXPORT_SYMBOL(alloc_buffer_head);

void free_buffer_head(struct buffer_head *bh)
{
        BUG_ON(!list_empty(&bh->b_assoc_buffers));
        kmem_cache_free(bh_cachep, bh);
        preempt_disable();
        __this_cpu_dec(bh_accounting.nr);
        recalc_bh_state();
        preempt_enable();
}
EXPORT_SYMBOL(free_buffer_head);

static int buffer_exit_cpu_dead(unsigned int cpu)
{
        int i;
        struct bh_lru *b = &per_cpu(bh_lrus, cpu);

        for (i = 0; i < BH_LRU_SIZE; i++) {
                brelse(b->bhs[i]);
                b->bhs[i] = NULL;
        }
        this_cpu_add(bh_accounting.nr, per_cpu(bh_accounting, cpu).nr);
        per_cpu(bh_accounting, cpu).nr = 0;
        return 0;
}

/**
 * bh_uptodate_or_lock - Test whether the buffer is uptodate
 * @bh: struct buffer_head
 *
 * Return true if the buffer is up-to-date and false,
 * with the buffer locked, if not.
 */
int bh_uptodate_or_lock(struct buffer_head *bh)
{
        if (!buffer_uptodate(bh)) {
                lock_buffer(bh);
                if (!buffer_uptodate(bh))
                        return 0;
                unlock_buffer(bh);
        }
        return 1;
}
EXPORT_SYMBOL(bh_uptodate_or_lock);

/**
 * __bh_read - Submit read for a locked buffer
 * @bh: struct buffer_head
 * @op_flags: appending REQ_OP_* flags besides REQ_OP_READ
 * @wait: wait until reading finish
 *
 * Returns zero on success or don't wait, and -EIO on error.
 */
int __bh_read(struct buffer_head *bh, blk_opf_t op_flags, bool wait)
{
        int ret = 0;

        BUG_ON(!buffer_locked(bh));

        get_bh(bh);
        bh->b_end_io = end_buffer_read_sync;
        submit_bh(REQ_OP_READ | op_flags, bh);
        if (wait) {
                wait_on_buffer(bh);
                if (!buffer_uptodate(bh))
                        ret = -EIO;
        }
        return ret;
}
EXPORT_SYMBOL(__bh_read);

/**
 * __bh_read_batch - Submit read for a batch of unlocked buffers
 * @nr: entry number of the buffer batch
 * @bhs: a batch of struct buffer_head
 * @op_flags: appending REQ_OP_* flags besides REQ_OP_READ
 * @force_lock: force to get a lock on the buffer if set, otherwise drops any
 *              buffer that cannot lock.
 *
 * Returns zero on success or don't wait, and -EIO on error.
 */
void __bh_read_batch(int nr, struct buffer_head *bhs[],
                     blk_opf_t op_flags, bool force_lock)
{
        int i;

        for (i = 0; i < nr; i++) {
                struct buffer_head *bh = bhs[i];

                if (buffer_uptodate(bh))
                        continue;

                if (force_lock)
                        lock_buffer(bh);
                else
                        if (!trylock_buffer(bh))
                                continue;

                if (buffer_uptodate(bh)) {
                        unlock_buffer(bh);
                        continue;
                }

                bh->b_end_io = end_buffer_read_sync;
                get_bh(bh);
                submit_bh(REQ_OP_READ | op_flags, bh);
        }
}
EXPORT_SYMBOL(__bh_read_batch);

void __init buffer_init(void)
{
        unsigned long nrpages;
        int ret;

        bh_cachep = KMEM_CACHE(buffer_head,
                                SLAB_RECLAIM_ACCOUNT|SLAB_PANIC);
        /*
         * Limit the bh occupancy to 10% of ZONE_NORMAL
         */
        nrpages = (nr_free_buffer_pages() * 10) / 100;
        max_buffer_heads = nrpages * (PAGE_SIZE / sizeof(struct buffer_head));
        ret = cpuhp_setup_state_nocalls(CPUHP_FS_BUFF_DEAD, "fs/buffer:dead",
                                        NULL, buffer_exit_cpu_dead);
        WARN_ON(ret < 0);
}









































































































    4 








































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
// SPDX-License-Identifier: GPL-2.0-only
/*
 * Common code for control of lockd and nfsv4 grace periods.
 *
 * Transplanted from lockd code
 */

#include <linux/module.h>
#include <net/net_namespace.h>
#include <net/netns/generic.h>
#include <linux/fs.h>
#include <linux/filelock.h>

static unsigned int grace_net_id;
static DEFINE_SPINLOCK(grace_lock);

/**
 * locks_start_grace
 * @net: net namespace that this lock manager belongs to
 * @lm: who this grace period is for
 *
 * A grace period is a period during which locks should not be given
 * out.  Currently grace periods are only enforced by the two lock
 * managers (lockd and nfsd), using the locks_in_grace() function to
 * check when they are in a grace period.
 *
 * This function is called to start a grace period.
 */
void
locks_start_grace(struct net *net, struct lock_manager *lm)
{
        struct list_head *grace_list = net_generic(net, grace_net_id);

        spin_lock(&grace_lock);
        if (list_empty(&lm->list))
                list_add(&lm->list, grace_list);
        else
                WARN(1, "double list_add attempt detected in net %x %s\n",
                     net->ns.inum, (net == &init_net) ? "(init_net)" : "");
        spin_unlock(&grace_lock);
}
EXPORT_SYMBOL_GPL(locks_start_grace);

/**
 * locks_end_grace
 * @lm: who this grace period is for
 *
 * Call this function to state that the given lock manager is ready to
 * resume regular locking.  The grace period will not end until all lock
 * managers that called locks_start_grace() also call locks_end_grace().
 * Note that callers count on it being safe to call this more than once,
 * and the second call should be a no-op.
 */
void
locks_end_grace(struct lock_manager *lm)
{
        spin_lock(&grace_lock);
        list_del_init(&lm->list);
        spin_unlock(&grace_lock);
}
EXPORT_SYMBOL_GPL(locks_end_grace);

static bool
__state_in_grace(struct net *net, bool open)
{
        struct list_head *grace_list = net_generic(net, grace_net_id);
        struct lock_manager *lm;

        if (!open)
                return !list_empty(grace_list);

        spin_lock(&grace_lock);
        list_for_each_entry(lm, grace_list, list) {
                if (lm->block_opens) {
                        spin_unlock(&grace_lock);
                        return true;
                }
        }
        spin_unlock(&grace_lock);
        return false;
}

/**
 * locks_in_grace
 * @net: network namespace
 *
 * Lock managers call this function to determine when it is OK for them
 * to answer ordinary lock requests, and when they should accept only
 * lock reclaims.
 */
bool locks_in_grace(struct net *net)
{
        return __state_in_grace(net, false);
}
EXPORT_SYMBOL_GPL(locks_in_grace);

bool opens_in_grace(struct net *net)
{
        return __state_in_grace(net, true);
}
EXPORT_SYMBOL_GPL(opens_in_grace);

static int __net_init
grace_init_net(struct net *net)
{
        struct list_head *grace_list = net_generic(net, grace_net_id);

        INIT_LIST_HEAD(grace_list);
        return 0;
}

static void __net_exit
grace_exit_net(struct net *net)
{
        struct list_head *grace_list = net_generic(net, grace_net_id);

        WARN_ONCE(!list_empty(grace_list),
                  "net %x %s: grace_list is not empty\n",
                  net->ns.inum, __func__);
}

static struct pernet_operations grace_net_ops = {
        .init = grace_init_net,
        .exit = grace_exit_net,
        .id   = &grace_net_id,
        .size = sizeof(struct list_head),
};

static int __init
init_grace(void)
{
        return register_pernet_subsys(&grace_net_ops);
}

static void __exit
exit_grace(void)
{
        unregister_pernet_subsys(&grace_net_ops);
}

MODULE_AUTHOR("Jeff Layton <jlayton@primarydata.com>");
MODULE_DESCRIPTION("NFS client and server infrastructure");
MODULE_LICENSE("GPL");
module_init(init_grace)
module_exit(exit_grace)




































   15 





    5 


   10 









   10 

















    3 








    3 


    3 













































































   17 

































   17 




   17 













    2 









    2 






    2 



    2 




    1 

















































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
// SPDX-License-Identifier: GPL-2.0-only
/*
 *
 * Generic part shared by ipv4 and ipv6 backends.
 */

#include <linux/kernel.h>
#include <linux/init.h>
#include <linux/module.h>
#include <linux/netlink.h>
#include <linux/netfilter.h>
#include <linux/netfilter/nf_tables.h>
#include <net/netfilter/nf_tables_core.h>
#include <net/netfilter/nf_tables.h>
#include <linux/in.h>
#include <net/xfrm.h>

static const struct nla_policy nft_xfrm_policy[NFTA_XFRM_MAX + 1] = {
        [NFTA_XFRM_KEY]                = NLA_POLICY_MAX(NLA_BE32, 255),
        [NFTA_XFRM_DIR]                = { .type = NLA_U8 },
        [NFTA_XFRM_SPNUM]        = NLA_POLICY_MAX(NLA_BE32, 255),
        [NFTA_XFRM_DREG]        = { .type = NLA_U32 },
};

struct nft_xfrm {
        enum nft_xfrm_keys        key:8;
        u8                        dreg;
        u8                        dir;
        u8                        spnum;
        u8                        len;
};

static int nft_xfrm_get_init(const struct nft_ctx *ctx,
                             const struct nft_expr *expr,
                             const struct nlattr * const tb[])
{
        struct nft_xfrm *priv = nft_expr_priv(expr);
        unsigned int len = 0;
        u32 spnum = 0;
        u8 dir;

        if (!tb[NFTA_XFRM_KEY] || !tb[NFTA_XFRM_DIR] || !tb[NFTA_XFRM_DREG])
                return -EINVAL;

        switch (ctx->family) {
        case NFPROTO_IPV4:
        case NFPROTO_IPV6:
        case NFPROTO_INET:
                break;
        default:
                return -EOPNOTSUPP;
        }

        priv->key = ntohl(nla_get_be32(tb[NFTA_XFRM_KEY]));
        switch (priv->key) {
        case NFT_XFRM_KEY_REQID:
        case NFT_XFRM_KEY_SPI:
                len = sizeof(u32);
                break;
        case NFT_XFRM_KEY_DADDR_IP4:
        case NFT_XFRM_KEY_SADDR_IP4:
                len = sizeof(struct in_addr);
                break;
        case NFT_XFRM_KEY_DADDR_IP6:
        case NFT_XFRM_KEY_SADDR_IP6:
                len = sizeof(struct in6_addr);
                break;
        default:
                return -EINVAL;
        }

        dir = nla_get_u8(tb[NFTA_XFRM_DIR]);
        switch (dir) {
        case XFRM_POLICY_IN:
        case XFRM_POLICY_OUT:
                priv->dir = dir;
                break;
        default:
                return -EINVAL;
        }

        if (tb[NFTA_XFRM_SPNUM])
                spnum = ntohl(nla_get_be32(tb[NFTA_XFRM_SPNUM]));

        if (spnum >= XFRM_MAX_DEPTH)
                return -ERANGE;

        priv->spnum = spnum;

        priv->len = len;
        return nft_parse_register_store(ctx, tb[NFTA_XFRM_DREG], &priv->dreg,
                                        NULL, NFT_DATA_VALUE, len);
}

/* Return true if key asks for daddr/saddr and current
 * state does have a valid address (BEET, TUNNEL).
 */
static bool xfrm_state_addr_ok(enum nft_xfrm_keys k, u8 family, u8 mode)
{
        switch (k) {
        case NFT_XFRM_KEY_DADDR_IP4:
        case NFT_XFRM_KEY_SADDR_IP4:
                if (family == NFPROTO_IPV4)
                        break;
                return false;
        case NFT_XFRM_KEY_DADDR_IP6:
        case NFT_XFRM_KEY_SADDR_IP6:
                if (family == NFPROTO_IPV6)
                        break;
                return false;
        default:
                return true;
        }

        return mode == XFRM_MODE_BEET || mode == XFRM_MODE_TUNNEL;
}

static void nft_xfrm_state_get_key(const struct nft_xfrm *priv,
                                   struct nft_regs *regs,
                                   const struct xfrm_state *state)
{
        u32 *dest = &regs->data[priv->dreg];

        if (!xfrm_state_addr_ok(priv->key,
                                state->props.family,
                                state->props.mode)) {
                regs->verdict.code = NFT_BREAK;
                return;
        }

        switch (priv->key) {
        case NFT_XFRM_KEY_UNSPEC:
        case __NFT_XFRM_KEY_MAX:
                WARN_ON_ONCE(1);
                break;
        case NFT_XFRM_KEY_DADDR_IP4:
                *dest = (__force __u32)state->id.daddr.a4;
                return;
        case NFT_XFRM_KEY_DADDR_IP6:
                memcpy(dest, &state->id.daddr.in6, sizeof(struct in6_addr));
                return;
        case NFT_XFRM_KEY_SADDR_IP4:
                *dest = (__force __u32)state->props.saddr.a4;
                return;
        case NFT_XFRM_KEY_SADDR_IP6:
                memcpy(dest, &state->props.saddr.in6, sizeof(struct in6_addr));
                return;
        case NFT_XFRM_KEY_REQID:
                *dest = state->props.reqid;
                return;
        case NFT_XFRM_KEY_SPI:
                *dest = (__force __u32)state->id.spi;
                return;
        }

        regs->verdict.code = NFT_BREAK;
}

static void nft_xfrm_get_eval_in(const struct nft_xfrm *priv,
                                    struct nft_regs *regs,
                                    const struct nft_pktinfo *pkt)
{
        const struct sec_path *sp = skb_sec_path(pkt->skb);
        const struct xfrm_state *state;

        if (sp == NULL || sp->len <= priv->spnum) {
                regs->verdict.code = NFT_BREAK;
                return;
        }

        state = sp->xvec[priv->spnum];
        nft_xfrm_state_get_key(priv, regs, state);
}

static void nft_xfrm_get_eval_out(const struct nft_xfrm *priv,
                                  struct nft_regs *regs,
                                  const struct nft_pktinfo *pkt)
{
        const struct dst_entry *dst = skb_dst(pkt->skb);
        int i;

        for (i = 0; dst && dst->xfrm;
             dst = ((const struct xfrm_dst *)dst)->child, i++) {
                if (i < priv->spnum)
                        continue;

                nft_xfrm_state_get_key(priv, regs, dst->xfrm);
                return;
        }

        regs->verdict.code = NFT_BREAK;
}

static void nft_xfrm_get_eval(const struct nft_expr *expr,
                              struct nft_regs *regs,
                              const struct nft_pktinfo *pkt)
{
        const struct nft_xfrm *priv = nft_expr_priv(expr);

        switch (priv->dir) {
        case XFRM_POLICY_IN:
                nft_xfrm_get_eval_in(priv, regs, pkt);
                break;
        case XFRM_POLICY_OUT:
                nft_xfrm_get_eval_out(priv, regs, pkt);
                break;
        default:
                WARN_ON_ONCE(1);
                regs->verdict.code = NFT_BREAK;
                break;
        }
}

static int nft_xfrm_get_dump(struct sk_buff *skb,
                             const struct nft_expr *expr, bool reset)
{
        const struct nft_xfrm *priv = nft_expr_priv(expr);

        if (nft_dump_register(skb, NFTA_XFRM_DREG, priv->dreg))
                return -1;

        if (nla_put_be32(skb, NFTA_XFRM_KEY, htonl(priv->key)))
                return -1;
        if (nla_put_u8(skb, NFTA_XFRM_DIR, priv->dir))
                return -1;
        if (nla_put_be32(skb, NFTA_XFRM_SPNUM, htonl(priv->spnum)))
                return -1;

        return 0;
}

static int nft_xfrm_validate(const struct nft_ctx *ctx, const struct nft_expr *expr)
{
        const struct nft_xfrm *priv = nft_expr_priv(expr);
        unsigned int hooks;

        if (ctx->family != NFPROTO_IPV4 &&
            ctx->family != NFPROTO_IPV6 &&
            ctx->family != NFPROTO_INET)
                return -EOPNOTSUPP;

        switch (priv->dir) {
        case XFRM_POLICY_IN:
                hooks = (1 << NF_INET_FORWARD) |
                        (1 << NF_INET_LOCAL_IN) |
                        (1 << NF_INET_PRE_ROUTING);
                break;
        case XFRM_POLICY_OUT:
                hooks = (1 << NF_INET_FORWARD) |
                        (1 << NF_INET_LOCAL_OUT) |
                        (1 << NF_INET_POST_ROUTING);
                break;
        default:
                WARN_ON_ONCE(1);
                return -EINVAL;
        }

        return nft_chain_validate_hooks(ctx->chain, hooks);
}

static bool nft_xfrm_reduce(struct nft_regs_track *track,
                            const struct nft_expr *expr)
{
        const struct nft_xfrm *priv = nft_expr_priv(expr);
        const struct nft_xfrm *xfrm;

        if (!nft_reg_track_cmp(track, expr, priv->dreg)) {
                nft_reg_track_update(track, expr, priv->dreg, priv->len);
                return false;
        }

        xfrm = nft_expr_priv(track->regs[priv->dreg].selector);
        if (priv->key != xfrm->key ||
            priv->dreg != xfrm->dreg ||
            priv->dir != xfrm->dir ||
            priv->spnum != xfrm->spnum) {
                nft_reg_track_update(track, expr, priv->dreg, priv->len);
                return false;
        }

        if (!track->regs[priv->dreg].bitwise)
                return true;

        return nft_expr_reduce_bitwise(track, expr);
}

static struct nft_expr_type nft_xfrm_type;
static const struct nft_expr_ops nft_xfrm_get_ops = {
        .type                = &nft_xfrm_type,
        .size                = NFT_EXPR_SIZE(sizeof(struct nft_xfrm)),
        .eval                = nft_xfrm_get_eval,
        .init                = nft_xfrm_get_init,
        .dump                = nft_xfrm_get_dump,
        .validate        = nft_xfrm_validate,
        .reduce                = nft_xfrm_reduce,
};

static struct nft_expr_type nft_xfrm_type __read_mostly = {
        .name                = "xfrm",
        .ops                = &nft_xfrm_get_ops,
        .policy                = nft_xfrm_policy,
        .maxattr        = NFTA_XFRM_MAX,
        .owner                = THIS_MODULE,
};

static int __init nft_xfrm_module_init(void)
{
        return nft_register_expr(&nft_xfrm_type);
}

static void __exit nft_xfrm_module_exit(void)
{
        nft_unregister_expr(&nft_xfrm_type);
}

module_init(nft_xfrm_module_init);
module_exit(nft_xfrm_module_exit);

MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("nf_tables: xfrm/IPSec matching");
MODULE_AUTHOR("Florian Westphal <fw@strlen.de>");
MODULE_AUTHOR("Máté Eckl <ecklm94@gmail.com>");
MODULE_ALIAS_NFT_EXPR("xfrm");
































































































































































    1 



    1 

    1 








    1 

























































































































































    5 
    1 



















    3 








    1 





    2 




    1 





    3 



























    2 

































    1 





    1 





    1 


    1 




    1 




















    2 


    2 
    2 






    2 












   13 
   13 


    1 























    1 




    1 
    1 











    1 





















    3 


    2 






    1 


    2 


    1 
    1 

    1 


































































































































































































































































































































































































































































































































































































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * CALIPSO - Common Architecture Label IPv6 Security Option
 *
 * This is an implementation of the CALIPSO protocol as specified in
 * RFC 5570.
 *
 * Authors: Paul Moore <paul.moore@hp.com>
 *          Huw Davies <huw@codeweavers.com>
 */

/* (c) Copyright Hewlett-Packard Development Company, L.P., 2006, 2008
 * (c) Copyright Huw Davies <huw@codeweavers.com>, 2015
 */

#include <linux/init.h>
#include <linux/types.h>
#include <linux/rcupdate.h>
#include <linux/list.h>
#include <linux/spinlock.h>
#include <linux/string.h>
#include <linux/jhash.h>
#include <linux/audit.h>
#include <linux/slab.h>
#include <net/ip.h>
#include <net/icmp.h>
#include <net/tcp.h>
#include <net/netlabel.h>
#include <net/calipso.h>
#include <linux/atomic.h>
#include <linux/bug.h>
#include <linux/unaligned.h>
#include <linux/crc-ccitt.h>

/* Maximium size of the calipso option including
 * the two-byte TLV header.
 */
#define CALIPSO_OPT_LEN_MAX (2 + 252)

/* Size of the minimum calipso option including
 * the two-byte TLV header.
 */
#define CALIPSO_HDR_LEN (2 + 8)

/* Maximium size of the calipso option including
 * the two-byte TLV header and upto 3 bytes of
 * leading pad and 7 bytes of trailing pad.
 */
#define CALIPSO_OPT_LEN_MAX_WITH_PAD (3 + CALIPSO_OPT_LEN_MAX + 7)

 /* Maximium size of u32 aligned buffer required to hold calipso
  * option.  Max of 3 initial pad bytes starting from buffer + 3.
  * i.e. the worst case is when the previous tlv finishes on 4n + 3.
  */
#define CALIPSO_MAX_BUFFER (6 + CALIPSO_OPT_LEN_MAX)

/* List of available DOI definitions */
static DEFINE_SPINLOCK(calipso_doi_list_lock);
static LIST_HEAD(calipso_doi_list);

/* Label mapping cache */
int calipso_cache_enabled = 1;
int calipso_cache_bucketsize = 10;
#define CALIPSO_CACHE_BUCKETBITS     7
#define CALIPSO_CACHE_BUCKETS        BIT(CALIPSO_CACHE_BUCKETBITS)
#define CALIPSO_CACHE_REORDERLIMIT   10
struct calipso_map_cache_bkt {
        spinlock_t lock;
        u32 size;
        struct list_head list;
};

struct calipso_map_cache_entry {
        u32 hash;
        unsigned char *key;
        size_t key_len;

        struct netlbl_lsm_cache *lsm_data;

        u32 activity;
        struct list_head list;
};

static struct calipso_map_cache_bkt *calipso_cache;

static void calipso_cache_invalidate(void);
static void calipso_doi_putdef(struct calipso_doi *doi_def);

/* Label Mapping Cache Functions
 */

/**
 * calipso_cache_entry_free - Frees a cache entry
 * @entry: the entry to free
 *
 * Description:
 * This function frees the memory associated with a cache entry including the
 * LSM cache data if there are no longer any users, i.e. reference count == 0.
 *
 */
static void calipso_cache_entry_free(struct calipso_map_cache_entry *entry)
{
        if (entry->lsm_data)
                netlbl_secattr_cache_free(entry->lsm_data);
        kfree(entry->key);
        kfree(entry);
}

/**
 * calipso_map_cache_hash - Hashing function for the CALIPSO cache
 * @key: the hash key
 * @key_len: the length of the key in bytes
 *
 * Description:
 * The CALIPSO tag hashing function.  Returns a 32-bit hash value.
 *
 */
static u32 calipso_map_cache_hash(const unsigned char *key, u32 key_len)
{
        return jhash(key, key_len, 0);
}

/**
 * calipso_cache_init - Initialize the CALIPSO cache
 *
 * Description:
 * Initializes the CALIPSO label mapping cache, this function should be called
 * before any of the other functions defined in this file.  Returns zero on
 * success, negative values on error.
 *
 */
static int __init calipso_cache_init(void)
{
        u32 iter;

        calipso_cache = kcalloc(CALIPSO_CACHE_BUCKETS,
                                sizeof(struct calipso_map_cache_bkt),
                                GFP_KERNEL);
        if (!calipso_cache)
                return -ENOMEM;

        for (iter = 0; iter < CALIPSO_CACHE_BUCKETS; iter++) {
                spin_lock_init(&calipso_cache[iter].lock);
                calipso_cache[iter].size = 0;
                INIT_LIST_HEAD(&calipso_cache[iter].list);
        }

        return 0;
}

/**
 * calipso_cache_invalidate - Invalidates the current CALIPSO cache
 *
 * Description:
 * Invalidates and frees any entries in the CALIPSO cache.  Returns zero on
 * success and negative values on failure.
 *
 */
static void calipso_cache_invalidate(void)
{
        struct calipso_map_cache_entry *entry, *tmp_entry;
        u32 iter;

        for (iter = 0; iter < CALIPSO_CACHE_BUCKETS; iter++) {
                spin_lock_bh(&calipso_cache[iter].lock);
                list_for_each_entry_safe(entry,
                                         tmp_entry,
                                         &calipso_cache[iter].list, list) {
                        list_del(&entry->list);
                        calipso_cache_entry_free(entry);
                }
                calipso_cache[iter].size = 0;
                spin_unlock_bh(&calipso_cache[iter].lock);
        }
}

/**
 * calipso_cache_check - Check the CALIPSO cache for a label mapping
 * @key: the buffer to check
 * @key_len: buffer length in bytes
 * @secattr: the security attribute struct to use
 *
 * Description:
 * This function checks the cache to see if a label mapping already exists for
 * the given key.  If there is a match then the cache is adjusted and the
 * @secattr struct is populated with the correct LSM security attributes.  The
 * cache is adjusted in the following manner if the entry is not already the
 * first in the cache bucket:
 *
 *  1. The cache entry's activity counter is incremented
 *  2. The previous (higher ranking) entry's activity counter is decremented
 *  3. If the difference between the two activity counters is geater than
 *     CALIPSO_CACHE_REORDERLIMIT the two entries are swapped
 *
 * Returns zero on success, -ENOENT for a cache miss, and other negative values
 * on error.
 *
 */
static int calipso_cache_check(const unsigned char *key,
                               u32 key_len,
                               struct netlbl_lsm_secattr *secattr)
{
        u32 bkt;
        struct calipso_map_cache_entry *entry;
        struct calipso_map_cache_entry *prev_entry = NULL;
        u32 hash;

        if (!calipso_cache_enabled)
                return -ENOENT;

        hash = calipso_map_cache_hash(key, key_len);
        bkt = hash & (CALIPSO_CACHE_BUCKETS - 1);
        spin_lock_bh(&calipso_cache[bkt].lock);
        list_for_each_entry(entry, &calipso_cache[bkt].list, list) {
                if (entry->hash == hash &&
                    entry->key_len == key_len &&
                    memcmp(entry->key, key, key_len) == 0) {
                        entry->activity += 1;
                        refcount_inc(&entry->lsm_data->refcount);
                        secattr->cache = entry->lsm_data;
                        secattr->flags |= NETLBL_SECATTR_CACHE;
                        secattr->type = NETLBL_NLTYPE_CALIPSO;
                        if (!prev_entry) {
                                spin_unlock_bh(&calipso_cache[bkt].lock);
                                return 0;
                        }

                        if (prev_entry->activity > 0)
                                prev_entry->activity -= 1;
                        if (entry->activity > prev_entry->activity &&
                            entry->activity - prev_entry->activity >
                            CALIPSO_CACHE_REORDERLIMIT) {
                                __list_del(entry->list.prev, entry->list.next);
                                __list_add(&entry->list,
                                           prev_entry->list.prev,
                                           &prev_entry->list);
                        }

                        spin_unlock_bh(&calipso_cache[bkt].lock);
                        return 0;
                }
                prev_entry = entry;
        }
        spin_unlock_bh(&calipso_cache[bkt].lock);

        return -ENOENT;
}

/**
 * calipso_cache_add - Add an entry to the CALIPSO cache
 * @calipso_ptr: the CALIPSO option
 * @secattr: the packet's security attributes
 *
 * Description:
 * Add a new entry into the CALIPSO label mapping cache.  Add the new entry to
 * head of the cache bucket's list, if the cache bucket is out of room remove
 * the last entry in the list first.  It is important to note that there is
 * currently no checking for duplicate keys.  Returns zero on success,
 * negative values on failure.  The key stored starts at calipso_ptr + 2,
 * i.e. the type and length bytes are not stored, this corresponds to
 * calipso_ptr[1] bytes of data.
 *
 */
static int calipso_cache_add(const unsigned char *calipso_ptr,
                             const struct netlbl_lsm_secattr *secattr)
{
        int ret_val = -EPERM;
        u32 bkt;
        struct calipso_map_cache_entry *entry = NULL;
        struct calipso_map_cache_entry *old_entry = NULL;
        u32 calipso_ptr_len;

        if (!calipso_cache_enabled || calipso_cache_bucketsize <= 0)
                return 0;

        calipso_ptr_len = calipso_ptr[1];

        entry = kzalloc(sizeof(*entry), GFP_ATOMIC);
        if (!entry)
                return -ENOMEM;
        entry->key = kmemdup(calipso_ptr + 2, calipso_ptr_len, GFP_ATOMIC);
        if (!entry->key) {
                ret_val = -ENOMEM;
                goto cache_add_failure;
        }
        entry->key_len = calipso_ptr_len;
        entry->hash = calipso_map_cache_hash(calipso_ptr, calipso_ptr_len);
        refcount_inc(&secattr->cache->refcount);
        entry->lsm_data = secattr->cache;

        bkt = entry->hash & (CALIPSO_CACHE_BUCKETS - 1);
        spin_lock_bh(&calipso_cache[bkt].lock);
        if (calipso_cache[bkt].size < calipso_cache_bucketsize) {
                list_add(&entry->list, &calipso_cache[bkt].list);
                calipso_cache[bkt].size += 1;
        } else {
                old_entry = list_entry(calipso_cache[bkt].list.prev,
                                       struct calipso_map_cache_entry, list);
                list_del(&old_entry->list);
                list_add(&entry->list, &calipso_cache[bkt].list);
                calipso_cache_entry_free(old_entry);
        }
        spin_unlock_bh(&calipso_cache[bkt].lock);

        return 0;

cache_add_failure:
        if (entry)
                calipso_cache_entry_free(entry);
        return ret_val;
}

/* DOI List Functions
 */

/**
 * calipso_doi_search - Searches for a DOI definition
 * @doi: the DOI to search for
 *
 * Description:
 * Search the DOI definition list for a DOI definition with a DOI value that
 * matches @doi.  The caller is responsible for calling rcu_read_[un]lock().
 * Returns a pointer to the DOI definition on success and NULL on failure.
 */
static struct calipso_doi *calipso_doi_search(u32 doi)
{
        struct calipso_doi *iter;

        list_for_each_entry_rcu(iter, &calipso_doi_list, list)
                if (iter->doi == doi && refcount_read(&iter->refcount))
                        return iter;
        return NULL;
}

/**
 * calipso_doi_add - Add a new DOI to the CALIPSO protocol engine
 * @doi_def: the DOI structure
 * @audit_info: NetLabel audit information
 *
 * Description:
 * The caller defines a new DOI for use by the CALIPSO engine and calls this
 * function to add it to the list of acceptable domains.  The caller must
 * ensure that the mapping table specified in @doi_def->map meets all of the
 * requirements of the mapping type (see calipso.h for details).  Returns
 * zero on success and non-zero on failure.
 *
 */
static int calipso_doi_add(struct calipso_doi *doi_def,
                           struct netlbl_audit *audit_info)
{
        int ret_val = -EINVAL;
        u32 doi;
        u32 doi_type;
        struct audit_buffer *audit_buf;

        doi = doi_def->doi;
        doi_type = doi_def->type;

        if (doi_def->doi == CALIPSO_DOI_UNKNOWN)
                goto doi_add_return;

        refcount_set(&doi_def->refcount, 1);

        spin_lock(&calipso_doi_list_lock);
        if (calipso_doi_search(doi_def->doi)) {
                spin_unlock(&calipso_doi_list_lock);
                ret_val = -EEXIST;
                goto doi_add_return;
        }
        list_add_tail_rcu(&doi_def->list, &calipso_doi_list);
        spin_unlock(&calipso_doi_list_lock);
        ret_val = 0;

doi_add_return:
        audit_buf = netlbl_audit_start(AUDIT_MAC_CALIPSO_ADD, audit_info);
        if (audit_buf) {
                const char *type_str;

                switch (doi_type) {
                case CALIPSO_MAP_PASS:
                        type_str = "pass";
                        break;
                default:
                        type_str = "(unknown)";
                }
                audit_log_format(audit_buf,
                                 " calipso_doi=%u calipso_type=%s res=%u",
                                 doi, type_str, ret_val == 0 ? 1 : 0);
                audit_log_end(audit_buf);
        }

        return ret_val;
}

/**
 * calipso_doi_free - Frees a DOI definition
 * @doi_def: the DOI definition
 *
 * Description:
 * This function frees all of the memory associated with a DOI definition.
 *
 */
static void calipso_doi_free(struct calipso_doi *doi_def)
{
        kfree(doi_def);
}

/**
 * calipso_doi_free_rcu - Frees a DOI definition via the RCU pointer
 * @entry: the entry's RCU field
 *
 * Description:
 * This function is designed to be used as a callback to the call_rcu()
 * function so that the memory allocated to the DOI definition can be released
 * safely.
 *
 */
static void calipso_doi_free_rcu(struct rcu_head *entry)
{
        struct calipso_doi *doi_def;

        doi_def = container_of(entry, struct calipso_doi, rcu);
        calipso_doi_free(doi_def);
}

/**
 * calipso_doi_remove - Remove an existing DOI from the CALIPSO protocol engine
 * @doi: the DOI value
 * @audit_info: NetLabel audit information
 *
 * Description:
 * Removes a DOI definition from the CALIPSO engine.  The NetLabel routines will
 * be called to release their own LSM domain mappings as well as our own
 * domain list.  Returns zero on success and negative values on failure.
 *
 */
static int calipso_doi_remove(u32 doi, struct netlbl_audit *audit_info)
{
        int ret_val;
        struct calipso_doi *doi_def;
        struct audit_buffer *audit_buf;

        spin_lock(&calipso_doi_list_lock);
        doi_def = calipso_doi_search(doi);
        if (!doi_def) {
                spin_unlock(&calipso_doi_list_lock);
                ret_val = -ENOENT;
                goto doi_remove_return;
        }
        list_del_rcu(&doi_def->list);
        spin_unlock(&calipso_doi_list_lock);

        calipso_doi_putdef(doi_def);
        ret_val = 0;

doi_remove_return:
        audit_buf = netlbl_audit_start(AUDIT_MAC_CALIPSO_DEL, audit_info);
        if (audit_buf) {
                audit_log_format(audit_buf,
                                 " calipso_doi=%u res=%u",
                                 doi, ret_val == 0 ? 1 : 0);
                audit_log_end(audit_buf);
        }

        return ret_val;
}

/**
 * calipso_doi_getdef - Returns a reference to a valid DOI definition
 * @doi: the DOI value
 *
 * Description:
 * Searches for a valid DOI definition and if one is found it is returned to
 * the caller.  Otherwise NULL is returned.  The caller must ensure that
 * calipso_doi_putdef() is called when the caller is done.
 *
 */
static struct calipso_doi *calipso_doi_getdef(u32 doi)
{
        struct calipso_doi *doi_def;

        rcu_read_lock();
        doi_def = calipso_doi_search(doi);
        if (!doi_def)
                goto doi_getdef_return;
        if (!refcount_inc_not_zero(&doi_def->refcount))
                doi_def = NULL;

doi_getdef_return:
        rcu_read_unlock();
        return doi_def;
}

/**
 * calipso_doi_putdef - Releases a reference for the given DOI definition
 * @doi_def: the DOI definition
 *
 * Description:
 * Releases a DOI definition reference obtained from calipso_doi_getdef().
 *
 */
static void calipso_doi_putdef(struct calipso_doi *doi_def)
{
        if (!doi_def)
                return;

        if (!refcount_dec_and_test(&doi_def->refcount))
                return;

        calipso_cache_invalidate();
        call_rcu(&doi_def->rcu, calipso_doi_free_rcu);
}

/**
 * calipso_doi_walk - Iterate through the DOI definitions
 * @skip_cnt: skip past this number of DOI definitions, updated
 * @callback: callback for each DOI definition
 * @cb_arg: argument for the callback function
 *
 * Description:
 * Iterate over the DOI definition list, skipping the first @skip_cnt entries.
 * For each entry call @callback, if @callback returns a negative value stop
 * 'walking' through the list and return.  Updates the value in @skip_cnt upon
 * return.  Returns zero on success, negative values on failure.
 *
 */
static int calipso_doi_walk(u32 *skip_cnt,
                            int (*callback)(struct calipso_doi *doi_def,
                                            void *arg),
                            void *cb_arg)
{
        int ret_val = -ENOENT;
        u32 doi_cnt = 0;
        struct calipso_doi *iter_doi;

        rcu_read_lock();
        list_for_each_entry_rcu(iter_doi, &calipso_doi_list, list)
                if (refcount_read(&iter_doi->refcount) > 0) {
                        if (doi_cnt++ < *skip_cnt)
                                continue;
                        ret_val = callback(iter_doi, cb_arg);
                        if (ret_val < 0) {
                                doi_cnt--;
                                goto doi_walk_return;
                        }
                }

doi_walk_return:
        rcu_read_unlock();
        *skip_cnt = doi_cnt;
        return ret_val;
}

/**
 * calipso_validate - Validate a CALIPSO option
 * @skb: the packet
 * @option: the start of the option
 *
 * Description:
 * This routine is called to validate a CALIPSO option.
 * If the option is valid then %true is returned, otherwise
 * %false is returned.
 *
 * The caller should have already checked that the length of the
 * option (including the TLV header) is >= 10 and that the catmap
 * length is consistent with the option length.
 *
 * We leave checks on the level and categories to the socket layer.
 */
bool calipso_validate(const struct sk_buff *skb, const unsigned char *option)
{
        struct calipso_doi *doi_def;
        bool ret_val;
        u16 crc, len = option[1] + 2;
        static const u8 zero[2];

        /* The original CRC runs over the option including the TLV header
         * with the CRC-16 field (at offset 8) zeroed out. */
        crc = crc_ccitt(0xffff, option, 8);
        crc = crc_ccitt(crc, zero, sizeof(zero));
        if (len > 10)
                crc = crc_ccitt(crc, option + 10, len - 10);
        crc = ~crc;
        if (option[8] != (crc & 0xff) || option[9] != ((crc >> 8) & 0xff))
                return false;

        rcu_read_lock();
        doi_def = calipso_doi_search(get_unaligned_be32(option + 2));
        ret_val = !!doi_def;
        rcu_read_unlock();

        return ret_val;
}

/**
 * calipso_map_cat_hton - Perform a category mapping from host to network
 * @doi_def: the DOI definition
 * @secattr: the security attributes
 * @net_cat: the zero'd out category bitmap in network/CALIPSO format
 * @net_cat_len: the length of the CALIPSO bitmap in bytes
 *
 * Description:
 * Perform a label mapping to translate a local MLS category bitmap to the
 * correct CALIPSO bitmap using the given DOI definition.  Returns the minimum
 * size in bytes of the network bitmap on success, negative values otherwise.
 *
 */
static int calipso_map_cat_hton(const struct calipso_doi *doi_def,
                                const struct netlbl_lsm_secattr *secattr,
                                unsigned char *net_cat,
                                u32 net_cat_len)
{
        int spot = -1;
        u32 net_spot_max = 0;
        u32 net_clen_bits = net_cat_len * 8;

        for (;;) {
                spot = netlbl_catmap_walk(secattr->attr.mls.cat,
                                          spot + 1);
                if (spot < 0)
                        break;
                if (spot >= net_clen_bits)
                        return -ENOSPC;
                netlbl_bitmap_setbit(net_cat, spot, 1);

                if (spot > net_spot_max)
                        net_spot_max = spot;
        }

        return (net_spot_max / 32 + 1) * 4;
}

/**
 * calipso_map_cat_ntoh - Perform a category mapping from network to host
 * @doi_def: the DOI definition
 * @net_cat: the category bitmap in network/CALIPSO format
 * @net_cat_len: the length of the CALIPSO bitmap in bytes
 * @secattr: the security attributes
 *
 * Description:
 * Perform a label mapping to translate a CALIPSO bitmap to the correct local
 * MLS category bitmap using the given DOI definition.  Returns zero on
 * success, negative values on failure.
 *
 */
static int calipso_map_cat_ntoh(const struct calipso_doi *doi_def,
                                const unsigned char *net_cat,
                                u32 net_cat_len,
                                struct netlbl_lsm_secattr *secattr)
{
        int ret_val;
        int spot = -1;
        u32 net_clen_bits = net_cat_len * 8;

        for (;;) {
                spot = netlbl_bitmap_walk(net_cat,
                                          net_clen_bits,
                                          spot + 1,
                                          1);
                if (spot < 0)
                        return 0;

                ret_val = netlbl_catmap_setbit(&secattr->attr.mls.cat,
                                               spot,
                                               GFP_ATOMIC);
                if (ret_val != 0)
                        return ret_val;
        }

        return -EINVAL;
}

/**
 * calipso_pad_write - Writes pad bytes in TLV format
 * @buf: the buffer
 * @offset: offset from start of buffer to write padding
 * @count: number of pad bytes to write
 *
 * Description:
 * Write @count bytes of TLV padding into @buffer starting at offset @offset.
 * @count should be less than 8 - see RFC 4942.
 *
 */
static int calipso_pad_write(unsigned char *buf, unsigned int offset,
                             unsigned int count)
{
        if (WARN_ON_ONCE(count >= 8))
                return -EINVAL;

        switch (count) {
        case 0:
                break;
        case 1:
                buf[offset] = IPV6_TLV_PAD1;
                break;
        default:
                buf[offset] = IPV6_TLV_PADN;
                buf[offset + 1] = count - 2;
                if (count > 2)
                        memset(buf + offset + 2, 0, count - 2);
                break;
        }
        return 0;
}

/**
 * calipso_genopt - Generate a CALIPSO option
 * @buf: the option buffer
 * @start: offset from which to write
 * @buf_len: the size of opt_buf
 * @doi_def: the CALIPSO DOI to use
 * @secattr: the security attributes
 *
 * Description:
 * Generate a CALIPSO option using the DOI definition and security attributes
 * passed to the function. This also generates upto three bytes of leading
 * padding that ensures that the option is 4n + 2 aligned.  It returns the
 * number of bytes written (including any initial padding).
 */
static int calipso_genopt(unsigned char *buf, u32 start, u32 buf_len,
                          const struct calipso_doi *doi_def,
                          const struct netlbl_lsm_secattr *secattr)
{
        int ret_val;
        u32 len, pad;
        u16 crc;
        static const unsigned char padding[4] = {2, 1, 0, 3};
        unsigned char *calipso;

        /* CALIPSO has 4n + 2 alignment */
        pad = padding[start & 3];
        if (buf_len <= start + pad + CALIPSO_HDR_LEN)
                return -ENOSPC;

        if ((secattr->flags & NETLBL_SECATTR_MLS_LVL) == 0)
                return -EPERM;

        len = CALIPSO_HDR_LEN;

        if (secattr->flags & NETLBL_SECATTR_MLS_CAT) {
                ret_val = calipso_map_cat_hton(doi_def,
                                               secattr,
                                               buf + start + pad + len,
                                               buf_len - start - pad - len);
                if (ret_val < 0)
                        return ret_val;
                len += ret_val;
        }

        calipso_pad_write(buf, start, pad);
        calipso = buf + start + pad;

        calipso[0] = IPV6_TLV_CALIPSO;
        calipso[1] = len - 2;
        *(__be32 *)(calipso + 2) = htonl(doi_def->doi);
        calipso[6] = (len - CALIPSO_HDR_LEN) / 4;
        calipso[7] = secattr->attr.mls.lvl;
        crc = ~crc_ccitt(0xffff, calipso, len);
        calipso[8] = crc & 0xff;
        calipso[9] = (crc >> 8) & 0xff;
        return pad + len;
}

/* Hop-by-hop hdr helper functions
 */

/**
 * calipso_opt_update - Replaces socket's hop options with a new set
 * @sk: the socket
 * @hop: new hop options
 *
 * Description:
 * Replaces @sk's hop options with @hop.  @hop may be NULL to leave
 * the socket with no hop options.
 *
 */
static int calipso_opt_update(struct sock *sk, struct ipv6_opt_hdr *hop)
{
        struct ipv6_txoptions *old = txopt_get(inet6_sk(sk)), *txopts;

        txopts = ipv6_renew_options(sk, old, IPV6_HOPOPTS, hop);
        txopt_put(old);
        if (IS_ERR(txopts))
                return PTR_ERR(txopts);

        txopts = ipv6_update_options(sk, txopts);
        if (txopts) {
                atomic_sub(txopts->tot_len, &sk->sk_omem_alloc);
                txopt_put(txopts);
        }

        return 0;
}

/**
 * calipso_tlv_len - Returns the length of the TLV
 * @opt: the option header
 * @offset: offset of the TLV within the header
 *
 * Description:
 * Returns the length of the TLV option at offset @offset within
 * the option header @opt.  Checks that the entire TLV fits inside
 * the option header, returns a negative value if this is not the case.
 */
static int calipso_tlv_len(struct ipv6_opt_hdr *opt, unsigned int offset)
{
        unsigned char *tlv = (unsigned char *)opt;
        unsigned int opt_len = ipv6_optlen(opt), tlv_len;

        if (offset < sizeof(*opt) || offset >= opt_len)
                return -EINVAL;
        if (tlv[offset] == IPV6_TLV_PAD1)
                return 1;
        if (offset + 1 >= opt_len)
                return -EINVAL;
        tlv_len = tlv[offset + 1] + 2;
        if (offset + tlv_len > opt_len)
                return -EINVAL;
        return tlv_len;
}

/**
 * calipso_opt_find - Finds the CALIPSO option in an IPv6 hop options header
 * @hop: the hop options header
 * @start: on return holds the offset of any leading padding
 * @end: on return holds the offset of the first non-pad TLV after CALIPSO
 *
 * Description:
 * Finds the space occupied by a CALIPSO option (including any leading and
 * trailing padding).
 *
 * If a CALIPSO option exists set @start and @end to the
 * offsets within @hop of the start of padding before the first
 * CALIPSO option and the end of padding after the first CALIPSO
 * option.  In this case the function returns 0.
 *
 * In the absence of a CALIPSO option, @start and @end will be
 * set to the start and end of any trailing padding in the header.
 * This is useful when appending a new option, as the caller may want
 * to overwrite some of this padding.  In this case the function will
 * return -ENOENT.
 */
static int calipso_opt_find(struct ipv6_opt_hdr *hop, unsigned int *start,
                            unsigned int *end)
{
        int ret_val = -ENOENT, tlv_len;
        unsigned int opt_len, offset, offset_s = 0, offset_e = 0;
        unsigned char *opt = (unsigned char *)hop;

        opt_len = ipv6_optlen(hop);
        offset = sizeof(*hop);

        while (offset < opt_len) {
                tlv_len = calipso_tlv_len(hop, offset);
                if (tlv_len < 0)
                        return tlv_len;

                switch (opt[offset]) {
                case IPV6_TLV_PAD1:
                case IPV6_TLV_PADN:
                        if (offset_e)
                                offset_e = offset;
                        break;
                case IPV6_TLV_CALIPSO:
                        ret_val = 0;
                        offset_e = offset;
                        break;
                default:
                        if (offset_e == 0)
                                offset_s = offset;
                        else
                                goto out;
                }
                offset += tlv_len;
        }

out:
        if (offset_s)
                *start = offset_s + calipso_tlv_len(hop, offset_s);
        else
                *start = sizeof(*hop);
        if (offset_e)
                *end = offset_e + calipso_tlv_len(hop, offset_e);
        else
                *end = opt_len;

        return ret_val;
}

/**
 * calipso_opt_insert - Inserts a CALIPSO option into an IPv6 hop opt hdr
 * @hop: the original hop options header
 * @doi_def: the CALIPSO DOI to use
 * @secattr: the specific security attributes of the socket
 *
 * Description:
 * Creates a new hop options header based on @hop with a
 * CALIPSO option added to it.  If @hop already contains a CALIPSO
 * option this is overwritten, otherwise the new option is appended
 * after any existing options.  If @hop is NULL then the new header
 * will contain just the CALIPSO option and any needed padding.
 *
 */
static struct ipv6_opt_hdr *
calipso_opt_insert(struct ipv6_opt_hdr *hop,
                   const struct calipso_doi *doi_def,
                   const struct netlbl_lsm_secattr *secattr)
{
        unsigned int start, end, buf_len, pad, hop_len;
        struct ipv6_opt_hdr *new;
        int ret_val;

        if (hop) {
                hop_len = ipv6_optlen(hop);
                ret_val = calipso_opt_find(hop, &start, &end);
                if (ret_val && ret_val != -ENOENT)
                        return ERR_PTR(ret_val);
        } else {
                hop_len = 0;
                start = sizeof(*hop);
                end = 0;
        }

        buf_len = hop_len + start - end + CALIPSO_OPT_LEN_MAX_WITH_PAD;
        new = kzalloc(buf_len, GFP_ATOMIC);
        if (!new)
                return ERR_PTR(-ENOMEM);

        if (start > sizeof(*hop))
                memcpy(new, hop, start);
        ret_val = calipso_genopt((unsigned char *)new, start, buf_len, doi_def,
                                 secattr);
        if (ret_val < 0) {
                kfree(new);
                return ERR_PTR(ret_val);
        }

        buf_len = start + ret_val;
        /* At this point buf_len aligns to 4n, so (buf_len & 4) pads to 8n */
        pad = ((buf_len & 4) + (end & 7)) & 7;
        calipso_pad_write((unsigned char *)new, buf_len, pad);
        buf_len += pad;

        if (end != hop_len) {
                memcpy((char *)new + buf_len, (char *)hop + end, hop_len - end);
                buf_len += hop_len - end;
        }
        new->nexthdr = 0;
        new->hdrlen = buf_len / 8 - 1;

        return new;
}

/**
 * calipso_opt_del - Removes the CALIPSO option from an option header
 * @hop: the original header
 * @new: the new header
 *
 * Description:
 * Creates a new header based on @hop without any CALIPSO option.  If @hop
 * doesn't contain a CALIPSO option it returns -ENOENT.  If @hop contains
 * no other non-padding options, it returns zero with @new set to NULL.
 * Otherwise it returns zero, creates a new header without the CALIPSO
 * option (and removing as much padding as possible) and returns with
 * @new set to that header.
 *
 */
static int calipso_opt_del(struct ipv6_opt_hdr *hop,
                           struct ipv6_opt_hdr **new)
{
        int ret_val;
        unsigned int start, end, delta, pad, hop_len;

        ret_val = calipso_opt_find(hop, &start, &end);
        if (ret_val)
                return ret_val;

        hop_len = ipv6_optlen(hop);
        if (start == sizeof(*hop) && end == hop_len) {
                /* There's no other option in the header so return NULL */
                *new = NULL;
                return 0;
        }

        delta = (end - start) & ~7;
        *new = kzalloc(hop_len - delta, GFP_ATOMIC);
        if (!*new)
                return -ENOMEM;

        memcpy(*new, hop, start);
        (*new)->hdrlen -= delta / 8;
        pad = (end - start) & 7;
        calipso_pad_write((unsigned char *)*new, start, pad);
        if (end != hop_len)
                memcpy((char *)*new + start + pad, (char *)hop + end,
                       hop_len - end);

        return 0;
}

/**
 * calipso_opt_getattr - Get the security attributes from a memory block
 * @calipso: the CALIPSO option
 * @secattr: the security attributes
 *
 * Description:
 * Inspect @calipso and return the security attributes in @secattr.
 * Returns zero on success and negative values on failure.
 *
 */
static int calipso_opt_getattr(const unsigned char *calipso,
                               struct netlbl_lsm_secattr *secattr)
{
        int ret_val = -ENOMSG;
        u32 doi, len = calipso[1], cat_len = calipso[6] * 4;
        struct calipso_doi *doi_def;

        if (cat_len + 8 > len)
                return -EINVAL;

        if (calipso_cache_check(calipso + 2, calipso[1], secattr) == 0)
                return 0;

        doi = get_unaligned_be32(calipso + 2);
        rcu_read_lock();
        doi_def = calipso_doi_search(doi);
        if (!doi_def)
                goto getattr_return;

        secattr->attr.mls.lvl = calipso[7];
        secattr->flags |= NETLBL_SECATTR_MLS_LVL;

        if (cat_len) {
                ret_val = calipso_map_cat_ntoh(doi_def,
                                               calipso + 10,
                                               cat_len,
                                               secattr);
                if (ret_val != 0) {
                        netlbl_catmap_free(secattr->attr.mls.cat);
                        goto getattr_return;
                }

                if (secattr->attr.mls.cat)
                        secattr->flags |= NETLBL_SECATTR_MLS_CAT;
        }

        secattr->type = NETLBL_NLTYPE_CALIPSO;

getattr_return:
        rcu_read_unlock();
        return ret_val;
}

/* sock functions.
 */

/**
 * calipso_sock_getattr - Get the security attributes from a sock
 * @sk: the sock
 * @secattr: the security attributes
 *
 * Description:
 * Query @sk to see if there is a CALIPSO option attached to the sock and if
 * there is return the CALIPSO security attributes in @secattr.  This function
 * requires that @sk be locked, or privately held, but it does not do any
 * locking itself.  Returns zero on success and negative values on failure.
 *
 */
static int calipso_sock_getattr(struct sock *sk,
                                struct netlbl_lsm_secattr *secattr)
{
        struct ipv6_opt_hdr *hop;
        int opt_len, len, ret_val = -ENOMSG, offset;
        unsigned char *opt;
        struct ipv6_txoptions *txopts = txopt_get(inet6_sk(sk));

        if (!txopts || !txopts->hopopt)
                goto done;

        hop = txopts->hopopt;
        opt = (unsigned char *)hop;
        opt_len = ipv6_optlen(hop);
        offset = sizeof(*hop);
        while (offset < opt_len) {
                len = calipso_tlv_len(hop, offset);
                if (len < 0) {
                        ret_val = len;
                        goto done;
                }
                switch (opt[offset]) {
                case IPV6_TLV_CALIPSO:
                        if (len < CALIPSO_HDR_LEN)
                                ret_val = -EINVAL;
                        else
                                ret_val = calipso_opt_getattr(&opt[offset],
                                                              secattr);
                        goto done;
                default:
                        offset += len;
                        break;
                }
        }
done:
        txopt_put(txopts);
        return ret_val;
}

/**
 * calipso_sock_setattr - Add a CALIPSO option to a socket
 * @sk: the socket
 * @doi_def: the CALIPSO DOI to use
 * @secattr: the specific security attributes of the socket
 *
 * Description:
 * Set the CALIPSO option on the given socket using the DOI definition and
 * security attributes passed to the function.  This function requires
 * exclusive access to @sk, which means it either needs to be in the
 * process of being created or locked.  Returns zero on success and negative
 * values on failure.
 *
 */
static int calipso_sock_setattr(struct sock *sk,
                                const struct calipso_doi *doi_def,
                                const struct netlbl_lsm_secattr *secattr)
{
        int ret_val;
        struct ipv6_opt_hdr *old, *new;
        struct ipv6_txoptions *txopts = txopt_get(inet6_sk(sk));

        old = NULL;
        if (txopts)
                old = txopts->hopopt;

        new = calipso_opt_insert(old, doi_def, secattr);
        txopt_put(txopts);
        if (IS_ERR(new))
                return PTR_ERR(new);

        ret_val = calipso_opt_update(sk, new);

        kfree(new);
        return ret_val;
}

/**
 * calipso_sock_delattr - Delete the CALIPSO option from a socket
 * @sk: the socket
 *
 * Description:
 * Removes the CALIPSO option from a socket, if present.
 *
 */
static void calipso_sock_delattr(struct sock *sk)
{
        struct ipv6_opt_hdr *new_hop;
        struct ipv6_txoptions *txopts = txopt_get(inet6_sk(sk));

        if (!txopts || !txopts->hopopt)
                goto done;

        if (calipso_opt_del(txopts->hopopt, &new_hop))
                goto done;

        calipso_opt_update(sk, new_hop);
        kfree(new_hop);

done:
        txopt_put(txopts);
}

/* request sock functions.
 */

/**
 * calipso_req_setattr - Add a CALIPSO option to a connection request socket
 * @req: the connection request socket
 * @doi_def: the CALIPSO DOI to use
 * @secattr: the specific security attributes of the socket
 *
 * Description:
 * Set the CALIPSO option on the given socket using the DOI definition and
 * security attributes passed to the function.  Returns zero on success and
 * negative values on failure.
 *
 */
static int calipso_req_setattr(struct request_sock *req,
                               const struct calipso_doi *doi_def,
                               const struct netlbl_lsm_secattr *secattr)
{
        struct ipv6_txoptions *txopts;
        struct inet_request_sock *req_inet = inet_rsk(req);
        struct ipv6_opt_hdr *old, *new;
        struct sock *sk = sk_to_full_sk(req_to_sk(req));

        if (req_inet->ipv6_opt && req_inet->ipv6_opt->hopopt)
                old = req_inet->ipv6_opt->hopopt;
        else
                old = NULL;

        new = calipso_opt_insert(old, doi_def, secattr);
        if (IS_ERR(new))
                return PTR_ERR(new);

        txopts = ipv6_renew_options(sk, req_inet->ipv6_opt, IPV6_HOPOPTS, new);

        kfree(new);

        if (IS_ERR(txopts))
                return PTR_ERR(txopts);

        txopts = xchg(&req_inet->ipv6_opt, txopts);
        if (txopts) {
                atomic_sub(txopts->tot_len, &sk->sk_omem_alloc);
                txopt_put(txopts);
        }

        return 0;
}

/**
 * calipso_req_delattr - Delete the CALIPSO option from a request socket
 * @req: the request socket
 *
 * Description:
 * Removes the CALIPSO option from a request socket, if present.
 *
 */
static void calipso_req_delattr(struct request_sock *req)
{
        struct inet_request_sock *req_inet = inet_rsk(req);
        struct ipv6_opt_hdr *new;
        struct ipv6_txoptions *txopts;
        struct sock *sk = sk_to_full_sk(req_to_sk(req));

        if (!req_inet->ipv6_opt || !req_inet->ipv6_opt->hopopt)
                return;

        if (calipso_opt_del(req_inet->ipv6_opt->hopopt, &new))
                return; /* Nothing to do */

        txopts = ipv6_renew_options(sk, req_inet->ipv6_opt, IPV6_HOPOPTS, new);

        if (!IS_ERR(txopts)) {
                txopts = xchg(&req_inet->ipv6_opt, txopts);
                if (txopts) {
                        atomic_sub(txopts->tot_len, &sk->sk_omem_alloc);
                        txopt_put(txopts);
                }
        }
        kfree(new);
}

/* skbuff functions.
 */

/**
 * calipso_skbuff_optptr - Find the CALIPSO option in the packet
 * @skb: the packet
 *
 * Description:
 * Parse the packet's IP header looking for a CALIPSO option.  Returns a pointer
 * to the start of the CALIPSO option on success, NULL if one if not found.
 *
 */
static unsigned char *calipso_skbuff_optptr(const struct sk_buff *skb)
{
        const struct ipv6hdr *ip6_hdr = ipv6_hdr(skb);
        int offset;

        if (ip6_hdr->nexthdr != NEXTHDR_HOP)
                return NULL;

        offset = ipv6_find_tlv(skb, sizeof(*ip6_hdr), IPV6_TLV_CALIPSO);
        if (offset >= 0)
                return (unsigned char *)ip6_hdr + offset;

        return NULL;
}

/**
 * calipso_skbuff_setattr - Set the CALIPSO option on a packet
 * @skb: the packet
 * @doi_def: the CALIPSO DOI to use
 * @secattr: the security attributes
 *
 * Description:
 * Set the CALIPSO option on the given packet based on the security attributes.
 * Returns a pointer to the IP header on success and NULL on failure.
 *
 */
static int calipso_skbuff_setattr(struct sk_buff *skb,
                                  const struct calipso_doi *doi_def,
                                  const struct netlbl_lsm_secattr *secattr)
{
        int ret_val;
        struct ipv6hdr *ip6_hdr;
        struct ipv6_opt_hdr *hop;
        unsigned char buf[CALIPSO_MAX_BUFFER];
        int len_delta, new_end, pad, payload;
        unsigned int start, end;

        ip6_hdr = ipv6_hdr(skb);
        if (ip6_hdr->nexthdr == NEXTHDR_HOP) {
                hop = (struct ipv6_opt_hdr *)(ip6_hdr + 1);
                ret_val = calipso_opt_find(hop, &start, &end);
                if (ret_val && ret_val != -ENOENT)
                        return ret_val;
        } else {
                start = 0;
                end = 0;
        }

        memset(buf, 0, sizeof(buf));
        ret_val = calipso_genopt(buf, start & 3, sizeof(buf), doi_def, secattr);
        if (ret_val < 0)
                return ret_val;

        new_end = start + ret_val;
        /* At this point new_end aligns to 4n, so (new_end & 4) pads to 8n */
        pad = ((new_end & 4) + (end & 7)) & 7;
        len_delta = new_end - (int)end + pad;
        ret_val = skb_cow(skb, skb_headroom(skb) + len_delta);
        if (ret_val < 0)
                return ret_val;

        ip6_hdr = ipv6_hdr(skb); /* Reset as skb_cow() may have moved it */

        if (len_delta) {
                if (len_delta > 0)
                        skb_push(skb, len_delta);
                else
                        skb_pull(skb, -len_delta);
                memmove((char *)ip6_hdr - len_delta, ip6_hdr,
                        sizeof(*ip6_hdr) + start);
                skb_reset_network_header(skb);
                ip6_hdr = ipv6_hdr(skb);
                payload = ntohs(ip6_hdr->payload_len);
                ip6_hdr->payload_len = htons(payload + len_delta);
        }

        hop = (struct ipv6_opt_hdr *)(ip6_hdr + 1);
        if (start == 0) {
                struct ipv6_opt_hdr *new_hop = (struct ipv6_opt_hdr *)buf;

                new_hop->nexthdr = ip6_hdr->nexthdr;
                new_hop->hdrlen = len_delta / 8 - 1;
                ip6_hdr->nexthdr = NEXTHDR_HOP;
        } else {
                hop->hdrlen += len_delta / 8;
        }
        memcpy((char *)hop + start, buf + (start & 3), new_end - start);
        calipso_pad_write((unsigned char *)hop, new_end, pad);

        return 0;
}

/**
 * calipso_skbuff_delattr - Delete any CALIPSO options from a packet
 * @skb: the packet
 *
 * Description:
 * Removes any and all CALIPSO options from the given packet.  Returns zero on
 * success, negative values on failure.
 *
 */
static int calipso_skbuff_delattr(struct sk_buff *skb)
{
        int ret_val;
        struct ipv6hdr *ip6_hdr;
        struct ipv6_opt_hdr *old_hop;
        u32 old_hop_len, start = 0, end = 0, delta, size, pad;

        if (!calipso_skbuff_optptr(skb))
                return 0;

        /* since we are changing the packet we should make a copy */
        ret_val = skb_cow(skb, skb_headroom(skb));
        if (ret_val < 0)
                return ret_val;

        ip6_hdr = ipv6_hdr(skb);
        old_hop = (struct ipv6_opt_hdr *)(ip6_hdr + 1);
        old_hop_len = ipv6_optlen(old_hop);

        ret_val = calipso_opt_find(old_hop, &start, &end);
        if (ret_val)
                return ret_val;

        if (start == sizeof(*old_hop) && end == old_hop_len) {
                /* There's no other option in the header so we delete
                 * the whole thing. */
                delta = old_hop_len;
                size = sizeof(*ip6_hdr);
                ip6_hdr->nexthdr = old_hop->nexthdr;
        } else {
                delta = (end - start) & ~7;
                if (delta)
                        old_hop->hdrlen -= delta / 8;
                pad = (end - start) & 7;
                size = sizeof(*ip6_hdr) + start + pad;
                calipso_pad_write((unsigned char *)old_hop, start, pad);
        }

        if (delta) {
                skb_pull(skb, delta);
                memmove((char *)ip6_hdr + delta, ip6_hdr, size);
                skb_reset_network_header(skb);
        }

        return 0;
}

static const struct netlbl_calipso_ops ops = {
        .doi_add          = calipso_doi_add,
        .doi_free         = calipso_doi_free,
        .doi_remove       = calipso_doi_remove,
        .doi_getdef       = calipso_doi_getdef,
        .doi_putdef       = calipso_doi_putdef,
        .doi_walk         = calipso_doi_walk,
        .sock_getattr     = calipso_sock_getattr,
        .sock_setattr     = calipso_sock_setattr,
        .sock_delattr     = calipso_sock_delattr,
        .req_setattr      = calipso_req_setattr,
        .req_delattr      = calipso_req_delattr,
        .opt_getattr      = calipso_opt_getattr,
        .skbuff_optptr    = calipso_skbuff_optptr,
        .skbuff_setattr   = calipso_skbuff_setattr,
        .skbuff_delattr   = calipso_skbuff_delattr,
        .cache_invalidate = calipso_cache_invalidate,
        .cache_add        = calipso_cache_add
};

/**
 * calipso_init - Initialize the CALIPSO module
 *
 * Description:
 * Initialize the CALIPSO module and prepare it for use.  Returns zero on
 * success and negative values on failure.
 *
 */
int __init calipso_init(void)
{
        int ret_val;

        ret_val = calipso_cache_init();
        if (!ret_val)
                netlbl_calipso_ops_register(&ops);
        return ret_val;
}

void calipso_exit(void)
{
        netlbl_calipso_ops_register(NULL);
        calipso_cache_invalidate();
        kfree(calipso_cache);
}













































































































































    1 
























































    1 















    1 







    1 






















    1 











































































































































    3 





    3 
    3 










    3 
























































































































































    4 


    4 

























































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
// SPDX-License-Identifier: GPL-2.0
/*
 *  linux/mm/mempool.c
 *
 *  memory buffer pool support. Such pools are mostly used
 *  for guaranteed, deadlock-free memory allocations during
 *  extreme VM load.
 *
 *  started by Ingo Molnar, Copyright (C) 2001
 *  debugging by David Rientjes, Copyright (C) 2015
 */

#include <linux/mm.h>
#include <linux/slab.h>
#include <linux/highmem.h>
#include <linux/kasan.h>
#include <linux/kmemleak.h>
#include <linux/export.h>
#include <linux/mempool.h>
#include <linux/writeback.h>
#include "slab.h"

#ifdef CONFIG_SLUB_DEBUG_ON
static void poison_error(mempool_t *pool, void *element, size_t size,
                         size_t byte)
{
        const int nr = pool->curr_nr;
        const int start = max_t(int, byte - (BITS_PER_LONG / 8), 0);
        const int end = min_t(int, byte + (BITS_PER_LONG / 8), size);
        int i;

        pr_err("BUG: mempool element poison mismatch\n");
        pr_err("Mempool %p size %zu\n", pool, size);
        pr_err(" nr=%d @ %p: %s0x", nr, element, start > 0 ? "... " : "");
        for (i = start; i < end; i++)
                pr_cont("%x ", *(u8 *)(element + i));
        pr_cont("%s\n", end < size ? "..." : "");
        dump_stack();
}

static void __check_element(mempool_t *pool, void *element, size_t size)
{
        u8 *obj = element;
        size_t i;

        for (i = 0; i < size; i++) {
                u8 exp = (i < size - 1) ? POISON_FREE : POISON_END;

                if (obj[i] != exp) {
                        poison_error(pool, element, size, i);
                        return;
                }
        }
        memset(obj, POISON_INUSE, size);
}

static void check_element(mempool_t *pool, void *element)
{
        /* Skip checking: KASAN might save its metadata in the element. */
        if (kasan_enabled())
                return;

        /* Mempools backed by slab allocator */
        if (pool->free == mempool_kfree) {
                __check_element(pool, element, (size_t)pool->pool_data);
        } else if (pool->free == mempool_free_slab) {
                __check_element(pool, element, kmem_cache_size(pool->pool_data));
        } else if (pool->free == mempool_free_pages) {
                /* Mempools backed by page allocator */
                int order = (int)(long)pool->pool_data;
                void *addr = kmap_local_page((struct page *)element);

                __check_element(pool, addr, 1UL << (PAGE_SHIFT + order));
                kunmap_local(addr);
        }
}

static void __poison_element(void *element, size_t size)
{
        u8 *obj = element;

        memset(obj, POISON_FREE, size - 1);
        obj[size - 1] = POISON_END;
}

static void poison_element(mempool_t *pool, void *element)
{
        /* Skip poisoning: KASAN might save its metadata in the element. */
        if (kasan_enabled())
                return;

        /* Mempools backed by slab allocator */
        if (pool->alloc == mempool_kmalloc) {
                __poison_element(element, (size_t)pool->pool_data);
        } else if (pool->alloc == mempool_alloc_slab) {
                __poison_element(element, kmem_cache_size(pool->pool_data));
        } else if (pool->alloc == mempool_alloc_pages) {
                /* Mempools backed by page allocator */
                int order = (int)(long)pool->pool_data;
                void *addr = kmap_local_page((struct page *)element);

                __poison_element(addr, 1UL << (PAGE_SHIFT + order));
                kunmap_local(addr);
        }
}
#else /* CONFIG_SLUB_DEBUG_ON */
static inline void check_element(mempool_t *pool, void *element)
{
}
static inline void poison_element(mempool_t *pool, void *element)
{
}
#endif /* CONFIG_SLUB_DEBUG_ON */

static __always_inline bool kasan_poison_element(mempool_t *pool, void *element)
{
        if (pool->alloc == mempool_alloc_slab || pool->alloc == mempool_kmalloc)
                return kasan_mempool_poison_object(element);
        else if (pool->alloc == mempool_alloc_pages)
                return kasan_mempool_poison_pages(element,
                                                (unsigned long)pool->pool_data);
        return true;
}

static void kasan_unpoison_element(mempool_t *pool, void *element)
{
        if (pool->alloc == mempool_kmalloc)
                kasan_mempool_unpoison_object(element, (size_t)pool->pool_data);
        else if (pool->alloc == mempool_alloc_slab)
                kasan_mempool_unpoison_object(element,
                                              kmem_cache_size(pool->pool_data));
        else if (pool->alloc == mempool_alloc_pages)
                kasan_mempool_unpoison_pages(element,
                                             (unsigned long)pool->pool_data);
}

static __always_inline void add_element(mempool_t *pool, void *element)
{
        BUG_ON(pool->curr_nr >= pool->min_nr);
        poison_element(pool, element);
        if (kasan_poison_element(pool, element))
                pool->elements[pool->curr_nr++] = element;
}

static void *remove_element(mempool_t *pool)
{
        void *element = pool->elements[--pool->curr_nr];

        BUG_ON(pool->curr_nr < 0);
        kasan_unpoison_element(pool, element);
        check_element(pool, element);
        return element;
}

/**
 * mempool_exit - exit a mempool initialized with mempool_init()
 * @pool:      pointer to the memory pool which was initialized with
 *             mempool_init().
 *
 * Free all reserved elements in @pool and @pool itself.  This function
 * only sleeps if the free_fn() function sleeps.
 *
 * May be called on a zeroed but uninitialized mempool (i.e. allocated with
 * kzalloc()).
 */
void mempool_exit(mempool_t *pool)
{
        while (pool->curr_nr) {
                void *element = remove_element(pool);
                pool->free(element, pool->pool_data);
        }
        kfree(pool->elements);
        pool->elements = NULL;
}
EXPORT_SYMBOL(mempool_exit);

/**
 * mempool_destroy - deallocate a memory pool
 * @pool:      pointer to the memory pool which was allocated via
 *             mempool_create().
 *
 * Free all reserved elements in @pool and @pool itself.  This function
 * only sleeps if the free_fn() function sleeps.
 */
void mempool_destroy(mempool_t *pool)
{
        if (unlikely(!pool))
                return;

        mempool_exit(pool);
        kfree(pool);
}
EXPORT_SYMBOL(mempool_destroy);

int mempool_init_node(mempool_t *pool, int min_nr, mempool_alloc_t *alloc_fn,
                      mempool_free_t *free_fn, void *pool_data,
                      gfp_t gfp_mask, int node_id)
{
        spin_lock_init(&pool->lock);
        pool->min_nr        = min_nr;
        pool->pool_data = pool_data;
        pool->alloc        = alloc_fn;
        pool->free        = free_fn;
        init_waitqueue_head(&pool->wait);

        pool->elements = kmalloc_array_node(min_nr, sizeof(void *),
                                            gfp_mask, node_id);
        if (!pool->elements)
                return -ENOMEM;

        /*
         * First pre-allocate the guaranteed number of buffers.
         */
        while (pool->curr_nr < pool->min_nr) {
                void *element;

                element = pool->alloc(gfp_mask, pool->pool_data);
                if (unlikely(!element)) {
                        mempool_exit(pool);
                        return -ENOMEM;
                }
                add_element(pool, element);
        }

        return 0;
}
EXPORT_SYMBOL(mempool_init_node);

/**
 * mempool_init - initialize a memory pool
 * @pool:      pointer to the memory pool that should be initialized
 * @min_nr:    the minimum number of elements guaranteed to be
 *             allocated for this pool.
 * @alloc_fn:  user-defined element-allocation function.
 * @free_fn:   user-defined element-freeing function.
 * @pool_data: optional private data available to the user-defined functions.
 *
 * Like mempool_create(), but initializes the pool in (i.e. embedded in another
 * structure).
 *
 * Return: %0 on success, negative error code otherwise.
 */
int mempool_init_noprof(mempool_t *pool, int min_nr, mempool_alloc_t *alloc_fn,
                        mempool_free_t *free_fn, void *pool_data)
{
        return mempool_init_node(pool, min_nr, alloc_fn, free_fn,
                                 pool_data, GFP_KERNEL, NUMA_NO_NODE);

}
EXPORT_SYMBOL(mempool_init_noprof);

/**
 * mempool_create_node - create a memory pool
 * @min_nr:    the minimum number of elements guaranteed to be
 *             allocated for this pool.
 * @alloc_fn:  user-defined element-allocation function.
 * @free_fn:   user-defined element-freeing function.
 * @pool_data: optional private data available to the user-defined functions.
 * @gfp_mask:  memory allocation flags
 * @node_id:   numa node to allocate on
 *
 * this function creates and allocates a guaranteed size, preallocated
 * memory pool. The pool can be used from the mempool_alloc() and mempool_free()
 * functions. This function might sleep. Both the alloc_fn() and the free_fn()
 * functions might sleep - as long as the mempool_alloc() function is not called
 * from IRQ contexts.
 *
 * Return: pointer to the created memory pool object or %NULL on error.
 */
mempool_t *mempool_create_node_noprof(int min_nr, mempool_alloc_t *alloc_fn,
                                      mempool_free_t *free_fn, void *pool_data,
                                      gfp_t gfp_mask, int node_id)
{
        mempool_t *pool;

        pool = kmalloc_node_noprof(sizeof(*pool), gfp_mask | __GFP_ZERO, node_id);
        if (!pool)
                return NULL;

        if (mempool_init_node(pool, min_nr, alloc_fn, free_fn, pool_data,
                              gfp_mask, node_id)) {
                kfree(pool);
                return NULL;
        }

        return pool;
}
EXPORT_SYMBOL(mempool_create_node_noprof);

/**
 * mempool_resize - resize an existing memory pool
 * @pool:       pointer to the memory pool which was allocated via
 *              mempool_create().
 * @new_min_nr: the new minimum number of elements guaranteed to be
 *              allocated for this pool.
 *
 * This function shrinks/grows the pool. In the case of growing,
 * it cannot be guaranteed that the pool will be grown to the new
 * size immediately, but new mempool_free() calls will refill it.
 * This function may sleep.
 *
 * Note, the caller must guarantee that no mempool_destroy is called
 * while this function is running. mempool_alloc() & mempool_free()
 * might be called (eg. from IRQ contexts) while this function executes.
 *
 * Return: %0 on success, negative error code otherwise.
 */
int mempool_resize(mempool_t *pool, int new_min_nr)
{
        void *element;
        void **new_elements;
        unsigned long flags;

        BUG_ON(new_min_nr <= 0);
        might_sleep();

        spin_lock_irqsave(&pool->lock, flags);
        if (new_min_nr <= pool->min_nr) {
                while (new_min_nr < pool->curr_nr) {
                        element = remove_element(pool);
                        spin_unlock_irqrestore(&pool->lock, flags);
                        pool->free(element, pool->pool_data);
                        spin_lock_irqsave(&pool->lock, flags);
                }
                pool->min_nr = new_min_nr;
                goto out_unlock;
        }
        spin_unlock_irqrestore(&pool->lock, flags);

        /* Grow the pool */
        new_elements = kmalloc_array(new_min_nr, sizeof(*new_elements),
                                     GFP_KERNEL);
        if (!new_elements)
                return -ENOMEM;

        spin_lock_irqsave(&pool->lock, flags);
        if (unlikely(new_min_nr <= pool->min_nr)) {
                /* Raced, other resize will do our work */
                spin_unlock_irqrestore(&pool->lock, flags);
                kfree(new_elements);
                goto out;
        }
        memcpy(new_elements, pool->elements,
                        pool->curr_nr * sizeof(*new_elements));
        kfree(pool->elements);
        pool->elements = new_elements;
        pool->min_nr = new_min_nr;

        while (pool->curr_nr < pool->min_nr) {
                spin_unlock_irqrestore(&pool->lock, flags);
                element = pool->alloc(GFP_KERNEL, pool->pool_data);
                if (!element)
                        goto out;
                spin_lock_irqsave(&pool->lock, flags);
                if (pool->curr_nr < pool->min_nr) {
                        add_element(pool, element);
                } else {
                        spin_unlock_irqrestore(&pool->lock, flags);
                        pool->free(element, pool->pool_data);        /* Raced */
                        goto out;
                }
        }
out_unlock:
        spin_unlock_irqrestore(&pool->lock, flags);
out:
        return 0;
}
EXPORT_SYMBOL(mempool_resize);

/**
 * mempool_alloc - allocate an element from a specific memory pool
 * @pool:      pointer to the memory pool which was allocated via
 *             mempool_create().
 * @gfp_mask:  the usual allocation bitmask.
 *
 * this function only sleeps if the alloc_fn() function sleeps or
 * returns NULL. Note that due to preallocation, this function
 * *never* fails when called from process contexts. (it might
 * fail if called from an IRQ context.)
 * Note: using __GFP_ZERO is not supported.
 *
 * Return: pointer to the allocated element or %NULL on error.
 */
void *mempool_alloc_noprof(mempool_t *pool, gfp_t gfp_mask)
{
        void *element;
        unsigned long flags;
        wait_queue_entry_t wait;
        gfp_t gfp_temp;

        VM_WARN_ON_ONCE(gfp_mask & __GFP_ZERO);
        might_alloc(gfp_mask);

        gfp_mask |= __GFP_NOMEMALLOC;        /* don't allocate emergency reserves */
        gfp_mask |= __GFP_NORETRY;        /* don't loop in __alloc_pages */
        gfp_mask |= __GFP_NOWARN;        /* failures are OK */

        gfp_temp = gfp_mask & ~(__GFP_DIRECT_RECLAIM|__GFP_IO);

repeat_alloc:

        element = pool->alloc(gfp_temp, pool->pool_data);
        if (likely(element != NULL))
                return element;

        spin_lock_irqsave(&pool->lock, flags);
        if (likely(pool->curr_nr)) {
                element = remove_element(pool);
                spin_unlock_irqrestore(&pool->lock, flags);
                /* paired with rmb in mempool_free(), read comment there */
                smp_wmb();
                /*
                 * Update the allocation stack trace as this is more useful
                 * for debugging.
                 */
                kmemleak_update_trace(element);
                return element;
        }

        /*
         * We use gfp mask w/o direct reclaim or IO for the first round.  If
         * alloc failed with that and @pool was empty, retry immediately.
         */
        if (gfp_temp != gfp_mask) {
                spin_unlock_irqrestore(&pool->lock, flags);
                gfp_temp = gfp_mask;
                goto repeat_alloc;
        }

        /* We must not sleep if !__GFP_DIRECT_RECLAIM */
        if (!(gfp_mask & __GFP_DIRECT_RECLAIM)) {
                spin_unlock_irqrestore(&pool->lock, flags);
                return NULL;
        }

        /* Let's wait for someone else to return an element to @pool */
        init_wait(&wait);
        prepare_to_wait(&pool->wait, &wait, TASK_UNINTERRUPTIBLE);

        spin_unlock_irqrestore(&pool->lock, flags);

        /*
         * FIXME: this should be io_schedule().  The timeout is there as a
         * workaround for some DM problems in 2.6.18.
         */
        io_schedule_timeout(5*HZ);

        finish_wait(&pool->wait, &wait);
        goto repeat_alloc;
}
EXPORT_SYMBOL(mempool_alloc_noprof);

/**
 * mempool_alloc_preallocated - allocate an element from preallocated elements
 *                              belonging to a specific memory pool
 * @pool:      pointer to the memory pool which was allocated via
 *             mempool_create().
 *
 * This function is similar to mempool_alloc, but it only attempts allocating
 * an element from the preallocated elements. It does not sleep and immediately
 * returns if no preallocated elements are available.
 *
 * Return: pointer to the allocated element or %NULL if no elements are
 * available.
 */
void *mempool_alloc_preallocated(mempool_t *pool)
{
        void *element;
        unsigned long flags;

        spin_lock_irqsave(&pool->lock, flags);
        if (likely(pool->curr_nr)) {
                element = remove_element(pool);
                spin_unlock_irqrestore(&pool->lock, flags);
                /* paired with rmb in mempool_free(), read comment there */
                smp_wmb();
                /*
                 * Update the allocation stack trace as this is more useful
                 * for debugging.
                 */
                kmemleak_update_trace(element);
                return element;
        }
        spin_unlock_irqrestore(&pool->lock, flags);

        return NULL;
}
EXPORT_SYMBOL(mempool_alloc_preallocated);

/**
 * mempool_free - return an element to the pool.
 * @element:   pool element pointer.
 * @pool:      pointer to the memory pool which was allocated via
 *             mempool_create().
 *
 * this function only sleeps if the free_fn() function sleeps.
 */
void mempool_free(void *element, mempool_t *pool)
{
        unsigned long flags;

        if (unlikely(element == NULL))
                return;

        /*
         * Paired with the wmb in mempool_alloc().  The preceding read is
         * for @element and the following @pool->curr_nr.  This ensures
         * that the visible value of @pool->curr_nr is from after the
         * allocation of @element.  This is necessary for fringe cases
         * where @element was passed to this task without going through
         * barriers.
         *
         * For example, assume @p is %NULL at the beginning and one task
         * performs "p = mempool_alloc(...);" while another task is doing
         * "while (!p) cpu_relax(); mempool_free(p, ...);".  This function
         * may end up using curr_nr value which is from before allocation
         * of @p without the following rmb.
         */
        smp_rmb();

        /*
         * For correctness, we need a test which is guaranteed to trigger
         * if curr_nr + #allocated == min_nr.  Testing curr_nr < min_nr
         * without locking achieves that and refilling as soon as possible
         * is desirable.
         *
         * Because curr_nr visible here is always a value after the
         * allocation of @element, any task which decremented curr_nr below
         * min_nr is guaranteed to see curr_nr < min_nr unless curr_nr gets
         * incremented to min_nr afterwards.  If curr_nr gets incremented
         * to min_nr after the allocation of @element, the elements
         * allocated after that are subject to the same guarantee.
         *
         * Waiters happen iff curr_nr is 0 and the above guarantee also
         * ensures that there will be frees which return elements to the
         * pool waking up the waiters.
         */
        if (unlikely(READ_ONCE(pool->curr_nr) < pool->min_nr)) {
                spin_lock_irqsave(&pool->lock, flags);
                if (likely(pool->curr_nr < pool->min_nr)) {
                        add_element(pool, element);
                        spin_unlock_irqrestore(&pool->lock, flags);
                        wake_up(&pool->wait);
                        return;
                }
                spin_unlock_irqrestore(&pool->lock, flags);
        }
        pool->free(element, pool->pool_data);
}
EXPORT_SYMBOL(mempool_free);

/*
 * A commonly used alloc and free fn.
 */
void *mempool_alloc_slab(gfp_t gfp_mask, void *pool_data)
{
        struct kmem_cache *mem = pool_data;
        VM_BUG_ON(mem->ctor);
        return kmem_cache_alloc_noprof(mem, gfp_mask);
}
EXPORT_SYMBOL(mempool_alloc_slab);

void mempool_free_slab(void *element, void *pool_data)
{
        struct kmem_cache *mem = pool_data;
        kmem_cache_free(mem, element);
}
EXPORT_SYMBOL(mempool_free_slab);

/*
 * A commonly used alloc and free fn that kmalloc/kfrees the amount of memory
 * specified by pool_data
 */
void *mempool_kmalloc(gfp_t gfp_mask, void *pool_data)
{
        size_t size = (size_t)pool_data;
        return kmalloc_noprof(size, gfp_mask);
}
EXPORT_SYMBOL(mempool_kmalloc);

void mempool_kfree(void *element, void *pool_data)
{
        kfree(element);
}
EXPORT_SYMBOL(mempool_kfree);

void *mempool_kvmalloc(gfp_t gfp_mask, void *pool_data)
{
        size_t size = (size_t)pool_data;
        return kvmalloc(size, gfp_mask);
}
EXPORT_SYMBOL(mempool_kvmalloc);

void mempool_kvfree(void *element, void *pool_data)
{
        kvfree(element);
}
EXPORT_SYMBOL(mempool_kvfree);

/*
 * A simple mempool-backed page allocator that allocates pages
 * of the order specified by pool_data.
 */
void *mempool_alloc_pages(gfp_t gfp_mask, void *pool_data)
{
        int order = (int)(long)pool_data;
        return alloc_pages_noprof(gfp_mask, order);
}
EXPORT_SYMBOL(mempool_alloc_pages);

void mempool_free_pages(void *element, void *pool_data)
{
        int order = (int)(long)pool_data;
        __free_pages(element, order);
}
EXPORT_SYMBOL(mempool_free_pages);











































    2 


    1 
    2 


























































    8 


    8 






























































    1 
















    1 





    1 
    1 


    1 










1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
// SPDX-License-Identifier: GPL-2.0
/* Copyright (C) B.A.T.M.A.N. contributors:
 *
 * Marek Lindner, Simon Wunderlich
 */

#include "main.h"

#include <linux/errno.h>
#include <linux/list.h>
#include <linux/moduleparam.h>
#include <linux/netlink.h>
#include <linux/printk.h>
#include <linux/skbuff.h>
#include <linux/stddef.h>
#include <linux/string.h>
#include <net/genetlink.h>
#include <net/netlink.h>
#include <uapi/linux/batman_adv.h>

#include "bat_algo.h"
#include "netlink.h"

char batadv_routing_algo[20] = "BATMAN_IV";
static struct hlist_head batadv_algo_list;

/**
 * batadv_algo_init() - Initialize batman-adv algorithm management data
 *  structures
 */
void batadv_algo_init(void)
{
        INIT_HLIST_HEAD(&batadv_algo_list);
}

/**
 * batadv_algo_get() - Search for algorithm with specific name
 * @name: algorithm name to find
 *
 * Return: Pointer to batadv_algo_ops on success, NULL otherwise
 */
struct batadv_algo_ops *batadv_algo_get(const char *name)
{
        struct batadv_algo_ops *bat_algo_ops = NULL, *bat_algo_ops_tmp;

        hlist_for_each_entry(bat_algo_ops_tmp, &batadv_algo_list, list) {
                if (strcmp(bat_algo_ops_tmp->name, name) != 0)
                        continue;

                bat_algo_ops = bat_algo_ops_tmp;
                break;
        }

        return bat_algo_ops;
}

/**
 * batadv_algo_register() - Register callbacks for a mesh algorithm
 * @bat_algo_ops: mesh algorithm callbacks to add
 *
 * Return: 0 on success or negative error number in case of failure
 */
int batadv_algo_register(struct batadv_algo_ops *bat_algo_ops)
{
        struct batadv_algo_ops *bat_algo_ops_tmp;

        bat_algo_ops_tmp = batadv_algo_get(bat_algo_ops->name);
        if (bat_algo_ops_tmp) {
                pr_info("Trying to register already registered routing algorithm: %s\n",
                        bat_algo_ops->name);
                return -EEXIST;
        }

        /* all algorithms must implement all ops (for now) */
        if (!bat_algo_ops->iface.enable ||
            !bat_algo_ops->iface.disable ||
            !bat_algo_ops->iface.update_mac ||
            !bat_algo_ops->iface.primary_set ||
            !bat_algo_ops->neigh.cmp ||
            !bat_algo_ops->neigh.is_similar_or_better) {
                pr_info("Routing algo '%s' does not implement required ops\n",
                        bat_algo_ops->name);
                return -EINVAL;
        }

        INIT_HLIST_NODE(&bat_algo_ops->list);
        hlist_add_head(&bat_algo_ops->list, &batadv_algo_list);

        return 0;
}

/**
 * batadv_algo_select() - Select algorithm of soft interface
 * @bat_priv: the bat priv with all the soft interface information
 * @name: name of the algorithm to select
 *
 * The algorithm callbacks for the soft interface will be set when the algorithm
 * with the correct name was found. Any previous selected algorithm will not be
 * deinitialized and the new selected algorithm will also not be initialized.
 * It is therefore not allowed to call batadv_algo_select outside the creation
 * function of the soft interface.
 *
 * Return: 0 on success or negative error number in case of failure
 */
int batadv_algo_select(struct batadv_priv *bat_priv, const char *name)
{
        struct batadv_algo_ops *bat_algo_ops;

        bat_algo_ops = batadv_algo_get(name);
        if (!bat_algo_ops)
                return -EINVAL;

        bat_priv->algo_ops = bat_algo_ops;

        return 0;
}

static int batadv_param_set_ra(const char *val, const struct kernel_param *kp)
{
        struct batadv_algo_ops *bat_algo_ops;
        char *algo_name = (char *)val;
        size_t name_len = strlen(algo_name);

        if (name_len > 0 && algo_name[name_len - 1] == '\n')
                algo_name[name_len - 1] = '\0';

        bat_algo_ops = batadv_algo_get(algo_name);
        if (!bat_algo_ops) {
                pr_err("Routing algorithm '%s' is not supported\n", algo_name);
                return -EINVAL;
        }

        return param_set_copystring(algo_name, kp);
}

static const struct kernel_param_ops batadv_param_ops_ra = {
        .set = batadv_param_set_ra,
        .get = param_get_string,
};

static struct kparam_string batadv_param_string_ra = {
        .maxlen = sizeof(batadv_routing_algo),
        .string = batadv_routing_algo,
};

module_param_cb(routing_algo, &batadv_param_ops_ra, &batadv_param_string_ra,
                0644);

/**
 * batadv_algo_dump_entry() - fill in information about one supported routing
 *  algorithm
 * @msg: netlink message to be sent back
 * @portid: Port to reply to
 * @seq: Sequence number of message
 * @bat_algo_ops: Algorithm to be dumped
 *
 * Return: Error number, or 0 on success
 */
static int batadv_algo_dump_entry(struct sk_buff *msg, u32 portid, u32 seq,
                                  struct batadv_algo_ops *bat_algo_ops)
{
        void *hdr;

        hdr = genlmsg_put(msg, portid, seq, &batadv_netlink_family,
                          NLM_F_MULTI, BATADV_CMD_GET_ROUTING_ALGOS);
        if (!hdr)
                return -EMSGSIZE;

        if (nla_put_string(msg, BATADV_ATTR_ALGO_NAME, bat_algo_ops->name))
                goto nla_put_failure;

        genlmsg_end(msg, hdr);
        return 0;

 nla_put_failure:
        genlmsg_cancel(msg, hdr);
        return -EMSGSIZE;
}

/**
 * batadv_algo_dump() - fill in information about supported routing
 *  algorithms
 * @msg: netlink message to be sent back
 * @cb: Parameters to the netlink request
 *
 * Return: Length of reply message.
 */
int batadv_algo_dump(struct sk_buff *msg, struct netlink_callback *cb)
{
        int portid = NETLINK_CB(cb->skb).portid;
        struct batadv_algo_ops *bat_algo_ops;
        int skip = cb->args[0];
        int i = 0;

        hlist_for_each_entry(bat_algo_ops, &batadv_algo_list, list) {
                if (i++ < skip)
                        continue;

                if (batadv_algo_dump_entry(msg, portid, cb->nlh->nlmsg_seq,
                                           bat_algo_ops)) {
                        i--;
                        break;
                }
        }

        cb->args[0] = i;

        return msg->len;
}






















































































































































































































































































































































































































































































































































































































































































    1 


    1 































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
// SPDX-License-Identifier: GPL-2.0
/* net/atm/svc.c - ATM SVC sockets */

/* Written 1995-2000 by Werner Almesberger, EPFL LRC/ICA */

#define pr_fmt(fmt) KBUILD_MODNAME ":%s: " fmt, __func__

#include <linux/string.h>
#include <linux/net.h>                /* struct socket, struct proto_ops */
#include <linux/errno.h>        /* error codes */
#include <linux/kernel.h>        /* printk */
#include <linux/skbuff.h>
#include <linux/wait.h>
#include <linux/sched/signal.h>
#include <linux/fcntl.h>        /* O_NONBLOCK */
#include <linux/init.h>
#include <linux/atm.h>                /* ATM stuff */
#include <linux/atmsap.h>
#include <linux/atmsvc.h>
#include <linux/atmdev.h>
#include <linux/bitops.h>
#include <net/sock.h>                /* for sock_no_* */
#include <linux/uaccess.h>
#include <linux/export.h>

#include "resources.h"
#include "common.h"                /* common for PVCs and SVCs */
#include "signaling.h"
#include "addr.h"

#ifdef CONFIG_COMPAT
/* It actually takes struct sockaddr_atmsvc, not struct atm_iobuf */
#define COMPAT_ATM_ADDPARTY _IOW('a', ATMIOC_SPECIAL + 4, struct compat_atm_iobuf)
#endif

static int svc_create(struct net *net, struct socket *sock, int protocol,
                      int kern);

/*
 * Note: since all this is still nicely synchronized with the signaling demon,
 *       there's no need to protect sleep loops with clis. If signaling is
 *       moved into the kernel, that would change.
 */


static int svc_shutdown(struct socket *sock, int how)
{
        return 0;
}

static void svc_disconnect(struct atm_vcc *vcc)
{
        DEFINE_WAIT(wait);
        struct sk_buff *skb;
        struct sock *sk = sk_atm(vcc);

        pr_debug("%p\n", vcc);
        if (test_bit(ATM_VF_REGIS, &vcc->flags)) {
                sigd_enq(vcc, as_close, NULL, NULL, NULL);
                for (;;) {
                        prepare_to_wait(sk_sleep(sk), &wait, TASK_UNINTERRUPTIBLE);
                        if (test_bit(ATM_VF_RELEASED, &vcc->flags) || !sigd)
                                break;
                        schedule();
                }
                finish_wait(sk_sleep(sk), &wait);
        }
        /* beware - socket is still in use by atmsigd until the last
           as_indicate has been answered */
        while ((skb = skb_dequeue(&sk->sk_receive_queue)) != NULL) {
                atm_return(vcc, skb->truesize);
                pr_debug("LISTEN REL\n");
                sigd_enq2(NULL, as_reject, vcc, NULL, NULL, &vcc->qos, 0);
                dev_kfree_skb(skb);
        }
        clear_bit(ATM_VF_REGIS, &vcc->flags);
        /* ... may retry later */
}

static int svc_release(struct socket *sock)
{
        struct sock *sk = sock->sk;
        struct atm_vcc *vcc;

        if (sk) {
                vcc = ATM_SD(sock);
                pr_debug("%p\n", vcc);
                clear_bit(ATM_VF_READY, &vcc->flags);
                /*
                 * VCC pointer is used as a reference,
                 * so we must not free it (thereby subjecting it to re-use)
                 * before all pending connections are closed
                 */
                svc_disconnect(vcc);
                vcc_release(sock);
        }
        return 0;
}

static int svc_bind(struct socket *sock, struct sockaddr *sockaddr,
                    int sockaddr_len)
{
        DEFINE_WAIT(wait);
        struct sock *sk = sock->sk;
        struct sockaddr_atmsvc *addr;
        struct atm_vcc *vcc;
        int error;

        if (sockaddr_len != sizeof(struct sockaddr_atmsvc))
                return -EINVAL;
        lock_sock(sk);
        if (sock->state == SS_CONNECTED) {
                error = -EISCONN;
                goto out;
        }
        if (sock->state != SS_UNCONNECTED) {
                error = -EINVAL;
                goto out;
        }
        vcc = ATM_SD(sock);
        addr = (struct sockaddr_atmsvc *) sockaddr;
        if (addr->sas_family != AF_ATMSVC) {
                error = -EAFNOSUPPORT;
                goto out;
        }
        clear_bit(ATM_VF_BOUND, &vcc->flags);
            /* failing rebind will kill old binding */
        /* @@@ check memory (de)allocation on rebind */
        if (!test_bit(ATM_VF_HASQOS, &vcc->flags)) {
                error = -EBADFD;
                goto out;
        }
        vcc->local = *addr;
        set_bit(ATM_VF_WAITING, &vcc->flags);
        sigd_enq(vcc, as_bind, NULL, NULL, &vcc->local);
        for (;;) {
                prepare_to_wait(sk_sleep(sk), &wait, TASK_UNINTERRUPTIBLE);
                if (!test_bit(ATM_VF_WAITING, &vcc->flags) || !sigd)
                        break;
                schedule();
        }
        finish_wait(sk_sleep(sk), &wait);
        clear_bit(ATM_VF_REGIS, &vcc->flags); /* doesn't count */
        if (!sigd) {
                error = -EUNATCH;
                goto out;
        }
        if (!sk->sk_err)
                set_bit(ATM_VF_BOUND, &vcc->flags);
        error = -sk->sk_err;
out:
        release_sock(sk);
        return error;
}

static int svc_connect(struct socket *sock, struct sockaddr *sockaddr,
                       int sockaddr_len, int flags)
{
        DEFINE_WAIT(wait);
        struct sock *sk = sock->sk;
        struct sockaddr_atmsvc *addr;
        struct atm_vcc *vcc = ATM_SD(sock);
        int error;

        pr_debug("%p\n", vcc);
        lock_sock(sk);
        if (sockaddr_len != sizeof(struct sockaddr_atmsvc)) {
                error = -EINVAL;
                goto out;
        }

        switch (sock->state) {
        default:
                error = -EINVAL;
                goto out;
        case SS_CONNECTED:
                error = -EISCONN;
                goto out;
        case SS_CONNECTING:
                if (test_bit(ATM_VF_WAITING, &vcc->flags)) {
                        error = -EALREADY;
                        goto out;
                }
                sock->state = SS_UNCONNECTED;
                if (sk->sk_err) {
                        error = -sk->sk_err;
                        goto out;
                }
                break;
        case SS_UNCONNECTED:
                addr = (struct sockaddr_atmsvc *) sockaddr;
                if (addr->sas_family != AF_ATMSVC) {
                        error = -EAFNOSUPPORT;
                        goto out;
                }
                if (!test_bit(ATM_VF_HASQOS, &vcc->flags)) {
                        error = -EBADFD;
                        goto out;
                }
                if (vcc->qos.txtp.traffic_class == ATM_ANYCLASS ||
                    vcc->qos.rxtp.traffic_class == ATM_ANYCLASS) {
                        error = -EINVAL;
                        goto out;
                }
                if (!vcc->qos.txtp.traffic_class &&
                    !vcc->qos.rxtp.traffic_class) {
                        error = -EINVAL;
                        goto out;
                }
                vcc->remote = *addr;
                set_bit(ATM_VF_WAITING, &vcc->flags);
                sigd_enq(vcc, as_connect, NULL, NULL, &vcc->remote);
                if (flags & O_NONBLOCK) {
                        sock->state = SS_CONNECTING;
                        error = -EINPROGRESS;
                        goto out;
                }
                error = 0;
                prepare_to_wait(sk_sleep(sk), &wait, TASK_INTERRUPTIBLE);
                while (test_bit(ATM_VF_WAITING, &vcc->flags) && sigd) {
                        schedule();
                        if (!signal_pending(current)) {
                                prepare_to_wait(sk_sleep(sk), &wait,
                                                TASK_INTERRUPTIBLE);
                                continue;
                        }
                        pr_debug("*ABORT*\n");
                        /*
                         * This is tricky:
                         *   Kernel ---close--> Demon
                         *   Kernel <--close--- Demon
                         * or
                         *   Kernel ---close--> Demon
                         *   Kernel <--error--- Demon
                         * or
                         *   Kernel ---close--> Demon
                         *   Kernel <--okay---- Demon
                         *   Kernel <--close--- Demon
                         */
                        sigd_enq(vcc, as_close, NULL, NULL, NULL);
                        while (test_bit(ATM_VF_WAITING, &vcc->flags) && sigd) {
                                prepare_to_wait(sk_sleep(sk), &wait,
                                                TASK_INTERRUPTIBLE);
                                schedule();
                        }
                        if (!sk->sk_err)
                                while (!test_bit(ATM_VF_RELEASED, &vcc->flags) &&
                                       sigd) {
                                        prepare_to_wait(sk_sleep(sk), &wait,
                                                        TASK_INTERRUPTIBLE);
                                        schedule();
                                }
                        clear_bit(ATM_VF_REGIS, &vcc->flags);
                        clear_bit(ATM_VF_RELEASED, &vcc->flags);
                        clear_bit(ATM_VF_CLOSE, &vcc->flags);
                            /* we're gone now but may connect later */
                        error = -EINTR;
                        break;
                }
                finish_wait(sk_sleep(sk), &wait);
                if (error)
                        goto out;
                if (!sigd) {
                        error = -EUNATCH;
                        goto out;
                }
                if (sk->sk_err) {
                        error = -sk->sk_err;
                        goto out;
                }
        }

        vcc->qos.txtp.max_pcr = SELECT_TOP_PCR(vcc->qos.txtp);
        vcc->qos.txtp.pcr = 0;
        vcc->qos.txtp.min_pcr = 0;

        error = vcc_connect(sock, vcc->itf, vcc->vpi, vcc->vci);
        if (!error)
                sock->state = SS_CONNECTED;
        else
                (void)svc_disconnect(vcc);
out:
        release_sock(sk);
        return error;
}

static int svc_listen(struct socket *sock, int backlog)
{
        DEFINE_WAIT(wait);
        struct sock *sk = sock->sk;
        struct atm_vcc *vcc = ATM_SD(sock);
        int error;

        pr_debug("%p\n", vcc);
        lock_sock(sk);
        /* let server handle listen on unbound sockets */
        if (test_bit(ATM_VF_SESSION, &vcc->flags)) {
                error = -EINVAL;
                goto out;
        }
        if (test_bit(ATM_VF_LISTEN, &vcc->flags)) {
                error = -EADDRINUSE;
                goto out;
        }
        set_bit(ATM_VF_WAITING, &vcc->flags);
        sigd_enq(vcc, as_listen, NULL, NULL, &vcc->local);
        for (;;) {
                prepare_to_wait(sk_sleep(sk), &wait, TASK_UNINTERRUPTIBLE);
                if (!test_bit(ATM_VF_WAITING, &vcc->flags) || !sigd)
                        break;
                schedule();
        }
        finish_wait(sk_sleep(sk), &wait);
        if (!sigd) {
                error = -EUNATCH;
                goto out;
        }
        set_bit(ATM_VF_LISTEN, &vcc->flags);
        vcc_insert_socket(sk);
        sk->sk_max_ack_backlog = backlog > 0 ? backlog : ATM_BACKLOG_DEFAULT;
        error = -sk->sk_err;
out:
        release_sock(sk);
        return error;
}

static int svc_accept(struct socket *sock, struct socket *newsock,
                      struct proto_accept_arg *arg)
{
        struct sock *sk = sock->sk;
        struct sk_buff *skb;
        struct atmsvc_msg *msg;
        struct atm_vcc *old_vcc = ATM_SD(sock);
        struct atm_vcc *new_vcc;
        int error;

        lock_sock(sk);

        error = svc_create(sock_net(sk), newsock, 0, arg->kern);
        if (error)
                goto out;

        new_vcc = ATM_SD(newsock);

        pr_debug("%p -> %p\n", old_vcc, new_vcc);
        while (1) {
                DEFINE_WAIT(wait);

                prepare_to_wait(sk_sleep(sk), &wait, TASK_INTERRUPTIBLE);
                while (!(skb = skb_dequeue(&sk->sk_receive_queue)) &&
                       sigd) {
                        if (test_bit(ATM_VF_RELEASED, &old_vcc->flags))
                                break;
                        if (test_bit(ATM_VF_CLOSE, &old_vcc->flags)) {
                                error = -sk->sk_err;
                                break;
                        }
                        if (arg->flags & O_NONBLOCK) {
                                error = -EAGAIN;
                                break;
                        }
                        release_sock(sk);
                        schedule();
                        lock_sock(sk);
                        if (signal_pending(current)) {
                                error = -ERESTARTSYS;
                                break;
                        }
                        prepare_to_wait(sk_sleep(sk), &wait,
                                        TASK_INTERRUPTIBLE);
                }
                finish_wait(sk_sleep(sk), &wait);
                if (error)
                        goto out;
                if (!skb) {
                        error = -EUNATCH;
                        goto out;
                }
                msg = (struct atmsvc_msg *)skb->data;
                new_vcc->qos = msg->qos;
                set_bit(ATM_VF_HASQOS, &new_vcc->flags);
                new_vcc->remote = msg->svc;
                new_vcc->local = msg->local;
                new_vcc->sap = msg->sap;
                error = vcc_connect(newsock, msg->pvc.sap_addr.itf,
                                    msg->pvc.sap_addr.vpi,
                                    msg->pvc.sap_addr.vci);
                dev_kfree_skb(skb);
                sk_acceptq_removed(sk);
                if (error) {
                        sigd_enq2(NULL, as_reject, old_vcc, NULL, NULL,
                                  &old_vcc->qos, error);
                        error = error == -EAGAIN ? -EBUSY : error;
                        goto out;
                }
                /* wait should be short, so we ignore the non-blocking flag */
                set_bit(ATM_VF_WAITING, &new_vcc->flags);
                sigd_enq(new_vcc, as_accept, old_vcc, NULL, NULL);
                for (;;) {
                        prepare_to_wait(sk_sleep(sk_atm(new_vcc)), &wait,
                                        TASK_UNINTERRUPTIBLE);
                        if (!test_bit(ATM_VF_WAITING, &new_vcc->flags) || !sigd)
                                break;
                        release_sock(sk);
                        schedule();
                        lock_sock(sk);
                }
                finish_wait(sk_sleep(sk_atm(new_vcc)), &wait);
                if (!sigd) {
                        error = -EUNATCH;
                        goto out;
                }
                if (!sk_atm(new_vcc)->sk_err)
                        break;
                if (sk_atm(new_vcc)->sk_err != ERESTARTSYS) {
                        error = -sk_atm(new_vcc)->sk_err;
                        goto out;
                }
        }
        newsock->state = SS_CONNECTED;
out:
        release_sock(sk);
        return error;
}

static int svc_getname(struct socket *sock, struct sockaddr *sockaddr,
                       int peer)
{
        struct sockaddr_atmsvc *addr;

        addr = (struct sockaddr_atmsvc *) sockaddr;
        memcpy(addr, peer ? &ATM_SD(sock)->remote : &ATM_SD(sock)->local,
               sizeof(struct sockaddr_atmsvc));
        return sizeof(struct sockaddr_atmsvc);
}

int svc_change_qos(struct atm_vcc *vcc, struct atm_qos *qos)
{
        struct sock *sk = sk_atm(vcc);
        DEFINE_WAIT(wait);

        set_bit(ATM_VF_WAITING, &vcc->flags);
        sigd_enq2(vcc, as_modify, NULL, NULL, &vcc->local, qos, 0);
        for (;;) {
                prepare_to_wait(sk_sleep(sk), &wait, TASK_UNINTERRUPTIBLE);
                if (!test_bit(ATM_VF_WAITING, &vcc->flags) ||
                    test_bit(ATM_VF_RELEASED, &vcc->flags) || !sigd) {
                        break;
                }
                schedule();
        }
        finish_wait(sk_sleep(sk), &wait);
        if (!sigd)
                return -EUNATCH;
        return -sk->sk_err;
}

static int svc_setsockopt(struct socket *sock, int level, int optname,
                          sockptr_t optval, unsigned int optlen)
{
        struct sock *sk = sock->sk;
        struct atm_vcc *vcc = ATM_SD(sock);
        int value, error = 0;

        lock_sock(sk);
        switch (optname) {
        case SO_ATMSAP:
                if (level != SOL_ATM || optlen != sizeof(struct atm_sap)) {
                        error = -EINVAL;
                        goto out;
                }
                if (copy_from_sockptr(&vcc->sap, optval, optlen)) {
                        error = -EFAULT;
                        goto out;
                }
                set_bit(ATM_VF_HASSAP, &vcc->flags);
                break;
        case SO_MULTIPOINT:
                if (level != SOL_ATM || optlen != sizeof(int)) {
                        error = -EINVAL;
                        goto out;
                }
                if (copy_from_sockptr(&value, optval, sizeof(int))) {
                        error = -EFAULT;
                        goto out;
                }
                if (value == 1)
                        set_bit(ATM_VF_SESSION, &vcc->flags);
                else if (value == 0)
                        clear_bit(ATM_VF_SESSION, &vcc->flags);
                else
                        error = -EINVAL;
                break;
        default:
                error = vcc_setsockopt(sock, level, optname, optval, optlen);
        }

out:
        release_sock(sk);
        return error;
}

static int svc_getsockopt(struct socket *sock, int level, int optname,
                          char __user *optval, int __user *optlen)
{
        struct sock *sk = sock->sk;
        int error = 0, len;

        lock_sock(sk);
        if (!__SO_LEVEL_MATCH(optname, level) || optname != SO_ATMSAP) {
                error = vcc_getsockopt(sock, level, optname, optval, optlen);
                goto out;
        }
        if (get_user(len, optlen)) {
                error = -EFAULT;
                goto out;
        }
        if (len != sizeof(struct atm_sap)) {
                error = -EINVAL;
                goto out;
        }
        if (copy_to_user(optval, &ATM_SD(sock)->sap, sizeof(struct atm_sap))) {
                error = -EFAULT;
                goto out;
        }
out:
        release_sock(sk);
        return error;
}

static int svc_addparty(struct socket *sock, struct sockaddr *sockaddr,
                        int sockaddr_len, int flags)
{
        DEFINE_WAIT(wait);
        struct sock *sk = sock->sk;
        struct atm_vcc *vcc = ATM_SD(sock);
        int error;

        lock_sock(sk);
        set_bit(ATM_VF_WAITING, &vcc->flags);
        sigd_enq(vcc, as_addparty, NULL, NULL,
                 (struct sockaddr_atmsvc *) sockaddr);
        if (flags & O_NONBLOCK) {
                error = -EINPROGRESS;
                goto out;
        }
        pr_debug("added wait queue\n");
        for (;;) {
                prepare_to_wait(sk_sleep(sk), &wait, TASK_INTERRUPTIBLE);
                if (!test_bit(ATM_VF_WAITING, &vcc->flags) || !sigd)
                        break;
                schedule();
        }
        finish_wait(sk_sleep(sk), &wait);
        error = -xchg(&sk->sk_err_soft, 0);
out:
        release_sock(sk);
        return error;
}

static int svc_dropparty(struct socket *sock, int ep_ref)
{
        DEFINE_WAIT(wait);
        struct sock *sk = sock->sk;
        struct atm_vcc *vcc = ATM_SD(sock);
        int error;

        lock_sock(sk);
        set_bit(ATM_VF_WAITING, &vcc->flags);
        sigd_enq2(vcc, as_dropparty, NULL, NULL, NULL, NULL, ep_ref);
        for (;;) {
                prepare_to_wait(sk_sleep(sk), &wait, TASK_INTERRUPTIBLE);
                if (!test_bit(ATM_VF_WAITING, &vcc->flags) || !sigd)
                        break;
                schedule();
        }
        finish_wait(sk_sleep(sk), &wait);
        if (!sigd) {
                error = -EUNATCH;
                goto out;
        }
        error = -xchg(&sk->sk_err_soft, 0);
out:
        release_sock(sk);
        return error;
}

static int svc_ioctl(struct socket *sock, unsigned int cmd, unsigned long arg)
{
        int error, ep_ref;
        struct sockaddr_atmsvc sa;
        struct atm_vcc *vcc = ATM_SD(sock);

        switch (cmd) {
        case ATM_ADDPARTY:
                if (!test_bit(ATM_VF_SESSION, &vcc->flags))
                        return -EINVAL;
                if (copy_from_user(&sa, (void __user *) arg, sizeof(sa)))
                        return -EFAULT;
                error = svc_addparty(sock, (struct sockaddr *)&sa, sizeof(sa),
                                     0);
                break;
        case ATM_DROPPARTY:
                if (!test_bit(ATM_VF_SESSION, &vcc->flags))
                        return -EINVAL;
                if (copy_from_user(&ep_ref, (void __user *) arg, sizeof(int)))
                        return -EFAULT;
                error = svc_dropparty(sock, ep_ref);
                break;
        default:
                error = vcc_ioctl(sock, cmd, arg);
        }

        return error;
}

#ifdef CONFIG_COMPAT
static int svc_compat_ioctl(struct socket *sock, unsigned int cmd,
                            unsigned long arg)
{
        /* The definition of ATM_ADDPARTY uses the size of struct atm_iobuf.
           But actually it takes a struct sockaddr_atmsvc, which doesn't need
           compat handling. So all we have to do is fix up cmd... */
        if (cmd == COMPAT_ATM_ADDPARTY)
                cmd = ATM_ADDPARTY;

        if (cmd == ATM_ADDPARTY || cmd == ATM_DROPPARTY)
                return svc_ioctl(sock, cmd, arg);
        else
                return vcc_compat_ioctl(sock, cmd, arg);
}
#endif /* CONFIG_COMPAT */

static const struct proto_ops svc_proto_ops = {
        .family =        PF_ATMSVC,
        .owner =        THIS_MODULE,

        .release =        svc_release,
        .bind =                svc_bind,
        .connect =        svc_connect,
        .socketpair =        sock_no_socketpair,
        .accept =        svc_accept,
        .getname =        svc_getname,
        .poll =                vcc_poll,
        .ioctl =        svc_ioctl,
#ifdef CONFIG_COMPAT
        .compat_ioctl =        svc_compat_ioctl,
#endif
        .gettstamp =        sock_gettstamp,
        .listen =        svc_listen,
        .shutdown =        svc_shutdown,
        .setsockopt =        svc_setsockopt,
        .getsockopt =        svc_getsockopt,
        .sendmsg =        vcc_sendmsg,
        .recvmsg =        vcc_recvmsg,
        .mmap =                sock_no_mmap,
};


static int svc_create(struct net *net, struct socket *sock, int protocol,
                      int kern)
{
        int error;

        if (!net_eq(net, &init_net))
                return -EAFNOSUPPORT;

        sock->ops = &svc_proto_ops;
        error = vcc_create(net, sock, protocol, AF_ATMSVC, kern);
        if (error)
                return error;
        ATM_SD(sock)->local.sas_family = AF_ATMSVC;
        ATM_SD(sock)->remote.sas_family = AF_ATMSVC;
        return 0;
}

static const struct net_proto_family svc_family_ops = {
        .family = PF_ATMSVC,
        .create = svc_create,
        .owner = THIS_MODULE,
};


/*
 *        Initialize the ATM SVC protocol family
 */

int __init atmsvc_init(void)
{
        return sock_register(&svc_family_ops);
}

void atmsvc_exit(void)
{
        sock_unregister(PF_ATMSVC);
}

















































































































 2283 





































































































































































































































































































































































































































































































 2282 






































































































































































































































































































































































































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
// SPDX-License-Identifier: GPL-2.0
/*
 * Detect hard and soft lockups on a system
 *
 * started by Don Zickus, Copyright (C) 2010 Red Hat, Inc.
 *
 * Note: Most of this code is borrowed heavily from the original softlockup
 * detector, so thanks to Ingo for the initial implementation.
 * Some chunks also taken from the old x86-specific nmi watchdog code, thanks
 * to those contributors as well.
 */

#define pr_fmt(fmt) "watchdog: " fmt

#include <linux/cpu.h>
#include <linux/init.h>
#include <linux/irq.h>
#include <linux/irqdesc.h>
#include <linux/kernel_stat.h>
#include <linux/kvm_para.h>
#include <linux/math64.h>
#include <linux/mm.h>
#include <linux/module.h>
#include <linux/nmi.h>
#include <linux/stop_machine.h>
#include <linux/sysctl.h>
#include <linux/tick.h>

#include <linux/sched/clock.h>
#include <linux/sched/debug.h>
#include <linux/sched/isolation.h>

#include <asm/irq_regs.h>

static DEFINE_MUTEX(watchdog_mutex);

#if defined(CONFIG_HARDLOCKUP_DETECTOR) || defined(CONFIG_HARDLOCKUP_DETECTOR_SPARC64)
# define WATCHDOG_HARDLOCKUP_DEFAULT        1
#else
# define WATCHDOG_HARDLOCKUP_DEFAULT        0
#endif

#define NUM_SAMPLE_PERIODS        5

unsigned long __read_mostly watchdog_enabled;
int __read_mostly watchdog_user_enabled = 1;
static int __read_mostly watchdog_hardlockup_user_enabled = WATCHDOG_HARDLOCKUP_DEFAULT;
static int __read_mostly watchdog_softlockup_user_enabled = 1;
int __read_mostly watchdog_thresh = 10;
static int __read_mostly watchdog_hardlockup_available;

struct cpumask watchdog_cpumask __read_mostly;
unsigned long *watchdog_cpumask_bits = cpumask_bits(&watchdog_cpumask);

#ifdef CONFIG_HARDLOCKUP_DETECTOR

# ifdef CONFIG_SMP
int __read_mostly sysctl_hardlockup_all_cpu_backtrace;
# endif /* CONFIG_SMP */

/*
 * Should we panic when a soft-lockup or hard-lockup occurs:
 */
unsigned int __read_mostly hardlockup_panic =
                        IS_ENABLED(CONFIG_BOOTPARAM_HARDLOCKUP_PANIC);
/*
 * We may not want to enable hard lockup detection by default in all cases,
 * for example when running the kernel as a guest on a hypervisor. In these
 * cases this function can be called to disable hard lockup detection. This
 * function should only be executed once by the boot processor before the
 * kernel command line parameters are parsed, because otherwise it is not
 * possible to override this in hardlockup_panic_setup().
 */
void __init hardlockup_detector_disable(void)
{
        watchdog_hardlockup_user_enabled = 0;
}

static int __init hardlockup_panic_setup(char *str)
{
next:
        if (!strncmp(str, "panic", 5))
                hardlockup_panic = 1;
        else if (!strncmp(str, "nopanic", 7))
                hardlockup_panic = 0;
        else if (!strncmp(str, "0", 1))
                watchdog_hardlockup_user_enabled = 0;
        else if (!strncmp(str, "1", 1))
                watchdog_hardlockup_user_enabled = 1;
        else if (!strncmp(str, "r", 1))
                hardlockup_config_perf_event(str + 1);
        while (*(str++)) {
                if (*str == ',') {
                        str++;
                        goto next;
                }
        }
        return 1;
}
__setup("nmi_watchdog=", hardlockup_panic_setup);

#endif /* CONFIG_HARDLOCKUP_DETECTOR */

#if defined(CONFIG_HARDLOCKUP_DETECTOR_COUNTS_HRTIMER)

static DEFINE_PER_CPU(atomic_t, hrtimer_interrupts);
static DEFINE_PER_CPU(int, hrtimer_interrupts_saved);
static DEFINE_PER_CPU(bool, watchdog_hardlockup_warned);
static DEFINE_PER_CPU(bool, watchdog_hardlockup_touched);
static unsigned long hard_lockup_nmi_warn;

notrace void arch_touch_nmi_watchdog(void)
{
        /*
         * Using __raw here because some code paths have
         * preemption enabled.  If preemption is enabled
         * then interrupts should be enabled too, in which
         * case we shouldn't have to worry about the watchdog
         * going off.
         */
        raw_cpu_write(watchdog_hardlockup_touched, true);
}
EXPORT_SYMBOL(arch_touch_nmi_watchdog);

void watchdog_hardlockup_touch_cpu(unsigned int cpu)
{
        per_cpu(watchdog_hardlockup_touched, cpu) = true;
}

static bool is_hardlockup(unsigned int cpu)
{
        int hrint = atomic_read(&per_cpu(hrtimer_interrupts, cpu));

        if (per_cpu(hrtimer_interrupts_saved, cpu) == hrint)
                return true;

        /*
         * NOTE: we don't need any fancy atomic_t or READ_ONCE/WRITE_ONCE
         * for hrtimer_interrupts_saved. hrtimer_interrupts_saved is
         * written/read by a single CPU.
         */
        per_cpu(hrtimer_interrupts_saved, cpu) = hrint;

        return false;
}

static void watchdog_hardlockup_kick(void)
{
        int new_interrupts;

        new_interrupts = atomic_inc_return(this_cpu_ptr(&hrtimer_interrupts));
        watchdog_buddy_check_hardlockup(new_interrupts);
}

void watchdog_hardlockup_check(unsigned int cpu, struct pt_regs *regs)
{
        if (per_cpu(watchdog_hardlockup_touched, cpu)) {
                per_cpu(watchdog_hardlockup_touched, cpu) = false;
                return;
        }

        /*
         * Check for a hardlockup by making sure the CPU's timer
         * interrupt is incrementing. The timer interrupt should have
         * fired multiple times before we overflow'd. If it hasn't
         * then this is a good indication the cpu is stuck
         */
        if (is_hardlockup(cpu)) {
                unsigned int this_cpu = smp_processor_id();
                unsigned long flags;

                /* Only print hardlockups once. */
                if (per_cpu(watchdog_hardlockup_warned, cpu))
                        return;

                /*
                 * Prevent multiple hard-lockup reports if one cpu is already
                 * engaged in dumping all cpu back traces.
                 */
                if (sysctl_hardlockup_all_cpu_backtrace) {
                        if (test_and_set_bit_lock(0, &hard_lockup_nmi_warn))
                                return;
                }

                /*
                 * NOTE: we call printk_cpu_sync_get_irqsave() after printing
                 * the lockup message. While it would be nice to serialize
                 * that printout, we really want to make sure that if some
                 * other CPU somehow locked up while holding the lock associated
                 * with printk_cpu_sync_get_irqsave() that we can still at least
                 * get the message about the lockup out.
                 */
                pr_emerg("Watchdog detected hard LOCKUP on cpu %d\n", cpu);
                printk_cpu_sync_get_irqsave(flags);

                print_modules();
                print_irqtrace_events(current);
                if (cpu == this_cpu) {
                        if (regs)
                                show_regs(regs);
                        else
                                dump_stack();
                        printk_cpu_sync_put_irqrestore(flags);
                } else {
                        printk_cpu_sync_put_irqrestore(flags);
                        trigger_single_cpu_backtrace(cpu);
                }

                if (sysctl_hardlockup_all_cpu_backtrace) {
                        trigger_allbutcpu_cpu_backtrace(cpu);
                        if (!hardlockup_panic)
                                clear_bit_unlock(0, &hard_lockup_nmi_warn);
                }

                if (hardlockup_panic)
                        nmi_panic(regs, "Hard LOCKUP");

                per_cpu(watchdog_hardlockup_warned, cpu) = true;
        } else {
                per_cpu(watchdog_hardlockup_warned, cpu) = false;
        }
}

#else /* CONFIG_HARDLOCKUP_DETECTOR_COUNTS_HRTIMER */

static inline void watchdog_hardlockup_kick(void) { }

#endif /* !CONFIG_HARDLOCKUP_DETECTOR_COUNTS_HRTIMER */

/*
 * These functions can be overridden based on the configured hardlockdup detector.
 *
 * watchdog_hardlockup_enable/disable can be implemented to start and stop when
 * softlockup watchdog start and stop. The detector must select the
 * SOFTLOCKUP_DETECTOR Kconfig.
 */
void __weak watchdog_hardlockup_enable(unsigned int cpu) { }

void __weak watchdog_hardlockup_disable(unsigned int cpu) { }

/*
 * Watchdog-detector specific API.
 *
 * Return 0 when hardlockup watchdog is available, negative value otherwise.
 * Note that the negative value means that a delayed probe might
 * succeed later.
 */
int __weak __init watchdog_hardlockup_probe(void)
{
        return -ENODEV;
}

/**
 * watchdog_hardlockup_stop - Stop the watchdog for reconfiguration
 *
 * The reconfiguration steps are:
 * watchdog_hardlockup_stop();
 * update_variables();
 * watchdog_hardlockup_start();
 */
void __weak watchdog_hardlockup_stop(void) { }

/**
 * watchdog_hardlockup_start - Start the watchdog after reconfiguration
 *
 * Counterpart to watchdog_hardlockup_stop().
 *
 * The following variables have been updated in update_variables() and
 * contain the currently valid configuration:
 * - watchdog_enabled
 * - watchdog_thresh
 * - watchdog_cpumask
 */
void __weak watchdog_hardlockup_start(void) { }

/**
 * lockup_detector_update_enable - Update the sysctl enable bit
 *
 * Caller needs to make sure that the hard watchdogs are off, so this
 * can't race with watchdog_hardlockup_disable().
 */
static void lockup_detector_update_enable(void)
{
        watchdog_enabled = 0;
        if (!watchdog_user_enabled)
                return;
        if (watchdog_hardlockup_available && watchdog_hardlockup_user_enabled)
                watchdog_enabled |= WATCHDOG_HARDLOCKUP_ENABLED;
        if (watchdog_softlockup_user_enabled)
                watchdog_enabled |= WATCHDOG_SOFTOCKUP_ENABLED;
}

#ifdef CONFIG_SOFTLOCKUP_DETECTOR

/*
 * Delay the soflockup report when running a known slow code.
 * It does _not_ affect the timestamp of the last successdul reschedule.
 */
#define SOFTLOCKUP_DELAY_REPORT        ULONG_MAX

#ifdef CONFIG_SMP
int __read_mostly sysctl_softlockup_all_cpu_backtrace;
#endif

static struct cpumask watchdog_allowed_mask __read_mostly;

/* Global variables, exported for sysctl */
unsigned int __read_mostly softlockup_panic =
                        IS_ENABLED(CONFIG_BOOTPARAM_SOFTLOCKUP_PANIC);

static bool softlockup_initialized __read_mostly;
static u64 __read_mostly sample_period;

/* Timestamp taken after the last successful reschedule. */
static DEFINE_PER_CPU(unsigned long, watchdog_touch_ts);
/* Timestamp of the last softlockup report. */
static DEFINE_PER_CPU(unsigned long, watchdog_report_ts);
static DEFINE_PER_CPU(struct hrtimer, watchdog_hrtimer);
static DEFINE_PER_CPU(bool, softlockup_touch_sync);
static unsigned long soft_lockup_nmi_warn;

static int __init softlockup_panic_setup(char *str)
{
        softlockup_panic = simple_strtoul(str, NULL, 0);
        return 1;
}
__setup("softlockup_panic=", softlockup_panic_setup);

static int __init nowatchdog_setup(char *str)
{
        watchdog_user_enabled = 0;
        return 1;
}
__setup("nowatchdog", nowatchdog_setup);

static int __init nosoftlockup_setup(char *str)
{
        watchdog_softlockup_user_enabled = 0;
        return 1;
}
__setup("nosoftlockup", nosoftlockup_setup);

static int __init watchdog_thresh_setup(char *str)
{
        get_option(&str, &watchdog_thresh);
        return 1;
}
__setup("watchdog_thresh=", watchdog_thresh_setup);

static void __lockup_detector_cleanup(void);

#ifdef CONFIG_SOFTLOCKUP_DETECTOR_INTR_STORM
enum stats_per_group {
        STATS_SYSTEM,
        STATS_SOFTIRQ,
        STATS_HARDIRQ,
        STATS_IDLE,
        NUM_STATS_PER_GROUP,
};

static const enum cpu_usage_stat tracked_stats[NUM_STATS_PER_GROUP] = {
        CPUTIME_SYSTEM,
        CPUTIME_SOFTIRQ,
        CPUTIME_IRQ,
        CPUTIME_IDLE,
};

static DEFINE_PER_CPU(u16, cpustat_old[NUM_STATS_PER_GROUP]);
static DEFINE_PER_CPU(u8, cpustat_util[NUM_SAMPLE_PERIODS][NUM_STATS_PER_GROUP]);
static DEFINE_PER_CPU(u8, cpustat_tail);

/*
 * We don't need nanosecond resolution. A granularity of 16ms is
 * sufficient for our precision, allowing us to use u16 to store
 * cpustats, which will roll over roughly every ~1000 seconds.
 * 2^24 ~= 16 * 10^6
 */
static u16 get_16bit_precision(u64 data_ns)
{
        return data_ns >> 24LL; /* 2^24ns ~= 16.8ms */
}

static void update_cpustat(void)
{
        int i;
        u8 util;
        u16 old_stat, new_stat;
        struct kernel_cpustat kcpustat;
        u64 *cpustat = kcpustat.cpustat;
        u8 tail = __this_cpu_read(cpustat_tail);
        u16 sample_period_16 = get_16bit_precision(sample_period);

        kcpustat_cpu_fetch(&kcpustat, smp_processor_id());

        for (i = 0; i < NUM_STATS_PER_GROUP; i++) {
                old_stat = __this_cpu_read(cpustat_old[i]);
                new_stat = get_16bit_precision(cpustat[tracked_stats[i]]);
                util = DIV_ROUND_UP(100 * (new_stat - old_stat), sample_period_16);
                __this_cpu_write(cpustat_util[tail][i], util);
                __this_cpu_write(cpustat_old[i], new_stat);
        }

        __this_cpu_write(cpustat_tail, (tail + 1) % NUM_SAMPLE_PERIODS);
}

static void print_cpustat(void)
{
        int i, group;
        u8 tail = __this_cpu_read(cpustat_tail);
        u64 sample_period_second = sample_period;

        do_div(sample_period_second, NSEC_PER_SEC);

        /*
         * Outputting the "watchdog" prefix on every line is redundant and not
         * concise, and the original alarm information is sufficient for
         * positioning in logs, hence here printk() is used instead of pr_crit().
         */
        printk(KERN_CRIT "CPU#%d Utilization every %llus during lockup:\n",
               smp_processor_id(), sample_period_second);

        for (i = 0; i < NUM_SAMPLE_PERIODS; i++) {
                group = (tail + i) % NUM_SAMPLE_PERIODS;
                printk(KERN_CRIT "\t#%d: %3u%% system,\t%3u%% softirq,\t"
                        "%3u%% hardirq,\t%3u%% idle\n", i + 1,
                        __this_cpu_read(cpustat_util[group][STATS_SYSTEM]),
                        __this_cpu_read(cpustat_util[group][STATS_SOFTIRQ]),
                        __this_cpu_read(cpustat_util[group][STATS_HARDIRQ]),
                        __this_cpu_read(cpustat_util[group][STATS_IDLE]));
        }
}

#define HARDIRQ_PERCENT_THRESH          50
#define NUM_HARDIRQ_REPORT              5
struct irq_counts {
        int irq;
        u32 counts;
};

static DEFINE_PER_CPU(bool, snapshot_taken);

/* Tabulate the most frequent interrupts. */
static void tabulate_irq_count(struct irq_counts *irq_counts, int irq, u32 counts, int rank)
{
        int i;
        struct irq_counts new_count = {irq, counts};

        for (i = 0; i < rank; i++) {
                if (counts > irq_counts[i].counts)
                        swap(new_count, irq_counts[i]);
        }
}

/*
 * If the hardirq time exceeds HARDIRQ_PERCENT_THRESH% of the sample_period,
 * then the cause of softlockup might be interrupt storm. In this case, it
 * would be useful to start interrupt counting.
 */
static bool need_counting_irqs(void)
{
        u8 util;
        int tail = __this_cpu_read(cpustat_tail);

        tail = (tail + NUM_HARDIRQ_REPORT - 1) % NUM_HARDIRQ_REPORT;
        util = __this_cpu_read(cpustat_util[tail][STATS_HARDIRQ]);
        return util > HARDIRQ_PERCENT_THRESH;
}

static void start_counting_irqs(void)
{
        if (!__this_cpu_read(snapshot_taken)) {
                kstat_snapshot_irqs();
                __this_cpu_write(snapshot_taken, true);
        }
}

static void stop_counting_irqs(void)
{
        __this_cpu_write(snapshot_taken, false);
}

static void print_irq_counts(void)
{
        unsigned int i, count;
        struct irq_counts irq_counts_sorted[NUM_HARDIRQ_REPORT] = {
                {-1, 0}, {-1, 0}, {-1, 0}, {-1, 0}, {-1, 0}
        };

        if (__this_cpu_read(snapshot_taken)) {
                for_each_active_irq(i) {
                        count = kstat_get_irq_since_snapshot(i);
                        tabulate_irq_count(irq_counts_sorted, i, count, NUM_HARDIRQ_REPORT);
                }

                /*
                 * Outputting the "watchdog" prefix on every line is redundant and not
                 * concise, and the original alarm information is sufficient for
                 * positioning in logs, hence here printk() is used instead of pr_crit().
                 */
                printk(KERN_CRIT "CPU#%d Detect HardIRQ Time exceeds %d%%. Most frequent HardIRQs:\n",
                       smp_processor_id(), HARDIRQ_PERCENT_THRESH);

                for (i = 0; i < NUM_HARDIRQ_REPORT; i++) {
                        if (irq_counts_sorted[i].irq == -1)
                                break;

                        printk(KERN_CRIT "\t#%u: %-10u\tirq#%d\n",
                               i + 1, irq_counts_sorted[i].counts,
                               irq_counts_sorted[i].irq);
                }

                /*
                 * If the hardirq time is less than HARDIRQ_PERCENT_THRESH% in the last
                 * sample_period, then we suspect the interrupt storm might be subsiding.
                 */
                if (!need_counting_irqs())
                        stop_counting_irqs();
        }
}

static void report_cpu_status(void)
{
        print_cpustat();
        print_irq_counts();
}
#else
static inline void update_cpustat(void) { }
static inline void report_cpu_status(void) { }
static inline bool need_counting_irqs(void) { return false; }
static inline void start_counting_irqs(void) { }
static inline void stop_counting_irqs(void) { }
#endif

/*
 * Hard-lockup warnings should be triggered after just a few seconds. Soft-
 * lockups can have false positives under extreme conditions. So we generally
 * want a higher threshold for soft lockups than for hard lockups. So we couple
 * the thresholds with a factor: we make the soft threshold twice the amount of
 * time the hard threshold is.
 */
static int get_softlockup_thresh(void)
{
        return watchdog_thresh * 2;
}

/*
 * Returns seconds, approximately.  We don't need nanosecond
 * resolution, and we don't need to waste time with a big divide when
 * 2^30ns == 1.074s.
 */
static unsigned long get_timestamp(void)
{
        return running_clock() >> 30LL;  /* 2^30 ~= 10^9 */
}

static void set_sample_period(void)
{
        /*
         * convert watchdog_thresh from seconds to ns
         * the divide by 5 is to give hrtimer several chances (two
         * or three with the current relation between the soft
         * and hard thresholds) to increment before the
         * hardlockup detector generates a warning
         */
        sample_period = get_softlockup_thresh() * ((u64)NSEC_PER_SEC / NUM_SAMPLE_PERIODS);
        watchdog_update_hrtimer_threshold(sample_period);
}

static void update_report_ts(void)
{
        __this_cpu_write(watchdog_report_ts, get_timestamp());
}

/* Commands for resetting the watchdog */
static void update_touch_ts(void)
{
        __this_cpu_write(watchdog_touch_ts, get_timestamp());
        update_report_ts();
}

/**
 * touch_softlockup_watchdog_sched - touch watchdog on scheduler stalls
 *
 * Call when the scheduler may have stalled for legitimate reasons
 * preventing the watchdog task from executing - e.g. the scheduler
 * entering idle state.  This should only be used for scheduler events.
 * Use touch_softlockup_watchdog() for everything else.
 */
notrace void touch_softlockup_watchdog_sched(void)
{
        /*
         * Preemption can be enabled.  It doesn't matter which CPU's watchdog
         * report period gets restarted here, so use the raw_ operation.
         */
        raw_cpu_write(watchdog_report_ts, SOFTLOCKUP_DELAY_REPORT);
}

notrace void touch_softlockup_watchdog(void)
{
        touch_softlockup_watchdog_sched();
        wq_watchdog_touch(raw_smp_processor_id());
}
EXPORT_SYMBOL(touch_softlockup_watchdog);

void touch_all_softlockup_watchdogs(void)
{
        int cpu;

        /*
         * watchdog_mutex cannpt be taken here, as this might be called
         * from (soft)interrupt context, so the access to
         * watchdog_allowed_cpumask might race with a concurrent update.
         *
         * The watchdog time stamp can race against a concurrent real
         * update as well, the only side effect might be a cycle delay for
         * the softlockup check.
         */
        for_each_cpu(cpu, &watchdog_allowed_mask) {
                per_cpu(watchdog_report_ts, cpu) = SOFTLOCKUP_DELAY_REPORT;
                wq_watchdog_touch(cpu);
        }
}

void touch_softlockup_watchdog_sync(void)
{
        __this_cpu_write(softlockup_touch_sync, true);
        __this_cpu_write(watchdog_report_ts, SOFTLOCKUP_DELAY_REPORT);
}

static int is_softlockup(unsigned long touch_ts,
                         unsigned long period_ts,
                         unsigned long now)
{
        if ((watchdog_enabled & WATCHDOG_SOFTOCKUP_ENABLED) && watchdog_thresh) {
                /*
                 * If period_ts has not been updated during a sample_period, then
                 * in the subsequent few sample_periods, period_ts might also not
                 * be updated, which could indicate a potential softlockup. In
                 * this case, if we suspect the cause of the potential softlockup
                 * might be interrupt storm, then we need to count the interrupts
                 * to find which interrupt is storming.
                 */
                if (time_after_eq(now, period_ts + get_softlockup_thresh() / NUM_SAMPLE_PERIODS) &&
                    need_counting_irqs())
                        start_counting_irqs();

                /*
                 * A poorly behaving BPF scheduler can live-lock the system into
                 * soft lockups. Tell sched_ext to try ejecting the BPF
                 * scheduler when close to a soft lockup.
                 */
                if (time_after_eq(now, period_ts + get_softlockup_thresh() * 3 / 4))
                        scx_softlockup(now - touch_ts);

                /* Warn about unreasonable delays. */
                if (time_after(now, period_ts + get_softlockup_thresh()))
                        return now - touch_ts;
        }
        return 0;
}

/* watchdog detector functions */
static DEFINE_PER_CPU(struct completion, softlockup_completion);
static DEFINE_PER_CPU(struct cpu_stop_work, softlockup_stop_work);

/*
 * The watchdog feed function - touches the timestamp.
 *
 * It only runs once every sample_period seconds (4 seconds by
 * default) to reset the softlockup timestamp. If this gets delayed
 * for more than 2*watchdog_thresh seconds then the debug-printout
 * triggers in watchdog_timer_fn().
 */
static int softlockup_fn(void *data)
{
        update_touch_ts();
        stop_counting_irqs();
        complete(this_cpu_ptr(&softlockup_completion));

        return 0;
}

/* watchdog kicker functions */
static enum hrtimer_restart watchdog_timer_fn(struct hrtimer *hrtimer)
{
        unsigned long touch_ts, period_ts, now;
        struct pt_regs *regs = get_irq_regs();
        int duration;
        int softlockup_all_cpu_backtrace = sysctl_softlockup_all_cpu_backtrace;
        unsigned long flags;

        if (!watchdog_enabled)
                return HRTIMER_NORESTART;

        watchdog_hardlockup_kick();

        /* kick the softlockup detector */
        if (completion_done(this_cpu_ptr(&softlockup_completion))) {
                reinit_completion(this_cpu_ptr(&softlockup_completion));
                stop_one_cpu_nowait(smp_processor_id(),
                                softlockup_fn, NULL,
                                this_cpu_ptr(&softlockup_stop_work));
        }

        /* .. and repeat */
        hrtimer_forward_now(hrtimer, ns_to_ktime(sample_period));

        /*
         * Read the current timestamp first. It might become invalid anytime
         * when a virtual machine is stopped by the host or when the watchog
         * is touched from NMI.
         */
        now = get_timestamp();
        /*
         * If a virtual machine is stopped by the host it can look to
         * the watchdog like a soft lockup. This function touches the watchdog.
         */
        kvm_check_and_clear_guest_paused();
        /*
         * The stored timestamp is comparable with @now only when not touched.
         * It might get touched anytime from NMI. Make sure that is_softlockup()
         * uses the same (valid) value.
         */
        period_ts = READ_ONCE(*this_cpu_ptr(&watchdog_report_ts));

        update_cpustat();

        /* Reset the interval when touched by known problematic code. */
        if (period_ts == SOFTLOCKUP_DELAY_REPORT) {
                if (unlikely(__this_cpu_read(softlockup_touch_sync))) {
                        /*
                         * If the time stamp was touched atomically
                         * make sure the scheduler tick is up to date.
                         */
                        __this_cpu_write(softlockup_touch_sync, false);
                        sched_clock_tick();
                }

                update_report_ts();
                return HRTIMER_RESTART;
        }

        /* Check for a softlockup. */
        touch_ts = __this_cpu_read(watchdog_touch_ts);
        duration = is_softlockup(touch_ts, period_ts, now);
        if (unlikely(duration)) {
                /*
                 * Prevent multiple soft-lockup reports if one cpu is already
                 * engaged in dumping all cpu back traces.
                 */
                if (softlockup_all_cpu_backtrace) {
                        if (test_and_set_bit_lock(0, &soft_lockup_nmi_warn))
                                return HRTIMER_RESTART;
                }

                /* Start period for the next softlockup warning. */
                update_report_ts();

                printk_cpu_sync_get_irqsave(flags);
                pr_emerg("BUG: soft lockup - CPU#%d stuck for %us! [%s:%d]\n",
                        smp_processor_id(), duration,
                        current->comm, task_pid_nr(current));
                report_cpu_status();
                print_modules();
                print_irqtrace_events(current);
                if (regs)
                        show_regs(regs);
                else
                        dump_stack();
                printk_cpu_sync_put_irqrestore(flags);

                if (softlockup_all_cpu_backtrace) {
                        trigger_allbutcpu_cpu_backtrace(smp_processor_id());
                        if (!softlockup_panic)
                                clear_bit_unlock(0, &soft_lockup_nmi_warn);
                }

                add_taint(TAINT_SOFTLOCKUP, LOCKDEP_STILL_OK);
                if (softlockup_panic)
                        panic("softlockup: hung tasks");
        }

        return HRTIMER_RESTART;
}

static void watchdog_enable(unsigned int cpu)
{
        struct hrtimer *hrtimer = this_cpu_ptr(&watchdog_hrtimer);
        struct completion *done = this_cpu_ptr(&softlockup_completion);

        WARN_ON_ONCE(cpu != smp_processor_id());

        init_completion(done);
        complete(done);

        /*
         * Start the timer first to prevent the hardlockup watchdog triggering
         * before the timer has a chance to fire.
         */
        hrtimer_init(hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_HARD);
        hrtimer->function = watchdog_timer_fn;
        hrtimer_start(hrtimer, ns_to_ktime(sample_period),
                      HRTIMER_MODE_REL_PINNED_HARD);

        /* Initialize timestamp */
        update_touch_ts();
        /* Enable the hardlockup detector */
        if (watchdog_enabled & WATCHDOG_HARDLOCKUP_ENABLED)
                watchdog_hardlockup_enable(cpu);
}

static void watchdog_disable(unsigned int cpu)
{
        struct hrtimer *hrtimer = this_cpu_ptr(&watchdog_hrtimer);

        WARN_ON_ONCE(cpu != smp_processor_id());

        /*
         * Disable the hardlockup detector first. That prevents that a large
         * delay between disabling the timer and disabling the hardlockup
         * detector causes a false positive.
         */
        watchdog_hardlockup_disable(cpu);
        hrtimer_cancel(hrtimer);
        wait_for_completion(this_cpu_ptr(&softlockup_completion));
}

static int softlockup_stop_fn(void *data)
{
        watchdog_disable(smp_processor_id());
        return 0;
}

static void softlockup_stop_all(void)
{
        int cpu;

        if (!softlockup_initialized)
                return;

        for_each_cpu(cpu, &watchdog_allowed_mask)
                smp_call_on_cpu(cpu, softlockup_stop_fn, NULL, false);

        cpumask_clear(&watchdog_allowed_mask);
}

static int softlockup_start_fn(void *data)
{
        watchdog_enable(smp_processor_id());
        return 0;
}

static void softlockup_start_all(void)
{
        int cpu;

        cpumask_copy(&watchdog_allowed_mask, &watchdog_cpumask);
        for_each_cpu(cpu, &watchdog_allowed_mask)
                smp_call_on_cpu(cpu, softlockup_start_fn, NULL, false);
}

int lockup_detector_online_cpu(unsigned int cpu)
{
        if (cpumask_test_cpu(cpu, &watchdog_allowed_mask))
                watchdog_enable(cpu);
        return 0;
}

int lockup_detector_offline_cpu(unsigned int cpu)
{
        if (cpumask_test_cpu(cpu, &watchdog_allowed_mask))
                watchdog_disable(cpu);
        return 0;
}

static void __lockup_detector_reconfigure(void)
{
        cpus_read_lock();
        watchdog_hardlockup_stop();

        softlockup_stop_all();
        set_sample_period();
        lockup_detector_update_enable();
        if (watchdog_enabled && watchdog_thresh)
                softlockup_start_all();

        watchdog_hardlockup_start();
        cpus_read_unlock();
        /*
         * Must be called outside the cpus locked section to prevent
         * recursive locking in the perf code.
         */
        __lockup_detector_cleanup();
}

void lockup_detector_reconfigure(void)
{
        mutex_lock(&watchdog_mutex);
        __lockup_detector_reconfigure();
        mutex_unlock(&watchdog_mutex);
}

/*
 * Create the watchdog infrastructure and configure the detector(s).
 */
static __init void lockup_detector_setup(void)
{
        /*
         * If sysctl is off and watchdog got disabled on the command line,
         * nothing to do here.
         */
        lockup_detector_update_enable();

        if (!IS_ENABLED(CONFIG_SYSCTL) &&
            !(watchdog_enabled && watchdog_thresh))
                return;

        mutex_lock(&watchdog_mutex);
        __lockup_detector_reconfigure();
        softlockup_initialized = true;
        mutex_unlock(&watchdog_mutex);
}

#else /* CONFIG_SOFTLOCKUP_DETECTOR */
static void __lockup_detector_reconfigure(void)
{
        cpus_read_lock();
        watchdog_hardlockup_stop();
        lockup_detector_update_enable();
        watchdog_hardlockup_start();
        cpus_read_unlock();
}
void lockup_detector_reconfigure(void)
{
        __lockup_detector_reconfigure();
}
static inline void lockup_detector_setup(void)
{
        __lockup_detector_reconfigure();
}
#endif /* !CONFIG_SOFTLOCKUP_DETECTOR */

static void __lockup_detector_cleanup(void)
{
        lockdep_assert_held(&watchdog_mutex);
        hardlockup_detector_perf_cleanup();
}

/**
 * lockup_detector_cleanup - Cleanup after cpu hotplug or sysctl changes
 *
 * Caller must not hold the cpu hotplug rwsem.
 */
void lockup_detector_cleanup(void)
{
        mutex_lock(&watchdog_mutex);
        __lockup_detector_cleanup();
        mutex_unlock(&watchdog_mutex);
}

/**
 * lockup_detector_soft_poweroff - Interface to stop lockup detector(s)
 *
 * Special interface for parisc. It prevents lockup detector warnings from
 * the default pm_poweroff() function which busy loops forever.
 */
void lockup_detector_soft_poweroff(void)
{
        watchdog_enabled = 0;
}

#ifdef CONFIG_SYSCTL

/* Propagate any changes to the watchdog infrastructure */
static void proc_watchdog_update(void)
{
        /* Remove impossible cpus to keep sysctl output clean. */
        cpumask_and(&watchdog_cpumask, &watchdog_cpumask, cpu_possible_mask);
        __lockup_detector_reconfigure();
}

/*
 * common function for watchdog, nmi_watchdog and soft_watchdog parameter
 *
 * caller             | table->data points to            | 'which'
 * -------------------|----------------------------------|-------------------------------
 * proc_watchdog      | watchdog_user_enabled            | WATCHDOG_HARDLOCKUP_ENABLED |
 *                    |                                  | WATCHDOG_SOFTOCKUP_ENABLED
 * -------------------|----------------------------------|-------------------------------
 * proc_nmi_watchdog  | watchdog_hardlockup_user_enabled | WATCHDOG_HARDLOCKUP_ENABLED
 * -------------------|----------------------------------|-------------------------------
 * proc_soft_watchdog | watchdog_softlockup_user_enabled | WATCHDOG_SOFTOCKUP_ENABLED
 */
static int proc_watchdog_common(int which, const struct ctl_table *table, int write,
                                void *buffer, size_t *lenp, loff_t *ppos)
{
        int err, old, *param = table->data;

        mutex_lock(&watchdog_mutex);

        if (!write) {
                /*
                 * On read synchronize the userspace interface. This is a
                 * racy snapshot.
                 */
                *param = (watchdog_enabled & which) != 0;
                err = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
        } else {
                old = READ_ONCE(*param);
                err = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
                if (!err && old != READ_ONCE(*param))
                        proc_watchdog_update();
        }
        mutex_unlock(&watchdog_mutex);
        return err;
}

/*
 * /proc/sys/kernel/watchdog
 */
static int proc_watchdog(const struct ctl_table *table, int write,
                         void *buffer, size_t *lenp, loff_t *ppos)
{
        return proc_watchdog_common(WATCHDOG_HARDLOCKUP_ENABLED |
                                    WATCHDOG_SOFTOCKUP_ENABLED,
                                    table, write, buffer, lenp, ppos);
}

/*
 * /proc/sys/kernel/nmi_watchdog
 */
static int proc_nmi_watchdog(const struct ctl_table *table, int write,
                             void *buffer, size_t *lenp, loff_t *ppos)
{
        if (!watchdog_hardlockup_available && write)
                return -ENOTSUPP;
        return proc_watchdog_common(WATCHDOG_HARDLOCKUP_ENABLED,
                                    table, write, buffer, lenp, ppos);
}

#ifdef CONFIG_SOFTLOCKUP_DETECTOR
/*
 * /proc/sys/kernel/soft_watchdog
 */
static int proc_soft_watchdog(const struct ctl_table *table, int write,
                              void *buffer, size_t *lenp, loff_t *ppos)
{
        return proc_watchdog_common(WATCHDOG_SOFTOCKUP_ENABLED,
                                    table, write, buffer, lenp, ppos);
}
#endif

/*
 * /proc/sys/kernel/watchdog_thresh
 */
static int proc_watchdog_thresh(const struct ctl_table *table, int write,
                                void *buffer, size_t *lenp, loff_t *ppos)
{
        int err, old;

        mutex_lock(&watchdog_mutex);

        old = READ_ONCE(watchdog_thresh);
        err = proc_dointvec_minmax(table, write, buffer, lenp, ppos);

        if (!err && write && old != READ_ONCE(watchdog_thresh))
                proc_watchdog_update();

        mutex_unlock(&watchdog_mutex);
        return err;
}

/*
 * The cpumask is the mask of possible cpus that the watchdog can run
 * on, not the mask of cpus it is actually running on.  This allows the
 * user to specify a mask that will include cpus that have not yet
 * been brought online, if desired.
 */
static int proc_watchdog_cpumask(const struct ctl_table *table, int write,
                                 void *buffer, size_t *lenp, loff_t *ppos)
{
        int err;

        mutex_lock(&watchdog_mutex);

        err = proc_do_large_bitmap(table, write, buffer, lenp, ppos);
        if (!err && write)
                proc_watchdog_update();

        mutex_unlock(&watchdog_mutex);
        return err;
}

static const int sixty = 60;

static struct ctl_table watchdog_sysctls[] = {
        {
                .procname       = "watchdog",
                .data                = &watchdog_user_enabled,
                .maxlen                = sizeof(int),
                .mode                = 0644,
                .proc_handler   = proc_watchdog,
                .extra1                = SYSCTL_ZERO,
                .extra2                = SYSCTL_ONE,
        },
        {
                .procname        = "watchdog_thresh",
                .data                = &watchdog_thresh,
                .maxlen                = sizeof(int),
                .mode                = 0644,
                .proc_handler        = proc_watchdog_thresh,
                .extra1                = SYSCTL_ZERO,
                .extra2                = (void *)&sixty,
        },
        {
                .procname        = "watchdog_cpumask",
                .data                = &watchdog_cpumask_bits,
                .maxlen                = NR_CPUS,
                .mode                = 0644,
                .proc_handler        = proc_watchdog_cpumask,
        },
#ifdef CONFIG_SOFTLOCKUP_DETECTOR
        {
                .procname       = "soft_watchdog",
                .data                = &watchdog_softlockup_user_enabled,
                .maxlen                = sizeof(int),
                .mode                = 0644,
                .proc_handler   = proc_soft_watchdog,
                .extra1                = SYSCTL_ZERO,
                .extra2                = SYSCTL_ONE,
        },
        {
                .procname        = "softlockup_panic",
                .data                = &softlockup_panic,
                .maxlen                = sizeof(int),
                .mode                = 0644,
                .proc_handler        = proc_dointvec_minmax,
                .extra1                = SYSCTL_ZERO,
                .extra2                = SYSCTL_ONE,
        },
#ifdef CONFIG_SMP
        {
                .procname        = "softlockup_all_cpu_backtrace",
                .data                = &sysctl_softlockup_all_cpu_backtrace,
                .maxlen                = sizeof(int),
                .mode                = 0644,
                .proc_handler        = proc_dointvec_minmax,
                .extra1                = SYSCTL_ZERO,
                .extra2                = SYSCTL_ONE,
        },
#endif /* CONFIG_SMP */
#endif
#ifdef CONFIG_HARDLOCKUP_DETECTOR
        {
                .procname        = "hardlockup_panic",
                .data                = &hardlockup_panic,
                .maxlen                = sizeof(int),
                .mode                = 0644,
                .proc_handler        = proc_dointvec_minmax,
                .extra1                = SYSCTL_ZERO,
                .extra2                = SYSCTL_ONE,
        },
#ifdef CONFIG_SMP
        {
                .procname        = "hardlockup_all_cpu_backtrace",
                .data                = &sysctl_hardlockup_all_cpu_backtrace,
                .maxlen                = sizeof(int),
                .mode                = 0644,
                .proc_handler        = proc_dointvec_minmax,
                .extra1                = SYSCTL_ZERO,
                .extra2                = SYSCTL_ONE,
        },
#endif /* CONFIG_SMP */
#endif
};

static struct ctl_table watchdog_hardlockup_sysctl[] = {
        {
                .procname       = "nmi_watchdog",
                .data                = &watchdog_hardlockup_user_enabled,
                .maxlen                = sizeof(int),
                .mode                = 0444,
                .proc_handler   = proc_nmi_watchdog,
                .extra1                = SYSCTL_ZERO,
                .extra2                = SYSCTL_ONE,
        },
};

static void __init watchdog_sysctl_init(void)
{
        register_sysctl_init("kernel", watchdog_sysctls);

        if (watchdog_hardlockup_available)
                watchdog_hardlockup_sysctl[0].mode = 0644;
        register_sysctl_init("kernel", watchdog_hardlockup_sysctl);
}

#else
#define watchdog_sysctl_init() do { } while (0)
#endif /* CONFIG_SYSCTL */

static void __init lockup_detector_delay_init(struct work_struct *work);
static bool allow_lockup_detector_init_retry __initdata;

static struct work_struct detector_work __initdata =
                __WORK_INITIALIZER(detector_work, lockup_detector_delay_init);

static void __init lockup_detector_delay_init(struct work_struct *work)
{
        int ret;

        ret = watchdog_hardlockup_probe();
        if (ret) {
                if (ret == -ENODEV)
                        pr_info("NMI not fully supported\n");
                else
                        pr_info("Delayed init of the lockup detector failed: %d\n", ret);
                pr_info("Hard watchdog permanently disabled\n");
                return;
        }

        allow_lockup_detector_init_retry = false;

        watchdog_hardlockup_available = true;
        lockup_detector_setup();
}

/*
 * lockup_detector_retry_init - retry init lockup detector if possible.
 *
 * Retry hardlockup detector init. It is useful when it requires some
 * functionality that has to be initialized later on a particular
 * platform.
 */
void __init lockup_detector_retry_init(void)
{
        /* Must be called before late init calls */
        if (!allow_lockup_detector_init_retry)
                return;

        schedule_work(&detector_work);
}

/*
 * Ensure that optional delayed hardlockup init is proceed before
 * the init code and memory is freed.
 */
static int __init lockup_detector_check(void)
{
        /* Prevent any later retry. */
        allow_lockup_detector_init_retry = false;

        /* Make sure no work is pending. */
        flush_work(&detector_work);

        watchdog_sysctl_init();

        return 0;

}
late_initcall_sync(lockup_detector_check);

void __init lockup_detector_init(void)
{
        if (tick_nohz_full_enabled())
                pr_info("Disabling watchdog on nohz_full cores by default\n");

        cpumask_copy(&watchdog_cpumask,
                     housekeeping_cpumask(HK_TYPE_TIMER));

        if (!watchdog_hardlockup_probe())
                watchdog_hardlockup_available = true;
        else
                allow_lockup_detector_init_retry = true;

        lockup_detector_setup();
}

























   49 


   48 



    1 


    1 


















   26 
    2 




   12 




    3 







    1 














    2 






    1 

    1 




















   12 












    4 





    1 






   12 
















   12 



























   28 











   28 












    3 
    1 




















   26 




















































































    4 


    3 







    4 

    4 

    4 












    4 







    4 




    4 

































































   12 



   12 


















   12 
   12 

    1 
   11 







    5 
    1 





    4 

    4 


    1 
    3 



























    1 







    1 



    1 
    1 



    1 

















    7 



    1 


    1 






    1 


















   10 





   10 
   10 


    2 


    8 





    7 





    1 




    8 



   10 
















    3 







    3 

    3 
    3 







































    3 









    8 






    2 





    2 





    5 

    5 










    4 





    4 






    1 



    3 




    3 



    1 


    2 



1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
// SPDX-License-Identifier: GPL-2.0
/*
 * To speed up listener socket lookup, create an array to store all sockets
 * listening on the same port.  This allows a decision to be made after finding
 * the first socket.  An optional BPF program can also be configured for
 * selecting the socket index from the array of available sockets.
 */

#include <net/ip.h>
#include <net/sock_reuseport.h>
#include <linux/bpf.h>
#include <linux/idr.h>
#include <linux/filter.h>
#include <linux/rcupdate.h>

#define INIT_SOCKS 128

DEFINE_SPINLOCK(reuseport_lock);

static DEFINE_IDA(reuseport_ida);
static int reuseport_resurrect(struct sock *sk, struct sock_reuseport *old_reuse,
                               struct sock_reuseport *reuse, bool bind_inany);

void reuseport_has_conns_set(struct sock *sk)
{
        struct sock_reuseport *reuse;

        if (!rcu_access_pointer(sk->sk_reuseport_cb))
                return;

        spin_lock_bh(&reuseport_lock);
        reuse = rcu_dereference_protected(sk->sk_reuseport_cb,
                                          lockdep_is_held(&reuseport_lock));
        if (likely(reuse))
                reuse->has_conns = 1;
        spin_unlock_bh(&reuseport_lock);
}
EXPORT_SYMBOL(reuseport_has_conns_set);

static void __reuseport_get_incoming_cpu(struct sock_reuseport *reuse)
{
        /* Paired with READ_ONCE() in reuseport_select_sock_by_hash(). */
        WRITE_ONCE(reuse->incoming_cpu, reuse->incoming_cpu + 1);
}

static void __reuseport_put_incoming_cpu(struct sock_reuseport *reuse)
{
        /* Paired with READ_ONCE() in reuseport_select_sock_by_hash(). */
        WRITE_ONCE(reuse->incoming_cpu, reuse->incoming_cpu - 1);
}

static void reuseport_get_incoming_cpu(struct sock *sk, struct sock_reuseport *reuse)
{
        if (sk->sk_incoming_cpu >= 0)
                __reuseport_get_incoming_cpu(reuse);
}

static void reuseport_put_incoming_cpu(struct sock *sk, struct sock_reuseport *reuse)
{
        if (sk->sk_incoming_cpu >= 0)
                __reuseport_put_incoming_cpu(reuse);
}

void reuseport_update_incoming_cpu(struct sock *sk, int val)
{
        struct sock_reuseport *reuse;
        int old_sk_incoming_cpu;

        if (unlikely(!rcu_access_pointer(sk->sk_reuseport_cb))) {
                /* Paired with REAE_ONCE() in sk_incoming_cpu_update()
                 * and compute_score().
                 */
                WRITE_ONCE(sk->sk_incoming_cpu, val);
                return;
        }

        spin_lock_bh(&reuseport_lock);

        /* This must be done under reuseport_lock to avoid a race with
         * reuseport_grow(), which accesses sk->sk_incoming_cpu without
         * lock_sock() when detaching a shutdown()ed sk.
         *
         * Paired with READ_ONCE() in reuseport_select_sock_by_hash().
         */
        old_sk_incoming_cpu = sk->sk_incoming_cpu;
        WRITE_ONCE(sk->sk_incoming_cpu, val);

        reuse = rcu_dereference_protected(sk->sk_reuseport_cb,
                                          lockdep_is_held(&reuseport_lock));

        /* reuseport_grow() has detached a closed sk. */
        if (!reuse)
                goto out;

        if (old_sk_incoming_cpu < 0 && val >= 0)
                __reuseport_get_incoming_cpu(reuse);
        else if (old_sk_incoming_cpu >= 0 && val < 0)
                __reuseport_put_incoming_cpu(reuse);

out:
        spin_unlock_bh(&reuseport_lock);
}

static int reuseport_sock_index(struct sock *sk,
                                const struct sock_reuseport *reuse,
                                bool closed)
{
        int left, right;

        if (!closed) {
                left = 0;
                right = reuse->num_socks;
        } else {
                left = reuse->max_socks - reuse->num_closed_socks;
                right = reuse->max_socks;
        }

        for (; left < right; left++)
                if (reuse->socks[left] == sk)
                        return left;
        return -1;
}

static void __reuseport_add_sock(struct sock *sk,
                                 struct sock_reuseport *reuse)
{
        reuse->socks[reuse->num_socks] = sk;
        /* paired with smp_rmb() in reuseport_(select|migrate)_sock() */
        smp_wmb();
        reuse->num_socks++;
        reuseport_get_incoming_cpu(sk, reuse);
}

static bool __reuseport_detach_sock(struct sock *sk,
                                    struct sock_reuseport *reuse)
{
        int i = reuseport_sock_index(sk, reuse, false);

        if (i == -1)
                return false;

        reuse->socks[i] = reuse->socks[reuse->num_socks - 1];
        reuse->num_socks--;
        reuseport_put_incoming_cpu(sk, reuse);

        return true;
}

static void __reuseport_add_closed_sock(struct sock *sk,
                                        struct sock_reuseport *reuse)
{
        reuse->socks[reuse->max_socks - reuse->num_closed_socks - 1] = sk;
        /* paired with READ_ONCE() in inet_csk_bind_conflict() */
        WRITE_ONCE(reuse->num_closed_socks, reuse->num_closed_socks + 1);
        reuseport_get_incoming_cpu(sk, reuse);
}

static bool __reuseport_detach_closed_sock(struct sock *sk,
                                           struct sock_reuseport *reuse)
{
        int i = reuseport_sock_index(sk, reuse, true);

        if (i == -1)
                return false;

        reuse->socks[i] = reuse->socks[reuse->max_socks - reuse->num_closed_socks];
        /* paired with READ_ONCE() in inet_csk_bind_conflict() */
        WRITE_ONCE(reuse->num_closed_socks, reuse->num_closed_socks - 1);
        reuseport_put_incoming_cpu(sk, reuse);

        return true;
}

static struct sock_reuseport *__reuseport_alloc(unsigned int max_socks)
{
        struct sock_reuseport *reuse;

        reuse = kzalloc(struct_size(reuse, socks, max_socks), GFP_ATOMIC);
        if (!reuse)
                return NULL;

        reuse->max_socks = max_socks;

        RCU_INIT_POINTER(reuse->prog, NULL);
        return reuse;
}

int reuseport_alloc(struct sock *sk, bool bind_inany)
{
        struct sock_reuseport *reuse;
        int id, ret = 0;

        /* bh lock used since this function call may precede hlist lock in
         * soft irq of receive path or setsockopt from process context
         */
        spin_lock_bh(&reuseport_lock);

        /* Allocation attempts can occur concurrently via the setsockopt path
         * and the bind/hash path.  Nothing to do when we lose the race.
         */
        reuse = rcu_dereference_protected(sk->sk_reuseport_cb,
                                          lockdep_is_held(&reuseport_lock));
        if (reuse) {
                if (reuse->num_closed_socks) {
                        /* sk was shutdown()ed before */
                        ret = reuseport_resurrect(sk, reuse, NULL, bind_inany);
                        goto out;
                }

                /* Only set reuse->bind_inany if the bind_inany is true.
                 * Otherwise, it will overwrite the reuse->bind_inany
                 * which was set by the bind/hash path.
                 */
                if (bind_inany)
                        reuse->bind_inany = bind_inany;
                goto out;
        }

        reuse = __reuseport_alloc(INIT_SOCKS);
        if (!reuse) {
                ret = -ENOMEM;
                goto out;
        }

        id = ida_alloc(&reuseport_ida, GFP_ATOMIC);
        if (id < 0) {
                kfree(reuse);
                ret = id;
                goto out;
        }

        reuse->reuseport_id = id;
        reuse->bind_inany = bind_inany;
        reuse->socks[0] = sk;
        reuse->num_socks = 1;
        reuseport_get_incoming_cpu(sk, reuse);
        rcu_assign_pointer(sk->sk_reuseport_cb, reuse);

out:
        spin_unlock_bh(&reuseport_lock);

        return ret;
}
EXPORT_SYMBOL(reuseport_alloc);

static struct sock_reuseport *reuseport_grow(struct sock_reuseport *reuse)
{
        struct sock_reuseport *more_reuse;
        u32 more_socks_size, i;

        more_socks_size = reuse->max_socks * 2U;
        if (more_socks_size > U16_MAX) {
                if (reuse->num_closed_socks) {
                        /* Make room by removing a closed sk.
                         * The child has already been migrated.
                         * Only reqsk left at this point.
                         */
                        struct sock *sk;

                        sk = reuse->socks[reuse->max_socks - reuse->num_closed_socks];
                        RCU_INIT_POINTER(sk->sk_reuseport_cb, NULL);
                        __reuseport_detach_closed_sock(sk, reuse);

                        return reuse;
                }

                return NULL;
        }

        more_reuse = __reuseport_alloc(more_socks_size);
        if (!more_reuse)
                return NULL;

        more_reuse->num_socks = reuse->num_socks;
        more_reuse->num_closed_socks = reuse->num_closed_socks;
        more_reuse->prog = reuse->prog;
        more_reuse->reuseport_id = reuse->reuseport_id;
        more_reuse->bind_inany = reuse->bind_inany;
        more_reuse->has_conns = reuse->has_conns;
        more_reuse->incoming_cpu = reuse->incoming_cpu;

        memcpy(more_reuse->socks, reuse->socks,
               reuse->num_socks * sizeof(struct sock *));
        memcpy(more_reuse->socks +
               (more_reuse->max_socks - more_reuse->num_closed_socks),
               reuse->socks + (reuse->max_socks - reuse->num_closed_socks),
               reuse->num_closed_socks * sizeof(struct sock *));
        more_reuse->synq_overflow_ts = READ_ONCE(reuse->synq_overflow_ts);

        for (i = 0; i < reuse->max_socks; ++i)
                rcu_assign_pointer(reuse->socks[i]->sk_reuseport_cb,
                                   more_reuse);

        /* Note: we use kfree_rcu here instead of reuseport_free_rcu so
         * that reuse and more_reuse can temporarily share a reference
         * to prog.
         */
        kfree_rcu(reuse, rcu);
        return more_reuse;
}

static void reuseport_free_rcu(struct rcu_head *head)
{
        struct sock_reuseport *reuse;

        reuse = container_of(head, struct sock_reuseport, rcu);
        sk_reuseport_prog_free(rcu_dereference_protected(reuse->prog, 1));
        ida_free(&reuseport_ida, reuse->reuseport_id);
        kfree(reuse);
}

/**
 *  reuseport_add_sock - Add a socket to the reuseport group of another.
 *  @sk:  New socket to add to the group.
 *  @sk2: Socket belonging to the existing reuseport group.
 *  @bind_inany: Whether or not the group is bound to a local INANY address.
 *
 *  May return ENOMEM and not add socket to group under memory pressure.
 */
int reuseport_add_sock(struct sock *sk, struct sock *sk2, bool bind_inany)
{
        struct sock_reuseport *old_reuse, *reuse;

        if (!rcu_access_pointer(sk2->sk_reuseport_cb)) {
                int err = reuseport_alloc(sk2, bind_inany);

                if (err)
                        return err;
        }

        spin_lock_bh(&reuseport_lock);
        reuse = rcu_dereference_protected(sk2->sk_reuseport_cb,
                                          lockdep_is_held(&reuseport_lock));
        old_reuse = rcu_dereference_protected(sk->sk_reuseport_cb,
                                              lockdep_is_held(&reuseport_lock));
        if (old_reuse && old_reuse->num_closed_socks) {
                /* sk was shutdown()ed before */
                int err = reuseport_resurrect(sk, old_reuse, reuse, reuse->bind_inany);

                spin_unlock_bh(&reuseport_lock);
                return err;
        }

        if (old_reuse && old_reuse->num_socks != 1) {
                spin_unlock_bh(&reuseport_lock);
                return -EBUSY;
        }

        if (reuse->num_socks + reuse->num_closed_socks == reuse->max_socks) {
                reuse = reuseport_grow(reuse);
                if (!reuse) {
                        spin_unlock_bh(&reuseport_lock);
                        return -ENOMEM;
                }
        }

        __reuseport_add_sock(sk, reuse);
        rcu_assign_pointer(sk->sk_reuseport_cb, reuse);

        spin_unlock_bh(&reuseport_lock);

        if (old_reuse)
                call_rcu(&old_reuse->rcu, reuseport_free_rcu);
        return 0;
}
EXPORT_SYMBOL(reuseport_add_sock);

static int reuseport_resurrect(struct sock *sk, struct sock_reuseport *old_reuse,
                               struct sock_reuseport *reuse, bool bind_inany)
{
        if (old_reuse == reuse) {
                /* If sk was in the same reuseport group, just pop sk out of
                 * the closed section and push sk into the listening section.
                 */
                __reuseport_detach_closed_sock(sk, old_reuse);
                __reuseport_add_sock(sk, old_reuse);
                return 0;
        }

        if (!reuse) {
                /* In bind()/listen() path, we cannot carry over the eBPF prog
                 * for the shutdown()ed socket. In setsockopt() path, we should
                 * not change the eBPF prog of listening sockets by attaching a
                 * prog to the shutdown()ed socket. Thus, we will allocate a new
                 * reuseport group and detach sk from the old group.
                 */
                int id;

                reuse = __reuseport_alloc(INIT_SOCKS);
                if (!reuse)
                        return -ENOMEM;

                id = ida_alloc(&reuseport_ida, GFP_ATOMIC);
                if (id < 0) {
                        kfree(reuse);
                        return id;
                }

                reuse->reuseport_id = id;
                reuse->bind_inany = bind_inany;
        } else {
                /* Move sk from the old group to the new one if
                 * - all the other listeners in the old group were close()d or
                 *   shutdown()ed, and then sk2 has listen()ed on the same port
                 * OR
                 * - sk listen()ed without bind() (or with autobind), was
                 *   shutdown()ed, and then listen()s on another port which
                 *   sk2 listen()s on.
                 */
                if (reuse->num_socks + reuse->num_closed_socks == reuse->max_socks) {
                        reuse = reuseport_grow(reuse);
                        if (!reuse)
                                return -ENOMEM;
                }
        }

        __reuseport_detach_closed_sock(sk, old_reuse);
        __reuseport_add_sock(sk, reuse);
        rcu_assign_pointer(sk->sk_reuseport_cb, reuse);

        if (old_reuse->num_socks + old_reuse->num_closed_socks == 0)
                call_rcu(&old_reuse->rcu, reuseport_free_rcu);

        return 0;
}

void reuseport_detach_sock(struct sock *sk)
{
        struct sock_reuseport *reuse;

        spin_lock_bh(&reuseport_lock);
        reuse = rcu_dereference_protected(sk->sk_reuseport_cb,
                                          lockdep_is_held(&reuseport_lock));

        /* reuseport_grow() has detached a closed sk */
        if (!reuse)
                goto out;

        /* Notify the bpf side. The sk may be added to a sockarray
         * map. If so, sockarray logic will remove it from the map.
         *
         * Other bpf map types that work with reuseport, like sockmap,
         * don't need an explicit callback from here. They override sk
         * unhash/close ops to remove the sk from the map before we
         * get to this point.
         */
        bpf_sk_reuseport_detach(sk);

        rcu_assign_pointer(sk->sk_reuseport_cb, NULL);

        if (!__reuseport_detach_closed_sock(sk, reuse))
                __reuseport_detach_sock(sk, reuse);

        if (reuse->num_socks + reuse->num_closed_socks == 0)
                call_rcu(&reuse->rcu, reuseport_free_rcu);

out:
        spin_unlock_bh(&reuseport_lock);
}
EXPORT_SYMBOL(reuseport_detach_sock);

void reuseport_stop_listen_sock(struct sock *sk)
{
        if (sk->sk_protocol == IPPROTO_TCP) {
                struct sock_reuseport *reuse;
                struct bpf_prog *prog;

                spin_lock_bh(&reuseport_lock);

                reuse = rcu_dereference_protected(sk->sk_reuseport_cb,
                                                  lockdep_is_held(&reuseport_lock));
                prog = rcu_dereference_protected(reuse->prog,
                                                 lockdep_is_held(&reuseport_lock));

                if (READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_migrate_req) ||
                    (prog && prog->expected_attach_type == BPF_SK_REUSEPORT_SELECT_OR_MIGRATE)) {
                        /* Migration capable, move sk from the listening section
                         * to the closed section.
                         */
                        bpf_sk_reuseport_detach(sk);

                        __reuseport_detach_sock(sk, reuse);
                        __reuseport_add_closed_sock(sk, reuse);

                        spin_unlock_bh(&reuseport_lock);
                        return;
                }

                spin_unlock_bh(&reuseport_lock);
        }

        /* Not capable to do migration, detach immediately */
        reuseport_detach_sock(sk);
}
EXPORT_SYMBOL(reuseport_stop_listen_sock);

static struct sock *run_bpf_filter(struct sock_reuseport *reuse, u16 socks,
                                   struct bpf_prog *prog, struct sk_buff *skb,
                                   int hdr_len)
{
        struct sk_buff *nskb = NULL;
        u32 index;

        if (skb_shared(skb)) {
                nskb = skb_clone(skb, GFP_ATOMIC);
                if (!nskb)
                        return NULL;
                skb = nskb;
        }

        /* temporarily advance data past protocol header */
        if (!pskb_pull(skb, hdr_len)) {
                kfree_skb(nskb);
                return NULL;
        }
        index = bpf_prog_run_save_cb(prog, skb);
        __skb_push(skb, hdr_len);

        consume_skb(nskb);

        if (index >= socks)
                return NULL;

        return reuse->socks[index];
}

static struct sock *reuseport_select_sock_by_hash(struct sock_reuseport *reuse,
                                                  u32 hash, u16 num_socks)
{
        struct sock *first_valid_sk = NULL;
        int i, j;

        i = j = reciprocal_scale(hash, num_socks);
        do {
                struct sock *sk = reuse->socks[i];

                if (sk->sk_state != TCP_ESTABLISHED) {
                        /* Paired with WRITE_ONCE() in __reuseport_(get|put)_incoming_cpu(). */
                        if (!READ_ONCE(reuse->incoming_cpu))
                                return sk;

                        /* Paired with WRITE_ONCE() in reuseport_update_incoming_cpu(). */
                        if (READ_ONCE(sk->sk_incoming_cpu) == raw_smp_processor_id())
                                return sk;

                        if (!first_valid_sk)
                                first_valid_sk = sk;
                }

                i++;
                if (i >= num_socks)
                        i = 0;
        } while (i != j);

        return first_valid_sk;
}

/**
 *  reuseport_select_sock - Select a socket from an SO_REUSEPORT group.
 *  @sk: First socket in the group.
 *  @hash: When no BPF filter is available, use this hash to select.
 *  @skb: skb to run through BPF filter.
 *  @hdr_len: BPF filter expects skb data pointer at payload data.  If
 *    the skb does not yet point at the payload, this parameter represents
 *    how far the pointer needs to advance to reach the payload.
 *  Returns a socket that should receive the packet (or NULL on error).
 */
struct sock *reuseport_select_sock(struct sock *sk,
                                   u32 hash,
                                   struct sk_buff *skb,
                                   int hdr_len)
{
        struct sock_reuseport *reuse;
        struct bpf_prog *prog;
        struct sock *sk2 = NULL;
        u16 socks;

        rcu_read_lock();
        reuse = rcu_dereference(sk->sk_reuseport_cb);

        /* if memory allocation failed or add call is not yet complete */
        if (!reuse)
                goto out;

        prog = rcu_dereference(reuse->prog);
        socks = READ_ONCE(reuse->num_socks);
        if (likely(socks)) {
                /* paired with smp_wmb() in __reuseport_add_sock() */
                smp_rmb();

                if (!prog || !skb)
                        goto select_by_hash;

                if (prog->type == BPF_PROG_TYPE_SK_REUSEPORT)
                        sk2 = bpf_run_sk_reuseport(reuse, sk, prog, skb, NULL, hash);
                else
                        sk2 = run_bpf_filter(reuse, socks, prog, skb, hdr_len);

select_by_hash:
                /* no bpf or invalid bpf result: fall back to hash usage */
                if (!sk2)
                        sk2 = reuseport_select_sock_by_hash(reuse, hash, socks);
        }

out:
        rcu_read_unlock();
        return sk2;
}
EXPORT_SYMBOL(reuseport_select_sock);

/**
 *  reuseport_migrate_sock - Select a socket from an SO_REUSEPORT group.
 *  @sk: close()ed or shutdown()ed socket in the group.
 *  @migrating_sk: ESTABLISHED/SYN_RECV full socket in the accept queue or
 *    NEW_SYN_RECV request socket during 3WHS.
 *  @skb: skb to run through BPF filter.
 *  Returns a socket (with sk_refcnt +1) that should accept the child socket
 *  (or NULL on error).
 */
struct sock *reuseport_migrate_sock(struct sock *sk,
                                    struct sock *migrating_sk,
                                    struct sk_buff *skb)
{
        struct sock_reuseport *reuse;
        struct sock *nsk = NULL;
        bool allocated = false;
        struct bpf_prog *prog;
        u16 socks;
        u32 hash;

        rcu_read_lock();

        reuse = rcu_dereference(sk->sk_reuseport_cb);
        if (!reuse)
                goto out;

        socks = READ_ONCE(reuse->num_socks);
        if (unlikely(!socks))
                goto failure;

        /* paired with smp_wmb() in __reuseport_add_sock() */
        smp_rmb();

        hash = migrating_sk->sk_hash;
        prog = rcu_dereference(reuse->prog);
        if (!prog || prog->expected_attach_type != BPF_SK_REUSEPORT_SELECT_OR_MIGRATE) {
                if (READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_migrate_req))
                        goto select_by_hash;
                goto failure;
        }

        if (!skb) {
                skb = alloc_skb(0, GFP_ATOMIC);
                if (!skb)
                        goto failure;
                allocated = true;
        }

        nsk = bpf_run_sk_reuseport(reuse, sk, prog, skb, migrating_sk, hash);

        if (allocated)
                kfree_skb(skb);

select_by_hash:
        if (!nsk)
                nsk = reuseport_select_sock_by_hash(reuse, hash, socks);

        if (IS_ERR_OR_NULL(nsk) || unlikely(!refcount_inc_not_zero(&nsk->sk_refcnt))) {
                nsk = NULL;
                goto failure;
        }

out:
        rcu_read_unlock();
        return nsk;

failure:
        __NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPMIGRATEREQFAILURE);
        goto out;
}
EXPORT_SYMBOL(reuseport_migrate_sock);

int reuseport_attach_prog(struct sock *sk, struct bpf_prog *prog)
{
        struct sock_reuseport *reuse;
        struct bpf_prog *old_prog;

        if (sk_unhashed(sk)) {
                int err;

                if (!sk->sk_reuseport)
                        return -EINVAL;

                err = reuseport_alloc(sk, false);
                if (err)
                        return err;
        } else if (!rcu_access_pointer(sk->sk_reuseport_cb)) {
                /* The socket wasn't bound with SO_REUSEPORT */
                return -EINVAL;
        }

        spin_lock_bh(&reuseport_lock);
        reuse = rcu_dereference_protected(sk->sk_reuseport_cb,
                                          lockdep_is_held(&reuseport_lock));
        old_prog = rcu_dereference_protected(reuse->prog,
                                             lockdep_is_held(&reuseport_lock));
        rcu_assign_pointer(reuse->prog, prog);
        spin_unlock_bh(&reuseport_lock);

        sk_reuseport_prog_free(old_prog);
        return 0;
}
EXPORT_SYMBOL(reuseport_attach_prog);

int reuseport_detach_prog(struct sock *sk)
{
        struct sock_reuseport *reuse;
        struct bpf_prog *old_prog;

        old_prog = NULL;
        spin_lock_bh(&reuseport_lock);
        reuse = rcu_dereference_protected(sk->sk_reuseport_cb,
                                          lockdep_is_held(&reuseport_lock));

        /* reuse must be checked after acquiring the reuseport_lock
         * because reuseport_grow() can detach a closed sk.
         */
        if (!reuse) {
                spin_unlock_bh(&reuseport_lock);
                return sk->sk_reuseport ? -ENOENT : -EINVAL;
        }

        if (sk_unhashed(sk) && reuse->num_closed_socks) {
                spin_unlock_bh(&reuseport_lock);
                return -ENOENT;
        }

        old_prog = rcu_replace_pointer(reuse->prog, old_prog,
                                       lockdep_is_held(&reuseport_lock));
        spin_unlock_bh(&reuseport_lock);

        if (!old_prog)
                return -ENOENT;

        sk_reuseport_prog_free(old_prog);
        return 0;
}
EXPORT_SYMBOL(reuseport_detach_prog);








































































































    2 



    2 



    2 






























    2 























    2 



    1 





































    2 



    1 



    1 































    1 



    2 






















    2 



    2 











    6 










    2 



    2 



    2 







    3 











    2 



    1 













































   39 






    1 
   38 

    2 

    1 


   30 
    6 





   28 
    2 





   25 
    3 





   21 
    4 





   16 
    3 

    2 



   10 
    4 

    2 



    4 


    6 



    1 


    3 






















































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
// SPDX-License-Identifier: GPL-2.0-or-later

#include <net/genetlink.h>

#include "br_private.h"
#include "br_private_cfm.h"

static const struct nla_policy
br_cfm_mep_create_policy[IFLA_BRIDGE_CFM_MEP_CREATE_MAX + 1] = {
        [IFLA_BRIDGE_CFM_MEP_CREATE_UNSPEC]        = { .type = NLA_REJECT },
        [IFLA_BRIDGE_CFM_MEP_CREATE_INSTANCE]        = { .type = NLA_U32 },
        [IFLA_BRIDGE_CFM_MEP_CREATE_DOMAIN]        = { .type = NLA_U32 },
        [IFLA_BRIDGE_CFM_MEP_CREATE_DIRECTION]        = { .type = NLA_U32 },
        [IFLA_BRIDGE_CFM_MEP_CREATE_IFINDEX]        = { .type = NLA_U32 },
};

static const struct nla_policy
br_cfm_mep_delete_policy[IFLA_BRIDGE_CFM_MEP_DELETE_MAX + 1] = {
        [IFLA_BRIDGE_CFM_MEP_DELETE_UNSPEC]        = { .type = NLA_REJECT },
        [IFLA_BRIDGE_CFM_MEP_DELETE_INSTANCE]        = { .type = NLA_U32 },
};

static const struct nla_policy
br_cfm_mep_config_policy[IFLA_BRIDGE_CFM_MEP_CONFIG_MAX + 1] = {
        [IFLA_BRIDGE_CFM_MEP_CONFIG_UNSPEC]         = { .type = NLA_REJECT },
        [IFLA_BRIDGE_CFM_MEP_CONFIG_INSTANCE]         = { .type = NLA_U32 },
        [IFLA_BRIDGE_CFM_MEP_CONFIG_UNICAST_MAC] = NLA_POLICY_ETH_ADDR,
        [IFLA_BRIDGE_CFM_MEP_CONFIG_MDLEVEL]         = NLA_POLICY_MAX(NLA_U32, 7),
        [IFLA_BRIDGE_CFM_MEP_CONFIG_MEPID]         = NLA_POLICY_MAX(NLA_U32, 0x1FFF),
};

static const struct nla_policy
br_cfm_cc_config_policy[IFLA_BRIDGE_CFM_CC_CONFIG_MAX + 1] = {
        [IFLA_BRIDGE_CFM_CC_CONFIG_UNSPEC]         = { .type = NLA_REJECT },
        [IFLA_BRIDGE_CFM_CC_CONFIG_INSTANCE]         = { .type = NLA_U32 },
        [IFLA_BRIDGE_CFM_CC_CONFIG_ENABLE]         = { .type = NLA_U32 },
        [IFLA_BRIDGE_CFM_CC_CONFIG_EXP_INTERVAL] = { .type = NLA_U32 },
        [IFLA_BRIDGE_CFM_CC_CONFIG_EXP_MAID]         = {
        .type = NLA_BINARY, .len = CFM_MAID_LENGTH },
};

static const struct nla_policy
br_cfm_cc_peer_mep_policy[IFLA_BRIDGE_CFM_CC_PEER_MEP_MAX + 1] = {
        [IFLA_BRIDGE_CFM_CC_PEER_MEP_UNSPEC]        = { .type = NLA_REJECT },
        [IFLA_BRIDGE_CFM_CC_PEER_MEP_INSTANCE]        = { .type = NLA_U32 },
        [IFLA_BRIDGE_CFM_CC_PEER_MEPID]                = NLA_POLICY_MAX(NLA_U32, 0x1FFF),
};

static const struct nla_policy
br_cfm_cc_rdi_policy[IFLA_BRIDGE_CFM_CC_RDI_MAX + 1] = {
        [IFLA_BRIDGE_CFM_CC_RDI_UNSPEC]                = { .type = NLA_REJECT },
        [IFLA_BRIDGE_CFM_CC_RDI_INSTANCE]        = { .type = NLA_U32 },
        [IFLA_BRIDGE_CFM_CC_RDI_RDI]                = { .type = NLA_U32 },
};

static const struct nla_policy
br_cfm_cc_ccm_tx_policy[IFLA_BRIDGE_CFM_CC_CCM_TX_MAX + 1] = {
        [IFLA_BRIDGE_CFM_CC_CCM_TX_UNSPEC]           = { .type = NLA_REJECT },
        [IFLA_BRIDGE_CFM_CC_CCM_TX_INSTANCE]           = { .type = NLA_U32 },
        [IFLA_BRIDGE_CFM_CC_CCM_TX_DMAC]           = NLA_POLICY_ETH_ADDR,
        [IFLA_BRIDGE_CFM_CC_CCM_TX_SEQ_NO_UPDATE]  = { .type = NLA_U32 },
        [IFLA_BRIDGE_CFM_CC_CCM_TX_PERIOD]           = { .type = NLA_U32 },
        [IFLA_BRIDGE_CFM_CC_CCM_TX_IF_TLV]           = { .type = NLA_U32 },
        [IFLA_BRIDGE_CFM_CC_CCM_TX_IF_TLV_VALUE]   = { .type = NLA_U8 },
        [IFLA_BRIDGE_CFM_CC_CCM_TX_PORT_TLV]           = { .type = NLA_U32 },
        [IFLA_BRIDGE_CFM_CC_CCM_TX_PORT_TLV_VALUE] = { .type = NLA_U8 },
};

static const struct nla_policy
br_cfm_policy[IFLA_BRIDGE_CFM_MAX + 1] = {
        [IFLA_BRIDGE_CFM_UNSPEC]                = { .type = NLA_REJECT },
        [IFLA_BRIDGE_CFM_MEP_CREATE]                =
                                NLA_POLICY_NESTED(br_cfm_mep_create_policy),
        [IFLA_BRIDGE_CFM_MEP_DELETE]                =
                                NLA_POLICY_NESTED(br_cfm_mep_delete_policy),
        [IFLA_BRIDGE_CFM_MEP_CONFIG]                =
                                NLA_POLICY_NESTED(br_cfm_mep_config_policy),
        [IFLA_BRIDGE_CFM_CC_CONFIG]                =
                                NLA_POLICY_NESTED(br_cfm_cc_config_policy),
        [IFLA_BRIDGE_CFM_CC_PEER_MEP_ADD]        =
                                NLA_POLICY_NESTED(br_cfm_cc_peer_mep_policy),
        [IFLA_BRIDGE_CFM_CC_PEER_MEP_REMOVE]        =
                                NLA_POLICY_NESTED(br_cfm_cc_peer_mep_policy),
        [IFLA_BRIDGE_CFM_CC_RDI]                =
                                NLA_POLICY_NESTED(br_cfm_cc_rdi_policy),
        [IFLA_BRIDGE_CFM_CC_CCM_TX]                =
                                NLA_POLICY_NESTED(br_cfm_cc_ccm_tx_policy),
};

static int br_mep_create_parse(struct net_bridge *br, struct nlattr *attr,
                               struct netlink_ext_ack *extack)
{
        struct nlattr *tb[IFLA_BRIDGE_CFM_MEP_CREATE_MAX + 1];
        struct br_cfm_mep_create create;
        u32 instance;
        int err;

        err = nla_parse_nested(tb, IFLA_BRIDGE_CFM_MEP_CREATE_MAX, attr,
                               br_cfm_mep_create_policy, extack);
        if (err)
                return err;

        if (!tb[IFLA_BRIDGE_CFM_MEP_CREATE_INSTANCE]) {
                NL_SET_ERR_MSG_MOD(extack, "Missing INSTANCE attribute");
                return -EINVAL;
        }
        if (!tb[IFLA_BRIDGE_CFM_MEP_CREATE_DOMAIN]) {
                NL_SET_ERR_MSG_MOD(extack, "Missing DOMAIN attribute");
                return -EINVAL;
        }
        if (!tb[IFLA_BRIDGE_CFM_MEP_CREATE_DIRECTION]) {
                NL_SET_ERR_MSG_MOD(extack, "Missing DIRECTION attribute");
                return -EINVAL;
        }
        if (!tb[IFLA_BRIDGE_CFM_MEP_CREATE_IFINDEX]) {
                NL_SET_ERR_MSG_MOD(extack, "Missing IFINDEX attribute");
                return -EINVAL;
        }

        memset(&create, 0, sizeof(create));

        instance =  nla_get_u32(tb[IFLA_BRIDGE_CFM_MEP_CREATE_INSTANCE]);
        create.domain = nla_get_u32(tb[IFLA_BRIDGE_CFM_MEP_CREATE_DOMAIN]);
        create.direction = nla_get_u32(tb[IFLA_BRIDGE_CFM_MEP_CREATE_DIRECTION]);
        create.ifindex = nla_get_u32(tb[IFLA_BRIDGE_CFM_MEP_CREATE_IFINDEX]);

        return br_cfm_mep_create(br, instance, &create, extack);
}

static int br_mep_delete_parse(struct net_bridge *br, struct nlattr *attr,
                               struct netlink_ext_ack *extack)
{
        struct nlattr *tb[IFLA_BRIDGE_CFM_MEP_DELETE_MAX + 1];
        u32 instance;
        int err;

        err = nla_parse_nested(tb, IFLA_BRIDGE_CFM_MEP_DELETE_MAX, attr,
                               br_cfm_mep_delete_policy, extack);
        if (err)
                return err;

        if (!tb[IFLA_BRIDGE_CFM_MEP_DELETE_INSTANCE]) {
                NL_SET_ERR_MSG_MOD(extack,
                                   "Missing INSTANCE attribute");
                return -EINVAL;
        }

        instance =  nla_get_u32(tb[IFLA_BRIDGE_CFM_MEP_DELETE_INSTANCE]);

        return br_cfm_mep_delete(br, instance, extack);
}

static int br_mep_config_parse(struct net_bridge *br, struct nlattr *attr,
                               struct netlink_ext_ack *extack)
{
        struct nlattr *tb[IFLA_BRIDGE_CFM_MEP_CONFIG_MAX + 1];
        struct br_cfm_mep_config config;
        u32 instance;
        int err;

        err = nla_parse_nested(tb, IFLA_BRIDGE_CFM_MEP_CONFIG_MAX, attr,
                               br_cfm_mep_config_policy, extack);
        if (err)
                return err;

        if (!tb[IFLA_BRIDGE_CFM_MEP_CONFIG_INSTANCE]) {
                NL_SET_ERR_MSG_MOD(extack, "Missing INSTANCE attribute");
                return -EINVAL;
        }
        if (!tb[IFLA_BRIDGE_CFM_MEP_CONFIG_UNICAST_MAC]) {
                NL_SET_ERR_MSG_MOD(extack, "Missing UNICAST_MAC attribute");
                return -EINVAL;
        }
        if (!tb[IFLA_BRIDGE_CFM_MEP_CONFIG_MDLEVEL]) {
                NL_SET_ERR_MSG_MOD(extack, "Missing MDLEVEL attribute");
                return -EINVAL;
        }
        if (!tb[IFLA_BRIDGE_CFM_MEP_CONFIG_MEPID]) {
                NL_SET_ERR_MSG_MOD(extack, "Missing MEPID attribute");
                return -EINVAL;
        }

        memset(&config, 0, sizeof(config));

        instance =  nla_get_u32(tb[IFLA_BRIDGE_CFM_MEP_CONFIG_INSTANCE]);
        nla_memcpy(&config.unicast_mac.addr,
                   tb[IFLA_BRIDGE_CFM_MEP_CONFIG_UNICAST_MAC],
                   sizeof(config.unicast_mac.addr));
        config.mdlevel = nla_get_u32(tb[IFLA_BRIDGE_CFM_MEP_CONFIG_MDLEVEL]);
        config.mepid = nla_get_u32(tb[IFLA_BRIDGE_CFM_MEP_CONFIG_MEPID]);

        return br_cfm_mep_config_set(br, instance, &config, extack);
}

static int br_cc_config_parse(struct net_bridge *br, struct nlattr *attr,
                              struct netlink_ext_ack *extack)
{
        struct nlattr *tb[IFLA_BRIDGE_CFM_CC_CONFIG_MAX + 1];
        struct br_cfm_cc_config config;
        u32 instance;
        int err;

        err = nla_parse_nested(tb, IFLA_BRIDGE_CFM_CC_CONFIG_MAX, attr,
                               br_cfm_cc_config_policy, extack);
        if (err)
                return err;

        if (!tb[IFLA_BRIDGE_CFM_CC_CONFIG_INSTANCE]) {
                NL_SET_ERR_MSG_MOD(extack, "Missing INSTANCE attribute");
                return -EINVAL;
        }
        if (!tb[IFLA_BRIDGE_CFM_CC_CONFIG_ENABLE]) {
                NL_SET_ERR_MSG_MOD(extack, "Missing ENABLE attribute");
                return -EINVAL;
        }
        if (!tb[IFLA_BRIDGE_CFM_CC_CONFIG_EXP_INTERVAL]) {
                NL_SET_ERR_MSG_MOD(extack, "Missing INTERVAL attribute");
                return -EINVAL;
        }
        if (!tb[IFLA_BRIDGE_CFM_CC_CONFIG_EXP_MAID]) {
                NL_SET_ERR_MSG_MOD(extack, "Missing MAID attribute");
                return -EINVAL;
        }

        memset(&config, 0, sizeof(config));

        instance =  nla_get_u32(tb[IFLA_BRIDGE_CFM_CC_CONFIG_INSTANCE]);
        config.enable = nla_get_u32(tb[IFLA_BRIDGE_CFM_CC_CONFIG_ENABLE]);
        config.exp_interval = nla_get_u32(tb[IFLA_BRIDGE_CFM_CC_CONFIG_EXP_INTERVAL]);
        nla_memcpy(&config.exp_maid.data, tb[IFLA_BRIDGE_CFM_CC_CONFIG_EXP_MAID],
                   sizeof(config.exp_maid.data));

        return br_cfm_cc_config_set(br, instance, &config, extack);
}

static int br_cc_peer_mep_add_parse(struct net_bridge *br, struct nlattr *attr,
                                    struct netlink_ext_ack *extack)
{
        struct nlattr *tb[IFLA_BRIDGE_CFM_CC_PEER_MEP_MAX + 1];
        u32 instance, peer_mep_id;
        int err;

        err = nla_parse_nested(tb, IFLA_BRIDGE_CFM_CC_PEER_MEP_MAX, attr,
                               br_cfm_cc_peer_mep_policy, extack);
        if (err)
                return err;

        if (!tb[IFLA_BRIDGE_CFM_CC_PEER_MEP_INSTANCE]) {
                NL_SET_ERR_MSG_MOD(extack, "Missing INSTANCE attribute");
                return -EINVAL;
        }
        if (!tb[IFLA_BRIDGE_CFM_CC_PEER_MEPID]) {
                NL_SET_ERR_MSG_MOD(extack, "Missing PEER_MEP_ID attribute");
                return -EINVAL;
        }

        instance =  nla_get_u32(tb[IFLA_BRIDGE_CFM_CC_PEER_MEP_INSTANCE]);
        peer_mep_id =  nla_get_u32(tb[IFLA_BRIDGE_CFM_CC_PEER_MEPID]);

        return br_cfm_cc_peer_mep_add(br, instance, peer_mep_id, extack);
}

static int br_cc_peer_mep_remove_parse(struct net_bridge *br, struct nlattr *attr,
                                       struct netlink_ext_ack *extack)
{
        struct nlattr *tb[IFLA_BRIDGE_CFM_CC_PEER_MEP_MAX + 1];
        u32 instance, peer_mep_id;
        int err;

        err = nla_parse_nested(tb, IFLA_BRIDGE_CFM_CC_PEER_MEP_MAX, attr,
                               br_cfm_cc_peer_mep_policy, extack);
        if (err)
                return err;

        if (!tb[IFLA_BRIDGE_CFM_CC_PEER_MEP_INSTANCE]) {
                NL_SET_ERR_MSG_MOD(extack, "Missing INSTANCE attribute");
                return -EINVAL;
        }
        if (!tb[IFLA_BRIDGE_CFM_CC_PEER_MEPID]) {
                NL_SET_ERR_MSG_MOD(extack, "Missing PEER_MEP_ID attribute");
                return -EINVAL;
        }

        instance =  nla_get_u32(tb[IFLA_BRIDGE_CFM_CC_PEER_MEP_INSTANCE]);
        peer_mep_id =  nla_get_u32(tb[IFLA_BRIDGE_CFM_CC_PEER_MEPID]);

        return br_cfm_cc_peer_mep_remove(br, instance, peer_mep_id, extack);
}

static int br_cc_rdi_parse(struct net_bridge *br, struct nlattr *attr,
                           struct netlink_ext_ack *extack)
{
        struct nlattr *tb[IFLA_BRIDGE_CFM_CC_RDI_MAX + 1];
        u32 instance, rdi;
        int err;

        err = nla_parse_nested(tb, IFLA_BRIDGE_CFM_CC_RDI_MAX, attr,
                               br_cfm_cc_rdi_policy, extack);
        if (err)
                return err;

        if (!tb[IFLA_BRIDGE_CFM_CC_RDI_INSTANCE]) {
                NL_SET_ERR_MSG_MOD(extack, "Missing INSTANCE attribute");
                return -EINVAL;
        }
        if (!tb[IFLA_BRIDGE_CFM_CC_RDI_RDI]) {
                NL_SET_ERR_MSG_MOD(extack, "Missing RDI attribute");
                return -EINVAL;
        }

        instance =  nla_get_u32(tb[IFLA_BRIDGE_CFM_CC_RDI_INSTANCE]);
        rdi =  nla_get_u32(tb[IFLA_BRIDGE_CFM_CC_RDI_RDI]);

        return br_cfm_cc_rdi_set(br, instance, rdi, extack);
}

static int br_cc_ccm_tx_parse(struct net_bridge *br, struct nlattr *attr,
                              struct netlink_ext_ack *extack)
{
        struct nlattr *tb[IFLA_BRIDGE_CFM_CC_CCM_TX_MAX + 1];
        struct br_cfm_cc_ccm_tx_info tx_info;
        u32 instance;
        int err;

        err = nla_parse_nested(tb, IFLA_BRIDGE_CFM_CC_CCM_TX_MAX, attr,
                               br_cfm_cc_ccm_tx_policy, extack);
        if (err)
                return err;

        if (!tb[IFLA_BRIDGE_CFM_CC_CCM_TX_INSTANCE]) {
                NL_SET_ERR_MSG_MOD(extack, "Missing INSTANCE attribute");
                return -EINVAL;
        }
        if (!tb[IFLA_BRIDGE_CFM_CC_CCM_TX_DMAC]) {
                NL_SET_ERR_MSG_MOD(extack, "Missing DMAC attribute");
                return -EINVAL;
        }
        if (!tb[IFLA_BRIDGE_CFM_CC_CCM_TX_SEQ_NO_UPDATE]) {
                NL_SET_ERR_MSG_MOD(extack, "Missing SEQ_NO_UPDATE attribute");
                return -EINVAL;
        }
        if (!tb[IFLA_BRIDGE_CFM_CC_CCM_TX_PERIOD]) {
                NL_SET_ERR_MSG_MOD(extack, "Missing PERIOD attribute");
                return -EINVAL;
        }
        if (!tb[IFLA_BRIDGE_CFM_CC_CCM_TX_IF_TLV]) {
                NL_SET_ERR_MSG_MOD(extack, "Missing IF_TLV attribute");
                return -EINVAL;
        }
        if (!tb[IFLA_BRIDGE_CFM_CC_CCM_TX_IF_TLV_VALUE]) {
                NL_SET_ERR_MSG_MOD(extack, "Missing IF_TLV_VALUE attribute");
                return -EINVAL;
        }
        if (!tb[IFLA_BRIDGE_CFM_CC_CCM_TX_PORT_TLV]) {
                NL_SET_ERR_MSG_MOD(extack, "Missing PORT_TLV attribute");
                return -EINVAL;
        }
        if (!tb[IFLA_BRIDGE_CFM_CC_CCM_TX_PORT_TLV_VALUE]) {
                NL_SET_ERR_MSG_MOD(extack, "Missing PORT_TLV_VALUE attribute");
                return -EINVAL;
        }

        memset(&tx_info, 0, sizeof(tx_info));

        instance = nla_get_u32(tb[IFLA_BRIDGE_CFM_CC_CCM_TX_INSTANCE]);
        nla_memcpy(&tx_info.dmac.addr,
                   tb[IFLA_BRIDGE_CFM_CC_CCM_TX_DMAC],
                   sizeof(tx_info.dmac.addr));
        tx_info.seq_no_update = nla_get_u32(tb[IFLA_BRIDGE_CFM_CC_CCM_TX_SEQ_NO_UPDATE]);
        tx_info.period = nla_get_u32(tb[IFLA_BRIDGE_CFM_CC_CCM_TX_PERIOD]);
        tx_info.if_tlv = nla_get_u32(tb[IFLA_BRIDGE_CFM_CC_CCM_TX_IF_TLV]);
        tx_info.if_tlv_value = nla_get_u8(tb[IFLA_BRIDGE_CFM_CC_CCM_TX_IF_TLV_VALUE]);
        tx_info.port_tlv = nla_get_u32(tb[IFLA_BRIDGE_CFM_CC_CCM_TX_PORT_TLV]);
        tx_info.port_tlv_value = nla_get_u8(tb[IFLA_BRIDGE_CFM_CC_CCM_TX_PORT_TLV_VALUE]);

        return br_cfm_cc_ccm_tx(br, instance, &tx_info, extack);
}

int br_cfm_parse(struct net_bridge *br, struct net_bridge_port *p,
                 struct nlattr *attr, int cmd, struct netlink_ext_ack *extack)
{
        struct nlattr *tb[IFLA_BRIDGE_CFM_MAX + 1];
        int err;

        /* When this function is called for a port then the br pointer is
         * invalid, therefor set the br to point correctly
         */
        if (p)
                br = p->br;

        err = nla_parse_nested(tb, IFLA_BRIDGE_CFM_MAX, attr,
                               br_cfm_policy, extack);
        if (err)
                return err;

        if (tb[IFLA_BRIDGE_CFM_MEP_CREATE]) {
                err = br_mep_create_parse(br, tb[IFLA_BRIDGE_CFM_MEP_CREATE],
                                          extack);
                if (err)
                        return err;
        }

        if (tb[IFLA_BRIDGE_CFM_MEP_DELETE]) {
                err = br_mep_delete_parse(br, tb[IFLA_BRIDGE_CFM_MEP_DELETE],
                                          extack);
                if (err)
                        return err;
        }

        if (tb[IFLA_BRIDGE_CFM_MEP_CONFIG]) {
                err = br_mep_config_parse(br, tb[IFLA_BRIDGE_CFM_MEP_CONFIG],
                                          extack);
                if (err)
                        return err;
        }

        if (tb[IFLA_BRIDGE_CFM_CC_CONFIG]) {
                err = br_cc_config_parse(br, tb[IFLA_BRIDGE_CFM_CC_CONFIG],
                                         extack);
                if (err)
                        return err;
        }

        if (tb[IFLA_BRIDGE_CFM_CC_PEER_MEP_ADD]) {
                err = br_cc_peer_mep_add_parse(br, tb[IFLA_BRIDGE_CFM_CC_PEER_MEP_ADD],
                                               extack);
                if (err)
                        return err;
        }

        if (tb[IFLA_BRIDGE_CFM_CC_PEER_MEP_REMOVE]) {
                err = br_cc_peer_mep_remove_parse(br, tb[IFLA_BRIDGE_CFM_CC_PEER_MEP_REMOVE],
                                                  extack);
                if (err)
                        return err;
        }

        if (tb[IFLA_BRIDGE_CFM_CC_RDI]) {
                err = br_cc_rdi_parse(br, tb[IFLA_BRIDGE_CFM_CC_RDI],
                                      extack);
                if (err)
                        return err;
        }

        if (tb[IFLA_BRIDGE_CFM_CC_CCM_TX]) {
                err = br_cc_ccm_tx_parse(br, tb[IFLA_BRIDGE_CFM_CC_CCM_TX],
                                         extack);
                if (err)
                        return err;
        }

        return 0;
}

int br_cfm_config_fill_info(struct sk_buff *skb, struct net_bridge *br)
{
        struct br_cfm_peer_mep *peer_mep;
        struct br_cfm_mep *mep;
        struct nlattr *tb;

        hlist_for_each_entry_rcu(mep, &br->mep_list, head) {
                tb = nla_nest_start(skb, IFLA_BRIDGE_CFM_MEP_CREATE_INFO);
                if (!tb)
                        goto nla_info_failure;

                if (nla_put_u32(skb, IFLA_BRIDGE_CFM_MEP_CREATE_INSTANCE,
                                mep->instance))
                        goto nla_put_failure;

                if (nla_put_u32(skb, IFLA_BRIDGE_CFM_MEP_CREATE_DOMAIN,
                                mep->create.domain))
                        goto nla_put_failure;

                if (nla_put_u32(skb, IFLA_BRIDGE_CFM_MEP_CREATE_DIRECTION,
                                mep->create.direction))
                        goto nla_put_failure;

                if (nla_put_u32(skb, IFLA_BRIDGE_CFM_MEP_CREATE_IFINDEX,
                                mep->create.ifindex))
                        goto nla_put_failure;

                nla_nest_end(skb, tb);

                tb = nla_nest_start(skb, IFLA_BRIDGE_CFM_MEP_CONFIG_INFO);

                if (!tb)
                        goto nla_info_failure;

                if (nla_put_u32(skb, IFLA_BRIDGE_CFM_MEP_CONFIG_INSTANCE,
                                mep->instance))
                        goto nla_put_failure;

                if (nla_put(skb, IFLA_BRIDGE_CFM_MEP_CONFIG_UNICAST_MAC,
                            sizeof(mep->config.unicast_mac.addr),
                            mep->config.unicast_mac.addr))
                        goto nla_put_failure;

                if (nla_put_u32(skb, IFLA_BRIDGE_CFM_MEP_CONFIG_MDLEVEL,
                                mep->config.mdlevel))
                        goto nla_put_failure;

                if (nla_put_u32(skb, IFLA_BRIDGE_CFM_MEP_CONFIG_MEPID,
                                mep->config.mepid))
                        goto nla_put_failure;

                nla_nest_end(skb, tb);

                tb = nla_nest_start(skb, IFLA_BRIDGE_CFM_CC_CONFIG_INFO);

                if (!tb)
                        goto nla_info_failure;

                if (nla_put_u32(skb, IFLA_BRIDGE_CFM_CC_CONFIG_INSTANCE,
                                mep->instance))
                        goto nla_put_failure;

                if (nla_put_u32(skb, IFLA_BRIDGE_CFM_CC_CONFIG_ENABLE,
                                mep->cc_config.enable))
                        goto nla_put_failure;

                if (nla_put_u32(skb, IFLA_BRIDGE_CFM_CC_CONFIG_EXP_INTERVAL,
                                mep->cc_config.exp_interval))
                        goto nla_put_failure;

                if (nla_put(skb, IFLA_BRIDGE_CFM_CC_CONFIG_EXP_MAID,
                            sizeof(mep->cc_config.exp_maid.data),
                            mep->cc_config.exp_maid.data))
                        goto nla_put_failure;

                nla_nest_end(skb, tb);

                tb = nla_nest_start(skb, IFLA_BRIDGE_CFM_CC_RDI_INFO);

                if (!tb)
                        goto nla_info_failure;

                if (nla_put_u32(skb, IFLA_BRIDGE_CFM_CC_RDI_INSTANCE,
                                mep->instance))
                        goto nla_put_failure;

                if (nla_put_u32(skb, IFLA_BRIDGE_CFM_CC_RDI_RDI,
                                mep->rdi))
                        goto nla_put_failure;

                nla_nest_end(skb, tb);

                tb = nla_nest_start(skb, IFLA_BRIDGE_CFM_CC_CCM_TX_INFO);

                if (!tb)
                        goto nla_info_failure;

                if (nla_put_u32(skb, IFLA_BRIDGE_CFM_CC_CCM_TX_INSTANCE,
                                mep->instance))
                        goto nla_put_failure;

                if (nla_put(skb, IFLA_BRIDGE_CFM_CC_CCM_TX_DMAC,
                            sizeof(mep->cc_ccm_tx_info.dmac),
                            mep->cc_ccm_tx_info.dmac.addr))
                        goto nla_put_failure;

                if (nla_put_u32(skb, IFLA_BRIDGE_CFM_CC_CCM_TX_SEQ_NO_UPDATE,
                                mep->cc_ccm_tx_info.seq_no_update))
                        goto nla_put_failure;

                if (nla_put_u32(skb, IFLA_BRIDGE_CFM_CC_CCM_TX_PERIOD,
                                mep->cc_ccm_tx_info.period))
                        goto nla_put_failure;

                if (nla_put_u32(skb, IFLA_BRIDGE_CFM_CC_CCM_TX_IF_TLV,
                                mep->cc_ccm_tx_info.if_tlv))
                        goto nla_put_failure;

                if (nla_put_u8(skb, IFLA_BRIDGE_CFM_CC_CCM_TX_IF_TLV_VALUE,
                               mep->cc_ccm_tx_info.if_tlv_value))
                        goto nla_put_failure;

                if (nla_put_u32(skb, IFLA_BRIDGE_CFM_CC_CCM_TX_PORT_TLV,
                                mep->cc_ccm_tx_info.port_tlv))
                        goto nla_put_failure;

                if (nla_put_u8(skb, IFLA_BRIDGE_CFM_CC_CCM_TX_PORT_TLV_VALUE,
                               mep->cc_ccm_tx_info.port_tlv_value))
                        goto nla_put_failure;

                nla_nest_end(skb, tb);

                hlist_for_each_entry_rcu(peer_mep, &mep->peer_mep_list, head) {
                        tb = nla_nest_start(skb,
                                            IFLA_BRIDGE_CFM_CC_PEER_MEP_INFO);

                        if (!tb)
                                goto nla_info_failure;

                        if (nla_put_u32(skb,
                                        IFLA_BRIDGE_CFM_CC_PEER_MEP_INSTANCE,
                                        mep->instance))
                                goto nla_put_failure;

                        if (nla_put_u32(skb, IFLA_BRIDGE_CFM_CC_PEER_MEPID,
                                        peer_mep->mepid))
                                goto nla_put_failure;

                        nla_nest_end(skb, tb);
                }
        }

        return 0;

nla_put_failure:
        nla_nest_cancel(skb, tb);

nla_info_failure:
        return -EMSGSIZE;
}

int br_cfm_status_fill_info(struct sk_buff *skb,
                            struct net_bridge *br,
                            bool getlink)
{
        struct br_cfm_peer_mep *peer_mep;
        struct br_cfm_mep *mep;
        struct nlattr *tb;

        hlist_for_each_entry_rcu(mep, &br->mep_list, head) {
                tb = nla_nest_start(skb, IFLA_BRIDGE_CFM_MEP_STATUS_INFO);
                if (!tb)
                        goto nla_info_failure;

                if (nla_put_u32(skb, IFLA_BRIDGE_CFM_MEP_STATUS_INSTANCE,
                                mep->instance))
                        goto nla_put_failure;

                if (nla_put_u32(skb,
                                IFLA_BRIDGE_CFM_MEP_STATUS_OPCODE_UNEXP_SEEN,
                                mep->status.opcode_unexp_seen))
                        goto nla_put_failure;

                if (nla_put_u32(skb,
                                IFLA_BRIDGE_CFM_MEP_STATUS_VERSION_UNEXP_SEEN,
                                mep->status.version_unexp_seen))
                        goto nla_put_failure;

                if (nla_put_u32(skb,
                                IFLA_BRIDGE_CFM_MEP_STATUS_RX_LEVEL_LOW_SEEN,
                                mep->status.rx_level_low_seen))
                        goto nla_put_failure;

                /* Only clear if this is a GETLINK */
                if (getlink) {
                        /* Clear all 'seen' indications */
                        mep->status.opcode_unexp_seen = false;
                        mep->status.version_unexp_seen = false;
                        mep->status.rx_level_low_seen = false;
                }

                nla_nest_end(skb, tb);

                hlist_for_each_entry_rcu(peer_mep, &mep->peer_mep_list, head) {
                        tb = nla_nest_start(skb,
                                            IFLA_BRIDGE_CFM_CC_PEER_STATUS_INFO);
                        if (!tb)
                                goto nla_info_failure;

                        if (nla_put_u32(skb,
                                        IFLA_BRIDGE_CFM_CC_PEER_STATUS_INSTANCE,
                                        mep->instance))
                                goto nla_put_failure;

                        if (nla_put_u32(skb,
                                        IFLA_BRIDGE_CFM_CC_PEER_STATUS_PEER_MEPID,
                                        peer_mep->mepid))
                                goto nla_put_failure;

                        if (nla_put_u32(skb,
                                        IFLA_BRIDGE_CFM_CC_PEER_STATUS_CCM_DEFECT,
                                        peer_mep->cc_status.ccm_defect))
                                goto nla_put_failure;

                        if (nla_put_u32(skb, IFLA_BRIDGE_CFM_CC_PEER_STATUS_RDI,
                                        peer_mep->cc_status.rdi))
                                goto nla_put_failure;

                        if (nla_put_u8(skb,
                                       IFLA_BRIDGE_CFM_CC_PEER_STATUS_PORT_TLV_VALUE,
                                       peer_mep->cc_status.port_tlv_value))
                                goto nla_put_failure;

                        if (nla_put_u8(skb,
                                       IFLA_BRIDGE_CFM_CC_PEER_STATUS_IF_TLV_VALUE,
                                       peer_mep->cc_status.if_tlv_value))
                                goto nla_put_failure;

                        if (nla_put_u32(skb,
                                        IFLA_BRIDGE_CFM_CC_PEER_STATUS_SEEN,
                                        peer_mep->cc_status.seen))
                                goto nla_put_failure;

                        if (nla_put_u32(skb,
                                        IFLA_BRIDGE_CFM_CC_PEER_STATUS_TLV_SEEN,
                                        peer_mep->cc_status.tlv_seen))
                                goto nla_put_failure;

                        if (nla_put_u32(skb,
                                        IFLA_BRIDGE_CFM_CC_PEER_STATUS_SEQ_UNEXP_SEEN,
                                        peer_mep->cc_status.seq_unexp_seen))
                                goto nla_put_failure;

                        if (getlink) { /* Only clear if this is a GETLINK */
                                /* Clear all 'seen' indications */
                                peer_mep->cc_status.seen = false;
                                peer_mep->cc_status.tlv_seen = false;
                                peer_mep->cc_status.seq_unexp_seen = false;
                        }

                        nla_nest_end(skb, tb);
                }
        }

        return 0;

nla_put_failure:
        nla_nest_cancel(skb, tb);

nla_info_failure:
        return -EMSGSIZE;
}


































































   12 

















   57 








   57 
   16 
    4 



   12 


   11 




   55 
    6 




   26 




   15 






   57 
   52 
   35 


    1 


   17 




   17 



   55 
   28 
   25 

    1 





   51 



   50 


    6 
    6 



    6 


















    3 
   71 








   50 

   35 

   50 




   12 

   14 





   56 






   56 



   37 
   34 
    4 
    6 

    1 








   15 







    1 

   14 



    4 



   14 














   14 




    1 






   45 










   27 









   11 
    9 


    3 
   15 




   43 


    4 





   45 


















    2 











   10 


   10 
























   19 




    6 
   12 






    1 




   18 















   18 

   18 






   17 


    2 

    1 
    1 





   16 




    6 

    9 


    2 
   14 





    1 
    2 





































   26 





    6 


   19 















   25 




   24 

   25 


   17 
   17 

    9 


   25 
    5 

    9 
    5 
    8 

    1 








    2 

    2 

    2 








    2 

    1 

















   22 
   10 




    1 












    1 




















































































   77 
















   75 




   26 


   52 
   12 




   41 





    2 
   19 
   19 








   21 







   21 




   21 





   21 












   36 


    2 

    6 








   44 

    1 








    2 






   40 







    2 




   11 





   10 
   14 


    2 



   24 









   24 

   10 

   10 





    4 





   24 






   24 





















    7 
    7 




    1 
    6 


    7 





    7 






    1 







   24 





   19 

    1 
    1 











   16 
    2 
    2 
    1 
   15 
    1 
   15 





    1 
   11 







    4 

    2 











    1 








    5 















    1 


    1 

    1 
















   44 


    1 




   44 
   43 









   16 






   16 















   42 






   43 











   25 
   19 

    1 















   42 
   42 




    8 







    5 





    1 
    4 






  164 







  169 








    4 








    4 









    4 
    4 









    4 


















































   47 










   15 


   32 







    1 


   14 
   17 


   29 


    1 
    1 








   31 











   19 





   17 





    2 
























   45 
























   30 
   15 
   45 





    8 





    8 
    8 








   70 







1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright (c) 2013 Nicira, Inc.
 */

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#include <linux/capability.h>
#include <linux/module.h>
#include <linux/types.h>
#include <linux/kernel.h>
#include <linux/slab.h>
#include <linux/uaccess.h>
#include <linux/skbuff.h>
#include <linux/netdevice.h>
#include <linux/in.h>
#include <linux/tcp.h>
#include <linux/udp.h>
#include <linux/if_arp.h>
#include <linux/init.h>
#include <linux/in6.h>
#include <linux/inetdevice.h>
#include <linux/igmp.h>
#include <linux/netfilter_ipv4.h>
#include <linux/etherdevice.h>
#include <linux/if_ether.h>
#include <linux/if_vlan.h>
#include <linux/rculist.h>
#include <linux/err.h>

#include <net/sock.h>
#include <net/ip.h>
#include <net/icmp.h>
#include <net/protocol.h>
#include <net/ip_tunnels.h>
#include <net/arp.h>
#include <net/checksum.h>
#include <net/dsfield.h>
#include <net/inet_ecn.h>
#include <net/xfrm.h>
#include <net/net_namespace.h>
#include <net/netns/generic.h>
#include <net/rtnetlink.h>
#include <net/udp.h>
#include <net/dst_metadata.h>
#include <net/inet_dscp.h>

#if IS_ENABLED(CONFIG_IPV6)
#include <net/ipv6.h>
#include <net/ip6_fib.h>
#include <net/ip6_route.h>
#endif

static unsigned int ip_tunnel_hash(__be32 key, __be32 remote)
{
        return hash_32((__force u32)key ^ (__force u32)remote,
                         IP_TNL_HASH_BITS);
}

static bool ip_tunnel_key_match(const struct ip_tunnel_parm_kern *p,
                                const unsigned long *flags, __be32 key)
{
        if (!test_bit(IP_TUNNEL_KEY_BIT, flags))
                return !test_bit(IP_TUNNEL_KEY_BIT, p->i_flags);

        return test_bit(IP_TUNNEL_KEY_BIT, p->i_flags) && p->i_key == key;
}

/* Fallback tunnel: no source, no destination, no key, no options

   Tunnel hash table:
   We require exact key match i.e. if a key is present in packet
   it will match only tunnel with the same key; if it is not present,
   it will match only keyless tunnel.

   All keysless packets, if not matched configured keyless tunnels
   will match fallback tunnel.
   Given src, dst and key, find appropriate for input tunnel.
*/
struct ip_tunnel *ip_tunnel_lookup(struct ip_tunnel_net *itn,
                                   int link, const unsigned long *flags,
                                   __be32 remote, __be32 local,
                                   __be32 key)
{
        struct ip_tunnel *t, *cand = NULL;
        struct hlist_head *head;
        struct net_device *ndev;
        unsigned int hash;

        hash = ip_tunnel_hash(key, remote);
        head = &itn->tunnels[hash];

        hlist_for_each_entry_rcu(t, head, hash_node) {
                if (local != t->parms.iph.saddr ||
                    remote != t->parms.iph.daddr ||
                    !(t->dev->flags & IFF_UP))
                        continue;

                if (!ip_tunnel_key_match(&t->parms, flags, key))
                        continue;

                if (READ_ONCE(t->parms.link) == link)
                        return t;
                cand = t;
        }

        hlist_for_each_entry_rcu(t, head, hash_node) {
                if (remote != t->parms.iph.daddr ||
                    t->parms.iph.saddr != 0 ||
                    !(t->dev->flags & IFF_UP))
                        continue;

                if (!ip_tunnel_key_match(&t->parms, flags, key))
                        continue;

                if (READ_ONCE(t->parms.link) == link)
                        return t;
                if (!cand)
                        cand = t;
        }

        hash = ip_tunnel_hash(key, 0);
        head = &itn->tunnels[hash];

        hlist_for_each_entry_rcu(t, head, hash_node) {
                if ((local != t->parms.iph.saddr || t->parms.iph.daddr != 0) &&
                    (local != t->parms.iph.daddr || !ipv4_is_multicast(local)))
                        continue;

                if (!(t->dev->flags & IFF_UP))
                        continue;

                if (!ip_tunnel_key_match(&t->parms, flags, key))
                        continue;

                if (READ_ONCE(t->parms.link) == link)
                        return t;
                if (!cand)
                        cand = t;
        }

        hlist_for_each_entry_rcu(t, head, hash_node) {
                if ((!test_bit(IP_TUNNEL_NO_KEY_BIT, flags) &&
                     t->parms.i_key != key) ||
                    t->parms.iph.saddr != 0 ||
                    t->parms.iph.daddr != 0 ||
                    !(t->dev->flags & IFF_UP))
                        continue;

                if (READ_ONCE(t->parms.link) == link)
                        return t;
                if (!cand)
                        cand = t;
        }

        if (cand)
                return cand;

        t = rcu_dereference(itn->collect_md_tun);
        if (t && t->dev->flags & IFF_UP)
                return t;

        ndev = READ_ONCE(itn->fb_tunnel_dev);
        if (ndev && ndev->flags & IFF_UP)
                return netdev_priv(ndev);

        return NULL;
}
EXPORT_SYMBOL_GPL(ip_tunnel_lookup);

static struct hlist_head *ip_bucket(struct ip_tunnel_net *itn,
                                    struct ip_tunnel_parm_kern *parms)
{
        unsigned int h;
        __be32 remote;
        __be32 i_key = parms->i_key;

        if (parms->iph.daddr && !ipv4_is_multicast(parms->iph.daddr))
                remote = parms->iph.daddr;
        else
                remote = 0;

        if (!test_bit(IP_TUNNEL_KEY_BIT, parms->i_flags) &&
            test_bit(IP_TUNNEL_VTI_BIT, parms->i_flags))
                i_key = 0;

        h = ip_tunnel_hash(i_key, remote);
        return &itn->tunnels[h];
}

static void ip_tunnel_add(struct ip_tunnel_net *itn, struct ip_tunnel *t)
{
        struct hlist_head *head = ip_bucket(itn, &t->parms);

        if (t->collect_md)
                rcu_assign_pointer(itn->collect_md_tun, t);
        hlist_add_head_rcu(&t->hash_node, head);
}

static void ip_tunnel_del(struct ip_tunnel_net *itn, struct ip_tunnel *t)
{
        if (t->collect_md)
                rcu_assign_pointer(itn->collect_md_tun, NULL);
        hlist_del_init_rcu(&t->hash_node);
}

static struct ip_tunnel *ip_tunnel_find(struct ip_tunnel_net *itn,
                                        struct ip_tunnel_parm_kern *parms,
                                        int type)
{
        __be32 remote = parms->iph.daddr;
        __be32 local = parms->iph.saddr;
        IP_TUNNEL_DECLARE_FLAGS(flags);
        __be32 key = parms->i_key;
        int link = parms->link;
        struct ip_tunnel *t = NULL;
        struct hlist_head *head = ip_bucket(itn, parms);

        ip_tunnel_flags_copy(flags, parms->i_flags);

        hlist_for_each_entry_rcu(t, head, hash_node, lockdep_rtnl_is_held()) {
                if (local == t->parms.iph.saddr &&
                    remote == t->parms.iph.daddr &&
                    link == READ_ONCE(t->parms.link) &&
                    type == t->dev->type &&
                    ip_tunnel_key_match(&t->parms, flags, key))
                        break;
        }
        return t;
}

static struct net_device *__ip_tunnel_create(struct net *net,
                                             const struct rtnl_link_ops *ops,
                                             struct ip_tunnel_parm_kern *parms)
{
        int err;
        struct ip_tunnel *tunnel;
        struct net_device *dev;
        char name[IFNAMSIZ];

        err = -E2BIG;
        if (parms->name[0]) {
                if (!dev_valid_name(parms->name))
                        goto failed;
                strscpy(name, parms->name, IFNAMSIZ);
        } else {
                if (strlen(ops->kind) > (IFNAMSIZ - 3))
                        goto failed;
                strcpy(name, ops->kind);
                strcat(name, "%d");
        }

        ASSERT_RTNL();
        dev = alloc_netdev(ops->priv_size, name, NET_NAME_UNKNOWN, ops->setup);
        if (!dev) {
                err = -ENOMEM;
                goto failed;
        }
        dev_net_set(dev, net);

        dev->rtnl_link_ops = ops;

        tunnel = netdev_priv(dev);
        tunnel->parms = *parms;
        tunnel->net = net;

        err = register_netdevice(dev);
        if (err)
                goto failed_free;

        return dev;

failed_free:
        free_netdev(dev);
failed:
        return ERR_PTR(err);
}

static int ip_tunnel_bind_dev(struct net_device *dev)
{
        struct net_device *tdev = NULL;
        struct ip_tunnel *tunnel = netdev_priv(dev);
        const struct iphdr *iph;
        int hlen = LL_MAX_HEADER;
        int mtu = ETH_DATA_LEN;
        int t_hlen = tunnel->hlen + sizeof(struct iphdr);

        iph = &tunnel->parms.iph;

        /* Guess output device to choose reasonable mtu and needed_headroom */
        if (iph->daddr) {
                struct flowi4 fl4;
                struct rtable *rt;

                ip_tunnel_init_flow(&fl4, iph->protocol, iph->daddr,
                                    iph->saddr, tunnel->parms.o_key,
                                    iph->tos & INET_DSCP_MASK, dev_net(dev),
                                    tunnel->parms.link, tunnel->fwmark, 0, 0);
                rt = ip_route_output_key(tunnel->net, &fl4);

                if (!IS_ERR(rt)) {
                        tdev = rt->dst.dev;
                        ip_rt_put(rt);
                }
                if (dev->type != ARPHRD_ETHER)
                        dev->flags |= IFF_POINTOPOINT;

                dst_cache_reset(&tunnel->dst_cache);
        }

        if (!tdev && tunnel->parms.link)
                tdev = __dev_get_by_index(tunnel->net, tunnel->parms.link);

        if (tdev) {
                hlen = tdev->hard_header_len + tdev->needed_headroom;
                mtu = min(tdev->mtu, IP_MAX_MTU);
        }

        dev->needed_headroom = t_hlen + hlen;
        mtu -= t_hlen + (dev->type == ARPHRD_ETHER ? dev->hard_header_len : 0);

        if (mtu < IPV4_MIN_MTU)
                mtu = IPV4_MIN_MTU;

        return mtu;
}

static struct ip_tunnel *ip_tunnel_create(struct net *net,
                                          struct ip_tunnel_net *itn,
                                          struct ip_tunnel_parm_kern *parms)
{
        struct ip_tunnel *nt;
        struct net_device *dev;
        int t_hlen;
        int mtu;
        int err;

        dev = __ip_tunnel_create(net, itn->rtnl_link_ops, parms);
        if (IS_ERR(dev))
                return ERR_CAST(dev);

        mtu = ip_tunnel_bind_dev(dev);
        err = dev_set_mtu(dev, mtu);
        if (err)
                goto err_dev_set_mtu;

        nt = netdev_priv(dev);
        t_hlen = nt->hlen + sizeof(struct iphdr);
        dev->min_mtu = ETH_MIN_MTU;
        dev->max_mtu = IP_MAX_MTU - t_hlen;
        if (dev->type == ARPHRD_ETHER)
                dev->max_mtu -= dev->hard_header_len;

        ip_tunnel_add(itn, nt);
        return nt;

err_dev_set_mtu:
        unregister_netdevice(dev);
        return ERR_PTR(err);
}

void ip_tunnel_md_udp_encap(struct sk_buff *skb, struct ip_tunnel_info *info)
{
        const struct iphdr *iph = ip_hdr(skb);
        const struct udphdr *udph;

        if (iph->protocol != IPPROTO_UDP)
                return;

        udph = (struct udphdr *)((__u8 *)iph + (iph->ihl << 2));
        info->encap.sport = udph->source;
        info->encap.dport = udph->dest;
}
EXPORT_SYMBOL(ip_tunnel_md_udp_encap);

int ip_tunnel_rcv(struct ip_tunnel *tunnel, struct sk_buff *skb,
                  const struct tnl_ptk_info *tpi, struct metadata_dst *tun_dst,
                  bool log_ecn_error)
{
        const struct iphdr *iph = ip_hdr(skb);
        int nh, err;

#ifdef CONFIG_NET_IPGRE_BROADCAST
        if (ipv4_is_multicast(iph->daddr)) {
                DEV_STATS_INC(tunnel->dev, multicast);
                skb->pkt_type = PACKET_BROADCAST;
        }
#endif

        if (test_bit(IP_TUNNEL_CSUM_BIT, tunnel->parms.i_flags) !=
            test_bit(IP_TUNNEL_CSUM_BIT, tpi->flags)) {
                DEV_STATS_INC(tunnel->dev, rx_crc_errors);
                DEV_STATS_INC(tunnel->dev, rx_errors);
                goto drop;
        }

        if (test_bit(IP_TUNNEL_SEQ_BIT, tunnel->parms.i_flags)) {
                if (!test_bit(IP_TUNNEL_SEQ_BIT, tpi->flags) ||
                    (tunnel->i_seqno && (s32)(ntohl(tpi->seq) - tunnel->i_seqno) < 0)) {
                        DEV_STATS_INC(tunnel->dev, rx_fifo_errors);
                        DEV_STATS_INC(tunnel->dev, rx_errors);
                        goto drop;
                }
                tunnel->i_seqno = ntohl(tpi->seq) + 1;
        }

        /* Save offset of outer header relative to skb->head,
         * because we are going to reset the network header to the inner header
         * and might change skb->head.
         */
        nh = skb_network_header(skb) - skb->head;

        skb_set_network_header(skb, (tunnel->dev->type == ARPHRD_ETHER) ? ETH_HLEN : 0);

        if (!pskb_inet_may_pull(skb)) {
                DEV_STATS_INC(tunnel->dev, rx_length_errors);
                DEV_STATS_INC(tunnel->dev, rx_errors);
                goto drop;
        }
        iph = (struct iphdr *)(skb->head + nh);

        err = IP_ECN_decapsulate(iph, skb);
        if (unlikely(err)) {
                if (log_ecn_error)
                        net_info_ratelimited("non-ECT from %pI4 with TOS=%#x\n",
                                        &iph->saddr, iph->tos);
                if (err > 1) {
                        DEV_STATS_INC(tunnel->dev, rx_frame_errors);
                        DEV_STATS_INC(tunnel->dev, rx_errors);
                        goto drop;
                }
        }

        dev_sw_netstats_rx_add(tunnel->dev, skb->len);
        skb_scrub_packet(skb, !net_eq(tunnel->net, dev_net(tunnel->dev)));

        if (tunnel->dev->type == ARPHRD_ETHER) {
                skb->protocol = eth_type_trans(skb, tunnel->dev);
                skb_postpull_rcsum(skb, eth_hdr(skb), ETH_HLEN);
        } else {
                skb->dev = tunnel->dev;
        }

        if (tun_dst)
                skb_dst_set(skb, (struct dst_entry *)tun_dst);

        gro_cells_receive(&tunnel->gro_cells, skb);
        return 0;

drop:
        if (tun_dst)
                dst_release((struct dst_entry *)tun_dst);
        kfree_skb(skb);
        return 0;
}
EXPORT_SYMBOL_GPL(ip_tunnel_rcv);

int ip_tunnel_encap_add_ops(const struct ip_tunnel_encap_ops *ops,
                            unsigned int num)
{
        if (num >= MAX_IPTUN_ENCAP_OPS)
                return -ERANGE;

        return !cmpxchg((const struct ip_tunnel_encap_ops **)
                        &iptun_encaps[num],
                        NULL, ops) ? 0 : -1;
}
EXPORT_SYMBOL(ip_tunnel_encap_add_ops);

int ip_tunnel_encap_del_ops(const struct ip_tunnel_encap_ops *ops,
                            unsigned int num)
{
        int ret;

        if (num >= MAX_IPTUN_ENCAP_OPS)
                return -ERANGE;

        ret = (cmpxchg((const struct ip_tunnel_encap_ops **)
                       &iptun_encaps[num],
                       ops, NULL) == ops) ? 0 : -1;

        synchronize_net();

        return ret;
}
EXPORT_SYMBOL(ip_tunnel_encap_del_ops);

int ip_tunnel_encap_setup(struct ip_tunnel *t,
                          struct ip_tunnel_encap *ipencap)
{
        int hlen;

        memset(&t->encap, 0, sizeof(t->encap));

        hlen = ip_encap_hlen(ipencap);
        if (hlen < 0)
                return hlen;

        t->encap.type = ipencap->type;
        t->encap.sport = ipencap->sport;
        t->encap.dport = ipencap->dport;
        t->encap.flags = ipencap->flags;

        t->encap_hlen = hlen;
        t->hlen = t->encap_hlen + t->tun_hlen;

        return 0;
}
EXPORT_SYMBOL_GPL(ip_tunnel_encap_setup);

static int tnl_update_pmtu(struct net_device *dev, struct sk_buff *skb,
                            struct rtable *rt, __be16 df,
                            const struct iphdr *inner_iph,
                            int tunnel_hlen, __be32 dst, bool md)
{
        struct ip_tunnel *tunnel = netdev_priv(dev);
        int pkt_size;
        int mtu;

        tunnel_hlen = md ? tunnel_hlen : tunnel->hlen;
        pkt_size = skb->len - tunnel_hlen;
        pkt_size -= dev->type == ARPHRD_ETHER ? dev->hard_header_len : 0;

        if (df) {
                mtu = dst_mtu(&rt->dst) - (sizeof(struct iphdr) + tunnel_hlen);
                mtu -= dev->type == ARPHRD_ETHER ? dev->hard_header_len : 0;
        } else {
                mtu = skb_valid_dst(skb) ? dst_mtu(skb_dst(skb)) : dev->mtu;
        }

        if (skb_valid_dst(skb))
                skb_dst_update_pmtu_no_confirm(skb, mtu);

        if (skb->protocol == htons(ETH_P_IP)) {
                if (!skb_is_gso(skb) &&
                    (inner_iph->frag_off & htons(IP_DF)) &&
                    mtu < pkt_size) {
                        icmp_ndo_send(skb, ICMP_DEST_UNREACH, ICMP_FRAG_NEEDED, htonl(mtu));
                        return -E2BIG;
                }
        }
#if IS_ENABLED(CONFIG_IPV6)
        else if (skb->protocol == htons(ETH_P_IPV6)) {
                struct rt6_info *rt6;
                __be32 daddr;

                rt6 = skb_valid_dst(skb) ? dst_rt6_info(skb_dst(skb)) :
                                           NULL;
                daddr = md ? dst : tunnel->parms.iph.daddr;

                if (rt6 && mtu < dst_mtu(skb_dst(skb)) &&
                           mtu >= IPV6_MIN_MTU) {
                        if ((daddr && !ipv4_is_multicast(daddr)) ||
                            rt6->rt6i_dst.plen == 128) {
                                rt6->rt6i_flags |= RTF_MODIFIED;
                                dst_metric_set(skb_dst(skb), RTAX_MTU, mtu);
                        }
                }

                if (!skb_is_gso(skb) && mtu >= IPV6_MIN_MTU &&
                                        mtu < pkt_size) {
                        icmpv6_ndo_send(skb, ICMPV6_PKT_TOOBIG, 0, mtu);
                        return -E2BIG;
                }
        }
#endif
        return 0;
}

static void ip_tunnel_adj_headroom(struct net_device *dev, unsigned int headroom)
{
        /* we must cap headroom to some upperlimit, else pskb_expand_head
         * will overflow header offsets in skb_headers_offset_update().
         */
        static const unsigned int max_allowed = 512;

        if (headroom > max_allowed)
                headroom = max_allowed;

        if (headroom > READ_ONCE(dev->needed_headroom))
                WRITE_ONCE(dev->needed_headroom, headroom);
}

void ip_md_tunnel_xmit(struct sk_buff *skb, struct net_device *dev,
                       u8 proto, int tunnel_hlen)
{
        struct ip_tunnel *tunnel = netdev_priv(dev);
        u32 headroom = sizeof(struct iphdr);
        struct ip_tunnel_info *tun_info;
        const struct ip_tunnel_key *key;
        const struct iphdr *inner_iph;
        struct rtable *rt = NULL;
        struct flowi4 fl4;
        __be16 df = 0;
        u8 tos, ttl;
        bool use_cache;

        tun_info = skb_tunnel_info(skb);
        if (unlikely(!tun_info || !(tun_info->mode & IP_TUNNEL_INFO_TX) ||
                     ip_tunnel_info_af(tun_info) != AF_INET))
                goto tx_error;
        key = &tun_info->key;
        memset(&(IPCB(skb)->opt), 0, sizeof(IPCB(skb)->opt));
        inner_iph = (const struct iphdr *)skb_inner_network_header(skb);
        tos = key->tos;
        if (tos == 1) {
                if (skb->protocol == htons(ETH_P_IP))
                        tos = inner_iph->tos;
                else if (skb->protocol == htons(ETH_P_IPV6))
                        tos = ipv6_get_dsfield((const struct ipv6hdr *)inner_iph);
        }
        ip_tunnel_init_flow(&fl4, proto, key->u.ipv4.dst, key->u.ipv4.src,
                            tunnel_id_to_key32(key->tun_id),
                            tos & INET_DSCP_MASK, dev_net(dev), 0, skb->mark,
                            skb_get_hash(skb), key->flow_flags);

        if (!tunnel_hlen)
                tunnel_hlen = ip_encap_hlen(&tun_info->encap);

        if (ip_tunnel_encap(skb, &tun_info->encap, &proto, &fl4) < 0)
                goto tx_error;

        use_cache = ip_tunnel_dst_cache_usable(skb, tun_info);
        if (use_cache)
                rt = dst_cache_get_ip4(&tun_info->dst_cache, &fl4.saddr);
        if (!rt) {
                rt = ip_route_output_key(tunnel->net, &fl4);
                if (IS_ERR(rt)) {
                        DEV_STATS_INC(dev, tx_carrier_errors);
                        goto tx_error;
                }
                if (use_cache)
                        dst_cache_set_ip4(&tun_info->dst_cache, &rt->dst,
                                          fl4.saddr);
        }
        if (rt->dst.dev == dev) {
                ip_rt_put(rt);
                DEV_STATS_INC(dev, collisions);
                goto tx_error;
        }

        if (test_bit(IP_TUNNEL_DONT_FRAGMENT_BIT, key->tun_flags))
                df = htons(IP_DF);
        if (tnl_update_pmtu(dev, skb, rt, df, inner_iph, tunnel_hlen,
                            key->u.ipv4.dst, true)) {
                ip_rt_put(rt);
                goto tx_error;
        }

        tos = ip_tunnel_ecn_encap(tos, inner_iph, skb);
        ttl = key->ttl;
        if (ttl == 0) {
                if (skb->protocol == htons(ETH_P_IP))
                        ttl = inner_iph->ttl;
                else if (skb->protocol == htons(ETH_P_IPV6))
                        ttl = ((const struct ipv6hdr *)inner_iph)->hop_limit;
                else
                        ttl = ip4_dst_hoplimit(&rt->dst);
        }

        headroom += LL_RESERVED_SPACE(rt->dst.dev) + rt->dst.header_len;
        if (skb_cow_head(skb, headroom)) {
                ip_rt_put(rt);
                goto tx_dropped;
        }

        ip_tunnel_adj_headroom(dev, headroom);

        iptunnel_xmit(NULL, rt, skb, fl4.saddr, fl4.daddr, proto, tos, ttl,
                      df, !net_eq(tunnel->net, dev_net(dev)));
        return;
tx_error:
        DEV_STATS_INC(dev, tx_errors);
        goto kfree;
tx_dropped:
        DEV_STATS_INC(dev, tx_dropped);
kfree:
        kfree_skb(skb);
}
EXPORT_SYMBOL_GPL(ip_md_tunnel_xmit);

void ip_tunnel_xmit(struct sk_buff *skb, struct net_device *dev,
                    const struct iphdr *tnl_params, u8 protocol)
{
        struct ip_tunnel *tunnel = netdev_priv(dev);
        struct ip_tunnel_info *tun_info = NULL;
        const struct iphdr *inner_iph;
        unsigned int max_headroom;        /* The extra header space needed */
        struct rtable *rt = NULL;                /* Route to the other host */
        __be16 payload_protocol;
        bool use_cache = false;
        struct flowi4 fl4;
        bool md = false;
        bool connected;
        u8 tos, ttl;
        __be32 dst;
        __be16 df;

        inner_iph = (const struct iphdr *)skb_inner_network_header(skb);
        connected = (tunnel->parms.iph.daddr != 0);
        payload_protocol = skb_protocol(skb, true);

        memset(&(IPCB(skb)->opt), 0, sizeof(IPCB(skb)->opt));

        dst = tnl_params->daddr;
        if (dst == 0) {
                /* NBMA tunnel */

                if (!skb_dst(skb)) {
                        DEV_STATS_INC(dev, tx_fifo_errors);
                        goto tx_error;
                }

                tun_info = skb_tunnel_info(skb);
                if (tun_info && (tun_info->mode & IP_TUNNEL_INFO_TX) &&
                    ip_tunnel_info_af(tun_info) == AF_INET &&
                    tun_info->key.u.ipv4.dst) {
                        dst = tun_info->key.u.ipv4.dst;
                        md = true;
                        connected = true;
                } else if (payload_protocol == htons(ETH_P_IP)) {
                        rt = skb_rtable(skb);
                        dst = rt_nexthop(rt, inner_iph->daddr);
                }
#if IS_ENABLED(CONFIG_IPV6)
                else if (payload_protocol == htons(ETH_P_IPV6)) {
                        const struct in6_addr *addr6;
                        struct neighbour *neigh;
                        bool do_tx_error_icmp;
                        int addr_type;

                        neigh = dst_neigh_lookup(skb_dst(skb),
                                                 &ipv6_hdr(skb)->daddr);
                        if (!neigh)
                                goto tx_error;

                        addr6 = (const struct in6_addr *)&neigh->primary_key;
                        addr_type = ipv6_addr_type(addr6);

                        if (addr_type == IPV6_ADDR_ANY) {
                                addr6 = &ipv6_hdr(skb)->daddr;
                                addr_type = ipv6_addr_type(addr6);
                        }

                        if ((addr_type & IPV6_ADDR_COMPATv4) == 0)
                                do_tx_error_icmp = true;
                        else {
                                do_tx_error_icmp = false;
                                dst = addr6->s6_addr32[3];
                        }
                        neigh_release(neigh);
                        if (do_tx_error_icmp)
                                goto tx_error_icmp;
                }
#endif
                else
                        goto tx_error;

                if (!md)
                        connected = false;
        }

        tos = tnl_params->tos;
        if (tos & 0x1) {
                tos &= ~0x1;
                if (payload_protocol == htons(ETH_P_IP)) {
                        tos = inner_iph->tos;
                        connected = false;
                } else if (payload_protocol == htons(ETH_P_IPV6)) {
                        tos = ipv6_get_dsfield((const struct ipv6hdr *)inner_iph);
                        connected = false;
                }
        }

        ip_tunnel_init_flow(&fl4, protocol, dst, tnl_params->saddr,
                            tunnel->parms.o_key, tos & INET_DSCP_MASK,
                            dev_net(dev), READ_ONCE(tunnel->parms.link),
                            tunnel->fwmark, skb_get_hash(skb), 0);

        if (ip_tunnel_encap(skb, &tunnel->encap, &protocol, &fl4) < 0)
                goto tx_error;

        if (connected && md) {
                use_cache = ip_tunnel_dst_cache_usable(skb, tun_info);
                if (use_cache)
                        rt = dst_cache_get_ip4(&tun_info->dst_cache,
                                               &fl4.saddr);
        } else {
                rt = connected ? dst_cache_get_ip4(&tunnel->dst_cache,
                                                &fl4.saddr) : NULL;
        }

        if (!rt) {
                rt = ip_route_output_key(tunnel->net, &fl4);

                if (IS_ERR(rt)) {
                        DEV_STATS_INC(dev, tx_carrier_errors);
                        goto tx_error;
                }
                if (use_cache)
                        dst_cache_set_ip4(&tun_info->dst_cache, &rt->dst,
                                          fl4.saddr);
                else if (!md && connected)
                        dst_cache_set_ip4(&tunnel->dst_cache, &rt->dst,
                                          fl4.saddr);
        }

        if (rt->dst.dev == dev) {
                ip_rt_put(rt);
                DEV_STATS_INC(dev, collisions);
                goto tx_error;
        }

        df = tnl_params->frag_off;
        if (payload_protocol == htons(ETH_P_IP) && !tunnel->ignore_df)
                df |= (inner_iph->frag_off & htons(IP_DF));

        if (tnl_update_pmtu(dev, skb, rt, df, inner_iph, 0, 0, false)) {
                ip_rt_put(rt);
                goto tx_error;
        }

        if (tunnel->err_count > 0) {
                if (time_before(jiffies,
                                tunnel->err_time + IPTUNNEL_ERR_TIMEO)) {
                        tunnel->err_count--;

                        dst_link_failure(skb);
                } else
                        tunnel->err_count = 0;
        }

        tos = ip_tunnel_ecn_encap(tos, inner_iph, skb);
        ttl = tnl_params->ttl;
        if (ttl == 0) {
                if (payload_protocol == htons(ETH_P_IP))
                        ttl = inner_iph->ttl;
#if IS_ENABLED(CONFIG_IPV6)
                else if (payload_protocol == htons(ETH_P_IPV6))
                        ttl = ((const struct ipv6hdr *)inner_iph)->hop_limit;
#endif
                else
                        ttl = ip4_dst_hoplimit(&rt->dst);
        }

        max_headroom = LL_RESERVED_SPACE(rt->dst.dev) + sizeof(struct iphdr)
                        + rt->dst.header_len + ip_encap_hlen(&tunnel->encap);

        if (skb_cow_head(skb, max_headroom)) {
                ip_rt_put(rt);
                DEV_STATS_INC(dev, tx_dropped);
                kfree_skb(skb);
                return;
        }

        ip_tunnel_adj_headroom(dev, max_headroom);

        iptunnel_xmit(NULL, rt, skb, fl4.saddr, fl4.daddr, protocol, tos, ttl,
                      df, !net_eq(tunnel->net, dev_net(dev)));
        return;

#if IS_ENABLED(CONFIG_IPV6)
tx_error_icmp:
        dst_link_failure(skb);
#endif
tx_error:
        DEV_STATS_INC(dev, tx_errors);
        kfree_skb(skb);
}
EXPORT_SYMBOL_GPL(ip_tunnel_xmit);

static void ip_tunnel_update(struct ip_tunnel_net *itn,
                             struct ip_tunnel *t,
                             struct net_device *dev,
                             struct ip_tunnel_parm_kern *p,
                             bool set_mtu,
                             __u32 fwmark)
{
        ip_tunnel_del(itn, t);
        t->parms.iph.saddr = p->iph.saddr;
        t->parms.iph.daddr = p->iph.daddr;
        t->parms.i_key = p->i_key;
        t->parms.o_key = p->o_key;
        if (dev->type != ARPHRD_ETHER) {
                __dev_addr_set(dev, &p->iph.saddr, 4);
                memcpy(dev->broadcast, &p->iph.daddr, 4);
        }
        ip_tunnel_add(itn, t);

        t->parms.iph.ttl = p->iph.ttl;
        t->parms.iph.tos = p->iph.tos;
        t->parms.iph.frag_off = p->iph.frag_off;

        if (t->parms.link != p->link || t->fwmark != fwmark) {
                int mtu;

                WRITE_ONCE(t->parms.link, p->link);
                t->fwmark = fwmark;
                mtu = ip_tunnel_bind_dev(dev);
                if (set_mtu)
                        WRITE_ONCE(dev->mtu, mtu);
        }
        dst_cache_reset(&t->dst_cache);
        netdev_state_change(dev);
}

int ip_tunnel_ctl(struct net_device *dev, struct ip_tunnel_parm_kern *p,
                  int cmd)
{
        int err = 0;
        struct ip_tunnel *t = netdev_priv(dev);
        struct net *net = t->net;
        struct ip_tunnel_net *itn = net_generic(net, t->ip_tnl_net_id);

        switch (cmd) {
        case SIOCGETTUNNEL:
                if (dev == itn->fb_tunnel_dev) {
                        t = ip_tunnel_find(itn, p, itn->fb_tunnel_dev->type);
                        if (!t)
                                t = netdev_priv(dev);
                }
                memcpy(p, &t->parms, sizeof(*p));
                break;

        case SIOCADDTUNNEL:
        case SIOCCHGTUNNEL:
                err = -EPERM;
                if (!ns_capable(net->user_ns, CAP_NET_ADMIN))
                        goto done;
                if (p->iph.ttl)
                        p->iph.frag_off |= htons(IP_DF);
                if (!test_bit(IP_TUNNEL_VTI_BIT, p->i_flags)) {
                        if (!test_bit(IP_TUNNEL_KEY_BIT, p->i_flags))
                                p->i_key = 0;
                        if (!test_bit(IP_TUNNEL_KEY_BIT, p->o_flags))
                                p->o_key = 0;
                }

                t = ip_tunnel_find(itn, p, itn->type);

                if (cmd == SIOCADDTUNNEL) {
                        if (!t) {
                                t = ip_tunnel_create(net, itn, p);
                                err = PTR_ERR_OR_ZERO(t);
                                break;
                        }

                        err = -EEXIST;
                        break;
                }
                if (dev != itn->fb_tunnel_dev && cmd == SIOCCHGTUNNEL) {
                        if (t) {
                                if (t->dev != dev) {
                                        err = -EEXIST;
                                        break;
                                }
                        } else {
                                unsigned int nflags = 0;

                                if (ipv4_is_multicast(p->iph.daddr))
                                        nflags = IFF_BROADCAST;
                                else if (p->iph.daddr)
                                        nflags = IFF_POINTOPOINT;

                                if ((dev->flags^nflags)&(IFF_POINTOPOINT|IFF_BROADCAST)) {
                                        err = -EINVAL;
                                        break;
                                }

                                t = netdev_priv(dev);
                        }
                }

                if (t) {
                        err = 0;
                        ip_tunnel_update(itn, t, dev, p, true, 0);
                } else {
                        err = -ENOENT;
                }
                break;

        case SIOCDELTUNNEL:
                err = -EPERM;
                if (!ns_capable(net->user_ns, CAP_NET_ADMIN))
                        goto done;

                if (dev == itn->fb_tunnel_dev) {
                        err = -ENOENT;
                        t = ip_tunnel_find(itn, p, itn->fb_tunnel_dev->type);
                        if (!t)
                                goto done;
                        err = -EPERM;
                        if (t == netdev_priv(itn->fb_tunnel_dev))
                                goto done;
                        dev = t->dev;
                }
                unregister_netdevice(dev);
                err = 0;
                break;

        default:
                err = -EINVAL;
        }

done:
        return err;
}
EXPORT_SYMBOL_GPL(ip_tunnel_ctl);

bool ip_tunnel_parm_from_user(struct ip_tunnel_parm_kern *kp,
                              const void __user *data)
{
        struct ip_tunnel_parm p;

        if (copy_from_user(&p, data, sizeof(p)))
                return false;

        strscpy(kp->name, p.name);
        kp->link = p.link;
        ip_tunnel_flags_from_be16(kp->i_flags, p.i_flags);
        ip_tunnel_flags_from_be16(kp->o_flags, p.o_flags);
        kp->i_key = p.i_key;
        kp->o_key = p.o_key;
        memcpy(&kp->iph, &p.iph, min(sizeof(kp->iph), sizeof(p.iph)));

        return true;
}
EXPORT_SYMBOL_GPL(ip_tunnel_parm_from_user);

bool ip_tunnel_parm_to_user(void __user *data, struct ip_tunnel_parm_kern *kp)
{
        struct ip_tunnel_parm p;

        if (!ip_tunnel_flags_is_be16_compat(kp->i_flags) ||
            !ip_tunnel_flags_is_be16_compat(kp->o_flags))
                return false;

        memset(&p, 0, sizeof(p));

        strscpy(p.name, kp->name);
        p.link = kp->link;
        p.i_flags = ip_tunnel_flags_to_be16(kp->i_flags);
        p.o_flags = ip_tunnel_flags_to_be16(kp->o_flags);
        p.i_key = kp->i_key;
        p.o_key = kp->o_key;
        memcpy(&p.iph, &kp->iph, min(sizeof(p.iph), sizeof(kp->iph)));

        return !copy_to_user(data, &p, sizeof(p));
}
EXPORT_SYMBOL_GPL(ip_tunnel_parm_to_user);

int ip_tunnel_siocdevprivate(struct net_device *dev, struct ifreq *ifr,
                             void __user *data, int cmd)
{
        struct ip_tunnel_parm_kern p;
        int err;

        if (!ip_tunnel_parm_from_user(&p, data))
                return -EFAULT;
        err = dev->netdev_ops->ndo_tunnel_ctl(dev, &p, cmd);
        if (!err && !ip_tunnel_parm_to_user(data, &p))
                return -EFAULT;
        return err;
}
EXPORT_SYMBOL_GPL(ip_tunnel_siocdevprivate);

int __ip_tunnel_change_mtu(struct net_device *dev, int new_mtu, bool strict)
{
        struct ip_tunnel *tunnel = netdev_priv(dev);
        int t_hlen = tunnel->hlen + sizeof(struct iphdr);
        int max_mtu = IP_MAX_MTU - t_hlen;

        if (dev->type == ARPHRD_ETHER)
                max_mtu -= dev->hard_header_len;

        if (new_mtu < ETH_MIN_MTU)
                return -EINVAL;

        if (new_mtu > max_mtu) {
                if (strict)
                        return -EINVAL;

                new_mtu = max_mtu;
        }

        WRITE_ONCE(dev->mtu, new_mtu);
        return 0;
}
EXPORT_SYMBOL_GPL(__ip_tunnel_change_mtu);

int ip_tunnel_change_mtu(struct net_device *dev, int new_mtu)
{
        return __ip_tunnel_change_mtu(dev, new_mtu, true);
}
EXPORT_SYMBOL_GPL(ip_tunnel_change_mtu);

static void ip_tunnel_dev_free(struct net_device *dev)
{
        struct ip_tunnel *tunnel = netdev_priv(dev);

        gro_cells_destroy(&tunnel->gro_cells);
        dst_cache_destroy(&tunnel->dst_cache);
}

void ip_tunnel_dellink(struct net_device *dev, struct list_head *head)
{
        struct ip_tunnel *tunnel = netdev_priv(dev);
        struct ip_tunnel_net *itn;

        itn = net_generic(tunnel->net, tunnel->ip_tnl_net_id);

        if (itn->fb_tunnel_dev != dev) {
                ip_tunnel_del(itn, netdev_priv(dev));
                unregister_netdevice_queue(dev, head);
        }
}
EXPORT_SYMBOL_GPL(ip_tunnel_dellink);

struct net *ip_tunnel_get_link_net(const struct net_device *dev)
{
        struct ip_tunnel *tunnel = netdev_priv(dev);

        return READ_ONCE(tunnel->net);
}
EXPORT_SYMBOL(ip_tunnel_get_link_net);

int ip_tunnel_get_iflink(const struct net_device *dev)
{
        const struct ip_tunnel *tunnel = netdev_priv(dev);

        return READ_ONCE(tunnel->parms.link);
}
EXPORT_SYMBOL(ip_tunnel_get_iflink);

int ip_tunnel_init_net(struct net *net, unsigned int ip_tnl_net_id,
                                  struct rtnl_link_ops *ops, char *devname)
{
        struct ip_tunnel_net *itn = net_generic(net, ip_tnl_net_id);
        struct ip_tunnel_parm_kern parms;
        unsigned int i;

        itn->rtnl_link_ops = ops;
        for (i = 0; i < IP_TNL_HASH_SIZE; i++)
                INIT_HLIST_HEAD(&itn->tunnels[i]);

        if (!ops || !net_has_fallback_tunnels(net)) {
                struct ip_tunnel_net *it_init_net;

                it_init_net = net_generic(&init_net, ip_tnl_net_id);
                itn->type = it_init_net->type;
                itn->fb_tunnel_dev = NULL;
                return 0;
        }

        memset(&parms, 0, sizeof(parms));
        if (devname)
                strscpy(parms.name, devname, IFNAMSIZ);

        rtnl_lock();
        itn->fb_tunnel_dev = __ip_tunnel_create(net, ops, &parms);
        /* FB netdevice is special: we have one, and only one per netns.
         * Allowing to move it to another netns is clearly unsafe.
         */
        if (!IS_ERR(itn->fb_tunnel_dev)) {
                itn->fb_tunnel_dev->netns_local = true;
                itn->fb_tunnel_dev->mtu = ip_tunnel_bind_dev(itn->fb_tunnel_dev);
                ip_tunnel_add(itn, netdev_priv(itn->fb_tunnel_dev));
                itn->type = itn->fb_tunnel_dev->type;
        }
        rtnl_unlock();

        return PTR_ERR_OR_ZERO(itn->fb_tunnel_dev);
}
EXPORT_SYMBOL_GPL(ip_tunnel_init_net);

static void ip_tunnel_destroy(struct net *net, struct ip_tunnel_net *itn,
                              struct list_head *head,
                              struct rtnl_link_ops *ops)
{
        struct net_device *dev, *aux;
        int h;

        for_each_netdev_safe(net, dev, aux)
                if (dev->rtnl_link_ops == ops)
                        unregister_netdevice_queue(dev, head);

        for (h = 0; h < IP_TNL_HASH_SIZE; h++) {
                struct ip_tunnel *t;
                struct hlist_node *n;
                struct hlist_head *thead = &itn->tunnels[h];

                hlist_for_each_entry_safe(t, n, thead, hash_node)
                        /* If dev is in the same netns, it has already
                         * been added to the list by the previous loop.
                         */
                        if (!net_eq(dev_net(t->dev), net))
                                unregister_netdevice_queue(t->dev, head);
        }
}

void ip_tunnel_delete_nets(struct list_head *net_list, unsigned int id,
                           struct rtnl_link_ops *ops,
                           struct list_head *dev_to_kill)
{
        struct ip_tunnel_net *itn;
        struct net *net;

        ASSERT_RTNL();
        list_for_each_entry(net, net_list, exit_list) {
                itn = net_generic(net, id);
                ip_tunnel_destroy(net, itn, dev_to_kill, ops);
        }
}
EXPORT_SYMBOL_GPL(ip_tunnel_delete_nets);

int ip_tunnel_newlink(struct net_device *dev, struct nlattr *tb[],
                      struct ip_tunnel_parm_kern *p, __u32 fwmark)
{
        struct ip_tunnel *nt;
        struct net *net = dev_net(dev);
        struct ip_tunnel_net *itn;
        int mtu;
        int err;

        nt = netdev_priv(dev);
        itn = net_generic(net, nt->ip_tnl_net_id);

        if (nt->collect_md) {
                if (rtnl_dereference(itn->collect_md_tun))
                        return -EEXIST;
        } else {
                if (ip_tunnel_find(itn, p, dev->type))
                        return -EEXIST;
        }

        nt->net = net;
        nt->parms = *p;
        nt->fwmark = fwmark;
        err = register_netdevice(dev);
        if (err)
                goto err_register_netdevice;

        if (dev->type == ARPHRD_ETHER && !tb[IFLA_ADDRESS])
                eth_hw_addr_random(dev);

        mtu = ip_tunnel_bind_dev(dev);
        if (tb[IFLA_MTU]) {
                unsigned int max = IP_MAX_MTU - (nt->hlen + sizeof(struct iphdr));

                if (dev->type == ARPHRD_ETHER)
                        max -= dev->hard_header_len;

                mtu = clamp(dev->mtu, (unsigned int)ETH_MIN_MTU, max);
        }

        err = dev_set_mtu(dev, mtu);
        if (err)
                goto err_dev_set_mtu;

        ip_tunnel_add(itn, nt);
        return 0;

err_dev_set_mtu:
        unregister_netdevice(dev);
err_register_netdevice:
        return err;
}
EXPORT_SYMBOL_GPL(ip_tunnel_newlink);

int ip_tunnel_changelink(struct net_device *dev, struct nlattr *tb[],
                         struct ip_tunnel_parm_kern *p, __u32 fwmark)
{
        struct ip_tunnel *t;
        struct ip_tunnel *tunnel = netdev_priv(dev);
        struct net *net = tunnel->net;
        struct ip_tunnel_net *itn = net_generic(net, tunnel->ip_tnl_net_id);

        if (dev == itn->fb_tunnel_dev)
                return -EINVAL;

        t = ip_tunnel_find(itn, p, dev->type);

        if (t) {
                if (t->dev != dev)
                        return -EEXIST;
        } else {
                t = tunnel;

                if (dev->type != ARPHRD_ETHER) {
                        unsigned int nflags = 0;

                        if (ipv4_is_multicast(p->iph.daddr))
                                nflags = IFF_BROADCAST;
                        else if (p->iph.daddr)
                                nflags = IFF_POINTOPOINT;

                        if ((dev->flags ^ nflags) &
                            (IFF_POINTOPOINT | IFF_BROADCAST))
                                return -EINVAL;
                }
        }

        ip_tunnel_update(itn, t, dev, p, !tb[IFLA_MTU], fwmark);
        return 0;
}
EXPORT_SYMBOL_GPL(ip_tunnel_changelink);

int ip_tunnel_init(struct net_device *dev)
{
        struct ip_tunnel *tunnel = netdev_priv(dev);
        struct iphdr *iph = &tunnel->parms.iph;
        int err;

        dev->needs_free_netdev = true;
        dev->priv_destructor = ip_tunnel_dev_free;
        dev->pcpu_stat_type = NETDEV_PCPU_STAT_TSTATS;

        err = dst_cache_init(&tunnel->dst_cache, GFP_KERNEL);
        if (err)
                return err;

        err = gro_cells_init(&tunnel->gro_cells, dev);
        if (err) {
                dst_cache_destroy(&tunnel->dst_cache);
                return err;
        }

        tunnel->dev = dev;
        tunnel->net = dev_net(dev);
        strscpy(tunnel->parms.name, dev->name);
        iph->version                = 4;
        iph->ihl                = 5;

        if (tunnel->collect_md)
                netif_keep_dst(dev);
        netdev_lockdep_set_classes(dev);
        return 0;
}
EXPORT_SYMBOL_GPL(ip_tunnel_init);

void ip_tunnel_uninit(struct net_device *dev)
{
        struct ip_tunnel *tunnel = netdev_priv(dev);
        struct net *net = tunnel->net;
        struct ip_tunnel_net *itn;

        itn = net_generic(net, tunnel->ip_tnl_net_id);
        ip_tunnel_del(itn, netdev_priv(dev));
        if (itn->fb_tunnel_dev == dev)
                WRITE_ONCE(itn->fb_tunnel_dev, NULL);

        dst_cache_reset(&tunnel->dst_cache);
}
EXPORT_SYMBOL_GPL(ip_tunnel_uninit);

/* Do least required initialization, rest of init is done in tunnel_init call */
void ip_tunnel_setup(struct net_device *dev, unsigned int net_id)
{
        struct ip_tunnel *tunnel = netdev_priv(dev);
        tunnel->ip_tnl_net_id = net_id;
}
EXPORT_SYMBOL_GPL(ip_tunnel_setup);

MODULE_DESCRIPTION("IPv4 tunnel implementation library");
MODULE_LICENSE("GPL");



















































































































































































































































































































































    6 
  223 



























    1 














  177 
    9 


























































































































































































    8 















    8 




















1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __LINUX_DCACHE_H
#define __LINUX_DCACHE_H

#include <linux/atomic.h>
#include <linux/list.h>
#include <linux/math.h>
#include <linux/rculist.h>
#include <linux/rculist_bl.h>
#include <linux/spinlock.h>
#include <linux/seqlock.h>
#include <linux/cache.h>
#include <linux/rcupdate.h>
#include <linux/lockref.h>
#include <linux/stringhash.h>
#include <linux/wait.h>

struct path;
struct file;
struct vfsmount;

/*
 * linux/include/linux/dcache.h
 *
 * Dirent cache data structures
 *
 * (C) Copyright 1997 Thomas Schoebel-Theuer,
 * with heavy changes by Linus Torvalds
 */

#define IS_ROOT(x) ((x) == (x)->d_parent)

/* The hash is always the low bits of hash_len */
#ifdef __LITTLE_ENDIAN
 #define HASH_LEN_DECLARE u32 hash; u32 len
 #define bytemask_from_count(cnt)        (~(~0ul << (cnt)*8))
#else
 #define HASH_LEN_DECLARE u32 len; u32 hash
 #define bytemask_from_count(cnt)        (~(~0ul >> (cnt)*8))
#endif

/*
 * "quick string" -- eases parameter passing, but more importantly
 * saves "metadata" about the string (ie length and the hash).
 *
 * hash comes first so it snuggles against d_parent in the
 * dentry.
 */
struct qstr {
        union {
                struct {
                        HASH_LEN_DECLARE;
                };
                u64 hash_len;
        };
        const unsigned char *name;
};

#define QSTR_INIT(n,l) { { { .len = l } }, .name = n }

extern const struct qstr empty_name;
extern const struct qstr slash_name;
extern const struct qstr dotdot_name;

/*
 * Try to keep struct dentry aligned on 64 byte cachelines (this will
 * give reasonable cacheline footprint with larger lines without the
 * large memory footprint increase).
 */
#ifdef CONFIG_64BIT
# define DNAME_INLINE_LEN 40 /* 192 bytes */
#else
# ifdef CONFIG_SMP
#  define DNAME_INLINE_LEN 36 /* 128 bytes */
# else
#  define DNAME_INLINE_LEN 44 /* 128 bytes */
# endif
#endif

#define d_lock        d_lockref.lock

struct dentry {
        /* RCU lookup touched fields */
        unsigned int d_flags;                /* protected by d_lock */
        seqcount_spinlock_t d_seq;        /* per dentry seqlock */
        struct hlist_bl_node d_hash;        /* lookup hash list */
        struct dentry *d_parent;        /* parent directory */
        struct qstr d_name;
        struct inode *d_inode;                /* Where the name belongs to - NULL is
                                         * negative */
        unsigned char d_iname[DNAME_INLINE_LEN];        /* small names */
        /* --- cacheline 1 boundary (64 bytes) was 32 bytes ago --- */

        /* Ref lookup also touches following */
        const struct dentry_operations *d_op;
        struct super_block *d_sb;        /* The root of the dentry tree */
        unsigned long d_time;                /* used by d_revalidate */
        void *d_fsdata;                        /* fs-specific data */
        /* --- cacheline 2 boundary (128 bytes) --- */
        struct lockref d_lockref;        /* per-dentry lock and refcount
                                         * keep separate from RCU lookup area if
                                         * possible!
                                         */

        union {
                struct list_head d_lru;                /* LRU list */
                wait_queue_head_t *d_wait;        /* in-lookup ones only */
        };
        struct hlist_node d_sib;        /* child of parent list */
        struct hlist_head d_children;        /* our children */
        /*
         * d_alias and d_rcu can share memory
         */
        union {
                struct hlist_node d_alias;        /* inode alias list */
                struct hlist_bl_node d_in_lookup_hash;        /* only for in-lookup ones */
                 struct rcu_head d_rcu;
        } d_u;
};

/*
 * dentry->d_lock spinlock nesting subclasses:
 *
 * 0: normal
 * 1: nested
 */
enum dentry_d_lock_class
{
        DENTRY_D_LOCK_NORMAL, /* implicitly used by plain spin_lock() APIs. */
        DENTRY_D_LOCK_NESTED
};

enum d_real_type {
        D_REAL_DATA,
        D_REAL_METADATA,
};

struct dentry_operations {
        int (*d_revalidate)(struct dentry *, unsigned int);
        int (*d_weak_revalidate)(struct dentry *, unsigned int);
        int (*d_hash)(const struct dentry *, struct qstr *);
        int (*d_compare)(const struct dentry *,
                        unsigned int, const char *, const struct qstr *);
        int (*d_delete)(const struct dentry *);
        int (*d_init)(struct dentry *);
        void (*d_release)(struct dentry *);
        void (*d_prune)(struct dentry *);
        void (*d_iput)(struct dentry *, struct inode *);
        char *(*d_dname)(struct dentry *, char *, int);
        struct vfsmount *(*d_automount)(struct path *);
        int (*d_manage)(const struct path *, bool);
        struct dentry *(*d_real)(struct dentry *, enum d_real_type type);
} ____cacheline_aligned;

/*
 * Locking rules for dentry_operations callbacks are to be found in
 * Documentation/filesystems/locking.rst. Keep it updated!
 *
 * FUrther descriptions are found in Documentation/filesystems/vfs.rst.
 * Keep it updated too!
 */

/* d_flags entries */
#define DCACHE_OP_HASH                        BIT(0)
#define DCACHE_OP_COMPARE                BIT(1)
#define DCACHE_OP_REVALIDATE                BIT(2)
#define DCACHE_OP_DELETE                BIT(3)
#define DCACHE_OP_PRUNE                        BIT(4)

#define        DCACHE_DISCONNECTED                BIT(5)
     /* This dentry is possibly not currently connected to the dcache tree, in
      * which case its parent will either be itself, or will have this flag as
      * well.  nfsd will not use a dentry with this bit set, but will first
      * endeavour to clear the bit either by discovering that it is connected,
      * or by performing lookup operations.   Any filesystem which supports
      * nfsd_operations MUST have a lookup function which, if it finds a
      * directory inode with a DCACHE_DISCONNECTED dentry, will d_move that
      * dentry into place and return that dentry rather than the passed one,
      * typically using d_splice_alias. */

#define DCACHE_REFERENCED                BIT(6) /* Recently used, don't discard. */

#define DCACHE_DONTCACHE                BIT(7) /* Purge from memory on final dput() */

#define DCACHE_CANT_MOUNT                BIT(8)
#define DCACHE_GENOCIDE                        BIT(9)
#define DCACHE_SHRINK_LIST                BIT(10)

#define DCACHE_OP_WEAK_REVALIDATE        BIT(11)

#define DCACHE_NFSFS_RENAMED                BIT(12)
     /* this dentry has been "silly renamed" and has to be deleted on the last
      * dput() */
#define DCACHE_FSNOTIFY_PARENT_WATCHED        BIT(14)
     /* Parent inode is watched by some fsnotify listener */

#define DCACHE_DENTRY_KILLED                BIT(15)

#define DCACHE_MOUNTED                        BIT(16) /* is a mountpoint */
#define DCACHE_NEED_AUTOMOUNT                BIT(17) /* handle automount on this dir */
#define DCACHE_MANAGE_TRANSIT                BIT(18) /* manage transit from this dirent */
#define DCACHE_MANAGED_DENTRY \
        (DCACHE_MOUNTED|DCACHE_NEED_AUTOMOUNT|DCACHE_MANAGE_TRANSIT)

#define DCACHE_LRU_LIST                        BIT(19)

#define DCACHE_ENTRY_TYPE                (7 << 20) /* bits 20..22 are for storing type: */
#define DCACHE_MISS_TYPE                (0 << 20) /* Negative dentry */
#define DCACHE_WHITEOUT_TYPE                (1 << 20) /* Whiteout dentry (stop pathwalk) */
#define DCACHE_DIRECTORY_TYPE                (2 << 20) /* Normal directory */
#define DCACHE_AUTODIR_TYPE                (3 << 20) /* Lookupless directory (presumed automount) */
#define DCACHE_REGULAR_TYPE                (4 << 20) /* Regular file type */
#define DCACHE_SPECIAL_TYPE                (5 << 20) /* Other file type */
#define DCACHE_SYMLINK_TYPE                (6 << 20) /* Symlink */

#define DCACHE_NOKEY_NAME                BIT(25) /* Encrypted name encoded without key */
#define DCACHE_OP_REAL                        BIT(26)

#define DCACHE_PAR_LOOKUP                BIT(28) /* being looked up (with parent locked shared) */
#define DCACHE_DENTRY_CURSOR                BIT(29)
#define DCACHE_NORCU                        BIT(30) /* No RCU delay for freeing */

extern seqlock_t rename_lock;

/*
 * These are the low-level FS interfaces to the dcache..
 */
extern void d_instantiate(struct dentry *, struct inode *);
extern void d_instantiate_new(struct dentry *, struct inode *);
extern void __d_drop(struct dentry *dentry);
extern void d_drop(struct dentry *dentry);
extern void d_delete(struct dentry *);
extern void d_set_d_op(struct dentry *dentry, const struct dentry_operations *op);

/* allocate/de-allocate */
extern struct dentry * d_alloc(struct dentry *, const struct qstr *);
extern struct dentry * d_alloc_anon(struct super_block *);
extern struct dentry * d_alloc_parallel(struct dentry *, const struct qstr *,
                                        wait_queue_head_t *);
extern struct dentry * d_splice_alias(struct inode *, struct dentry *);
extern struct dentry * d_add_ci(struct dentry *, struct inode *, struct qstr *);
extern bool d_same_name(const struct dentry *dentry, const struct dentry *parent,
                        const struct qstr *name);
extern struct dentry * d_exact_alias(struct dentry *, struct inode *);
extern struct dentry *d_find_any_alias(struct inode *inode);
extern struct dentry * d_obtain_alias(struct inode *);
extern struct dentry * d_obtain_root(struct inode *);
extern void shrink_dcache_sb(struct super_block *);
extern void shrink_dcache_parent(struct dentry *);
extern void d_invalidate(struct dentry *);

/* only used at mount-time */
extern struct dentry * d_make_root(struct inode *);

extern void d_mark_tmpfile(struct file *, struct inode *);
extern void d_tmpfile(struct file *, struct inode *);

extern struct dentry *d_find_alias(struct inode *);
extern void d_prune_aliases(struct inode *);

extern struct dentry *d_find_alias_rcu(struct inode *);

/* test whether we have any submounts in a subdir tree */
extern int path_has_submounts(const struct path *);

/*
 * This adds the entry to the hash queues.
 */
extern void d_rehash(struct dentry *);
 
extern void d_add(struct dentry *, struct inode *);

/* used for rename() and baskets */
extern void d_move(struct dentry *, struct dentry *);
extern void d_exchange(struct dentry *, struct dentry *);
extern struct dentry *d_ancestor(struct dentry *, struct dentry *);

extern struct dentry *d_lookup(const struct dentry *, const struct qstr *);
extern struct dentry *d_hash_and_lookup(struct dentry *, struct qstr *);

static inline unsigned d_count(const struct dentry *dentry)
{
        return dentry->d_lockref.count;
}

ino_t d_parent_ino(struct dentry *dentry);

/*
 * helper function for dentry_operations.d_dname() members
 */
extern __printf(3, 4)
char *dynamic_dname(char *, int, const char *, ...);

extern char *__d_path(const struct path *, const struct path *, char *, int);
extern char *d_absolute_path(const struct path *, char *, int);
extern char *d_path(const struct path *, char *, int);
extern char *dentry_path_raw(const struct dentry *, char *, int);
extern char *dentry_path(const struct dentry *, char *, int);

/* Allocation counts.. */

/**
 * dget_dlock -        get a reference to a dentry
 * @dentry: dentry to get a reference to
 *
 * Given a live dentry, increment the reference count and return the dentry.
 * Caller must hold @dentry->d_lock.  Making sure that dentry is alive is
 * caller's resonsibility.  There are many conditions sufficient to guarantee
 * that; e.g. anything with non-negative refcount is alive, so's anything
 * hashed, anything positive, anyone's parent, etc.
 */
static inline struct dentry *dget_dlock(struct dentry *dentry)
{
        dentry->d_lockref.count++;
        return dentry;
}


/**
 * dget - get a reference to a dentry
 * @dentry: dentry to get a reference to
 *
 * Given a dentry or %NULL pointer increment the reference count
 * if appropriate and return the dentry.  A dentry will not be
 * destroyed when it has references.  Conversely, a dentry with
 * no references can disappear for any number of reasons, starting
 * with memory pressure.  In other words, that primitive is
 * used to clone an existing reference; using it on something with
 * zero refcount is a bug.
 *
 * NOTE: it will spin if @dentry->d_lock is held.  From the deadlock
 * avoidance point of view it is equivalent to spin_lock()/increment
 * refcount/spin_unlock(), so calling it under @dentry->d_lock is
 * always a bug; so's calling it under ->d_lock on any of its descendents.
 *
 */
static inline struct dentry *dget(struct dentry *dentry)
{
        if (dentry)
                lockref_get(&dentry->d_lockref);
        return dentry;
}

extern struct dentry *dget_parent(struct dentry *dentry);

/**
 * d_unhashed - is dentry hashed
 * @dentry: entry to check
 *
 * Returns true if the dentry passed is not currently hashed.
 */
static inline int d_unhashed(const struct dentry *dentry)
{
        return hlist_bl_unhashed(&dentry->d_hash);
}

static inline int d_unlinked(const struct dentry *dentry)
{
        return d_unhashed(dentry) && !IS_ROOT(dentry);
}

static inline int cant_mount(const struct dentry *dentry)
{
        return (dentry->d_flags & DCACHE_CANT_MOUNT);
}

static inline void dont_mount(struct dentry *dentry)
{
        spin_lock(&dentry->d_lock);
        dentry->d_flags |= DCACHE_CANT_MOUNT;
        spin_unlock(&dentry->d_lock);
}

extern void __d_lookup_unhash_wake(struct dentry *dentry);

static inline int d_in_lookup(const struct dentry *dentry)
{
        return dentry->d_flags & DCACHE_PAR_LOOKUP;
}

static inline void d_lookup_done(struct dentry *dentry)
{
        if (unlikely(d_in_lookup(dentry)))
                __d_lookup_unhash_wake(dentry);
}

extern void dput(struct dentry *);

static inline bool d_managed(const struct dentry *dentry)
{
        return dentry->d_flags & DCACHE_MANAGED_DENTRY;
}

static inline bool d_mountpoint(const struct dentry *dentry)
{
        return dentry->d_flags & DCACHE_MOUNTED;
}

/*
 * Directory cache entry type accessor functions.
 */
static inline unsigned __d_entry_type(const struct dentry *dentry)
{
        return dentry->d_flags & DCACHE_ENTRY_TYPE;
}

static inline bool d_is_miss(const struct dentry *dentry)
{
        return __d_entry_type(dentry) == DCACHE_MISS_TYPE;
}

static inline bool d_is_whiteout(const struct dentry *dentry)
{
        return __d_entry_type(dentry) == DCACHE_WHITEOUT_TYPE;
}

static inline bool d_can_lookup(const struct dentry *dentry)
{
        return __d_entry_type(dentry) == DCACHE_DIRECTORY_TYPE;
}

static inline bool d_is_autodir(const struct dentry *dentry)
{
        return __d_entry_type(dentry) == DCACHE_AUTODIR_TYPE;
}

static inline bool d_is_dir(const struct dentry *dentry)
{
        return d_can_lookup(dentry) || d_is_autodir(dentry);
}

static inline bool d_is_symlink(const struct dentry *dentry)
{
        return __d_entry_type(dentry) == DCACHE_SYMLINK_TYPE;
}

static inline bool d_is_reg(const struct dentry *dentry)
{
        return __d_entry_type(dentry) == DCACHE_REGULAR_TYPE;
}

static inline bool d_is_special(const struct dentry *dentry)
{
        return __d_entry_type(dentry) == DCACHE_SPECIAL_TYPE;
}

static inline bool d_is_file(const struct dentry *dentry)
{
        return d_is_reg(dentry) || d_is_special(dentry);
}

static inline bool d_is_negative(const struct dentry *dentry)
{
        // TODO: check d_is_whiteout(dentry) also.
        return d_is_miss(dentry);
}

static inline bool d_flags_negative(unsigned flags)
{
        return (flags & DCACHE_ENTRY_TYPE) == DCACHE_MISS_TYPE;
}

static inline bool d_is_positive(const struct dentry *dentry)
{
        return !d_is_negative(dentry);
}

/**
 * d_really_is_negative - Determine if a dentry is really negative (ignoring fallthroughs)
 * @dentry: The dentry in question
 *
 * Returns true if the dentry represents either an absent name or a name that
 * doesn't map to an inode (ie. ->d_inode is NULL).  The dentry could represent
 * a true miss, a whiteout that isn't represented by a 0,0 chardev or a
 * fallthrough marker in an opaque directory.
 *
 * Note!  (1) This should be used *only* by a filesystem to examine its own
 * dentries.  It should not be used to look at some other filesystem's
 * dentries.  (2) It should also be used in combination with d_inode() to get
 * the inode.  (3) The dentry may have something attached to ->d_lower and the
 * type field of the flags may be set to something other than miss or whiteout.
 */
static inline bool d_really_is_negative(const struct dentry *dentry)
{
        return dentry->d_inode == NULL;
}

/**
 * d_really_is_positive - Determine if a dentry is really positive (ignoring fallthroughs)
 * @dentry: The dentry in question
 *
 * Returns true if the dentry represents a name that maps to an inode
 * (ie. ->d_inode is not NULL).  The dentry might still represent a whiteout if
 * that is represented on medium as a 0,0 chardev.
 *
 * Note!  (1) This should be used *only* by a filesystem to examine its own
 * dentries.  It should not be used to look at some other filesystem's
 * dentries.  (2) It should also be used in combination with d_inode() to get
 * the inode.
 */
static inline bool d_really_is_positive(const struct dentry *dentry)
{
        return dentry->d_inode != NULL;
}

static inline int simple_positive(const struct dentry *dentry)
{
        return d_really_is_positive(dentry) && !d_unhashed(dentry);
}

extern int sysctl_vfs_cache_pressure;

static inline unsigned long vfs_pressure_ratio(unsigned long val)
{
        return mult_frac(val, sysctl_vfs_cache_pressure, 100);
}

/**
 * d_inode - Get the actual inode of this dentry
 * @dentry: The dentry to query
 *
 * This is the helper normal filesystems should use to get at their own inodes
 * in their own dentries and ignore the layering superimposed upon them.
 */
static inline struct inode *d_inode(const struct dentry *dentry)
{
        return dentry->d_inode;
}

/**
 * d_inode_rcu - Get the actual inode of this dentry with READ_ONCE()
 * @dentry: The dentry to query
 *
 * This is the helper normal filesystems should use to get at their own inodes
 * in their own dentries and ignore the layering superimposed upon them.
 */
static inline struct inode *d_inode_rcu(const struct dentry *dentry)
{
        return READ_ONCE(dentry->d_inode);
}

/**
 * d_backing_inode - Get upper or lower inode we should be using
 * @upper: The upper layer
 *
 * This is the helper that should be used to get at the inode that will be used
 * if this dentry were to be opened as a file.  The inode may be on the upper
 * dentry or it may be on a lower dentry pinned by the upper.
 *
 * Normal filesystems should not use this to access their own inodes.
 */
static inline struct inode *d_backing_inode(const struct dentry *upper)
{
        struct inode *inode = upper->d_inode;

        return inode;
}

/**
 * d_real - Return the real dentry
 * @dentry: the dentry to query
 * @type: the type of real dentry (data or metadata)
 *
 * If dentry is on a union/overlay, then return the underlying, real dentry.
 * Otherwise return the dentry itself.
 *
 * See also: Documentation/filesystems/vfs.rst
 */
static inline struct dentry *d_real(struct dentry *dentry, enum d_real_type type)
{
        if (unlikely(dentry->d_flags & DCACHE_OP_REAL))
                return dentry->d_op->d_real(dentry, type);
        else
                return dentry;
}

/**
 * d_real_inode - Return the real inode hosting the data
 * @dentry: The dentry to query
 *
 * If dentry is on a union/overlay, then return the underlying, real inode.
 * Otherwise return d_inode().
 */
static inline struct inode *d_real_inode(const struct dentry *dentry)
{
        /* This usage of d_real() results in const dentry */
        return d_inode(d_real((struct dentry *) dentry, D_REAL_DATA));
}

struct name_snapshot {
        struct qstr name;
        unsigned char inline_name[DNAME_INLINE_LEN];
};
void take_dentry_name_snapshot(struct name_snapshot *, struct dentry *);
void release_dentry_name_snapshot(struct name_snapshot *);

static inline struct dentry *d_first_child(const struct dentry *dentry)
{
        return hlist_entry_safe(dentry->d_children.first, struct dentry, d_sib);
}

static inline struct dentry *d_next_sibling(const struct dentry *dentry)
{
        return hlist_entry_safe(dentry->d_sib.next, struct dentry, d_sib);
}

#endif        /* __LINUX_DCACHE_H */






































   15 






    7 


    8 
    8 






    8 















    8 

    8 







    8 





    6 

    6 

    4 


    1 
    1 






    9 

    9 


    6 
    3 





   66 




   67 
   67 
   67 
   67 





    2 

    1 


    1 




















   12 













   12 








   12 











    4 







    7 















   11 
    1 


   12 


    1 






    4 




    1 

    6 


    1 


    6 
















    7 




    7 









    7 










   63 

   64 






























1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
// SPDX-License-Identifier: GPL-2.0-only
/*
 * vxcan.c - Virtual CAN Tunnel for cross namespace communication
 *
 * This code is derived from drivers/net/can/vcan.c for the virtual CAN
 * specific parts and from drivers/net/veth.c to implement the netlink API
 * for network interface pairs in a common and established way.
 *
 * Copyright (c) 2017 Oliver Hartkopp <socketcan@hartkopp.net>
 */

#include <linux/ethtool.h>
#include <linux/module.h>
#include <linux/init.h>
#include <linux/netdevice.h>
#include <linux/if_arp.h>
#include <linux/if_ether.h>
#include <linux/can.h>
#include <linux/can/dev.h>
#include <linux/can/skb.h>
#include <linux/can/vxcan.h>
#include <linux/can/can-ml.h>
#include <linux/slab.h>
#include <net/rtnetlink.h>

#define DRV_NAME "vxcan"

MODULE_DESCRIPTION("Virtual CAN Tunnel");
MODULE_LICENSE("GPL");
MODULE_AUTHOR("Oliver Hartkopp <socketcan@hartkopp.net>");
MODULE_ALIAS_RTNL_LINK(DRV_NAME);

struct vxcan_priv {
        struct net_device __rcu        *peer;
};

static netdev_tx_t vxcan_xmit(struct sk_buff *oskb, struct net_device *dev)
{
        struct vxcan_priv *priv = netdev_priv(dev);
        struct net_device *peer;
        struct net_device_stats *peerstats, *srcstats = &dev->stats;
        struct sk_buff *skb;
        unsigned int len;

        if (can_dropped_invalid_skb(dev, oskb))
                return NETDEV_TX_OK;

        rcu_read_lock();
        peer = rcu_dereference(priv->peer);
        if (unlikely(!peer)) {
                kfree_skb(oskb);
                dev->stats.tx_dropped++;
                goto out_unlock;
        }

        skb_tx_timestamp(oskb);

        skb = skb_clone(oskb, GFP_ATOMIC);
        if (skb) {
                consume_skb(oskb);
        } else {
                kfree_skb(oskb);
                goto out_unlock;
        }

        /* reset CAN GW hop counter */
        skb->csum_start = 0;
        skb->pkt_type   = PACKET_BROADCAST;
        skb->dev        = peer;
        skb->ip_summed  = CHECKSUM_UNNECESSARY;

        len = can_skb_get_data_len(skb);
        if (netif_rx(skb) == NET_RX_SUCCESS) {
                srcstats->tx_packets++;
                srcstats->tx_bytes += len;
                peerstats = &peer->stats;
                peerstats->rx_packets++;
                peerstats->rx_bytes += len;
        }

out_unlock:
        rcu_read_unlock();
        return NETDEV_TX_OK;
}


static int vxcan_open(struct net_device *dev)
{
        struct vxcan_priv *priv = netdev_priv(dev);
        struct net_device *peer = rtnl_dereference(priv->peer);

        if (!peer)
                return -ENOTCONN;

        if (peer->flags & IFF_UP) {
                netif_carrier_on(dev);
                netif_carrier_on(peer);
        }
        return 0;
}

static int vxcan_close(struct net_device *dev)
{
        struct vxcan_priv *priv = netdev_priv(dev);
        struct net_device *peer = rtnl_dereference(priv->peer);

        netif_carrier_off(dev);
        if (peer)
                netif_carrier_off(peer);

        return 0;
}

static int vxcan_get_iflink(const struct net_device *dev)
{
        struct vxcan_priv *priv = netdev_priv(dev);
        struct net_device *peer;
        int iflink;

        rcu_read_lock();
        peer = rcu_dereference(priv->peer);
        iflink = peer ? READ_ONCE(peer->ifindex) : 0;
        rcu_read_unlock();

        return iflink;
}

static int vxcan_change_mtu(struct net_device *dev, int new_mtu)
{
        /* Do not allow changing the MTU while running */
        if (dev->flags & IFF_UP)
                return -EBUSY;

        if (new_mtu != CAN_MTU && new_mtu != CANFD_MTU &&
            !can_is_canxl_dev_mtu(new_mtu))
                return -EINVAL;

        WRITE_ONCE(dev->mtu, new_mtu);
        return 0;
}

static const struct net_device_ops vxcan_netdev_ops = {
        .ndo_open        = vxcan_open,
        .ndo_stop        = vxcan_close,
        .ndo_start_xmit        = vxcan_xmit,
        .ndo_get_iflink        = vxcan_get_iflink,
        .ndo_change_mtu = vxcan_change_mtu,
};

static const struct ethtool_ops vxcan_ethtool_ops = {
        .get_ts_info = ethtool_op_get_ts_info,
};

static void vxcan_setup(struct net_device *dev)
{
        struct can_ml_priv *can_ml;

        dev->type                = ARPHRD_CAN;
        dev->mtu                = CANFD_MTU;
        dev->hard_header_len        = 0;
        dev->addr_len                = 0;
        dev->tx_queue_len        = 0;
        dev->flags                = IFF_NOARP;
        dev->netdev_ops                = &vxcan_netdev_ops;
        dev->ethtool_ops        = &vxcan_ethtool_ops;
        dev->needs_free_netdev        = true;

        can_ml = netdev_priv(dev) + ALIGN(sizeof(struct vxcan_priv), NETDEV_ALIGN);
        can_set_ml_priv(dev, can_ml);
}

/* forward declaration for rtnl_create_link() */
static struct rtnl_link_ops vxcan_link_ops;

static int vxcan_newlink(struct net *net, struct net_device *dev,
                         struct nlattr *tb[], struct nlattr *data[],
                         struct netlink_ext_ack *extack)
{
        struct vxcan_priv *priv;
        struct net_device *peer;
        struct net *peer_net;

        struct nlattr *peer_tb[IFLA_MAX + 1], **tbp = tb;
        char ifname[IFNAMSIZ];
        unsigned char name_assign_type;
        struct ifinfomsg *ifmp = NULL;
        int err;

        /* register peer device */
        if (data && data[VXCAN_INFO_PEER]) {
                struct nlattr *nla_peer = data[VXCAN_INFO_PEER];

                ifmp = nla_data(nla_peer);
                rtnl_nla_parse_ifinfomsg(peer_tb, nla_peer, extack);
                tbp = peer_tb;
        }

        if (ifmp && tbp[IFLA_IFNAME]) {
                nla_strscpy(ifname, tbp[IFLA_IFNAME], IFNAMSIZ);
                name_assign_type = NET_NAME_USER;
        } else {
                snprintf(ifname, IFNAMSIZ, DRV_NAME "%%d");
                name_assign_type = NET_NAME_ENUM;
        }

        peer_net = rtnl_link_get_net(net, tbp);
        peer = rtnl_create_link(peer_net, ifname, name_assign_type,
                                &vxcan_link_ops, tbp, extack);
        if (IS_ERR(peer)) {
                put_net(peer_net);
                return PTR_ERR(peer);
        }

        if (ifmp && dev->ifindex)
                peer->ifindex = ifmp->ifi_index;

        err = register_netdevice(peer);
        put_net(peer_net);
        peer_net = NULL;
        if (err < 0) {
                free_netdev(peer);
                return err;
        }

        netif_carrier_off(peer);

        err = rtnl_configure_link(peer, ifmp, 0, NULL);
        if (err < 0)
                goto unregister_network_device;

        /* register first device */
        if (tb[IFLA_IFNAME])
                nla_strscpy(dev->name, tb[IFLA_IFNAME], IFNAMSIZ);
        else
                snprintf(dev->name, IFNAMSIZ, DRV_NAME "%%d");

        err = register_netdevice(dev);
        if (err < 0)
                goto unregister_network_device;

        netif_carrier_off(dev);

        /* cross link the device pair */
        priv = netdev_priv(dev);
        rcu_assign_pointer(priv->peer, peer);

        priv = netdev_priv(peer);
        rcu_assign_pointer(priv->peer, dev);

        return 0;

unregister_network_device:
        unregister_netdevice(peer);
        return err;
}

static void vxcan_dellink(struct net_device *dev, struct list_head *head)
{
        struct vxcan_priv *priv;
        struct net_device *peer;

        priv = netdev_priv(dev);
        peer = rtnl_dereference(priv->peer);

        /* Note : dellink() is called from default_device_exit_batch(),
         * before a rcu_synchronize() point. The devices are guaranteed
         * not being freed before one RCU grace period.
         */
        RCU_INIT_POINTER(priv->peer, NULL);
        unregister_netdevice_queue(dev, head);

        if (peer) {
                priv = netdev_priv(peer);
                RCU_INIT_POINTER(priv->peer, NULL);
                unregister_netdevice_queue(peer, head);
        }
}

static const struct nla_policy vxcan_policy[VXCAN_INFO_MAX + 1] = {
        [VXCAN_INFO_PEER] = { .len = sizeof(struct ifinfomsg) },
};

static struct net *vxcan_get_link_net(const struct net_device *dev)
{
        struct vxcan_priv *priv = netdev_priv(dev);
        struct net_device *peer = rtnl_dereference(priv->peer);

        return peer ? dev_net(peer) : dev_net(dev);
}

static struct rtnl_link_ops vxcan_link_ops = {
        .kind                = DRV_NAME,
        .priv_size        = ALIGN(sizeof(struct vxcan_priv), NETDEV_ALIGN) + sizeof(struct can_ml_priv),
        .setup                = vxcan_setup,
        .newlink        = vxcan_newlink,
        .dellink        = vxcan_dellink,
        .policy                = vxcan_policy,
        .peer_type        = VXCAN_INFO_PEER,
        .maxtype        = VXCAN_INFO_MAX,
        .get_link_net        = vxcan_get_link_net,
};

static __init int vxcan_init(void)
{
        pr_info("vxcan: Virtual CAN Tunnel driver\n");

        return rtnl_link_register(&vxcan_link_ops);
}

static __exit void vxcan_exit(void)
{
        rtnl_link_unregister(&vxcan_link_ops);
}

module_init(vxcan_init);
module_exit(vxcan_exit);










































  102 












































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
/* SPDX-License-Identifier: GPL-2.0 */
/*
 * Copyright (C) 1994 Linus Torvalds
 *
 * Pentium III FXSR, SSE support
 * General FPU state handling cleanups
 *        Gareth Hughes <gareth@valinux.com>, May 2000
 * x86-64 work by Andi Kleen 2002
 */

#ifndef _ASM_X86_FPU_API_H
#define _ASM_X86_FPU_API_H
#include <linux/bottom_half.h>

#include <asm/fpu/types.h>

/*
 * Use kernel_fpu_begin/end() if you intend to use FPU in kernel context. It
 * disables preemption so be careful if you intend to use it for long periods
 * of time.
 * If you intend to use the FPU in irq/softirq you need to check first with
 * irq_fpu_usable() if it is possible.
 */

/* Kernel FPU states to initialize in kernel_fpu_begin_mask() */
#define KFPU_387        _BITUL(0)        /* 387 state will be initialized */
#define KFPU_MXCSR        _BITUL(1)        /* MXCSR will be initialized */

extern void kernel_fpu_begin_mask(unsigned int kfpu_mask);
extern void kernel_fpu_end(void);
extern bool irq_fpu_usable(void);
extern void fpregs_mark_activate(void);

/* Code that is unaware of kernel_fpu_begin_mask() can use this */
static inline void kernel_fpu_begin(void)
{
#ifdef CONFIG_X86_64
        /*
         * Any 64-bit code that uses 387 instructions must explicitly request
         * KFPU_387.
         */
        kernel_fpu_begin_mask(KFPU_MXCSR);
#else
        /*
         * 32-bit kernel code may use 387 operations as well as SSE2, etc,
         * as long as it checks that the CPU has the required capability.
         */
        kernel_fpu_begin_mask(KFPU_387 | KFPU_MXCSR);
#endif
}

/*
 * Use fpregs_lock() while editing CPU's FPU registers or fpu->fpstate.
 * A context switch will (and softirq might) save CPU's FPU registers to
 * fpu->fpstate.regs and set TIF_NEED_FPU_LOAD leaving CPU's FPU registers in
 * a random state.
 *
 * local_bh_disable() protects against both preemption and soft interrupts
 * on !RT kernels.
 *
 * On RT kernels local_bh_disable() is not sufficient because it only
 * serializes soft interrupt related sections via a local lock, but stays
 * preemptible. Disabling preemption is the right choice here as bottom
 * half processing is always in thread context on RT kernels so it
 * implicitly prevents bottom half processing as well.
 *
 * Disabling preemption also serializes against kernel_fpu_begin().
 */
static inline void fpregs_lock(void)
{
        if (!IS_ENABLED(CONFIG_PREEMPT_RT))
                local_bh_disable();
        else
                preempt_disable();
}

static inline void fpregs_unlock(void)
{
        if (!IS_ENABLED(CONFIG_PREEMPT_RT))
                local_bh_enable();
        else
                preempt_enable();
}

/*
 * FPU state gets lazily restored before returning to userspace. So when in the
 * kernel, the valid FPU state may be kept in the buffer. This function will force
 * restore all the fpu state to the registers early if needed, and lock them from
 * being automatically saved/restored. Then FPU state can be modified safely in the
 * registers, before unlocking with fpregs_unlock().
 */
void fpregs_lock_and_load(void);

#ifdef CONFIG_X86_DEBUG_FPU
extern void fpregs_assert_state_consistent(void);
#else
static inline void fpregs_assert_state_consistent(void) { }
#endif

/*
 * Load the task FPU state before returning to userspace.
 */
extern void switch_fpu_return(void);

/*
 * Query the presence of one or more xfeatures. Works on any legacy CPU as well.
 *
 * If 'feature_name' is set then put a human-readable description of
 * the feature there as well - this can be used to print error (or success)
 * messages.
 */
extern int cpu_has_xfeatures(u64 xfeatures_mask, const char **feature_name);

/* Trap handling */
extern int  fpu__exception_code(struct fpu *fpu, int trap_nr);
extern void fpu_sync_fpstate(struct fpu *fpu);
extern void fpu_reset_from_exception_fixup(void);

/* Boot, hotplug and resume */
extern void fpu__init_cpu(void);
extern void fpu__init_system(void);
extern void fpu__init_check_bugs(void);
extern void fpu__resume_cpu(void);

#ifdef CONFIG_MATH_EMULATION
extern void fpstate_init_soft(struct swregs_state *soft);
#else
static inline void fpstate_init_soft(struct swregs_state *soft) {}
#endif

/* State tracking */
DECLARE_PER_CPU(struct fpu *, fpu_fpregs_owner_ctx);

/* Process cleanup */
#ifdef CONFIG_X86_64
extern void fpstate_free(struct fpu *fpu);
#else
static inline void fpstate_free(struct fpu *fpu) { }
#endif

/* fpstate-related functions which are exported to KVM */
extern void fpstate_clear_xstate_component(struct fpstate *fps, unsigned int xfeature);

extern u64 xstate_get_guest_group_perm(void);

extern void *get_xsave_addr(struct xregs_state *xsave, int xfeature_nr);


/* KVM specific functions */
extern bool fpu_alloc_guest_fpstate(struct fpu_guest *gfpu);
extern void fpu_free_guest_fpstate(struct fpu_guest *gfpu);
extern int fpu_swap_kvm_fpstate(struct fpu_guest *gfpu, bool enter_guest);
extern int fpu_enable_guest_xfd_features(struct fpu_guest *guest_fpu, u64 xfeatures);

#ifdef CONFIG_X86_64
extern void fpu_update_guest_xfd(struct fpu_guest *guest_fpu, u64 xfd);
extern void fpu_sync_guest_vmexit_xfd_state(void);
#else
static inline void fpu_update_guest_xfd(struct fpu_guest *guest_fpu, u64 xfd) { }
static inline void fpu_sync_guest_vmexit_xfd_state(void) { }
#endif

extern void fpu_copy_guest_fpstate_to_uabi(struct fpu_guest *gfpu, void *buf,
                                           unsigned int size, u64 xfeatures, u32 pkru);
extern int fpu_copy_uabi_to_guest_fpstate(struct fpu_guest *gfpu, const void *buf, u64 xcr0, u32 *vpkru);

static inline void fpstate_set_confidential(struct fpu_guest *gfpu)
{
        gfpu->fpstate->is_confidential = true;
}

static inline bool fpstate_is_confidential(struct fpu_guest *gfpu)
{
        return gfpu->fpstate->is_confidential;
}

/* prctl */
extern long fpu_xstate_prctl(int option, unsigned long arg2);

extern void fpu_idle_fpregs(void);

#endif /* _ASM_X86_FPU_API_H */
































































  158 


    1 














































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
/* SPDX-License-Identifier: GPL-2.0-or-later */
/* SCTP kernel implementation
 * (C) Copyright 2007 Hewlett-Packard Development Company, L.P.
 *
 * This file is part of the SCTP kernel implementation
 *
 * Please send any bug reports or fixes you make to the
 * email address(es):
 *    lksctp developers <linux-sctp@vger.kernel.org>
 *
 * Written or modified by:
 *   Vlad Yasevich     <vladislav.yasevich@hp.com>
 */

#ifndef __sctp_auth_h__
#define __sctp_auth_h__

#include <linux/list.h>
#include <linux/refcount.h>

struct sctp_endpoint;
struct sctp_association;
struct sctp_authkey;
struct sctp_hmacalgo;
struct crypto_shash;

/*
 * Define a generic struct that will hold all the info
 * necessary for an HMAC transform
 */
struct sctp_hmac {
        __u16 hmac_id;                /* one of the above ids */
        char *hmac_name;        /* name for loading */
        __u16 hmac_len;                /* length of the signature */
};

/* This is generic structure that containst authentication bytes used
 * as keying material.  It's a what is referred to as byte-vector all
 * over SCTP-AUTH
 */
struct sctp_auth_bytes {
        refcount_t refcnt;
        __u32 len;
        __u8  data[];
};

/* Definition for a shared key, weather endpoint or association */
struct sctp_shared_key {
        struct list_head key_list;
        struct sctp_auth_bytes *key;
        refcount_t refcnt;
        __u16 key_id;
        __u8 deactivated;
};

#define key_for_each(__key, __list_head) \
        list_for_each_entry(__key, __list_head, key_list)

#define key_for_each_safe(__key, __tmp, __list_head) \
        list_for_each_entry_safe(__key, __tmp, __list_head, key_list)

static inline void sctp_auth_key_hold(struct sctp_auth_bytes *key)
{
        if (!key)
                return;

        refcount_inc(&key->refcnt);
}

void sctp_auth_key_put(struct sctp_auth_bytes *key);
struct sctp_shared_key *sctp_auth_shkey_create(__u16 key_id, gfp_t gfp);
void sctp_auth_destroy_keys(struct list_head *keys);
int sctp_auth_asoc_init_active_key(struct sctp_association *asoc, gfp_t gfp);
struct sctp_shared_key *sctp_auth_get_shkey(
                                const struct sctp_association *asoc,
                                __u16 key_id);
int sctp_auth_asoc_copy_shkeys(const struct sctp_endpoint *ep,
                                struct sctp_association *asoc,
                                gfp_t gfp);
int sctp_auth_init_hmacs(struct sctp_endpoint *ep, gfp_t gfp);
void sctp_auth_destroy_hmacs(struct crypto_shash *auth_hmacs[]);
struct sctp_hmac *sctp_auth_get_hmac(__u16 hmac_id);
struct sctp_hmac *sctp_auth_asoc_get_hmac(const struct sctp_association *asoc);
void sctp_auth_asoc_set_default_hmac(struct sctp_association *asoc,
                                     struct sctp_hmac_algo_param *hmacs);
int sctp_auth_asoc_verify_hmac_id(const struct sctp_association *asoc,
                                    __be16 hmac_id);
int sctp_auth_send_cid(enum sctp_cid chunk,
                       const struct sctp_association *asoc);
int sctp_auth_recv_cid(enum sctp_cid chunk,
                       const struct sctp_association *asoc);
void sctp_auth_calculate_hmac(const struct sctp_association *asoc,
                              struct sk_buff *skb, struct sctp_auth_chunk *auth,
                              struct sctp_shared_key *ep_key, gfp_t gfp);
void sctp_auth_shkey_release(struct sctp_shared_key *sh_key);
void sctp_auth_shkey_hold(struct sctp_shared_key *sh_key);

/* API Helpers */
int sctp_auth_ep_add_chunkid(struct sctp_endpoint *ep, __u8 chunk_id);
int sctp_auth_ep_set_hmacs(struct sctp_endpoint *ep,
                            struct sctp_hmacalgo *hmacs);
int sctp_auth_set_key(struct sctp_endpoint *ep, struct sctp_association *asoc,
                      struct sctp_authkey *auth_key);
int sctp_auth_set_active_key(struct sctp_endpoint *ep,
                             struct sctp_association *asoc, __u16 key_id);
int sctp_auth_del_key_id(struct sctp_endpoint *ep,
                         struct sctp_association *asoc, __u16 key_id);
int sctp_auth_deact_key_id(struct sctp_endpoint *ep,
                           struct sctp_association *asoc, __u16 key_id);
int sctp_auth_init(struct sctp_endpoint *ep, gfp_t gfp);
void sctp_auth_free(struct sctp_endpoint *ep);

#endif



















































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _NET_ESP_H
#define _NET_ESP_H

#include <linux/skbuff.h>

struct ip_esp_hdr;
struct xfrm_state;

static inline struct ip_esp_hdr *ip_esp_hdr(const struct sk_buff *skb)
{
        return (struct ip_esp_hdr *)skb_transport_header(skb);
}

static inline void esp_output_fill_trailer(u8 *tail, int tfclen, int plen, __u8 proto)
{
        /* Fill padding... */
        if (tfclen) {
                memset(tail, 0, tfclen);
                tail += tfclen;
        }
        do {
                int i;
                for (i = 0; i < plen - 2; i++)
                        tail[i] = i + 1;
        } while (0);
        tail[plen - 2] = plen - 2;
        tail[plen - 1] = proto;
}

struct esp_info {
        struct        ip_esp_hdr *esph;
        __be64        seqno;
        int        tfclen;
        int        tailen;
        int        plen;
        int        clen;
        int         len;
        int         nfrags;
        __u8        proto;
        bool        inplace;
};

int esp_output_head(struct xfrm_state *x, struct sk_buff *skb, struct esp_info *esp);
int esp_output_tail(struct xfrm_state *x, struct sk_buff *skb, struct esp_info *esp);
int esp_input_done2(struct sk_buff *skb, int err);
int esp6_output_head(struct xfrm_state *x, struct sk_buff *skb, struct esp_info *esp);
int esp6_output_tail(struct xfrm_state *x, struct sk_buff *skb, struct esp_info *esp);
int esp6_input_done2(struct sk_buff *skb, int err);
#endif



































































































    4 





    4 












    2 


    5 
    3 













    1 




    1 



















    5 








    1 



    1 


    3 



























1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 *        X.25 Packet Layer release 002
 *
 *        This is ALPHA test software. This code may break your machine,
 *        randomly fail to work with new releases, misbehave and/or generally
 *        screw up. It might even work.
 *
 *        This code REQUIRES 2.1.15 or higher
 *
 *        History
 *        X.25 001        Jonathan Naylor        Started coding.
 */

#include <linux/if_arp.h>
#include <linux/init.h>
#include <linux/slab.h>
#include <net/x25.h>

LIST_HEAD(x25_route_list);
DEFINE_RWLOCK(x25_route_list_lock);

/*
 *        Add a new route.
 */
static int x25_add_route(struct x25_address *address, unsigned int sigdigits,
                         struct net_device *dev)
{
        struct x25_route *rt;
        int rc = -EINVAL;

        write_lock_bh(&x25_route_list_lock);

        list_for_each_entry(rt, &x25_route_list, node) {
                if (!memcmp(&rt->address, address, sigdigits) &&
                    rt->sigdigits == sigdigits)
                        goto out;
        }

        rt = kmalloc(sizeof(*rt), GFP_ATOMIC);
        rc = -ENOMEM;
        if (!rt)
                goto out;

        strcpy(rt->address.x25_addr, "000000000000000");
        memcpy(rt->address.x25_addr, address->x25_addr, sigdigits);

        rt->sigdigits = sigdigits;
        rt->dev       = dev;
        refcount_set(&rt->refcnt, 1);

        list_add(&rt->node, &x25_route_list);
        rc = 0;
out:
        write_unlock_bh(&x25_route_list_lock);
        return rc;
}

/**
 * __x25_remove_route - remove route from x25_route_list
 * @rt: route to remove
 *
 * Remove route from x25_route_list. If it was there.
 * Caller must hold x25_route_list_lock.
 */
static void __x25_remove_route(struct x25_route *rt)
{
        if (rt->node.next) {
                list_del(&rt->node);
                x25_route_put(rt);
        }
}

static int x25_del_route(struct x25_address *address, unsigned int sigdigits,
                         struct net_device *dev)
{
        struct x25_route *rt;
        int rc = -EINVAL;

        write_lock_bh(&x25_route_list_lock);

        list_for_each_entry(rt, &x25_route_list, node) {
                if (!memcmp(&rt->address, address, sigdigits) &&
                    rt->sigdigits == sigdigits && rt->dev == dev) {
                        __x25_remove_route(rt);
                        rc = 0;
                        break;
                }
        }

        write_unlock_bh(&x25_route_list_lock);
        return rc;
}

/*
 *        A device has been removed, remove its routes.
 */
void x25_route_device_down(struct net_device *dev)
{
        struct x25_route *rt;
        struct list_head *entry, *tmp;

        write_lock_bh(&x25_route_list_lock);

        list_for_each_safe(entry, tmp, &x25_route_list) {
                rt = list_entry(entry, struct x25_route, node);

                if (rt->dev == dev)
                        __x25_remove_route(rt);
        }
        write_unlock_bh(&x25_route_list_lock);
}

/*
 *        Check that the device given is a valid X.25 interface that is "up".
 */
struct net_device *x25_dev_get(char *devname)
{
        struct net_device *dev = dev_get_by_name(&init_net, devname);

        if (dev && (!(dev->flags & IFF_UP) || dev->type != ARPHRD_X25)) {
                dev_put(dev);
                dev = NULL;
        }

        return dev;
}

/**
 *         x25_get_route -        Find a route given an X.25 address.
 *        @addr: - address to find a route for
 *
 *         Find a route given an X.25 address.
 */
struct x25_route *x25_get_route(struct x25_address *addr)
{
        struct x25_route *rt, *use = NULL;

        read_lock_bh(&x25_route_list_lock);

        list_for_each_entry(rt, &x25_route_list, node) {
                if (!memcmp(&rt->address, addr, rt->sigdigits)) {
                        if (!use)
                                use = rt;
                        else if (rt->sigdigits > use->sigdigits)
                                use = rt;
                }
        }

        if (use)
                x25_route_hold(use);

        read_unlock_bh(&x25_route_list_lock);
        return use;
}

/*
 *        Handle the ioctls that control the routing functions.
 */
int x25_route_ioctl(unsigned int cmd, void __user *arg)
{
        struct x25_route_struct rt;
        struct net_device *dev;
        int rc = -EINVAL;

        if (cmd != SIOCADDRT && cmd != SIOCDELRT)
                goto out;

        rc = -EFAULT;
        if (copy_from_user(&rt, arg, sizeof(rt)))
                goto out;

        rc = -EINVAL;
        if (rt.sigdigits > 15)
                goto out;

        dev = x25_dev_get(rt.device);
        if (!dev)
                goto out;

        if (cmd == SIOCADDRT)
                rc = x25_add_route(&rt.address, rt.sigdigits, dev);
        else
                rc = x25_del_route(&rt.address, rt.sigdigits, dev);
        dev_put(dev);
out:
        return rc;
}

/*
 *        Release all memory associated with X.25 routing structures.
 */
void __exit x25_route_free(void)
{
        struct x25_route *rt;
        struct list_head *entry, *tmp;

        write_lock_bh(&x25_route_list_lock);
        list_for_each_safe(entry, tmp, &x25_route_list) {
                rt = list_entry(entry, struct x25_route, node);
                __x25_remove_route(rt);
        }
        write_unlock_bh(&x25_route_list_lock);
}


































































































































































































































































































































































































































































































































































































    1 




























































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
// SPDX-License-Identifier: GPL-2.0-only
/*
 * NFC Digital Protocol stack
 * Copyright (c) 2013, Intel Corporation.
 */

#define pr_fmt(fmt) "digital: %s: " fmt, __func__

#include <linux/module.h>

#include "digital.h"

#define DIGITAL_PROTO_NFCA_RF_TECH \
        (NFC_PROTO_JEWEL_MASK | NFC_PROTO_MIFARE_MASK | \
        NFC_PROTO_NFC_DEP_MASK | NFC_PROTO_ISO14443_MASK)

#define DIGITAL_PROTO_NFCB_RF_TECH        NFC_PROTO_ISO14443_B_MASK

#define DIGITAL_PROTO_NFCF_RF_TECH \
        (NFC_PROTO_FELICA_MASK | NFC_PROTO_NFC_DEP_MASK)

#define DIGITAL_PROTO_ISO15693_RF_TECH        NFC_PROTO_ISO15693_MASK

/* Delay between each poll frame (ms) */
#define DIGITAL_POLL_INTERVAL 10

struct digital_cmd {
        struct list_head queue;

        u8 type;
        u8 pending;

        u16 timeout;
        struct sk_buff *req;
        struct sk_buff *resp;
        struct digital_tg_mdaa_params *mdaa_params;

        nfc_digital_cmd_complete_t cmd_cb;
        void *cb_context;
};

struct sk_buff *digital_skb_alloc(struct nfc_digital_dev *ddev,
                                  unsigned int len)
{
        struct sk_buff *skb;

        skb = alloc_skb(len + ddev->tx_headroom + ddev->tx_tailroom,
                        GFP_KERNEL);
        if (skb)
                skb_reserve(skb, ddev->tx_headroom);

        return skb;
}

void digital_skb_add_crc(struct sk_buff *skb, crc_func_t crc_func, u16 init,
                         u8 bitwise_inv, u8 msb_first)
{
        u16 crc;

        crc = crc_func(init, skb->data, skb->len);

        if (bitwise_inv)
                crc = ~crc;

        if (msb_first)
                crc = __fswab16(crc);

        skb_put_u8(skb, crc & 0xFF);
        skb_put_u8(skb, (crc >> 8) & 0xFF);
}

int digital_skb_check_crc(struct sk_buff *skb, crc_func_t crc_func,
                          u16 crc_init, u8 bitwise_inv, u8 msb_first)
{
        int rc;
        u16 crc;

        if (skb->len <= 2)
                return -EIO;

        crc = crc_func(crc_init, skb->data, skb->len - 2);

        if (bitwise_inv)
                crc = ~crc;

        if (msb_first)
                crc = __swab16(crc);

        rc = (skb->data[skb->len - 2] - (crc & 0xFF)) +
             (skb->data[skb->len - 1] - ((crc >> 8) & 0xFF));

        if (rc)
                return -EIO;

        skb_trim(skb, skb->len - 2);

        return 0;
}

static inline void digital_switch_rf(struct nfc_digital_dev *ddev, bool on)
{
        ddev->ops->switch_rf(ddev, on);
}

static inline void digital_abort_cmd(struct nfc_digital_dev *ddev)
{
        ddev->ops->abort_cmd(ddev);
}

static void digital_wq_cmd_complete(struct work_struct *work)
{
        struct digital_cmd *cmd;
        struct nfc_digital_dev *ddev = container_of(work,
                                                    struct nfc_digital_dev,
                                                    cmd_complete_work);

        mutex_lock(&ddev->cmd_lock);

        cmd = list_first_entry_or_null(&ddev->cmd_queue, struct digital_cmd,
                                       queue);
        if (!cmd) {
                mutex_unlock(&ddev->cmd_lock);
                return;
        }

        list_del(&cmd->queue);

        mutex_unlock(&ddev->cmd_lock);

        if (!IS_ERR(cmd->resp))
                print_hex_dump_debug("DIGITAL RX: ", DUMP_PREFIX_NONE, 16, 1,
                                     cmd->resp->data, cmd->resp->len, false);

        cmd->cmd_cb(ddev, cmd->cb_context, cmd->resp);

        kfree(cmd->mdaa_params);
        kfree(cmd);

        schedule_work(&ddev->cmd_work);
}

static void digital_send_cmd_complete(struct nfc_digital_dev *ddev,
                                      void *arg, struct sk_buff *resp)
{
        struct digital_cmd *cmd = arg;

        cmd->resp = resp;

        schedule_work(&ddev->cmd_complete_work);
}

static void digital_wq_cmd(struct work_struct *work)
{
        int rc;
        struct digital_cmd *cmd;
        struct digital_tg_mdaa_params *params;
        struct nfc_digital_dev *ddev = container_of(work,
                                                    struct nfc_digital_dev,
                                                    cmd_work);

        mutex_lock(&ddev->cmd_lock);

        cmd = list_first_entry_or_null(&ddev->cmd_queue, struct digital_cmd,
                                       queue);
        if (!cmd || cmd->pending) {
                mutex_unlock(&ddev->cmd_lock);
                return;
        }

        cmd->pending = 1;

        mutex_unlock(&ddev->cmd_lock);

        if (cmd->req)
                print_hex_dump_debug("DIGITAL TX: ", DUMP_PREFIX_NONE, 16, 1,
                                     cmd->req->data, cmd->req->len, false);

        switch (cmd->type) {
        case DIGITAL_CMD_IN_SEND:
                rc = ddev->ops->in_send_cmd(ddev, cmd->req, cmd->timeout,
                                            digital_send_cmd_complete, cmd);
                break;

        case DIGITAL_CMD_TG_SEND:
                rc = ddev->ops->tg_send_cmd(ddev, cmd->req, cmd->timeout,
                                            digital_send_cmd_complete, cmd);
                break;

        case DIGITAL_CMD_TG_LISTEN:
                rc = ddev->ops->tg_listen(ddev, cmd->timeout,
                                          digital_send_cmd_complete, cmd);
                break;

        case DIGITAL_CMD_TG_LISTEN_MDAA:
                params = cmd->mdaa_params;

                rc = ddev->ops->tg_listen_mdaa(ddev, params, cmd->timeout,
                                               digital_send_cmd_complete, cmd);
                break;

        case DIGITAL_CMD_TG_LISTEN_MD:
                rc = ddev->ops->tg_listen_md(ddev, cmd->timeout,
                                               digital_send_cmd_complete, cmd);
                break;

        default:
                pr_err("Unknown cmd type %d\n", cmd->type);
                return;
        }

        if (!rc)
                return;

        pr_err("in_send_command returned err %d\n", rc);

        mutex_lock(&ddev->cmd_lock);
        list_del(&cmd->queue);
        mutex_unlock(&ddev->cmd_lock);

        kfree_skb(cmd->req);
        kfree(cmd->mdaa_params);
        kfree(cmd);

        schedule_work(&ddev->cmd_work);
}

int digital_send_cmd(struct nfc_digital_dev *ddev, u8 cmd_type,
                     struct sk_buff *skb, struct digital_tg_mdaa_params *params,
                     u16 timeout, nfc_digital_cmd_complete_t cmd_cb,
                     void *cb_context)
{
        struct digital_cmd *cmd;

        cmd = kzalloc(sizeof(*cmd), GFP_KERNEL);
        if (!cmd)
                return -ENOMEM;

        cmd->type = cmd_type;
        cmd->timeout = timeout;
        cmd->req = skb;
        cmd->mdaa_params = params;
        cmd->cmd_cb = cmd_cb;
        cmd->cb_context = cb_context;
        INIT_LIST_HEAD(&cmd->queue);

        mutex_lock(&ddev->cmd_lock);
        list_add_tail(&cmd->queue, &ddev->cmd_queue);
        mutex_unlock(&ddev->cmd_lock);

        schedule_work(&ddev->cmd_work);

        return 0;
}

int digital_in_configure_hw(struct nfc_digital_dev *ddev, int type, int param)
{
        int rc;

        rc = ddev->ops->in_configure_hw(ddev, type, param);
        if (rc)
                pr_err("in_configure_hw failed: %d\n", rc);

        return rc;
}

int digital_tg_configure_hw(struct nfc_digital_dev *ddev, int type, int param)
{
        int rc;

        rc = ddev->ops->tg_configure_hw(ddev, type, param);
        if (rc)
                pr_err("tg_configure_hw failed: %d\n", rc);

        return rc;
}

static int digital_tg_listen_mdaa(struct nfc_digital_dev *ddev, u8 rf_tech)
{
        struct digital_tg_mdaa_params *params;
        int rc;

        params = kzalloc(sizeof(*params), GFP_KERNEL);
        if (!params)
                return -ENOMEM;

        params->sens_res = DIGITAL_SENS_RES_NFC_DEP;
        get_random_bytes(params->nfcid1, sizeof(params->nfcid1));
        params->sel_res = DIGITAL_SEL_RES_NFC_DEP;

        params->nfcid2[0] = DIGITAL_SENSF_NFCID2_NFC_DEP_B1;
        params->nfcid2[1] = DIGITAL_SENSF_NFCID2_NFC_DEP_B2;
        get_random_bytes(params->nfcid2 + 2, NFC_NFCID2_MAXSIZE - 2);
        params->sc = DIGITAL_SENSF_FELICA_SC;

        rc = digital_send_cmd(ddev, DIGITAL_CMD_TG_LISTEN_MDAA, NULL, params,
                              500, digital_tg_recv_atr_req, NULL);
        if (rc)
                kfree(params);

        return rc;
}

static int digital_tg_listen_md(struct nfc_digital_dev *ddev, u8 rf_tech)
{
        return digital_send_cmd(ddev, DIGITAL_CMD_TG_LISTEN_MD, NULL, NULL, 500,
                                digital_tg_recv_md_req, NULL);
}

int digital_target_found(struct nfc_digital_dev *ddev,
                         struct nfc_target *target, u8 protocol)
{
        int rc;
        u8 framing;
        u8 rf_tech;
        u8 poll_tech_count;
        int (*check_crc)(struct sk_buff *skb);
        void (*add_crc)(struct sk_buff *skb);

        rf_tech = ddev->poll_techs[ddev->poll_tech_index].rf_tech;

        switch (protocol) {
        case NFC_PROTO_JEWEL:
                framing = NFC_DIGITAL_FRAMING_NFCA_T1T;
                check_crc = digital_skb_check_crc_b;
                add_crc = digital_skb_add_crc_b;
                break;

        case NFC_PROTO_MIFARE:
                framing = NFC_DIGITAL_FRAMING_NFCA_T2T;
                check_crc = digital_skb_check_crc_a;
                add_crc = digital_skb_add_crc_a;
                break;

        case NFC_PROTO_FELICA:
                framing = NFC_DIGITAL_FRAMING_NFCF_T3T;
                check_crc = digital_skb_check_crc_f;
                add_crc = digital_skb_add_crc_f;
                break;

        case NFC_PROTO_NFC_DEP:
                if (rf_tech == NFC_DIGITAL_RF_TECH_106A) {
                        framing = NFC_DIGITAL_FRAMING_NFCA_NFC_DEP;
                        check_crc = digital_skb_check_crc_a;
                        add_crc = digital_skb_add_crc_a;
                } else {
                        framing = NFC_DIGITAL_FRAMING_NFCF_NFC_DEP;
                        check_crc = digital_skb_check_crc_f;
                        add_crc = digital_skb_add_crc_f;
                }
                break;

        case NFC_PROTO_ISO15693:
                framing = NFC_DIGITAL_FRAMING_ISO15693_T5T;
                check_crc = digital_skb_check_crc_b;
                add_crc = digital_skb_add_crc_b;
                break;

        case NFC_PROTO_ISO14443:
                framing = NFC_DIGITAL_FRAMING_NFCA_T4T;
                check_crc = digital_skb_check_crc_a;
                add_crc = digital_skb_add_crc_a;
                break;

        case NFC_PROTO_ISO14443_B:
                framing = NFC_DIGITAL_FRAMING_NFCB_T4T;
                check_crc = digital_skb_check_crc_b;
                add_crc = digital_skb_add_crc_b;
                break;

        default:
                pr_err("Invalid protocol %d\n", protocol);
                return -EINVAL;
        }

        pr_debug("rf_tech=%d, protocol=%d\n", rf_tech, protocol);

        ddev->curr_rf_tech = rf_tech;

        if (DIGITAL_DRV_CAPS_IN_CRC(ddev)) {
                ddev->skb_add_crc = digital_skb_add_crc_none;
                ddev->skb_check_crc = digital_skb_check_crc_none;
        } else {
                ddev->skb_add_crc = add_crc;
                ddev->skb_check_crc = check_crc;
        }

        rc = digital_in_configure_hw(ddev, NFC_DIGITAL_CONFIG_FRAMING, framing);
        if (rc)
                return rc;

        target->supported_protocols = (1 << protocol);

        poll_tech_count = ddev->poll_tech_count;
        ddev->poll_tech_count = 0;

        rc = nfc_targets_found(ddev->nfc_dev, target, 1);
        if (rc) {
                ddev->poll_tech_count = poll_tech_count;
                return rc;
        }

        return 0;
}

void digital_poll_next_tech(struct nfc_digital_dev *ddev)
{
        u8 rand_mod;

        digital_switch_rf(ddev, 0);

        mutex_lock(&ddev->poll_lock);

        if (!ddev->poll_tech_count) {
                mutex_unlock(&ddev->poll_lock);
                return;
        }

        get_random_bytes(&rand_mod, sizeof(rand_mod));
        ddev->poll_tech_index = rand_mod % ddev->poll_tech_count;

        mutex_unlock(&ddev->poll_lock);

        schedule_delayed_work(&ddev->poll_work,
                              msecs_to_jiffies(DIGITAL_POLL_INTERVAL));
}

static void digital_wq_poll(struct work_struct *work)
{
        int rc;
        struct digital_poll_tech *poll_tech;
        struct nfc_digital_dev *ddev = container_of(work,
                                                    struct nfc_digital_dev,
                                                    poll_work.work);
        mutex_lock(&ddev->poll_lock);

        if (!ddev->poll_tech_count) {
                mutex_unlock(&ddev->poll_lock);
                return;
        }

        poll_tech = &ddev->poll_techs[ddev->poll_tech_index];

        mutex_unlock(&ddev->poll_lock);

        rc = poll_tech->poll_func(ddev, poll_tech->rf_tech);
        if (rc)
                digital_poll_next_tech(ddev);
}

static void digital_add_poll_tech(struct nfc_digital_dev *ddev, u8 rf_tech,
                                  digital_poll_t poll_func)
{
        struct digital_poll_tech *poll_tech;

        if (ddev->poll_tech_count >= NFC_DIGITAL_POLL_MODE_COUNT_MAX)
                return;

        poll_tech = &ddev->poll_techs[ddev->poll_tech_count++];

        poll_tech->rf_tech = rf_tech;
        poll_tech->poll_func = poll_func;
}

/**
 * digital_start_poll - start_poll operation
 * @nfc_dev: device to be polled
 * @im_protocols: bitset of nfc initiator protocols to be used for polling
 * @tm_protocols: bitset of nfc transport protocols to be used for polling
 *
 * For every supported protocol, the corresponding polling function is added
 * to the table of polling technologies (ddev->poll_techs[]) using
 * digital_add_poll_tech().
 * When a polling function fails (by timeout or protocol error) the next one is
 * schedule by digital_poll_next_tech() on the poll workqueue (ddev->poll_work).
 */
static int digital_start_poll(struct nfc_dev *nfc_dev, __u32 im_protocols,
                              __u32 tm_protocols)
{
        struct nfc_digital_dev *ddev = nfc_get_drvdata(nfc_dev);
        u32 matching_im_protocols, matching_tm_protocols;

        pr_debug("protocols: im 0x%x, tm 0x%x, supported 0x%x\n", im_protocols,
                 tm_protocols, ddev->protocols);

        matching_im_protocols = ddev->protocols & im_protocols;
        matching_tm_protocols = ddev->protocols & tm_protocols;

        if (!matching_im_protocols && !matching_tm_protocols) {
                pr_err("Unknown protocol\n");
                return -EINVAL;
        }

        if (ddev->poll_tech_count) {
                pr_err("Already polling\n");
                return -EBUSY;
        }

        if (ddev->curr_protocol) {
                pr_err("A target is already active\n");
                return -EBUSY;
        }

        ddev->poll_tech_count = 0;
        ddev->poll_tech_index = 0;

        if (matching_im_protocols & DIGITAL_PROTO_NFCA_RF_TECH)
                digital_add_poll_tech(ddev, NFC_DIGITAL_RF_TECH_106A,
                                      digital_in_send_sens_req);

        if (matching_im_protocols & DIGITAL_PROTO_NFCB_RF_TECH)
                digital_add_poll_tech(ddev, NFC_DIGITAL_RF_TECH_106B,
                                      digital_in_send_sensb_req);

        if (matching_im_protocols & DIGITAL_PROTO_NFCF_RF_TECH) {
                digital_add_poll_tech(ddev, NFC_DIGITAL_RF_TECH_212F,
                                      digital_in_send_sensf_req);

                digital_add_poll_tech(ddev, NFC_DIGITAL_RF_TECH_424F,
                                      digital_in_send_sensf_req);
        }

        if (matching_im_protocols & DIGITAL_PROTO_ISO15693_RF_TECH)
                digital_add_poll_tech(ddev, NFC_DIGITAL_RF_TECH_ISO15693,
                                      digital_in_send_iso15693_inv_req);

        if (matching_tm_protocols & NFC_PROTO_NFC_DEP_MASK) {
                if (ddev->ops->tg_listen_mdaa) {
                        digital_add_poll_tech(ddev, 0,
                                              digital_tg_listen_mdaa);
                } else if (ddev->ops->tg_listen_md) {
                        digital_add_poll_tech(ddev, 0,
                                              digital_tg_listen_md);
                } else {
                        digital_add_poll_tech(ddev, NFC_DIGITAL_RF_TECH_106A,
                                              digital_tg_listen_nfca);

                        digital_add_poll_tech(ddev, NFC_DIGITAL_RF_TECH_212F,
                                              digital_tg_listen_nfcf);

                        digital_add_poll_tech(ddev, NFC_DIGITAL_RF_TECH_424F,
                                              digital_tg_listen_nfcf);
                }
        }

        if (!ddev->poll_tech_count) {
                pr_err("Unsupported protocols: im=0x%x, tm=0x%x\n",
                       matching_im_protocols, matching_tm_protocols);
                return -EINVAL;
        }

        schedule_delayed_work(&ddev->poll_work, 0);

        return 0;
}

static void digital_stop_poll(struct nfc_dev *nfc_dev)
{
        struct nfc_digital_dev *ddev = nfc_get_drvdata(nfc_dev);

        mutex_lock(&ddev->poll_lock);

        if (!ddev->poll_tech_count) {
                pr_err("Polling operation was not running\n");
                mutex_unlock(&ddev->poll_lock);
                return;
        }

        ddev->poll_tech_count = 0;

        mutex_unlock(&ddev->poll_lock);

        cancel_delayed_work_sync(&ddev->poll_work);

        digital_abort_cmd(ddev);
}

static int digital_dev_up(struct nfc_dev *nfc_dev)
{
        struct nfc_digital_dev *ddev = nfc_get_drvdata(nfc_dev);

        digital_switch_rf(ddev, 1);

        return 0;
}

static int digital_dev_down(struct nfc_dev *nfc_dev)
{
        struct nfc_digital_dev *ddev = nfc_get_drvdata(nfc_dev);

        digital_switch_rf(ddev, 0);

        return 0;
}

static int digital_dep_link_up(struct nfc_dev *nfc_dev,
                               struct nfc_target *target,
                               __u8 comm_mode, __u8 *gb, size_t gb_len)
{
        struct nfc_digital_dev *ddev = nfc_get_drvdata(nfc_dev);
        int rc;

        rc = digital_in_send_atr_req(ddev, target, comm_mode, gb, gb_len);

        if (!rc)
                ddev->curr_protocol = NFC_PROTO_NFC_DEP;

        return rc;
}

static int digital_dep_link_down(struct nfc_dev *nfc_dev)
{
        struct nfc_digital_dev *ddev = nfc_get_drvdata(nfc_dev);

        digital_abort_cmd(ddev);

        ddev->curr_protocol = 0;

        return 0;
}

static int digital_activate_target(struct nfc_dev *nfc_dev,
                                   struct nfc_target *target, __u32 protocol)
{
        struct nfc_digital_dev *ddev = nfc_get_drvdata(nfc_dev);

        if (ddev->poll_tech_count) {
                pr_err("Can't activate a target while polling\n");
                return -EBUSY;
        }

        if (ddev->curr_protocol) {
                pr_err("A target is already active\n");
                return -EBUSY;
        }

        ddev->curr_protocol = protocol;

        return 0;
}

static void digital_deactivate_target(struct nfc_dev *nfc_dev,
                                      struct nfc_target *target,
                                      u8 mode)
{
        struct nfc_digital_dev *ddev = nfc_get_drvdata(nfc_dev);

        if (!ddev->curr_protocol) {
                pr_err("No active target\n");
                return;
        }

        digital_abort_cmd(ddev);
        ddev->curr_protocol = 0;
}

static int digital_tg_send(struct nfc_dev *dev, struct sk_buff *skb)
{
        struct nfc_digital_dev *ddev = nfc_get_drvdata(dev);

        return digital_tg_send_dep_res(ddev, skb);
}

static void digital_in_send_complete(struct nfc_digital_dev *ddev, void *arg,
                                     struct sk_buff *resp)
{
        struct digital_data_exch *data_exch = arg;
        int rc;

        if (IS_ERR(resp)) {
                rc = PTR_ERR(resp);
                resp = NULL;
                goto done;
        }

        if (ddev->curr_protocol == NFC_PROTO_MIFARE) {
                rc = digital_in_recv_mifare_res(resp);
                /* crc check is done in digital_in_recv_mifare_res() */
                goto done;
        }

        if ((ddev->curr_protocol == NFC_PROTO_ISO14443) ||
            (ddev->curr_protocol == NFC_PROTO_ISO14443_B)) {
                rc = digital_in_iso_dep_pull_sod(ddev, resp);
                if (rc)
                        goto done;
        }

        rc = ddev->skb_check_crc(resp);

done:
        if (rc) {
                kfree_skb(resp);
                resp = NULL;
        }

        data_exch->cb(data_exch->cb_context, resp, rc);

        kfree(data_exch);
}

static int digital_in_send(struct nfc_dev *nfc_dev, struct nfc_target *target,
                           struct sk_buff *skb, data_exchange_cb_t cb,
                           void *cb_context)
{
        struct nfc_digital_dev *ddev = nfc_get_drvdata(nfc_dev);
        struct digital_data_exch *data_exch;
        int rc;

        data_exch = kzalloc(sizeof(*data_exch), GFP_KERNEL);
        if (!data_exch)
                return -ENOMEM;

        data_exch->cb = cb;
        data_exch->cb_context = cb_context;

        if (ddev->curr_protocol == NFC_PROTO_NFC_DEP) {
                rc = digital_in_send_dep_req(ddev, target, skb, data_exch);
                goto exit;
        }

        if ((ddev->curr_protocol == NFC_PROTO_ISO14443) ||
            (ddev->curr_protocol == NFC_PROTO_ISO14443_B)) {
                rc = digital_in_iso_dep_push_sod(ddev, skb);
                if (rc)
                        goto exit;
        }

        ddev->skb_add_crc(skb);

        rc = digital_in_send_cmd(ddev, skb, 500, digital_in_send_complete,
                                 data_exch);

exit:
        if (rc)
                kfree(data_exch);

        return rc;
}

static const struct nfc_ops digital_nfc_ops = {
        .dev_up = digital_dev_up,
        .dev_down = digital_dev_down,
        .start_poll = digital_start_poll,
        .stop_poll = digital_stop_poll,
        .dep_link_up = digital_dep_link_up,
        .dep_link_down = digital_dep_link_down,
        .activate_target = digital_activate_target,
        .deactivate_target = digital_deactivate_target,
        .tm_send = digital_tg_send,
        .im_transceive = digital_in_send,
};

struct nfc_digital_dev *nfc_digital_allocate_device(const struct nfc_digital_ops *ops,
                                            __u32 supported_protocols,
                                            __u32 driver_capabilities,
                                            int tx_headroom, int tx_tailroom)
{
        struct nfc_digital_dev *ddev;

        if (!ops->in_configure_hw || !ops->in_send_cmd || !ops->tg_listen ||
            !ops->tg_configure_hw || !ops->tg_send_cmd || !ops->abort_cmd ||
            !ops->switch_rf || (ops->tg_listen_md && !ops->tg_get_rf_tech))
                return NULL;

        ddev = kzalloc(sizeof(*ddev), GFP_KERNEL);
        if (!ddev)
                return NULL;

        ddev->driver_capabilities = driver_capabilities;
        ddev->ops = ops;

        mutex_init(&ddev->cmd_lock);
        INIT_LIST_HEAD(&ddev->cmd_queue);

        INIT_WORK(&ddev->cmd_work, digital_wq_cmd);
        INIT_WORK(&ddev->cmd_complete_work, digital_wq_cmd_complete);

        mutex_init(&ddev->poll_lock);
        INIT_DELAYED_WORK(&ddev->poll_work, digital_wq_poll);

        if (supported_protocols & NFC_PROTO_JEWEL_MASK)
                ddev->protocols |= NFC_PROTO_JEWEL_MASK;
        if (supported_protocols & NFC_PROTO_MIFARE_MASK)
                ddev->protocols |= NFC_PROTO_MIFARE_MASK;
        if (supported_protocols & NFC_PROTO_FELICA_MASK)
                ddev->protocols |= NFC_PROTO_FELICA_MASK;
        if (supported_protocols & NFC_PROTO_NFC_DEP_MASK)
                ddev->protocols |= NFC_PROTO_NFC_DEP_MASK;
        if (supported_protocols & NFC_PROTO_ISO15693_MASK)
                ddev->protocols |= NFC_PROTO_ISO15693_MASK;
        if (supported_protocols & NFC_PROTO_ISO14443_MASK)
                ddev->protocols |= NFC_PROTO_ISO14443_MASK;
        if (supported_protocols & NFC_PROTO_ISO14443_B_MASK)
                ddev->protocols |= NFC_PROTO_ISO14443_B_MASK;

        ddev->tx_headroom = tx_headroom + DIGITAL_MAX_HEADER_LEN;
        ddev->tx_tailroom = tx_tailroom + DIGITAL_CRC_LEN;

        ddev->nfc_dev = nfc_allocate_device(&digital_nfc_ops, ddev->protocols,
                                            ddev->tx_headroom,
                                            ddev->tx_tailroom);
        if (!ddev->nfc_dev) {
                pr_err("nfc_allocate_device failed\n");
                goto free_dev;
        }

        nfc_set_drvdata(ddev->nfc_dev, ddev);

        return ddev;

free_dev:
        kfree(ddev);

        return NULL;
}
EXPORT_SYMBOL(nfc_digital_allocate_device);

void nfc_digital_free_device(struct nfc_digital_dev *ddev)
{
        nfc_free_device(ddev->nfc_dev);
        kfree(ddev);
}
EXPORT_SYMBOL(nfc_digital_free_device);

int nfc_digital_register_device(struct nfc_digital_dev *ddev)
{
        return nfc_register_device(ddev->nfc_dev);
}
EXPORT_SYMBOL(nfc_digital_register_device);

void nfc_digital_unregister_device(struct nfc_digital_dev *ddev)
{
        struct digital_cmd *cmd, *n;

        nfc_unregister_device(ddev->nfc_dev);

        mutex_lock(&ddev->poll_lock);
        ddev->poll_tech_count = 0;
        mutex_unlock(&ddev->poll_lock);

        cancel_delayed_work_sync(&ddev->poll_work);
        cancel_work_sync(&ddev->cmd_work);
        cancel_work_sync(&ddev->cmd_complete_work);

        list_for_each_entry_safe(cmd, n, &ddev->cmd_queue, queue) {
                list_del(&cmd->queue);

                /* Call the command callback if any and pass it a ENODEV error.
                 * This gives a chance to the command issuer to free any
                 * allocated buffer.
                 */
                if (cmd->cmd_cb)
                        cmd->cmd_cb(ddev, cmd->cb_context, ERR_PTR(-ENODEV));

                kfree(cmd->mdaa_params);
                kfree(cmd);
        }
}
EXPORT_SYMBOL(nfc_digital_unregister_device);

MODULE_DESCRIPTION("NFC Digital protocol stack");
MODULE_LICENSE("GPL");
























































































































































































































































































































































































 2696 





















 2342 













































  225 
 2489 
 2198 
 2342 


































































































   63 

   62 
   64 

   63 














 2489 




















































































































































 3222 







   62 




























































































 2566 











   30 













    3 

  129 







































  227 






















































































 2636 
   27 
 2646 




 2271 





 2273 
 2281 
 2123 
 2201 
 2268 































 1797 

 1808 



















 1797 
 1796 

 1808 
 1790 




 1795 
 1808 








































































































































































































































































































































































































































































































































  227 





    9 
    9 
    9 












  237 
  229 





















































































































    5 



   43 








































    9 












































































































































  175 





  175 





















































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
/* SPDX-License-Identifier: GPL-2.0-or-later */
/* memcontrol.h - Memory Controller
 *
 * Copyright IBM Corporation, 2007
 * Author Balbir Singh <balbir@linux.vnet.ibm.com>
 *
 * Copyright 2007 OpenVZ SWsoft Inc
 * Author: Pavel Emelianov <xemul@openvz.org>
 */

#ifndef _LINUX_MEMCONTROL_H
#define _LINUX_MEMCONTROL_H
#include <linux/cgroup.h>
#include <linux/vm_event_item.h>
#include <linux/hardirq.h>
#include <linux/jump_label.h>
#include <linux/kernel.h>
#include <linux/page_counter.h>
#include <linux/vmpressure.h>
#include <linux/eventfd.h>
#include <linux/mm.h>
#include <linux/vmstat.h>
#include <linux/writeback.h>
#include <linux/page-flags.h>
#include <linux/shrinker.h>

struct mem_cgroup;
struct obj_cgroup;
struct page;
struct mm_struct;
struct kmem_cache;

/* Cgroup-specific page state, on top of universal node page state */
enum memcg_stat_item {
        MEMCG_SWAP = NR_VM_NODE_STAT_ITEMS,
        MEMCG_SOCK,
        MEMCG_PERCPU_B,
        MEMCG_VMALLOC,
        MEMCG_KMEM,
        MEMCG_ZSWAP_B,
        MEMCG_ZSWAPPED,
        MEMCG_NR_STAT,
};

enum memcg_memory_event {
        MEMCG_LOW,
        MEMCG_HIGH,
        MEMCG_MAX,
        MEMCG_OOM,
        MEMCG_OOM_KILL,
        MEMCG_OOM_GROUP_KILL,
        MEMCG_SWAP_HIGH,
        MEMCG_SWAP_MAX,
        MEMCG_SWAP_FAIL,
        MEMCG_NR_MEMORY_EVENTS,
};

struct mem_cgroup_reclaim_cookie {
        pg_data_t *pgdat;
        int generation;
};

#ifdef CONFIG_MEMCG

#define MEM_CGROUP_ID_SHIFT        16

struct mem_cgroup_id {
        int id;
        refcount_t ref;
};

struct memcg_vmstats_percpu;
struct memcg1_events_percpu;
struct memcg_vmstats;
struct lruvec_stats_percpu;
struct lruvec_stats;

struct mem_cgroup_reclaim_iter {
        struct mem_cgroup *position;
        /* scan generation, increased every round-trip */
        atomic_t generation;
};

/*
 * per-node information in memory controller.
 */
struct mem_cgroup_per_node {
        /* Keep the read-only fields at the start */
        struct mem_cgroup        *memcg;                /* Back pointer, we cannot */
                                                /* use container_of           */

        struct lruvec_stats_percpu __percpu        *lruvec_stats_percpu;
        struct lruvec_stats                        *lruvec_stats;
        struct shrinker_info __rcu        *shrinker_info;

#ifdef CONFIG_MEMCG_V1
        /*
         * Memcg-v1 only stuff in middle as buffer between read mostly fields
         * and update often fields to avoid false sharing. If v1 stuff is
         * not present, an explicit padding is needed.
         */

        struct rb_node                tree_node;        /* RB tree node */
        unsigned long                usage_in_excess;/* Set to the value by which */
                                                /* the soft limit is exceeded*/
        bool                        on_tree;
#else
        CACHELINE_PADDING(_pad1_);
#endif

        /* Fields which get updated often at the end. */
        struct lruvec                lruvec;
        CACHELINE_PADDING(_pad2_);
        unsigned long                lru_zone_size[MAX_NR_ZONES][NR_LRU_LISTS];
        struct mem_cgroup_reclaim_iter        iter;
};

struct mem_cgroup_threshold {
        struct eventfd_ctx *eventfd;
        unsigned long threshold;
};

/* For threshold */
struct mem_cgroup_threshold_ary {
        /* An array index points to threshold just below or equal to usage. */
        int current_threshold;
        /* Size of entries[] */
        unsigned int size;
        /* Array of thresholds */
        struct mem_cgroup_threshold entries[] __counted_by(size);
};

struct mem_cgroup_thresholds {
        /* Primary thresholds array */
        struct mem_cgroup_threshold_ary *primary;
        /*
         * Spare threshold array.
         * This is needed to make mem_cgroup_unregister_event() "never fail".
         * It must be able to store at least primary->size - 1 entries.
         */
        struct mem_cgroup_threshold_ary *spare;
};

/*
 * Remember four most recent foreign writebacks with dirty pages in this
 * cgroup.  Inode sharing is expected to be uncommon and, even if we miss
 * one in a given round, we're likely to catch it later if it keeps
 * foreign-dirtying, so a fairly low count should be enough.
 *
 * See mem_cgroup_track_foreign_dirty_slowpath() for details.
 */
#define MEMCG_CGWB_FRN_CNT        4

struct memcg_cgwb_frn {
        u64 bdi_id;                        /* bdi->id of the foreign inode */
        int memcg_id;                        /* memcg->css.id of foreign inode */
        u64 at;                                /* jiffies_64 at the time of dirtying */
        struct wb_completion done;        /* tracks in-flight foreign writebacks */
};

/*
 * Bucket for arbitrarily byte-sized objects charged to a memory
 * cgroup. The bucket can be reparented in one piece when the cgroup
 * is destroyed, without having to round up the individual references
 * of all live memory objects in the wild.
 */
struct obj_cgroup {
        struct percpu_ref refcnt;
        struct mem_cgroup *memcg;
        atomic_t nr_charged_bytes;
        union {
                struct list_head list; /* protected by objcg_lock */
                struct rcu_head rcu;
        };
};

/*
 * The memory controller data structure. The memory controller controls both
 * page cache and RSS per cgroup. We would eventually like to provide
 * statistics based on the statistics developed by Rik Van Riel for clock-pro,
 * to help the administrator determine what knobs to tune.
 */
struct mem_cgroup {
        struct cgroup_subsys_state css;

        /* Private memcg ID. Used to ID objects that outlive the cgroup */
        struct mem_cgroup_id id;

        /* Accounted resources */
        struct page_counter memory;                /* Both v1 & v2 */

        union {
                struct page_counter swap;        /* v2 only */
                struct page_counter memsw;        /* v1 only */
        };

        /* registered local peak watchers */
        struct list_head memory_peaks;
        struct list_head swap_peaks;
        spinlock_t         peaks_lock;

        /* Range enforcement for interrupt charges */
        struct work_struct high_work;

#ifdef CONFIG_ZSWAP
        unsigned long zswap_max;

        /*
         * Prevent pages from this memcg from being written back from zswap to
         * swap, and from being swapped out on zswap store failures.
         */
        bool zswap_writeback;
#endif

        /* vmpressure notifications */
        struct vmpressure vmpressure;

        /*
         * Should the OOM killer kill all belonging tasks, had it kill one?
         */
        bool oom_group;

        int swappiness;

        /* memory.events and memory.events.local */
        struct cgroup_file events_file;
        struct cgroup_file events_local_file;

        /* handle for "memory.swap.events" */
        struct cgroup_file swap_events_file;

        /* memory.stat */
        struct memcg_vmstats        *vmstats;

        /* memory.events */
        atomic_long_t                memory_events[MEMCG_NR_MEMORY_EVENTS];
        atomic_long_t                memory_events_local[MEMCG_NR_MEMORY_EVENTS];

        /*
         * Hint of reclaim pressure for socket memroy management. Note
         * that this indicator should NOT be used in legacy cgroup mode
         * where socket memory is accounted/charged separately.
         */
        unsigned long                socket_pressure;

        int kmemcg_id;
        /*
         * memcg->objcg is wiped out as a part of the objcg repaprenting
         * process. memcg->orig_objcg preserves a pointer (and a reference)
         * to the original objcg until the end of live of memcg.
         */
        struct obj_cgroup __rcu        *objcg;
        struct obj_cgroup        *orig_objcg;
        /* list of inherited objcgs, protected by objcg_lock */
        struct list_head objcg_list;

        struct memcg_vmstats_percpu __percpu *vmstats_percpu;

#ifdef CONFIG_CGROUP_WRITEBACK
        struct list_head cgwb_list;
        struct wb_domain cgwb_domain;
        struct memcg_cgwb_frn cgwb_frn[MEMCG_CGWB_FRN_CNT];
#endif

#ifdef CONFIG_TRANSPARENT_HUGEPAGE
        struct deferred_split deferred_split_queue;
#endif

#ifdef CONFIG_LRU_GEN_WALKS_MMU
        /* per-memcg mm_struct list */
        struct lru_gen_mm_list mm_list;
#endif

#ifdef CONFIG_MEMCG_V1
        /* Legacy consumer-oriented counters */
        struct page_counter kmem;                /* v1 only */
        struct page_counter tcpmem;                /* v1 only */

        struct memcg1_events_percpu __percpu *events_percpu;

        unsigned long soft_limit;

        /* protected by memcg_oom_lock */
        bool oom_lock;
        int under_oom;

        /* OOM-Killer disable */
        int oom_kill_disable;

        /* protect arrays of thresholds */
        struct mutex thresholds_lock;

        /* thresholds for memory usage. RCU-protected */
        struct mem_cgroup_thresholds thresholds;

        /* thresholds for mem+swap usage. RCU-protected */
        struct mem_cgroup_thresholds memsw_thresholds;

        /* For oom notifier event fd */
        struct list_head oom_notify;

        /*
         * Should we move charges of a task when a task is moved into this
         * mem_cgroup ? And what type of charges should we move ?
         */
        unsigned long move_charge_at_immigrate;
        /* taken only while moving_account > 0 */
        spinlock_t move_lock;
        unsigned long move_lock_flags;

        /* Legacy tcp memory accounting */
        bool tcpmem_active;
        int tcpmem_pressure;

        /*
         * set > 0 if pages under this cgroup are moving to other cgroup.
         */
        atomic_t moving_account;
        struct task_struct *move_lock_task;

        /* List of events which userspace want to receive */
        struct list_head event_list;
        spinlock_t event_list_lock;
#endif /* CONFIG_MEMCG_V1 */

        struct mem_cgroup_per_node *nodeinfo[];
};

/*
 * size of first charge trial.
 * TODO: maybe necessary to use big numbers in big irons or dynamic based of the
 * workload.
 */
#define MEMCG_CHARGE_BATCH 64U

extern struct mem_cgroup *root_mem_cgroup;

enum page_memcg_data_flags {
        /* page->memcg_data is a pointer to an slabobj_ext vector */
        MEMCG_DATA_OBJEXTS = (1UL << 0),
        /* page has been accounted as a non-slab kernel page */
        MEMCG_DATA_KMEM = (1UL << 1),
        /* the next bit after the last actual flag */
        __NR_MEMCG_DATA_FLAGS  = (1UL << 2),
};

#define __FIRST_OBJEXT_FLAG        __NR_MEMCG_DATA_FLAGS

#else /* CONFIG_MEMCG */

#define __FIRST_OBJEXT_FLAG        (1UL << 0)

#endif /* CONFIG_MEMCG */

enum objext_flags {
        /* slabobj_ext vector failed to allocate */
        OBJEXTS_ALLOC_FAIL = __FIRST_OBJEXT_FLAG,
        /* the next bit after the last actual flag */
        __NR_OBJEXTS_FLAGS  = (__FIRST_OBJEXT_FLAG << 1),
};

#define OBJEXTS_FLAGS_MASK (__NR_OBJEXTS_FLAGS - 1)

#ifdef CONFIG_MEMCG

static inline bool folio_memcg_kmem(struct folio *folio);

/*
 * After the initialization objcg->memcg is always pointing at
 * a valid memcg, but can be atomically swapped to the parent memcg.
 *
 * The caller must ensure that the returned memcg won't be released.
 */
static inline struct mem_cgroup *obj_cgroup_memcg(struct obj_cgroup *objcg)
{
        lockdep_assert_once(rcu_read_lock_held() || lockdep_is_held(&cgroup_mutex));
        return READ_ONCE(objcg->memcg);
}

/*
 * __folio_memcg - Get the memory cgroup associated with a non-kmem folio
 * @folio: Pointer to the folio.
 *
 * Returns a pointer to the memory cgroup associated with the folio,
 * or NULL. This function assumes that the folio is known to have a
 * proper memory cgroup pointer. It's not safe to call this function
 * against some type of folios, e.g. slab folios or ex-slab folios or
 * kmem folios.
 */
static inline struct mem_cgroup *__folio_memcg(struct folio *folio)
{
        unsigned long memcg_data = folio->memcg_data;

        VM_BUG_ON_FOLIO(folio_test_slab(folio), folio);
        VM_BUG_ON_FOLIO(memcg_data & MEMCG_DATA_OBJEXTS, folio);
        VM_BUG_ON_FOLIO(memcg_data & MEMCG_DATA_KMEM, folio);

        return (struct mem_cgroup *)(memcg_data & ~OBJEXTS_FLAGS_MASK);
}

/*
 * __folio_objcg - get the object cgroup associated with a kmem folio.
 * @folio: Pointer to the folio.
 *
 * Returns a pointer to the object cgroup associated with the folio,
 * or NULL. This function assumes that the folio is known to have a
 * proper object cgroup pointer. It's not safe to call this function
 * against some type of folios, e.g. slab folios or ex-slab folios or
 * LRU folios.
 */
static inline struct obj_cgroup *__folio_objcg(struct folio *folio)
{
        unsigned long memcg_data = folio->memcg_data;

        VM_BUG_ON_FOLIO(folio_test_slab(folio), folio);
        VM_BUG_ON_FOLIO(memcg_data & MEMCG_DATA_OBJEXTS, folio);
        VM_BUG_ON_FOLIO(!(memcg_data & MEMCG_DATA_KMEM), folio);

        return (struct obj_cgroup *)(memcg_data & ~OBJEXTS_FLAGS_MASK);
}

/*
 * folio_memcg - Get the memory cgroup associated with a folio.
 * @folio: Pointer to the folio.
 *
 * Returns a pointer to the memory cgroup associated with the folio,
 * or NULL. This function assumes that the folio is known to have a
 * proper memory cgroup pointer. It's not safe to call this function
 * against some type of folios, e.g. slab folios or ex-slab folios.
 *
 * For a non-kmem folio any of the following ensures folio and memcg binding
 * stability:
 *
 * - the folio lock
 * - LRU isolation
 * - folio_memcg_lock()
 * - exclusive reference
 * - mem_cgroup_trylock_pages()
 *
 * For a kmem folio a caller should hold an rcu read lock to protect memcg
 * associated with a kmem folio from being released.
 */
static inline struct mem_cgroup *folio_memcg(struct folio *folio)
{
        if (folio_memcg_kmem(folio))
                return obj_cgroup_memcg(__folio_objcg(folio));
        return __folio_memcg(folio);
}

/*
 * folio_memcg_charged - If a folio is charged to a memory cgroup.
 * @folio: Pointer to the folio.
 *
 * Returns true if folio is charged to a memory cgroup, otherwise returns false.
 */
static inline bool folio_memcg_charged(struct folio *folio)
{
        if (folio_memcg_kmem(folio))
                return __folio_objcg(folio) != NULL;
        return __folio_memcg(folio) != NULL;
}

/**
 * folio_memcg_rcu - Locklessly get the memory cgroup associated with a folio.
 * @folio: Pointer to the folio.
 *
 * This function assumes that the folio is known to have a
 * proper memory cgroup pointer. It's not safe to call this function
 * against some type of folios, e.g. slab folios or ex-slab folios.
 *
 * Return: A pointer to the memory cgroup associated with the folio,
 * or NULL.
 */
static inline struct mem_cgroup *folio_memcg_rcu(struct folio *folio)
{
        unsigned long memcg_data = READ_ONCE(folio->memcg_data);

        VM_BUG_ON_FOLIO(folio_test_slab(folio), folio);

        if (memcg_data & MEMCG_DATA_KMEM) {
                struct obj_cgroup *objcg;

                objcg = (void *)(memcg_data & ~OBJEXTS_FLAGS_MASK);
                return obj_cgroup_memcg(objcg);
        }

        WARN_ON_ONCE(!rcu_read_lock_held());

        return (struct mem_cgroup *)(memcg_data & ~OBJEXTS_FLAGS_MASK);
}

/*
 * folio_memcg_check - Get the memory cgroup associated with a folio.
 * @folio: Pointer to the folio.
 *
 * Returns a pointer to the memory cgroup associated with the folio,
 * or NULL. This function unlike folio_memcg() can take any folio
 * as an argument. It has to be used in cases when it's not known if a folio
 * has an associated memory cgroup pointer or an object cgroups vector or
 * an object cgroup.
 *
 * For a non-kmem folio any of the following ensures folio and memcg binding
 * stability:
 *
 * - the folio lock
 * - LRU isolation
 * - lock_folio_memcg()
 * - exclusive reference
 * - mem_cgroup_trylock_pages()
 *
 * For a kmem folio a caller should hold an rcu read lock to protect memcg
 * associated with a kmem folio from being released.
 */
static inline struct mem_cgroup *folio_memcg_check(struct folio *folio)
{
        /*
         * Because folio->memcg_data might be changed asynchronously
         * for slabs, READ_ONCE() should be used here.
         */
        unsigned long memcg_data = READ_ONCE(folio->memcg_data);

        if (memcg_data & MEMCG_DATA_OBJEXTS)
                return NULL;

        if (memcg_data & MEMCG_DATA_KMEM) {
                struct obj_cgroup *objcg;

                objcg = (void *)(memcg_data & ~OBJEXTS_FLAGS_MASK);
                return obj_cgroup_memcg(objcg);
        }

        return (struct mem_cgroup *)(memcg_data & ~OBJEXTS_FLAGS_MASK);
}

static inline struct mem_cgroup *page_memcg_check(struct page *page)
{
        if (PageTail(page))
                return NULL;
        return folio_memcg_check((struct folio *)page);
}

static inline struct mem_cgroup *get_mem_cgroup_from_objcg(struct obj_cgroup *objcg)
{
        struct mem_cgroup *memcg;

        rcu_read_lock();
retry:
        memcg = obj_cgroup_memcg(objcg);
        if (unlikely(!css_tryget(&memcg->css)))
                goto retry;
        rcu_read_unlock();

        return memcg;
}

/*
 * folio_memcg_kmem - Check if the folio has the memcg_kmem flag set.
 * @folio: Pointer to the folio.
 *
 * Checks if the folio has MemcgKmem flag set. The caller must ensure
 * that the folio has an associated memory cgroup. It's not safe to call
 * this function against some types of folios, e.g. slab folios.
 */
static inline bool folio_memcg_kmem(struct folio *folio)
{
        VM_BUG_ON_PGFLAGS(PageTail(&folio->page), &folio->page);
        VM_BUG_ON_FOLIO(folio->memcg_data & MEMCG_DATA_OBJEXTS, folio);
        return folio->memcg_data & MEMCG_DATA_KMEM;
}

static inline bool PageMemcgKmem(struct page *page)
{
        return folio_memcg_kmem(page_folio(page));
}

static inline bool mem_cgroup_is_root(struct mem_cgroup *memcg)
{
        return (memcg == root_mem_cgroup);
}

static inline bool mem_cgroup_disabled(void)
{
        return !cgroup_subsys_enabled(memory_cgrp_subsys);
}

static inline void mem_cgroup_protection(struct mem_cgroup *root,
                                         struct mem_cgroup *memcg,
                                         unsigned long *min,
                                         unsigned long *low)
{
        *min = *low = 0;

        if (mem_cgroup_disabled())
                return;

        /*
         * There is no reclaim protection applied to a targeted reclaim.
         * We are special casing this specific case here because
         * mem_cgroup_calculate_protection is not robust enough to keep
         * the protection invariant for calculated effective values for
         * parallel reclaimers with different reclaim target. This is
         * especially a problem for tail memcgs (as they have pages on LRU)
         * which would want to have effective values 0 for targeted reclaim
         * but a different value for external reclaim.
         *
         * Example
         * Let's have global and A's reclaim in parallel:
         *  |
         *  A (low=2G, usage = 3G, max = 3G, children_low_usage = 1.5G)
         *  |\
         *  | C (low = 1G, usage = 2.5G)
         *  B (low = 1G, usage = 0.5G)
         *
         * For the global reclaim
         * A.elow = A.low
         * B.elow = min(B.usage, B.low) because children_low_usage <= A.elow
         * C.elow = min(C.usage, C.low)
         *
         * With the effective values resetting we have A reclaim
         * A.elow = 0
         * B.elow = B.low
         * C.elow = C.low
         *
         * If the global reclaim races with A's reclaim then
         * B.elow = C.elow = 0 because children_low_usage > A.elow)
         * is possible and reclaiming B would be violating the protection.
         *
         */
        if (root == memcg)
                return;

        *min = READ_ONCE(memcg->memory.emin);
        *low = READ_ONCE(memcg->memory.elow);
}

void mem_cgroup_calculate_protection(struct mem_cgroup *root,
                                     struct mem_cgroup *memcg);

static inline bool mem_cgroup_unprotected(struct mem_cgroup *target,
                                          struct mem_cgroup *memcg)
{
        /*
         * The root memcg doesn't account charges, and doesn't support
         * protection. The target memcg's protection is ignored, see
         * mem_cgroup_calculate_protection() and mem_cgroup_protection()
         */
        return mem_cgroup_disabled() || mem_cgroup_is_root(memcg) ||
                memcg == target;
}

static inline bool mem_cgroup_below_low(struct mem_cgroup *target,
                                        struct mem_cgroup *memcg)
{
        if (mem_cgroup_unprotected(target, memcg))
                return false;

        return READ_ONCE(memcg->memory.elow) >=
                page_counter_read(&memcg->memory);
}

static inline bool mem_cgroup_below_min(struct mem_cgroup *target,
                                        struct mem_cgroup *memcg)
{
        if (mem_cgroup_unprotected(target, memcg))
                return false;

        return READ_ONCE(memcg->memory.emin) >=
                page_counter_read(&memcg->memory);
}

void mem_cgroup_commit_charge(struct folio *folio, struct mem_cgroup *memcg);

int __mem_cgroup_charge(struct folio *folio, struct mm_struct *mm, gfp_t gfp);

/**
 * mem_cgroup_charge - Charge a newly allocated folio to a cgroup.
 * @folio: Folio to charge.
 * @mm: mm context of the allocating task.
 * @gfp: Reclaim mode.
 *
 * Try to charge @folio to the memcg that @mm belongs to, reclaiming
 * pages according to @gfp if necessary.  If @mm is NULL, try to
 * charge to the active memcg.
 *
 * Do not use this for folios allocated for swapin.
 *
 * Return: 0 on success. Otherwise, an error code is returned.
 */
static inline int mem_cgroup_charge(struct folio *folio, struct mm_struct *mm,
                                    gfp_t gfp)
{
        if (mem_cgroup_disabled())
                return 0;
        return __mem_cgroup_charge(folio, mm, gfp);
}

int mem_cgroup_hugetlb_try_charge(struct mem_cgroup *memcg, gfp_t gfp,
                long nr_pages);

int mem_cgroup_swapin_charge_folio(struct folio *folio, struct mm_struct *mm,
                                  gfp_t gfp, swp_entry_t entry);

void mem_cgroup_swapin_uncharge_swap(swp_entry_t entry, unsigned int nr_pages);

void __mem_cgroup_uncharge(struct folio *folio);

/**
 * mem_cgroup_uncharge - Uncharge a folio.
 * @folio: Folio to uncharge.
 *
 * Uncharge a folio previously charged with mem_cgroup_charge().
 */
static inline void mem_cgroup_uncharge(struct folio *folio)
{
        if (mem_cgroup_disabled())
                return;
        __mem_cgroup_uncharge(folio);
}

void __mem_cgroup_uncharge_folios(struct folio_batch *folios);
static inline void mem_cgroup_uncharge_folios(struct folio_batch *folios)
{
        if (mem_cgroup_disabled())
                return;
        __mem_cgroup_uncharge_folios(folios);
}

void mem_cgroup_cancel_charge(struct mem_cgroup *memcg, unsigned int nr_pages);
void mem_cgroup_replace_folio(struct folio *old, struct folio *new);
void mem_cgroup_migrate(struct folio *old, struct folio *new);

/**
 * mem_cgroup_lruvec - get the lru list vector for a memcg & node
 * @memcg: memcg of the wanted lruvec
 * @pgdat: pglist_data
 *
 * Returns the lru list vector holding pages for a given @memcg &
 * @pgdat combination. This can be the node lruvec, if the memory
 * controller is disabled.
 */
static inline struct lruvec *mem_cgroup_lruvec(struct mem_cgroup *memcg,
                                               struct pglist_data *pgdat)
{
        struct mem_cgroup_per_node *mz;
        struct lruvec *lruvec;

        if (mem_cgroup_disabled()) {
                lruvec = &pgdat->__lruvec;
                goto out;
        }

        if (!memcg)
                memcg = root_mem_cgroup;

        mz = memcg->nodeinfo[pgdat->node_id];
        lruvec = &mz->lruvec;
out:
        /*
         * Since a node can be onlined after the mem_cgroup was created,
         * we have to be prepared to initialize lruvec->pgdat here;
         * and if offlined then reonlined, we need to reinitialize it.
         */
        if (unlikely(lruvec->pgdat != pgdat))
                lruvec->pgdat = pgdat;
        return lruvec;
}

/**
 * folio_lruvec - return lruvec for isolating/putting an LRU folio
 * @folio: Pointer to the folio.
 *
 * This function relies on folio->mem_cgroup being stable.
 */
static inline struct lruvec *folio_lruvec(struct folio *folio)
{
        struct mem_cgroup *memcg = folio_memcg(folio);

        VM_WARN_ON_ONCE_FOLIO(!memcg && !mem_cgroup_disabled(), folio);
        return mem_cgroup_lruvec(memcg, folio_pgdat(folio));
}

struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p);

struct mem_cgroup *get_mem_cgroup_from_mm(struct mm_struct *mm);

struct mem_cgroup *get_mem_cgroup_from_current(void);

struct mem_cgroup *get_mem_cgroup_from_folio(struct folio *folio);

struct lruvec *folio_lruvec_lock(struct folio *folio);
struct lruvec *folio_lruvec_lock_irq(struct folio *folio);
struct lruvec *folio_lruvec_lock_irqsave(struct folio *folio,
                                                unsigned long *flags);

#ifdef CONFIG_DEBUG_VM
void lruvec_memcg_debug(struct lruvec *lruvec, struct folio *folio);
#else
static inline
void lruvec_memcg_debug(struct lruvec *lruvec, struct folio *folio)
{
}
#endif

static inline
struct mem_cgroup *mem_cgroup_from_css(struct cgroup_subsys_state *css){
        return css ? container_of(css, struct mem_cgroup, css) : NULL;
}

static inline bool obj_cgroup_tryget(struct obj_cgroup *objcg)
{
        return percpu_ref_tryget(&objcg->refcnt);
}

static inline void obj_cgroup_get(struct obj_cgroup *objcg)
{
        percpu_ref_get(&objcg->refcnt);
}

static inline void obj_cgroup_get_many(struct obj_cgroup *objcg,
                                       unsigned long nr)
{
        percpu_ref_get_many(&objcg->refcnt, nr);
}

static inline void obj_cgroup_put(struct obj_cgroup *objcg)
{
        if (objcg)
                percpu_ref_put(&objcg->refcnt);
}

static inline bool mem_cgroup_tryget(struct mem_cgroup *memcg)
{
        return !memcg || css_tryget(&memcg->css);
}

static inline bool mem_cgroup_tryget_online(struct mem_cgroup *memcg)
{
        return !memcg || css_tryget_online(&memcg->css);
}

static inline void mem_cgroup_put(struct mem_cgroup *memcg)
{
        if (memcg)
                css_put(&memcg->css);
}

#define mem_cgroup_from_counter(counter, member)        \
        container_of(counter, struct mem_cgroup, member)

struct mem_cgroup *mem_cgroup_iter(struct mem_cgroup *,
                                   struct mem_cgroup *,
                                   struct mem_cgroup_reclaim_cookie *);
void mem_cgroup_iter_break(struct mem_cgroup *, struct mem_cgroup *);
void mem_cgroup_scan_tasks(struct mem_cgroup *memcg,
                           int (*)(struct task_struct *, void *), void *arg);

static inline unsigned short mem_cgroup_id(struct mem_cgroup *memcg)
{
        if (mem_cgroup_disabled())
                return 0;

        return memcg->id.id;
}
struct mem_cgroup *mem_cgroup_from_id(unsigned short id);

#ifdef CONFIG_SHRINKER_DEBUG
static inline unsigned long mem_cgroup_ino(struct mem_cgroup *memcg)
{
        return memcg ? cgroup_ino(memcg->css.cgroup) : 0;
}

struct mem_cgroup *mem_cgroup_get_from_ino(unsigned long ino);
#endif

static inline struct mem_cgroup *mem_cgroup_from_seq(struct seq_file *m)
{
        return mem_cgroup_from_css(seq_css(m));
}

static inline struct mem_cgroup *lruvec_memcg(struct lruvec *lruvec)
{
        struct mem_cgroup_per_node *mz;

        if (mem_cgroup_disabled())
                return NULL;

        mz = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
        return mz->memcg;
}

/**
 * parent_mem_cgroup - find the accounting parent of a memcg
 * @memcg: memcg whose parent to find
 *
 * Returns the parent memcg, or NULL if this is the root.
 */
static inline struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *memcg)
{
        return mem_cgroup_from_css(memcg->css.parent);
}

static inline bool mem_cgroup_is_descendant(struct mem_cgroup *memcg,
                              struct mem_cgroup *root)
{
        if (root == memcg)
                return true;
        return cgroup_is_descendant(memcg->css.cgroup, root->css.cgroup);
}

static inline bool mm_match_cgroup(struct mm_struct *mm,
                                   struct mem_cgroup *memcg)
{
        struct mem_cgroup *task_memcg;
        bool match = false;

        rcu_read_lock();
        task_memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));
        if (task_memcg)
                match = mem_cgroup_is_descendant(task_memcg, memcg);
        rcu_read_unlock();
        return match;
}

struct cgroup_subsys_state *mem_cgroup_css_from_folio(struct folio *folio);
ino_t page_cgroup_ino(struct page *page);

static inline bool mem_cgroup_online(struct mem_cgroup *memcg)
{
        if (mem_cgroup_disabled())
                return true;
        return !!(memcg->css.flags & CSS_ONLINE);
}

void mem_cgroup_update_lru_size(struct lruvec *lruvec, enum lru_list lru,
                int zid, int nr_pages);

static inline
unsigned long mem_cgroup_get_zone_lru_size(struct lruvec *lruvec,
                enum lru_list lru, int zone_idx)
{
        struct mem_cgroup_per_node *mz;

        mz = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
        return READ_ONCE(mz->lru_zone_size[zone_idx][lru]);
}

void mem_cgroup_handle_over_high(gfp_t gfp_mask);

unsigned long mem_cgroup_get_max(struct mem_cgroup *memcg);

unsigned long mem_cgroup_size(struct mem_cgroup *memcg);

void mem_cgroup_print_oom_context(struct mem_cgroup *memcg,
                                struct task_struct *p);

void mem_cgroup_print_oom_meminfo(struct mem_cgroup *memcg);

struct mem_cgroup *mem_cgroup_get_oom_group(struct task_struct *victim,
                                            struct mem_cgroup *oom_domain);
void mem_cgroup_print_oom_group(struct mem_cgroup *memcg);

void __mod_memcg_state(struct mem_cgroup *memcg, enum memcg_stat_item idx,
                       int val);

/* idx can be of type enum memcg_stat_item or node_stat_item */
static inline void mod_memcg_state(struct mem_cgroup *memcg,
                                   enum memcg_stat_item idx, int val)
{
        unsigned long flags;

        local_irq_save(flags);
        __mod_memcg_state(memcg, idx, val);
        local_irq_restore(flags);
}

static inline void mod_memcg_page_state(struct page *page,
                                        enum memcg_stat_item idx, int val)
{
        struct mem_cgroup *memcg;

        if (mem_cgroup_disabled())
                return;

        rcu_read_lock();
        memcg = folio_memcg(page_folio(page));
        if (memcg)
                mod_memcg_state(memcg, idx, val);
        rcu_read_unlock();
}

unsigned long memcg_page_state(struct mem_cgroup *memcg, int idx);
unsigned long lruvec_page_state(struct lruvec *lruvec, enum node_stat_item idx);
unsigned long lruvec_page_state_local(struct lruvec *lruvec,
                                      enum node_stat_item idx);

void mem_cgroup_flush_stats(struct mem_cgroup *memcg);
void mem_cgroup_flush_stats_ratelimited(struct mem_cgroup *memcg);

void __mod_lruvec_kmem_state(void *p, enum node_stat_item idx, int val);

static inline void mod_lruvec_kmem_state(void *p, enum node_stat_item idx,
                                         int val)
{
        unsigned long flags;

        local_irq_save(flags);
        __mod_lruvec_kmem_state(p, idx, val);
        local_irq_restore(flags);
}

void __count_memcg_events(struct mem_cgroup *memcg, enum vm_event_item idx,
                          unsigned long count);

static inline void count_memcg_events(struct mem_cgroup *memcg,
                                      enum vm_event_item idx,
                                      unsigned long count)
{
        unsigned long flags;

        local_irq_save(flags);
        __count_memcg_events(memcg, idx, count);
        local_irq_restore(flags);
}

static inline void count_memcg_folio_events(struct folio *folio,
                enum vm_event_item idx, unsigned long nr)
{
        struct mem_cgroup *memcg = folio_memcg(folio);

        if (memcg)
                count_memcg_events(memcg, idx, nr);
}

static inline void count_memcg_events_mm(struct mm_struct *mm,
                                        enum vm_event_item idx, unsigned long count)
{
        struct mem_cgroup *memcg;

        if (mem_cgroup_disabled())
                return;

        rcu_read_lock();
        memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));
        if (likely(memcg))
                count_memcg_events(memcg, idx, count);
        rcu_read_unlock();
}

static inline void count_memcg_event_mm(struct mm_struct *mm,
                                        enum vm_event_item idx)
{
        count_memcg_events_mm(mm, idx, 1);
}

static inline void memcg_memory_event(struct mem_cgroup *memcg,
                                      enum memcg_memory_event event)
{
        bool swap_event = event == MEMCG_SWAP_HIGH || event == MEMCG_SWAP_MAX ||
                          event == MEMCG_SWAP_FAIL;

        atomic_long_inc(&memcg->memory_events_local[event]);
        if (!swap_event)
                cgroup_file_notify(&memcg->events_local_file);

        do {
                atomic_long_inc(&memcg->memory_events[event]);
                if (swap_event)
                        cgroup_file_notify(&memcg->swap_events_file);
                else
                        cgroup_file_notify(&memcg->events_file);

                if (!cgroup_subsys_on_dfl(memory_cgrp_subsys))
                        break;
                if (cgrp_dfl_root.flags & CGRP_ROOT_MEMORY_LOCAL_EVENTS)
                        break;
        } while ((memcg = parent_mem_cgroup(memcg)) &&
                 !mem_cgroup_is_root(memcg));
}

static inline void memcg_memory_event_mm(struct mm_struct *mm,
                                         enum memcg_memory_event event)
{
        struct mem_cgroup *memcg;

        if (mem_cgroup_disabled())
                return;

        rcu_read_lock();
        memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));
        if (likely(memcg))
                memcg_memory_event(memcg, event);
        rcu_read_unlock();
}

void split_page_memcg(struct page *head, int old_order, int new_order);

#else /* CONFIG_MEMCG */

#define MEM_CGROUP_ID_SHIFT        0

static inline struct mem_cgroup *folio_memcg(struct folio *folio)
{
        return NULL;
}

static inline struct mem_cgroup *folio_memcg_rcu(struct folio *folio)
{
        WARN_ON_ONCE(!rcu_read_lock_held());
        return NULL;
}

static inline struct mem_cgroup *folio_memcg_check(struct folio *folio)
{
        return NULL;
}

static inline struct mem_cgroup *page_memcg_check(struct page *page)
{
        return NULL;
}

static inline struct mem_cgroup *get_mem_cgroup_from_objcg(struct obj_cgroup *objcg)
{
        return NULL;
}

static inline bool folio_memcg_kmem(struct folio *folio)
{
        return false;
}

static inline bool PageMemcgKmem(struct page *page)
{
        return false;
}

static inline bool mem_cgroup_is_root(struct mem_cgroup *memcg)
{
        return true;
}

static inline bool mem_cgroup_disabled(void)
{
        return true;
}

static inline void memcg_memory_event(struct mem_cgroup *memcg,
                                      enum memcg_memory_event event)
{
}

static inline void memcg_memory_event_mm(struct mm_struct *mm,
                                         enum memcg_memory_event event)
{
}

static inline void mem_cgroup_protection(struct mem_cgroup *root,
                                         struct mem_cgroup *memcg,
                                         unsigned long *min,
                                         unsigned long *low)
{
        *min = *low = 0;
}

static inline void mem_cgroup_calculate_protection(struct mem_cgroup *root,
                                                   struct mem_cgroup *memcg)
{
}

static inline bool mem_cgroup_unprotected(struct mem_cgroup *target,
                                          struct mem_cgroup *memcg)
{
        return true;
}
static inline bool mem_cgroup_below_low(struct mem_cgroup *target,
                                        struct mem_cgroup *memcg)
{
        return false;
}

static inline bool mem_cgroup_below_min(struct mem_cgroup *target,
                                        struct mem_cgroup *memcg)
{
        return false;
}

static inline void mem_cgroup_commit_charge(struct folio *folio,
                struct mem_cgroup *memcg)
{
}

static inline int mem_cgroup_charge(struct folio *folio,
                struct mm_struct *mm, gfp_t gfp)
{
        return 0;
}

static inline int mem_cgroup_hugetlb_try_charge(struct mem_cgroup *memcg,
                gfp_t gfp, long nr_pages)
{
        return 0;
}

static inline int mem_cgroup_swapin_charge_folio(struct folio *folio,
                        struct mm_struct *mm, gfp_t gfp, swp_entry_t entry)
{
        return 0;
}

static inline void mem_cgroup_swapin_uncharge_swap(swp_entry_t entry, unsigned int nr)
{
}

static inline void mem_cgroup_uncharge(struct folio *folio)
{
}

static inline void mem_cgroup_uncharge_folios(struct folio_batch *folios)
{
}

static inline void mem_cgroup_cancel_charge(struct mem_cgroup *memcg,
                unsigned int nr_pages)
{
}

static inline void mem_cgroup_replace_folio(struct folio *old,
                struct folio *new)
{
}

static inline void mem_cgroup_migrate(struct folio *old, struct folio *new)
{
}

static inline struct lruvec *mem_cgroup_lruvec(struct mem_cgroup *memcg,
                                               struct pglist_data *pgdat)
{
        return &pgdat->__lruvec;
}

static inline struct lruvec *folio_lruvec(struct folio *folio)
{
        struct pglist_data *pgdat = folio_pgdat(folio);
        return &pgdat->__lruvec;
}

static inline
void lruvec_memcg_debug(struct lruvec *lruvec, struct folio *folio)
{
}

static inline struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *memcg)
{
        return NULL;
}

static inline bool mm_match_cgroup(struct mm_struct *mm,
                struct mem_cgroup *memcg)
{
        return true;
}

static inline struct mem_cgroup *get_mem_cgroup_from_mm(struct mm_struct *mm)
{
        return NULL;
}

static inline struct mem_cgroup *get_mem_cgroup_from_current(void)
{
        return NULL;
}

static inline struct mem_cgroup *get_mem_cgroup_from_folio(struct folio *folio)
{
        return NULL;
}

static inline
struct mem_cgroup *mem_cgroup_from_css(struct cgroup_subsys_state *css)
{
        return NULL;
}

static inline void obj_cgroup_put(struct obj_cgroup *objcg)
{
}

static inline bool mem_cgroup_tryget(struct mem_cgroup *memcg)
{
        return true;
}

static inline bool mem_cgroup_tryget_online(struct mem_cgroup *memcg)
{
        return true;
}

static inline void mem_cgroup_put(struct mem_cgroup *memcg)
{
}

static inline struct lruvec *folio_lruvec_lock(struct folio *folio)
{
        struct pglist_data *pgdat = folio_pgdat(folio);

        spin_lock(&pgdat->__lruvec.lru_lock);
        return &pgdat->__lruvec;
}

static inline struct lruvec *folio_lruvec_lock_irq(struct folio *folio)
{
        struct pglist_data *pgdat = folio_pgdat(folio);

        spin_lock_irq(&pgdat->__lruvec.lru_lock);
        return &pgdat->__lruvec;
}

static inline struct lruvec *folio_lruvec_lock_irqsave(struct folio *folio,
                unsigned long *flagsp)
{
        struct pglist_data *pgdat = folio_pgdat(folio);

        spin_lock_irqsave(&pgdat->__lruvec.lru_lock, *flagsp);
        return &pgdat->__lruvec;
}

static inline struct mem_cgroup *
mem_cgroup_iter(struct mem_cgroup *root,
                struct mem_cgroup *prev,
                struct mem_cgroup_reclaim_cookie *reclaim)
{
        return NULL;
}

static inline void mem_cgroup_iter_break(struct mem_cgroup *root,
                                         struct mem_cgroup *prev)
{
}

static inline void mem_cgroup_scan_tasks(struct mem_cgroup *memcg,
                int (*fn)(struct task_struct *, void *), void *arg)
{
}

static inline unsigned short mem_cgroup_id(struct mem_cgroup *memcg)
{
        return 0;
}

static inline struct mem_cgroup *mem_cgroup_from_id(unsigned short id)
{
        WARN_ON_ONCE(id);
        /* XXX: This should always return root_mem_cgroup */
        return NULL;
}

#ifdef CONFIG_SHRINKER_DEBUG
static inline unsigned long mem_cgroup_ino(struct mem_cgroup *memcg)
{
        return 0;
}

static inline struct mem_cgroup *mem_cgroup_get_from_ino(unsigned long ino)
{
        return NULL;
}
#endif

static inline struct mem_cgroup *mem_cgroup_from_seq(struct seq_file *m)
{
        return NULL;
}

static inline struct mem_cgroup *lruvec_memcg(struct lruvec *lruvec)
{
        return NULL;
}

static inline bool mem_cgroup_online(struct mem_cgroup *memcg)
{
        return true;
}

static inline
unsigned long mem_cgroup_get_zone_lru_size(struct lruvec *lruvec,
                enum lru_list lru, int zone_idx)
{
        return 0;
}

static inline unsigned long mem_cgroup_get_max(struct mem_cgroup *memcg)
{
        return 0;
}

static inline unsigned long mem_cgroup_size(struct mem_cgroup *memcg)
{
        return 0;
}

static inline void
mem_cgroup_print_oom_context(struct mem_cgroup *memcg, struct task_struct *p)
{
}

static inline void
mem_cgroup_print_oom_meminfo(struct mem_cgroup *memcg)
{
}

static inline void mem_cgroup_handle_over_high(gfp_t gfp_mask)
{
}

static inline struct mem_cgroup *mem_cgroup_get_oom_group(
        struct task_struct *victim, struct mem_cgroup *oom_domain)
{
        return NULL;
}

static inline void mem_cgroup_print_oom_group(struct mem_cgroup *memcg)
{
}

static inline void __mod_memcg_state(struct mem_cgroup *memcg,
                                     enum memcg_stat_item idx,
                                     int nr)
{
}

static inline void mod_memcg_state(struct mem_cgroup *memcg,
                                   enum memcg_stat_item idx,
                                   int nr)
{
}

static inline void mod_memcg_page_state(struct page *page,
                                        enum memcg_stat_item idx, int val)
{
}

static inline unsigned long memcg_page_state(struct mem_cgroup *memcg, int idx)
{
        return 0;
}

static inline unsigned long lruvec_page_state(struct lruvec *lruvec,
                                              enum node_stat_item idx)
{
        return node_page_state(lruvec_pgdat(lruvec), idx);
}

static inline unsigned long lruvec_page_state_local(struct lruvec *lruvec,
                                                    enum node_stat_item idx)
{
        return node_page_state(lruvec_pgdat(lruvec), idx);
}

static inline void mem_cgroup_flush_stats(struct mem_cgroup *memcg)
{
}

static inline void mem_cgroup_flush_stats_ratelimited(struct mem_cgroup *memcg)
{
}

static inline void __mod_lruvec_kmem_state(void *p, enum node_stat_item idx,
                                           int val)
{
        struct page *page = virt_to_head_page(p);

        __mod_node_page_state(page_pgdat(page), idx, val);
}

static inline void mod_lruvec_kmem_state(void *p, enum node_stat_item idx,
                                         int val)
{
        struct page *page = virt_to_head_page(p);

        mod_node_page_state(page_pgdat(page), idx, val);
}

static inline void count_memcg_events(struct mem_cgroup *memcg,
                                      enum vm_event_item idx,
                                      unsigned long count)
{
}

static inline void __count_memcg_events(struct mem_cgroup *memcg,
                                        enum vm_event_item idx,
                                        unsigned long count)
{
}

static inline void count_memcg_folio_events(struct folio *folio,
                enum vm_event_item idx, unsigned long nr)
{
}

static inline void count_memcg_events_mm(struct mm_struct *mm,
                                        enum vm_event_item idx, unsigned long count)
{
}

static inline
void count_memcg_event_mm(struct mm_struct *mm, enum vm_event_item idx)
{
}

static inline void split_page_memcg(struct page *head, int old_order, int new_order)
{
}
#endif /* CONFIG_MEMCG */

/*
 * Extended information for slab objects stored as an array in page->memcg_data
 * if MEMCG_DATA_OBJEXTS is set.
 */
struct slabobj_ext {
#ifdef CONFIG_MEMCG
        struct obj_cgroup *objcg;
#endif
#ifdef CONFIG_MEM_ALLOC_PROFILING
        union codetag_ref ref;
#endif
} __aligned(8);

static inline void __inc_lruvec_kmem_state(void *p, enum node_stat_item idx)
{
        __mod_lruvec_kmem_state(p, idx, 1);
}

static inline void __dec_lruvec_kmem_state(void *p, enum node_stat_item idx)
{
        __mod_lruvec_kmem_state(p, idx, -1);
}

static inline struct lruvec *parent_lruvec(struct lruvec *lruvec)
{
        struct mem_cgroup *memcg;

        memcg = lruvec_memcg(lruvec);
        if (!memcg)
                return NULL;
        memcg = parent_mem_cgroup(memcg);
        if (!memcg)
                return NULL;
        return mem_cgroup_lruvec(memcg, lruvec_pgdat(lruvec));
}

static inline void unlock_page_lruvec(struct lruvec *lruvec)
{
        spin_unlock(&lruvec->lru_lock);
}

static inline void unlock_page_lruvec_irq(struct lruvec *lruvec)
{
        spin_unlock_irq(&lruvec->lru_lock);
}

static inline void unlock_page_lruvec_irqrestore(struct lruvec *lruvec,
                unsigned long flags)
{
        spin_unlock_irqrestore(&lruvec->lru_lock, flags);
}

/* Test requires a stable folio->memcg binding, see folio_memcg() */
static inline bool folio_matches_lruvec(struct folio *folio,
                struct lruvec *lruvec)
{
        return lruvec_pgdat(lruvec) == folio_pgdat(folio) &&
               lruvec_memcg(lruvec) == folio_memcg(folio);
}

/* Don't lock again iff page's lruvec locked */
static inline struct lruvec *folio_lruvec_relock_irq(struct folio *folio,
                struct lruvec *locked_lruvec)
{
        if (locked_lruvec) {
                if (folio_matches_lruvec(folio, locked_lruvec))
                        return locked_lruvec;

                unlock_page_lruvec_irq(locked_lruvec);
        }

        return folio_lruvec_lock_irq(folio);
}

/* Don't lock again iff folio's lruvec locked */
static inline void folio_lruvec_relock_irqsave(struct folio *folio,
                struct lruvec **lruvecp, unsigned long *flags)
{
        if (*lruvecp) {
                if (folio_matches_lruvec(folio, *lruvecp))
                        return;

                unlock_page_lruvec_irqrestore(*lruvecp, *flags);
        }

        *lruvecp = folio_lruvec_lock_irqsave(folio, flags);
}

#ifdef CONFIG_CGROUP_WRITEBACK

struct wb_domain *mem_cgroup_wb_domain(struct bdi_writeback *wb);
void mem_cgroup_wb_stats(struct bdi_writeback *wb, unsigned long *pfilepages,
                         unsigned long *pheadroom, unsigned long *pdirty,
                         unsigned long *pwriteback);

void mem_cgroup_track_foreign_dirty_slowpath(struct folio *folio,
                                             struct bdi_writeback *wb);

static inline void mem_cgroup_track_foreign_dirty(struct folio *folio,
                                                  struct bdi_writeback *wb)
{
        struct mem_cgroup *memcg;

        if (mem_cgroup_disabled())
                return;

        memcg = folio_memcg(folio);
        if (unlikely(memcg && &memcg->css != wb->memcg_css))
                mem_cgroup_track_foreign_dirty_slowpath(folio, wb);
}

void mem_cgroup_flush_foreign(struct bdi_writeback *wb);

#else        /* CONFIG_CGROUP_WRITEBACK */

static inline struct wb_domain *mem_cgroup_wb_domain(struct bdi_writeback *wb)
{
        return NULL;
}

static inline void mem_cgroup_wb_stats(struct bdi_writeback *wb,
                                       unsigned long *pfilepages,
                                       unsigned long *pheadroom,
                                       unsigned long *pdirty,
                                       unsigned long *pwriteback)
{
}

static inline void mem_cgroup_track_foreign_dirty(struct folio *folio,
                                                  struct bdi_writeback *wb)
{
}

static inline void mem_cgroup_flush_foreign(struct bdi_writeback *wb)
{
}

#endif        /* CONFIG_CGROUP_WRITEBACK */

struct sock;
bool mem_cgroup_charge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages,
                             gfp_t gfp_mask);
void mem_cgroup_uncharge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages);
#ifdef CONFIG_MEMCG
extern struct static_key_false memcg_sockets_enabled_key;
#define mem_cgroup_sockets_enabled static_branch_unlikely(&memcg_sockets_enabled_key)
void mem_cgroup_sk_alloc(struct sock *sk);
void mem_cgroup_sk_free(struct sock *sk);
static inline bool mem_cgroup_under_socket_pressure(struct mem_cgroup *memcg)
{
#ifdef CONFIG_MEMCG_V1
        if (!cgroup_subsys_on_dfl(memory_cgrp_subsys))
                return !!memcg->tcpmem_pressure;
#endif /* CONFIG_MEMCG_V1 */
        do {
                if (time_before(jiffies, READ_ONCE(memcg->socket_pressure)))
                        return true;
        } while ((memcg = parent_mem_cgroup(memcg)));
        return false;
}

int alloc_shrinker_info(struct mem_cgroup *memcg);
void free_shrinker_info(struct mem_cgroup *memcg);
void set_shrinker_bit(struct mem_cgroup *memcg, int nid, int shrinker_id);
void reparent_shrinker_deferred(struct mem_cgroup *memcg);
#else
#define mem_cgroup_sockets_enabled 0
static inline void mem_cgroup_sk_alloc(struct sock *sk) { };
static inline void mem_cgroup_sk_free(struct sock *sk) { };
static inline bool mem_cgroup_under_socket_pressure(struct mem_cgroup *memcg)
{
        return false;
}

static inline void set_shrinker_bit(struct mem_cgroup *memcg,
                                    int nid, int shrinker_id)
{
}
#endif

#ifdef CONFIG_MEMCG
bool mem_cgroup_kmem_disabled(void);
int __memcg_kmem_charge_page(struct page *page, gfp_t gfp, int order);
void __memcg_kmem_uncharge_page(struct page *page, int order);

/*
 * The returned objcg pointer is safe to use without additional
 * protection within a scope. The scope is defined either by
 * the current task (similar to the "current" global variable)
 * or by set_active_memcg() pair.
 * Please, use obj_cgroup_get() to get a reference if the pointer
 * needs to be used outside of the local scope.
 */
struct obj_cgroup *current_obj_cgroup(void);
struct obj_cgroup *get_obj_cgroup_from_folio(struct folio *folio);

static inline struct obj_cgroup *get_obj_cgroup_from_current(void)
{
        struct obj_cgroup *objcg = current_obj_cgroup();

        if (objcg)
                obj_cgroup_get(objcg);

        return objcg;
}

int obj_cgroup_charge(struct obj_cgroup *objcg, gfp_t gfp, size_t size);
void obj_cgroup_uncharge(struct obj_cgroup *objcg, size_t size);

extern struct static_key_false memcg_bpf_enabled_key;
static inline bool memcg_bpf_enabled(void)
{
        return static_branch_likely(&memcg_bpf_enabled_key);
}

extern struct static_key_false memcg_kmem_online_key;

static inline bool memcg_kmem_online(void)
{
        return static_branch_likely(&memcg_kmem_online_key);
}

static inline int memcg_kmem_charge_page(struct page *page, gfp_t gfp,
                                         int order)
{
        if (memcg_kmem_online())
                return __memcg_kmem_charge_page(page, gfp, order);
        return 0;
}

static inline void memcg_kmem_uncharge_page(struct page *page, int order)
{
        if (memcg_kmem_online())
                __memcg_kmem_uncharge_page(page, order);
}

/*
 * A helper for accessing memcg's kmem_id, used for getting
 * corresponding LRU lists.
 */
static inline int memcg_kmem_id(struct mem_cgroup *memcg)
{
        return memcg ? memcg->kmemcg_id : -1;
}

struct mem_cgroup *mem_cgroup_from_slab_obj(void *p);

static inline void count_objcg_events(struct obj_cgroup *objcg,
                                      enum vm_event_item idx,
                                      unsigned long count)
{
        struct mem_cgroup *memcg;

        if (!memcg_kmem_online())
                return;

        rcu_read_lock();
        memcg = obj_cgroup_memcg(objcg);
        count_memcg_events(memcg, idx, count);
        rcu_read_unlock();
}

#else
static inline bool mem_cgroup_kmem_disabled(void)
{
        return true;
}

static inline int memcg_kmem_charge_page(struct page *page, gfp_t gfp,
                                         int order)
{
        return 0;
}

static inline void memcg_kmem_uncharge_page(struct page *page, int order)
{
}

static inline int __memcg_kmem_charge_page(struct page *page, gfp_t gfp,
                                           int order)
{
        return 0;
}

static inline void __memcg_kmem_uncharge_page(struct page *page, int order)
{
}

static inline struct obj_cgroup *get_obj_cgroup_from_folio(struct folio *folio)
{
        return NULL;
}

static inline bool memcg_bpf_enabled(void)
{
        return false;
}

static inline bool memcg_kmem_online(void)
{
        return false;
}

static inline int memcg_kmem_id(struct mem_cgroup *memcg)
{
        return -1;
}

static inline struct mem_cgroup *mem_cgroup_from_slab_obj(void *p)
{
        return NULL;
}

static inline void count_objcg_events(struct obj_cgroup *objcg,
                                      enum vm_event_item idx,
                                      unsigned long count)
{
}

#endif /* CONFIG_MEMCG */

#if defined(CONFIG_MEMCG) && defined(CONFIG_ZSWAP)
bool obj_cgroup_may_zswap(struct obj_cgroup *objcg);
void obj_cgroup_charge_zswap(struct obj_cgroup *objcg, size_t size);
void obj_cgroup_uncharge_zswap(struct obj_cgroup *objcg, size_t size);
bool mem_cgroup_zswap_writeback_enabled(struct mem_cgroup *memcg);
#else
static inline bool obj_cgroup_may_zswap(struct obj_cgroup *objcg)
{
        return true;
}
static inline void obj_cgroup_charge_zswap(struct obj_cgroup *objcg,
                                           size_t size)
{
}
static inline void obj_cgroup_uncharge_zswap(struct obj_cgroup *objcg,
                                             size_t size)
{
}
static inline bool mem_cgroup_zswap_writeback_enabled(struct mem_cgroup *memcg)
{
        /* if zswap is disabled, do not block pages going to the swapping device */
        return true;
}
#endif


/* Cgroup v1-related declarations */

#ifdef CONFIG_MEMCG_V1
unsigned long memcg1_soft_limit_reclaim(pg_data_t *pgdat, int order,
                                        gfp_t gfp_mask,
                                        unsigned long *total_scanned);

bool mem_cgroup_oom_synchronize(bool wait);

static inline bool task_in_memcg_oom(struct task_struct *p)
{
        return p->memcg_in_oom;
}

void folio_memcg_lock(struct folio *folio);
void folio_memcg_unlock(struct folio *folio);

/* try to stablize folio_memcg() for all the pages in a memcg */
static inline bool mem_cgroup_trylock_pages(struct mem_cgroup *memcg)
{
        rcu_read_lock();

        if (mem_cgroup_disabled() || !atomic_read(&memcg->moving_account))
                return true;

        rcu_read_unlock();
        return false;
}

static inline void mem_cgroup_unlock_pages(void)
{
        rcu_read_unlock();
}

static inline void mem_cgroup_enter_user_fault(void)
{
        WARN_ON(current->in_user_fault);
        current->in_user_fault = 1;
}

static inline void mem_cgroup_exit_user_fault(void)
{
        WARN_ON(!current->in_user_fault);
        current->in_user_fault = 0;
}

#else /* CONFIG_MEMCG_V1 */
static inline
unsigned long memcg1_soft_limit_reclaim(pg_data_t *pgdat, int order,
                                        gfp_t gfp_mask,
                                        unsigned long *total_scanned)
{
        return 0;
}

static inline void folio_memcg_lock(struct folio *folio)
{
}

static inline void folio_memcg_unlock(struct folio *folio)
{
}

static inline bool mem_cgroup_trylock_pages(struct mem_cgroup *memcg)
{
        /* to match folio_memcg_rcu() */
        rcu_read_lock();
        return true;
}

static inline void mem_cgroup_unlock_pages(void)
{
        rcu_read_unlock();
}

static inline bool task_in_memcg_oom(struct task_struct *p)
{
        return false;
}

static inline bool mem_cgroup_oom_synchronize(bool wait)
{
        return false;
}

static inline void mem_cgroup_enter_user_fault(void)
{
}

static inline void mem_cgroup_exit_user_fault(void)
{
}

#endif /* CONFIG_MEMCG_V1 */

#endif /* _LINUX_MEMCONTROL_H */




















































































































































































































































































































































































































































































































































































































































    2 



































   35 




































    5 





































































































   15 






























































































































































































    5 






























































































































































































































































































    2 







































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
// SPDX-License-Identifier: GPL-2.0
/*
 *  file.c - part of debugfs, a tiny little debug file system
 *
 *  Copyright (C) 2004 Greg Kroah-Hartman <greg@kroah.com>
 *  Copyright (C) 2004 IBM Inc.
 *
 *  debugfs is for people to use instead of /proc or /sys.
 *  See Documentation/filesystems/ for more details.
 */

#include <linux/module.h>
#include <linux/fs.h>
#include <linux/seq_file.h>
#include <linux/pagemap.h>
#include <linux/debugfs.h>
#include <linux/io.h>
#include <linux/slab.h>
#include <linux/atomic.h>
#include <linux/device.h>
#include <linux/pm_runtime.h>
#include <linux/poll.h>
#include <linux/security.h>

#include "internal.h"

struct poll_table_struct;

static ssize_t default_read_file(struct file *file, char __user *buf,
                                 size_t count, loff_t *ppos)
{
        return 0;
}

static ssize_t default_write_file(struct file *file, const char __user *buf,
                                   size_t count, loff_t *ppos)
{
        return count;
}

const struct file_operations debugfs_noop_file_operations = {
        .read =                default_read_file,
        .write =        default_write_file,
        .open =                simple_open,
        .llseek =        noop_llseek,
};

#define F_DENTRY(filp) ((filp)->f_path.dentry)

const struct file_operations *debugfs_real_fops(const struct file *filp)
{
        struct debugfs_fsdata *fsd = F_DENTRY(filp)->d_fsdata;

        if ((unsigned long)fsd & DEBUGFS_FSDATA_IS_REAL_FOPS_BIT) {
                /*
                 * Urgh, we've been called w/o a protecting
                 * debugfs_file_get().
                 */
                WARN_ON(1);
                return NULL;
        }

        return fsd->real_fops;
}
EXPORT_SYMBOL_GPL(debugfs_real_fops);

/**
 * debugfs_file_get - mark the beginning of file data access
 * @dentry: the dentry object whose data is being accessed.
 *
 * Up to a matching call to debugfs_file_put(), any successive call
 * into the file removing functions debugfs_remove() and
 * debugfs_remove_recursive() will block. Since associated private
 * file data may only get freed after a successful return of any of
 * the removal functions, you may safely access it after a successful
 * call to debugfs_file_get() without worrying about lifetime issues.
 *
 * If -%EIO is returned, the file has already been removed and thus,
 * it is not safe to access any of its data. If, on the other hand,
 * it is allowed to access the file data, zero is returned.
 */
int debugfs_file_get(struct dentry *dentry)
{
        struct debugfs_fsdata *fsd;
        void *d_fsd;

        /*
         * This could only happen if some debugfs user erroneously calls
         * debugfs_file_get() on a dentry that isn't even a file, let
         * them know about it.
         */
        if (WARN_ON(!d_is_reg(dentry)))
                return -EINVAL;

        d_fsd = READ_ONCE(dentry->d_fsdata);
        if (!((unsigned long)d_fsd & DEBUGFS_FSDATA_IS_REAL_FOPS_BIT)) {
                fsd = d_fsd;
        } else {
                fsd = kmalloc(sizeof(*fsd), GFP_KERNEL);
                if (!fsd)
                        return -ENOMEM;

                if ((unsigned long)d_fsd & DEBUGFS_FSDATA_IS_SHORT_FOPS_BIT) {
                        fsd->real_fops = NULL;
                        fsd->short_fops = (void *)((unsigned long)d_fsd &
                                                ~(DEBUGFS_FSDATA_IS_REAL_FOPS_BIT |
                                                  DEBUGFS_FSDATA_IS_SHORT_FOPS_BIT));
                } else {
                        fsd->real_fops = (void *)((unsigned long)d_fsd &
                                                ~DEBUGFS_FSDATA_IS_REAL_FOPS_BIT);
                        fsd->short_fops = NULL;
                }
                refcount_set(&fsd->active_users, 1);
                init_completion(&fsd->active_users_drained);
                INIT_LIST_HEAD(&fsd->cancellations);
                mutex_init(&fsd->cancellations_mtx);

                if (cmpxchg(&dentry->d_fsdata, d_fsd, fsd) != d_fsd) {
                        mutex_destroy(&fsd->cancellations_mtx);
                        kfree(fsd);
                        fsd = READ_ONCE(dentry->d_fsdata);
                }
        }

        /*
         * In case of a successful cmpxchg() above, this check is
         * strictly necessary and must follow it, see the comment in
         * __debugfs_remove_file().
         * OTOH, if the cmpxchg() hasn't been executed or wasn't
         * successful, this serves the purpose of not starving
         * removers.
         */
        if (d_unlinked(dentry))
                return -EIO;

        if (!refcount_inc_not_zero(&fsd->active_users))
                return -EIO;

        return 0;
}
EXPORT_SYMBOL_GPL(debugfs_file_get);

/**
 * debugfs_file_put - mark the end of file data access
 * @dentry: the dentry object formerly passed to
 *          debugfs_file_get().
 *
 * Allow any ongoing concurrent call into debugfs_remove() or
 * debugfs_remove_recursive() blocked by a former call to
 * debugfs_file_get() to proceed and return to its caller.
 */
void debugfs_file_put(struct dentry *dentry)
{
        struct debugfs_fsdata *fsd = READ_ONCE(dentry->d_fsdata);

        if (refcount_dec_and_test(&fsd->active_users))
                complete(&fsd->active_users_drained);
}
EXPORT_SYMBOL_GPL(debugfs_file_put);

/**
 * debugfs_enter_cancellation - enter a debugfs cancellation
 * @file: the file being accessed
 * @cancellation: the cancellation object, the cancel callback
 *        inside of it must be initialized
 *
 * When a debugfs file is removed it needs to wait for all active
 * operations to complete. However, the operation itself may need
 * to wait for hardware or completion of some asynchronous process
 * or similar. As such, it may need to be cancelled to avoid long
 * waits or even deadlocks.
 *
 * This function can be used inside a debugfs handler that may
 * need to be cancelled. As soon as this function is called, the
 * cancellation's 'cancel' callback may be called, at which point
 * the caller should proceed to call debugfs_leave_cancellation()
 * and leave the debugfs handler function as soon as possible.
 * Note that the 'cancel' callback is only ever called in the
 * context of some kind of debugfs_remove().
 *
 * This function must be paired with debugfs_leave_cancellation().
 */
void debugfs_enter_cancellation(struct file *file,
                                struct debugfs_cancellation *cancellation)
{
        struct debugfs_fsdata *fsd;
        struct dentry *dentry = F_DENTRY(file);

        INIT_LIST_HEAD(&cancellation->list);

        if (WARN_ON(!d_is_reg(dentry)))
                return;

        if (WARN_ON(!cancellation->cancel))
                return;

        fsd = READ_ONCE(dentry->d_fsdata);
        if (WARN_ON(!fsd ||
                    ((unsigned long)fsd & DEBUGFS_FSDATA_IS_REAL_FOPS_BIT)))
                return;

        mutex_lock(&fsd->cancellations_mtx);
        list_add(&cancellation->list, &fsd->cancellations);
        mutex_unlock(&fsd->cancellations_mtx);

        /* if we're already removing wake it up to cancel */
        if (d_unlinked(dentry))
                complete(&fsd->active_users_drained);
}
EXPORT_SYMBOL_GPL(debugfs_enter_cancellation);

/**
 * debugfs_leave_cancellation - leave cancellation section
 * @file: the file being accessed
 * @cancellation: the cancellation previously registered with
 *        debugfs_enter_cancellation()
 *
 * See the documentation of debugfs_enter_cancellation().
 */
void debugfs_leave_cancellation(struct file *file,
                                struct debugfs_cancellation *cancellation)
{
        struct debugfs_fsdata *fsd;
        struct dentry *dentry = F_DENTRY(file);

        if (WARN_ON(!d_is_reg(dentry)))
                return;

        fsd = READ_ONCE(dentry->d_fsdata);
        if (WARN_ON(!fsd ||
                    ((unsigned long)fsd & DEBUGFS_FSDATA_IS_REAL_FOPS_BIT)))
                return;

        mutex_lock(&fsd->cancellations_mtx);
        if (!list_empty(&cancellation->list))
                list_del(&cancellation->list);
        mutex_unlock(&fsd->cancellations_mtx);
}
EXPORT_SYMBOL_GPL(debugfs_leave_cancellation);

/*
 * Only permit access to world-readable files when the kernel is locked down.
 * We also need to exclude any file that has ways to write or alter it as root
 * can bypass the permissions check.
 */
static int debugfs_locked_down(struct inode *inode,
                               struct file *filp,
                               const struct file_operations *real_fops)
{
        if ((inode->i_mode & 07777 & ~0444) == 0 &&
            !(filp->f_mode & FMODE_WRITE) &&
            (!real_fops ||
             (!real_fops->unlocked_ioctl &&
              !real_fops->compat_ioctl &&
              !real_fops->mmap)))
                return 0;

        if (security_locked_down(LOCKDOWN_DEBUGFS))
                return -EPERM;

        return 0;
}

static int open_proxy_open(struct inode *inode, struct file *filp)
{
        struct dentry *dentry = F_DENTRY(filp);
        const struct file_operations *real_fops = NULL;
        int r;

        r = debugfs_file_get(dentry);
        if (r)
                return r == -EIO ? -ENOENT : r;

        real_fops = debugfs_real_fops(filp);

        r = debugfs_locked_down(inode, filp, real_fops);
        if (r)
                goto out;

        if (!fops_get(real_fops)) {
#ifdef CONFIG_MODULES
                if (real_fops->owner &&
                    real_fops->owner->state == MODULE_STATE_GOING) {
                        r = -ENXIO;
                        goto out;
                }
#endif

                /* Huh? Module did not clean up after itself at exit? */
                WARN(1, "debugfs file owner did not clean up at exit: %pd",
                        dentry);
                r = -ENXIO;
                goto out;
        }
        replace_fops(filp, real_fops);

        if (real_fops->open)
                r = real_fops->open(inode, filp);

out:
        debugfs_file_put(dentry);
        return r;
}

const struct file_operations debugfs_open_proxy_file_operations = {
        .open = open_proxy_open,
};

#define PROTO(args...) args
#define ARGS(args...) args

#define FULL_PROXY_FUNC(name, ret_type, filp, proto, args)                \
static ret_type full_proxy_ ## name(proto)                                \
{                                                                        \
        struct dentry *dentry = F_DENTRY(filp);                        \
        const struct file_operations *real_fops;                        \
        ret_type r;                                                        \
                                                                        \
        r = debugfs_file_get(dentry);                                        \
        if (unlikely(r))                                                \
                return r;                                                \
        real_fops = debugfs_real_fops(filp);                                \
        r = real_fops->name(args);                                        \
        debugfs_file_put(dentry);                                        \
        return r;                                                        \
}

#define FULL_PROXY_FUNC_BOTH(name, ret_type, filp, proto, args)                \
static ret_type full_proxy_ ## name(proto)                                \
{                                                                        \
        struct dentry *dentry = F_DENTRY(filp);                                \
        struct debugfs_fsdata *fsd;                                        \
        ret_type r;                                                        \
                                                                        \
        r = debugfs_file_get(dentry);                                        \
        if (unlikely(r))                                                \
                return r;                                                \
        fsd = dentry->d_fsdata;                                                \
        if (fsd->real_fops)                                                \
                r = fsd->real_fops->name(args);                                \
        else                                                                \
                r = fsd->short_fops->name(args);                        \
        debugfs_file_put(dentry);                                        \
        return r;                                                        \
}

FULL_PROXY_FUNC_BOTH(llseek, loff_t, filp,
                     PROTO(struct file *filp, loff_t offset, int whence),
                     ARGS(filp, offset, whence));

FULL_PROXY_FUNC_BOTH(read, ssize_t, filp,
                     PROTO(struct file *filp, char __user *buf, size_t size,
                           loff_t *ppos),
                     ARGS(filp, buf, size, ppos));

FULL_PROXY_FUNC_BOTH(write, ssize_t, filp,
                     PROTO(struct file *filp, const char __user *buf,
                           size_t size, loff_t *ppos),
                     ARGS(filp, buf, size, ppos));

FULL_PROXY_FUNC(unlocked_ioctl, long, filp,
                PROTO(struct file *filp, unsigned int cmd, unsigned long arg),
                ARGS(filp, cmd, arg));

static __poll_t full_proxy_poll(struct file *filp,
                                struct poll_table_struct *wait)
{
        struct dentry *dentry = F_DENTRY(filp);
        __poll_t r = 0;
        const struct file_operations *real_fops;

        if (debugfs_file_get(dentry))
                return EPOLLHUP;

        real_fops = debugfs_real_fops(filp);
        r = real_fops->poll(filp, wait);
        debugfs_file_put(dentry);
        return r;
}

static int full_proxy_release(struct inode *inode, struct file *filp)
{
        const struct dentry *dentry = F_DENTRY(filp);
        const struct file_operations *real_fops = debugfs_real_fops(filp);
        const struct file_operations *proxy_fops = filp->f_op;
        int r = 0;

        /*
         * We must not protect this against removal races here: the
         * original releaser should be called unconditionally in order
         * not to leak any resources. Releasers must not assume that
         * ->i_private is still being meaningful here.
         */
        if (real_fops && real_fops->release)
                r = real_fops->release(inode, filp);

        replace_fops(filp, d_inode(dentry)->i_fop);
        kfree(proxy_fops);
        fops_put(real_fops);
        return r;
}

static void __full_proxy_fops_init(struct file_operations *proxy_fops,
                                   struct debugfs_fsdata *fsd)
{
        proxy_fops->release = full_proxy_release;

        if ((fsd->real_fops && fsd->real_fops->llseek) ||
            (fsd->short_fops && fsd->short_fops->llseek))
                proxy_fops->llseek = full_proxy_llseek;

        if ((fsd->real_fops && fsd->real_fops->read) ||
            (fsd->short_fops && fsd->short_fops->read))
                proxy_fops->read = full_proxy_read;

        if ((fsd->real_fops && fsd->real_fops->write) ||
            (fsd->short_fops && fsd->short_fops->write))
                proxy_fops->write = full_proxy_write;

        if (fsd->real_fops && fsd->real_fops->poll)
                proxy_fops->poll = full_proxy_poll;

        if (fsd->real_fops && fsd->real_fops->unlocked_ioctl)
                proxy_fops->unlocked_ioctl = full_proxy_unlocked_ioctl;
}

static int full_proxy_open(struct inode *inode, struct file *filp)
{
        struct dentry *dentry = F_DENTRY(filp);
        const struct file_operations *real_fops;
        struct file_operations *proxy_fops = NULL;
        struct debugfs_fsdata *fsd;
        int r;

        r = debugfs_file_get(dentry);
        if (r)
                return r == -EIO ? -ENOENT : r;

        fsd = dentry->d_fsdata;
        real_fops = fsd->real_fops;
        r = debugfs_locked_down(inode, filp, real_fops);
        if (r)
                goto out;

        if (real_fops && !fops_get(real_fops)) {
#ifdef CONFIG_MODULES
                if (real_fops->owner &&
                    real_fops->owner->state == MODULE_STATE_GOING) {
                        r = -ENXIO;
                        goto out;
                }
#endif

                /* Huh? Module did not cleanup after itself at exit? */
                WARN(1, "debugfs file owner did not clean up at exit: %pd",
                        dentry);
                r = -ENXIO;
                goto out;
        }

        proxy_fops = kzalloc(sizeof(*proxy_fops), GFP_KERNEL);
        if (!proxy_fops) {
                r = -ENOMEM;
                goto free_proxy;
        }
        __full_proxy_fops_init(proxy_fops, fsd);
        replace_fops(filp, proxy_fops);

        if (!real_fops || real_fops->open) {
                if (real_fops)
                        r = real_fops->open(inode, filp);
                else
                        r = simple_open(inode, filp);
                if (r) {
                        replace_fops(filp, d_inode(dentry)->i_fop);
                        goto free_proxy;
                } else if (filp->f_op != proxy_fops) {
                        /* No protection against file removal anymore. */
                        WARN(1, "debugfs file owner replaced proxy fops: %pd",
                                dentry);
                        goto free_proxy;
                }
        }

        goto out;
free_proxy:
        kfree(proxy_fops);
        fops_put(real_fops);
out:
        debugfs_file_put(dentry);
        return r;
}

const struct file_operations debugfs_full_proxy_file_operations = {
        .open = full_proxy_open,
};

ssize_t debugfs_attr_read(struct file *file, char __user *buf,
                        size_t len, loff_t *ppos)
{
        struct dentry *dentry = F_DENTRY(file);
        ssize_t ret;

        ret = debugfs_file_get(dentry);
        if (unlikely(ret))
                return ret;
        ret = simple_attr_read(file, buf, len, ppos);
        debugfs_file_put(dentry);
        return ret;
}
EXPORT_SYMBOL_GPL(debugfs_attr_read);

static ssize_t debugfs_attr_write_xsigned(struct file *file, const char __user *buf,
                         size_t len, loff_t *ppos, bool is_signed)
{
        struct dentry *dentry = F_DENTRY(file);
        ssize_t ret;

        ret = debugfs_file_get(dentry);
        if (unlikely(ret))
                return ret;
        if (is_signed)
                ret = simple_attr_write_signed(file, buf, len, ppos);
        else
                ret = simple_attr_write(file, buf, len, ppos);
        debugfs_file_put(dentry);
        return ret;
}

ssize_t debugfs_attr_write(struct file *file, const char __user *buf,
                         size_t len, loff_t *ppos)
{
        return debugfs_attr_write_xsigned(file, buf, len, ppos, false);
}
EXPORT_SYMBOL_GPL(debugfs_attr_write);

ssize_t debugfs_attr_write_signed(struct file *file, const char __user *buf,
                         size_t len, loff_t *ppos)
{
        return debugfs_attr_write_xsigned(file, buf, len, ppos, true);
}
EXPORT_SYMBOL_GPL(debugfs_attr_write_signed);

static struct dentry *debugfs_create_mode_unsafe(const char *name, umode_t mode,
                                        struct dentry *parent, void *value,
                                        const struct file_operations *fops,
                                        const struct file_operations *fops_ro,
                                        const struct file_operations *fops_wo)
{
        /* if there are no write bits set, make read only */
        if (!(mode & S_IWUGO))
                return debugfs_create_file_unsafe(name, mode, parent, value,
                                                fops_ro);
        /* if there are no read bits set, make write only */
        if (!(mode & S_IRUGO))
                return debugfs_create_file_unsafe(name, mode, parent, value,
                                                fops_wo);

        return debugfs_create_file_unsafe(name, mode, parent, value, fops);
}

static int debugfs_u8_set(void *data, u64 val)
{
        *(u8 *)data = val;
        return 0;
}
static int debugfs_u8_get(void *data, u64 *val)
{
        *val = *(u8 *)data;
        return 0;
}
DEFINE_DEBUGFS_ATTRIBUTE(fops_u8, debugfs_u8_get, debugfs_u8_set, "%llu\n");
DEFINE_DEBUGFS_ATTRIBUTE(fops_u8_ro, debugfs_u8_get, NULL, "%llu\n");
DEFINE_DEBUGFS_ATTRIBUTE(fops_u8_wo, NULL, debugfs_u8_set, "%llu\n");

/**
 * debugfs_create_u8 - create a debugfs file that is used to read and write an unsigned 8-bit value
 * @name: a pointer to a string containing the name of the file to create.
 * @mode: the permission that the file should have
 * @parent: a pointer to the parent dentry for this file.  This should be a
 *          directory dentry if set.  If this parameter is %NULL, then the
 *          file will be created in the root of the debugfs filesystem.
 * @value: a pointer to the variable that the file should read to and write
 *         from.
 *
 * This function creates a file in debugfs with the given name that
 * contains the value of the variable @value.  If the @mode variable is so
 * set, it can be read from, and written to.
 */
void debugfs_create_u8(const char *name, umode_t mode, struct dentry *parent,
                       u8 *value)
{
        debugfs_create_mode_unsafe(name, mode, parent, value, &fops_u8,
                                   &fops_u8_ro, &fops_u8_wo);
}
EXPORT_SYMBOL_GPL(debugfs_create_u8);

static int debugfs_u16_set(void *data, u64 val)
{
        *(u16 *)data = val;
        return 0;
}
static int debugfs_u16_get(void *data, u64 *val)
{
        *val = *(u16 *)data;
        return 0;
}
DEFINE_DEBUGFS_ATTRIBUTE(fops_u16, debugfs_u16_get, debugfs_u16_set, "%llu\n");
DEFINE_DEBUGFS_ATTRIBUTE(fops_u16_ro, debugfs_u16_get, NULL, "%llu\n");
DEFINE_DEBUGFS_ATTRIBUTE(fops_u16_wo, NULL, debugfs_u16_set, "%llu\n");

/**
 * debugfs_create_u16 - create a debugfs file that is used to read and write an unsigned 16-bit value
 * @name: a pointer to a string containing the name of the file to create.
 * @mode: the permission that the file should have
 * @parent: a pointer to the parent dentry for this file.  This should be a
 *          directory dentry if set.  If this parameter is %NULL, then the
 *          file will be created in the root of the debugfs filesystem.
 * @value: a pointer to the variable that the file should read to and write
 *         from.
 *
 * This function creates a file in debugfs with the given name that
 * contains the value of the variable @value.  If the @mode variable is so
 * set, it can be read from, and written to.
 */
void debugfs_create_u16(const char *name, umode_t mode, struct dentry *parent,
                        u16 *value)
{
        debugfs_create_mode_unsafe(name, mode, parent, value, &fops_u16,
                                   &fops_u16_ro, &fops_u16_wo);
}
EXPORT_SYMBOL_GPL(debugfs_create_u16);

static int debugfs_u32_set(void *data, u64 val)
{
        *(u32 *)data = val;
        return 0;
}
static int debugfs_u32_get(void *data, u64 *val)
{
        *val = *(u32 *)data;
        return 0;
}
DEFINE_DEBUGFS_ATTRIBUTE(fops_u32, debugfs_u32_get, debugfs_u32_set, "%llu\n");
DEFINE_DEBUGFS_ATTRIBUTE(fops_u32_ro, debugfs_u32_get, NULL, "%llu\n");
DEFINE_DEBUGFS_ATTRIBUTE(fops_u32_wo, NULL, debugfs_u32_set, "%llu\n");

/**
 * debugfs_create_u32 - create a debugfs file that is used to read and write an unsigned 32-bit value
 * @name: a pointer to a string containing the name of the file to create.
 * @mode: the permission that the file should have
 * @parent: a pointer to the parent dentry for this file.  This should be a
 *          directory dentry if set.  If this parameter is %NULL, then the
 *          file will be created in the root of the debugfs filesystem.
 * @value: a pointer to the variable that the file should read to and write
 *         from.
 *
 * This function creates a file in debugfs with the given name that
 * contains the value of the variable @value.  If the @mode variable is so
 * set, it can be read from, and written to.
 */
void debugfs_create_u32(const char *name, umode_t mode, struct dentry *parent,
                        u32 *value)
{
        debugfs_create_mode_unsafe(name, mode, parent, value, &fops_u32,
                                   &fops_u32_ro, &fops_u32_wo);
}
EXPORT_SYMBOL_GPL(debugfs_create_u32);

static int debugfs_u64_set(void *data, u64 val)
{
        *(u64 *)data = val;
        return 0;
}

static int debugfs_u64_get(void *data, u64 *val)
{
        *val = *(u64 *)data;
        return 0;
}
DEFINE_DEBUGFS_ATTRIBUTE(fops_u64, debugfs_u64_get, debugfs_u64_set, "%llu\n");
DEFINE_DEBUGFS_ATTRIBUTE(fops_u64_ro, debugfs_u64_get, NULL, "%llu\n");
DEFINE_DEBUGFS_ATTRIBUTE(fops_u64_wo, NULL, debugfs_u64_set, "%llu\n");

/**
 * debugfs_create_u64 - create a debugfs file that is used to read and write an unsigned 64-bit value
 * @name: a pointer to a string containing the name of the file to create.
 * @mode: the permission that the file should have
 * @parent: a pointer to the parent dentry for this file.  This should be a
 *          directory dentry if set.  If this parameter is %NULL, then the
 *          file will be created in the root of the debugfs filesystem.
 * @value: a pointer to the variable that the file should read to and write
 *         from.
 *
 * This function creates a file in debugfs with the given name that
 * contains the value of the variable @value.  If the @mode variable is so
 * set, it can be read from, and written to.
 */
void debugfs_create_u64(const char *name, umode_t mode, struct dentry *parent,
                        u64 *value)
{
        debugfs_create_mode_unsafe(name, mode, parent, value, &fops_u64,
                                   &fops_u64_ro, &fops_u64_wo);
}
EXPORT_SYMBOL_GPL(debugfs_create_u64);

static int debugfs_ulong_set(void *data, u64 val)
{
        *(unsigned long *)data = val;
        return 0;
}

static int debugfs_ulong_get(void *data, u64 *val)
{
        *val = *(unsigned long *)data;
        return 0;
}
DEFINE_DEBUGFS_ATTRIBUTE(fops_ulong, debugfs_ulong_get, debugfs_ulong_set,
                        "%llu\n");
DEFINE_DEBUGFS_ATTRIBUTE(fops_ulong_ro, debugfs_ulong_get, NULL, "%llu\n");
DEFINE_DEBUGFS_ATTRIBUTE(fops_ulong_wo, NULL, debugfs_ulong_set, "%llu\n");

/**
 * debugfs_create_ulong - create a debugfs file that is used to read and write
 * an unsigned long value.
 * @name: a pointer to a string containing the name of the file to create.
 * @mode: the permission that the file should have
 * @parent: a pointer to the parent dentry for this file.  This should be a
 *          directory dentry if set.  If this parameter is %NULL, then the
 *          file will be created in the root of the debugfs filesystem.
 * @value: a pointer to the variable that the file should read to and write
 *         from.
 *
 * This function creates a file in debugfs with the given name that
 * contains the value of the variable @value.  If the @mode variable is so
 * set, it can be read from, and written to.
 */
void debugfs_create_ulong(const char *name, umode_t mode, struct dentry *parent,
                          unsigned long *value)
{
        debugfs_create_mode_unsafe(name, mode, parent, value, &fops_ulong,
                                   &fops_ulong_ro, &fops_ulong_wo);
}
EXPORT_SYMBOL_GPL(debugfs_create_ulong);

DEFINE_DEBUGFS_ATTRIBUTE(fops_x8, debugfs_u8_get, debugfs_u8_set, "0x%02llx\n");
DEFINE_DEBUGFS_ATTRIBUTE(fops_x8_ro, debugfs_u8_get, NULL, "0x%02llx\n");
DEFINE_DEBUGFS_ATTRIBUTE(fops_x8_wo, NULL, debugfs_u8_set, "0x%02llx\n");

DEFINE_DEBUGFS_ATTRIBUTE(fops_x16, debugfs_u16_get, debugfs_u16_set,
                        "0x%04llx\n");
DEFINE_DEBUGFS_ATTRIBUTE(fops_x16_ro, debugfs_u16_get, NULL, "0x%04llx\n");
DEFINE_DEBUGFS_ATTRIBUTE(fops_x16_wo, NULL, debugfs_u16_set, "0x%04llx\n");

DEFINE_DEBUGFS_ATTRIBUTE(fops_x32, debugfs_u32_get, debugfs_u32_set,
                        "0x%08llx\n");
DEFINE_DEBUGFS_ATTRIBUTE(fops_x32_ro, debugfs_u32_get, NULL, "0x%08llx\n");
DEFINE_DEBUGFS_ATTRIBUTE(fops_x32_wo, NULL, debugfs_u32_set, "0x%08llx\n");

DEFINE_DEBUGFS_ATTRIBUTE(fops_x64, debugfs_u64_get, debugfs_u64_set,
                        "0x%016llx\n");
DEFINE_DEBUGFS_ATTRIBUTE(fops_x64_ro, debugfs_u64_get, NULL, "0x%016llx\n");
DEFINE_DEBUGFS_ATTRIBUTE(fops_x64_wo, NULL, debugfs_u64_set, "0x%016llx\n");

/*
 * debugfs_create_x{8,16,32,64} - create a debugfs file that is used to read and write an unsigned {8,16,32,64}-bit value
 *
 * These functions are exactly the same as the above functions (but use a hex
 * output for the decimal challenged). For details look at the above unsigned
 * decimal functions.
 */

/**
 * debugfs_create_x8 - create a debugfs file that is used to read and write an unsigned 8-bit value
 * @name: a pointer to a string containing the name of the file to create.
 * @mode: the permission that the file should have
 * @parent: a pointer to the parent dentry for this file.  This should be a
 *          directory dentry if set.  If this parameter is %NULL, then the
 *          file will be created in the root of the debugfs filesystem.
 * @value: a pointer to the variable that the file should read to and write
 *         from.
 */
void debugfs_create_x8(const char *name, umode_t mode, struct dentry *parent,
                       u8 *value)
{
        debugfs_create_mode_unsafe(name, mode, parent, value, &fops_x8,
                                   &fops_x8_ro, &fops_x8_wo);
}
EXPORT_SYMBOL_GPL(debugfs_create_x8);

/**
 * debugfs_create_x16 - create a debugfs file that is used to read and write an unsigned 16-bit value
 * @name: a pointer to a string containing the name of the file to create.
 * @mode: the permission that the file should have
 * @parent: a pointer to the parent dentry for this file.  This should be a
 *          directory dentry if set.  If this parameter is %NULL, then the
 *          file will be created in the root of the debugfs filesystem.
 * @value: a pointer to the variable that the file should read to and write
 *         from.
 */
void debugfs_create_x16(const char *name, umode_t mode, struct dentry *parent,
                        u16 *value)
{
        debugfs_create_mode_unsafe(name, mode, parent, value, &fops_x16,
                                   &fops_x16_ro, &fops_x16_wo);
}
EXPORT_SYMBOL_GPL(debugfs_create_x16);

/**
 * debugfs_create_x32 - create a debugfs file that is used to read and write an unsigned 32-bit value
 * @name: a pointer to a string containing the name of the file to create.
 * @mode: the permission that the file should have
 * @parent: a pointer to the parent dentry for this file.  This should be a
 *          directory dentry if set.  If this parameter is %NULL, then the
 *          file will be created in the root of the debugfs filesystem.
 * @value: a pointer to the variable that the file should read to and write
 *         from.
 */
void debugfs_create_x32(const char *name, umode_t mode, struct dentry *parent,
                        u32 *value)
{
        debugfs_create_mode_unsafe(name, mode, parent, value, &fops_x32,
                                   &fops_x32_ro, &fops_x32_wo);
}
EXPORT_SYMBOL_GPL(debugfs_create_x32);

/**
 * debugfs_create_x64 - create a debugfs file that is used to read and write an unsigned 64-bit value
 * @name: a pointer to a string containing the name of the file to create.
 * @mode: the permission that the file should have
 * @parent: a pointer to the parent dentry for this file.  This should be a
 *          directory dentry if set.  If this parameter is %NULL, then the
 *          file will be created in the root of the debugfs filesystem.
 * @value: a pointer to the variable that the file should read to and write
 *         from.
 */
void debugfs_create_x64(const char *name, umode_t mode, struct dentry *parent,
                        u64 *value)
{
        debugfs_create_mode_unsafe(name, mode, parent, value, &fops_x64,
                                   &fops_x64_ro, &fops_x64_wo);
}
EXPORT_SYMBOL_GPL(debugfs_create_x64);


static int debugfs_size_t_set(void *data, u64 val)
{
        *(size_t *)data = val;
        return 0;
}
static int debugfs_size_t_get(void *data, u64 *val)
{
        *val = *(size_t *)data;
        return 0;
}
DEFINE_DEBUGFS_ATTRIBUTE(fops_size_t, debugfs_size_t_get, debugfs_size_t_set,
                        "%llu\n"); /* %llu and %zu are more or less the same */
DEFINE_DEBUGFS_ATTRIBUTE(fops_size_t_ro, debugfs_size_t_get, NULL, "%llu\n");
DEFINE_DEBUGFS_ATTRIBUTE(fops_size_t_wo, NULL, debugfs_size_t_set, "%llu\n");

/**
 * debugfs_create_size_t - create a debugfs file that is used to read and write an size_t value
 * @name: a pointer to a string containing the name of the file to create.
 * @mode: the permission that the file should have
 * @parent: a pointer to the parent dentry for this file.  This should be a
 *          directory dentry if set.  If this parameter is %NULL, then the
 *          file will be created in the root of the debugfs filesystem.
 * @value: a pointer to the variable that the file should read to and write
 *         from.
 */
void debugfs_create_size_t(const char *name, umode_t mode,
                           struct dentry *parent, size_t *value)
{
        debugfs_create_mode_unsafe(name, mode, parent, value, &fops_size_t,
                                   &fops_size_t_ro, &fops_size_t_wo);
}
EXPORT_SYMBOL_GPL(debugfs_create_size_t);

static int debugfs_atomic_t_set(void *data, u64 val)
{
        atomic_set((atomic_t *)data, val);
        return 0;
}
static int debugfs_atomic_t_get(void *data, u64 *val)
{
        *val = atomic_read((atomic_t *)data);
        return 0;
}
DEFINE_DEBUGFS_ATTRIBUTE_SIGNED(fops_atomic_t, debugfs_atomic_t_get,
                        debugfs_atomic_t_set, "%lld\n");
DEFINE_DEBUGFS_ATTRIBUTE_SIGNED(fops_atomic_t_ro, debugfs_atomic_t_get, NULL,
                        "%lld\n");
DEFINE_DEBUGFS_ATTRIBUTE_SIGNED(fops_atomic_t_wo, NULL, debugfs_atomic_t_set,
                        "%lld\n");

/**
 * debugfs_create_atomic_t - create a debugfs file that is used to read and
 * write an atomic_t value
 * @name: a pointer to a string containing the name of the file to create.
 * @mode: the permission that the file should have
 * @parent: a pointer to the parent dentry for this file.  This should be a
 *          directory dentry if set.  If this parameter is %NULL, then the
 *          file will be created in the root of the debugfs filesystem.
 * @value: a pointer to the variable that the file should read to and write
 *         from.
 */
void debugfs_create_atomic_t(const char *name, umode_t mode,
                             struct dentry *parent, atomic_t *value)
{
        debugfs_create_mode_unsafe(name, mode, parent, value, &fops_atomic_t,
                                   &fops_atomic_t_ro, &fops_atomic_t_wo);
}
EXPORT_SYMBOL_GPL(debugfs_create_atomic_t);

ssize_t debugfs_read_file_bool(struct file *file, char __user *user_buf,
                               size_t count, loff_t *ppos)
{
        char buf[2];
        bool val;
        int r;
        struct dentry *dentry = F_DENTRY(file);

        r = debugfs_file_get(dentry);
        if (unlikely(r))
                return r;
        val = *(bool *)file->private_data;
        debugfs_file_put(dentry);

        if (val)
                buf[0] = 'Y';
        else
                buf[0] = 'N';
        buf[1] = '\n';
        return simple_read_from_buffer(user_buf, count, ppos, buf, 2);
}
EXPORT_SYMBOL_GPL(debugfs_read_file_bool);

ssize_t debugfs_write_file_bool(struct file *file, const char __user *user_buf,
                                size_t count, loff_t *ppos)
{
        bool bv;
        int r;
        bool *val = file->private_data;
        struct dentry *dentry = F_DENTRY(file);

        r = kstrtobool_from_user(user_buf, count, &bv);
        if (!r) {
                r = debugfs_file_get(dentry);
                if (unlikely(r))
                        return r;
                *val = bv;
                debugfs_file_put(dentry);
        }

        return count;
}
EXPORT_SYMBOL_GPL(debugfs_write_file_bool);

static const struct file_operations fops_bool = {
        .read =                debugfs_read_file_bool,
        .write =        debugfs_write_file_bool,
        .open =                simple_open,
        .llseek =        default_llseek,
};

static const struct file_operations fops_bool_ro = {
        .read =                debugfs_read_file_bool,
        .open =                simple_open,
        .llseek =        default_llseek,
};

static const struct file_operations fops_bool_wo = {
        .write =        debugfs_write_file_bool,
        .open =                simple_open,
        .llseek =        default_llseek,
};

/**
 * debugfs_create_bool - create a debugfs file that is used to read and write a boolean value
 * @name: a pointer to a string containing the name of the file to create.
 * @mode: the permission that the file should have
 * @parent: a pointer to the parent dentry for this file.  This should be a
 *          directory dentry if set.  If this parameter is %NULL, then the
 *          file will be created in the root of the debugfs filesystem.
 * @value: a pointer to the variable that the file should read to and write
 *         from.
 *
 * This function creates a file in debugfs with the given name that
 * contains the value of the variable @value.  If the @mode variable is so
 * set, it can be read from, and written to.
 */
void debugfs_create_bool(const char *name, umode_t mode, struct dentry *parent,
                         bool *value)
{
        debugfs_create_mode_unsafe(name, mode, parent, value, &fops_bool,
                                   &fops_bool_ro, &fops_bool_wo);
}
EXPORT_SYMBOL_GPL(debugfs_create_bool);

ssize_t debugfs_read_file_str(struct file *file, char __user *user_buf,
                              size_t count, loff_t *ppos)
{
        struct dentry *dentry = F_DENTRY(file);
        char *str, *copy = NULL;
        int copy_len, len;
        ssize_t ret;

        ret = debugfs_file_get(dentry);
        if (unlikely(ret))
                return ret;

        str = *(char **)file->private_data;
        len = strlen(str) + 1;
        copy = kmalloc(len, GFP_KERNEL);
        if (!copy) {
                debugfs_file_put(dentry);
                return -ENOMEM;
        }

        copy_len = strscpy(copy, str, len);
        debugfs_file_put(dentry);
        if (copy_len < 0) {
                kfree(copy);
                return copy_len;
        }

        copy[copy_len] = '\n';

        ret = simple_read_from_buffer(user_buf, count, ppos, copy, len);
        kfree(copy);

        return ret;
}
EXPORT_SYMBOL_GPL(debugfs_create_str);

static ssize_t debugfs_write_file_str(struct file *file, const char __user *user_buf,
                                      size_t count, loff_t *ppos)
{
        struct dentry *dentry = F_DENTRY(file);
        char *old, *new = NULL;
        int pos = *ppos;
        int r;

        r = debugfs_file_get(dentry);
        if (unlikely(r))
                return r;

        old = *(char **)file->private_data;

        /* only allow strict concatenation */
        r = -EINVAL;
        if (pos && pos != strlen(old))
                goto error;

        r = -E2BIG;
        if (pos + count + 1 > PAGE_SIZE)
                goto error;

        r = -ENOMEM;
        new = kmalloc(pos + count + 1, GFP_KERNEL);
        if (!new)
                goto error;

        if (pos)
                memcpy(new, old, pos);

        r = -EFAULT;
        if (copy_from_user(new + pos, user_buf, count))
                goto error;

        new[pos + count] = '\0';
        strim(new);

        rcu_assign_pointer(*(char __rcu **)file->private_data, new);
        synchronize_rcu();
        kfree(old);

        debugfs_file_put(dentry);
        return count;

error:
        kfree(new);
        debugfs_file_put(dentry);
        return r;
}

static const struct file_operations fops_str = {
        .read =                debugfs_read_file_str,
        .write =        debugfs_write_file_str,
        .open =                simple_open,
        .llseek =        default_llseek,
};

static const struct file_operations fops_str_ro = {
        .read =                debugfs_read_file_str,
        .open =                simple_open,
        .llseek =        default_llseek,
};

static const struct file_operations fops_str_wo = {
        .write =        debugfs_write_file_str,
        .open =                simple_open,
        .llseek =        default_llseek,
};

/**
 * debugfs_create_str - create a debugfs file that is used to read and write a string value
 * @name: a pointer to a string containing the name of the file to create.
 * @mode: the permission that the file should have
 * @parent: a pointer to the parent dentry for this file.  This should be a
 *          directory dentry if set.  If this parameter is %NULL, then the
 *          file will be created in the root of the debugfs filesystem.
 * @value: a pointer to the variable that the file should read to and write
 *         from.
 *
 * This function creates a file in debugfs with the given name that
 * contains the value of the variable @value.  If the @mode variable is so
 * set, it can be read from, and written to.
 */
void debugfs_create_str(const char *name, umode_t mode,
                        struct dentry *parent, char **value)
{
        debugfs_create_mode_unsafe(name, mode, parent, value, &fops_str,
                                   &fops_str_ro, &fops_str_wo);
}

static ssize_t read_file_blob(struct file *file, char __user *user_buf,
                              size_t count, loff_t *ppos)
{
        struct debugfs_blob_wrapper *blob = file->private_data;
        struct dentry *dentry = F_DENTRY(file);
        ssize_t r;

        r = debugfs_file_get(dentry);
        if (unlikely(r))
                return r;
        r = simple_read_from_buffer(user_buf, count, ppos, blob->data,
                                blob->size);
        debugfs_file_put(dentry);
        return r;
}

static ssize_t write_file_blob(struct file *file, const char __user *user_buf,
                               size_t count, loff_t *ppos)
{
        struct debugfs_blob_wrapper *blob = file->private_data;
        struct dentry *dentry = F_DENTRY(file);
        ssize_t r;

        r = debugfs_file_get(dentry);
        if (unlikely(r))
                return r;
        r = simple_write_to_buffer(blob->data, blob->size, ppos, user_buf,
                                   count);

        debugfs_file_put(dentry);
        return r;
}

static const struct file_operations fops_blob = {
        .read =                read_file_blob,
        .write =        write_file_blob,
        .open =                simple_open,
        .llseek =        default_llseek,
};

/**
 * debugfs_create_blob - create a debugfs file that is used to read and write
 * a binary blob
 * @name: a pointer to a string containing the name of the file to create.
 * @mode: the permission that the file should have
 * @parent: a pointer to the parent dentry for this file.  This should be a
 *          directory dentry if set.  If this parameter is %NULL, then the
 *          file will be created in the root of the debugfs filesystem.
 * @blob: a pointer to a struct debugfs_blob_wrapper which contains a pointer
 *        to the blob data and the size of the data.
 *
 * This function creates a file in debugfs with the given name that exports
 * @blob->data as a binary blob. If the @mode variable is so set it can be
 * read from and written to.
 *
 * This function will return a pointer to a dentry if it succeeds.  This
 * pointer must be passed to the debugfs_remove() function when the file is
 * to be removed (no automatic cleanup happens if your module is unloaded,
 * you are responsible here.)  If an error occurs, ERR_PTR(-ERROR) will be
 * returned.
 *
 * If debugfs is not enabled in the kernel, the value ERR_PTR(-ENODEV) will
 * be returned.
 */
struct dentry *debugfs_create_blob(const char *name, umode_t mode,
                                   struct dentry *parent,
                                   struct debugfs_blob_wrapper *blob)
{
        return debugfs_create_file_unsafe(name, mode & 0644, parent, blob, &fops_blob);
}
EXPORT_SYMBOL_GPL(debugfs_create_blob);

static size_t u32_format_array(char *buf, size_t bufsize,
                               u32 *array, int array_size)
{
        size_t ret = 0;

        while (--array_size >= 0) {
                size_t len;
                char term = array_size ? ' ' : '\n';

                len = snprintf(buf, bufsize, "%u%c", *array++, term);
                ret += len;

                buf += len;
                bufsize -= len;
        }
        return ret;
}

static int u32_array_open(struct inode *inode, struct file *file)
{
        struct debugfs_u32_array *data = inode->i_private;
        int size, elements = data->n_elements;
        char *buf;

        /*
         * Max size:
         *  - 10 digits + ' '/'\n' = 11 bytes per number
         *  - terminating NUL character
         */
        size = elements*11;
        buf = kmalloc(size+1, GFP_KERNEL);
        if (!buf)
                return -ENOMEM;
        buf[size] = 0;

        file->private_data = buf;
        u32_format_array(buf, size, data->array, data->n_elements);

        return nonseekable_open(inode, file);
}

static ssize_t u32_array_read(struct file *file, char __user *buf, size_t len,
                              loff_t *ppos)
{
        size_t size = strlen(file->private_data);

        return simple_read_from_buffer(buf, len, ppos,
                                        file->private_data, size);
}

static int u32_array_release(struct inode *inode, struct file *file)
{
        kfree(file->private_data);

        return 0;
}

static const struct file_operations u32_array_fops = {
        .owner         = THIS_MODULE,
        .open         = u32_array_open,
        .release = u32_array_release,
        .read         = u32_array_read,
};

/**
 * debugfs_create_u32_array - create a debugfs file that is used to read u32
 * array.
 * @name: a pointer to a string containing the name of the file to create.
 * @mode: the permission that the file should have.
 * @parent: a pointer to the parent dentry for this file.  This should be a
 *          directory dentry if set.  If this parameter is %NULL, then the
 *          file will be created in the root of the debugfs filesystem.
 * @array: wrapper struct containing data pointer and size of the array.
 *
 * This function creates a file in debugfs with the given name that exports
 * @array as data. If the @mode variable is so set it can be read from.
 * Writing is not supported. Seek within the file is also not supported.
 * Once array is created its size can not be changed.
 */
void debugfs_create_u32_array(const char *name, umode_t mode,
                              struct dentry *parent,
                              struct debugfs_u32_array *array)
{
        debugfs_create_file_unsafe(name, mode, parent, array, &u32_array_fops);
}
EXPORT_SYMBOL_GPL(debugfs_create_u32_array);

#ifdef CONFIG_HAS_IOMEM

/*
 * The regset32 stuff is used to print 32-bit registers using the
 * seq_file utilities. We offer printing a register set in an already-opened
 * sequential file or create a debugfs file that only prints a regset32.
 */

/**
 * debugfs_print_regs32 - use seq_print to describe a set of registers
 * @s: the seq_file structure being used to generate output
 * @regs: an array if struct debugfs_reg32 structures
 * @nregs: the length of the above array
 * @base: the base address to be used in reading the registers
 * @prefix: a string to be prefixed to every output line
 *
 * This function outputs a text block describing the current values of
 * some 32-bit hardware registers. It is meant to be used within debugfs
 * files based on seq_file that need to show registers, intermixed with other
 * information. The prefix argument may be used to specify a leading string,
 * because some peripherals have several blocks of identical registers,
 * for example configuration of dma channels
 */
void debugfs_print_regs32(struct seq_file *s, const struct debugfs_reg32 *regs,
                          int nregs, void __iomem *base, char *prefix)
{
        int i;

        for (i = 0; i < nregs; i++, regs++) {
                if (prefix)
                        seq_printf(s, "%s", prefix);
                seq_printf(s, "%s = 0x%08x\n", regs->name,
                           readl(base + regs->offset));
                if (seq_has_overflowed(s))
                        break;
        }
}
EXPORT_SYMBOL_GPL(debugfs_print_regs32);

static int debugfs_regset32_show(struct seq_file *s, void *data)
{
        struct debugfs_regset32 *regset = s->private;

        if (regset->dev)
                pm_runtime_get_sync(regset->dev);

        debugfs_print_regs32(s, regset->regs, regset->nregs, regset->base, "");

        if (regset->dev)
                pm_runtime_put(regset->dev);

        return 0;
}

DEFINE_SHOW_ATTRIBUTE(debugfs_regset32);

/**
 * debugfs_create_regset32 - create a debugfs file that returns register values
 * @name: a pointer to a string containing the name of the file to create.
 * @mode: the permission that the file should have
 * @parent: a pointer to the parent dentry for this file.  This should be a
 *          directory dentry if set.  If this parameter is %NULL, then the
 *          file will be created in the root of the debugfs filesystem.
 * @regset: a pointer to a struct debugfs_regset32, which contains a pointer
 *          to an array of register definitions, the array size and the base
 *          address where the register bank is to be found.
 *
 * This function creates a file in debugfs with the given name that reports
 * the names and values of a set of 32-bit registers. If the @mode variable
 * is so set it can be read from. Writing is not supported.
 */
void debugfs_create_regset32(const char *name, umode_t mode,
                             struct dentry *parent,
                             struct debugfs_regset32 *regset)
{
        debugfs_create_file(name, mode, parent, regset, &debugfs_regset32_fops);
}
EXPORT_SYMBOL_GPL(debugfs_create_regset32);

#endif /* CONFIG_HAS_IOMEM */

struct debugfs_devm_entry {
        int (*read)(struct seq_file *seq, void *data);
        struct device *dev;
};

static int debugfs_devm_entry_open(struct inode *inode, struct file *f)
{
        struct debugfs_devm_entry *entry = inode->i_private;

        return single_open(f, entry->read, entry->dev);
}

static const struct file_operations debugfs_devm_entry_ops = {
        .owner = THIS_MODULE,
        .open = debugfs_devm_entry_open,
        .release = single_release,
        .read = seq_read,
        .llseek = seq_lseek
};

/**
 * debugfs_create_devm_seqfile - create a debugfs file that is bound to device.
 *
 * @dev: device related to this debugfs file.
 * @name: name of the debugfs file.
 * @parent: a pointer to the parent dentry for this file.  This should be a
 *        directory dentry if set.  If this parameter is %NULL, then the
 *        file will be created in the root of the debugfs filesystem.
 * @read_fn: function pointer called to print the seq_file content.
 */
void debugfs_create_devm_seqfile(struct device *dev, const char *name,
                                 struct dentry *parent,
                                 int (*read_fn)(struct seq_file *s, void *data))
{
        struct debugfs_devm_entry *entry;

        if (IS_ERR(parent))
                return;

        entry = devm_kzalloc(dev, sizeof(*entry), GFP_KERNEL);
        if (!entry)
                return;

        entry->read = read_fn;
        entry->dev = dev;

        debugfs_create_file(name, S_IRUGO, parent, entry,
                            &debugfs_devm_entry_ops);
}
EXPORT_SYMBOL_GPL(debugfs_create_devm_seqfile);












































  149 
  187 




   53 


  143 






  191 



  192 


   58 


  191 




  190 


  180 



  180 
   13 










   80 



   79 










































   83 













   82 





   84 






   83 
































   76 




























    3 








   76 

   76 







  189 















  180 


    1 


    2 

   83 






















  156 


    1 

   82 




  188 
    2 
  188 

    1 
   83 











   75 
    8 








  191 














  192 
  155 


   83 


   73 

    8 











   44 
   39 
















   32 



    6 
















   38 




   38 







   38 


















   26 



























   11 













   12 













































  152 














  153 













































   45 

  173 



































  151 






  153 
    3 





  150 








    1 


















   60 


    7 


   12 



   44 
   12 
   45 
   45 





   12 



    5 
    7 





















1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * Scatterlist Cryptographic API.
 *
 * Copyright (c) 2002 James Morris <jmorris@intercode.com.au>
 * Copyright (c) 2002 David S. Miller (davem@redhat.com)
 * Copyright (c) 2005 Herbert Xu <herbert@gondor.apana.org.au>
 *
 * Portions derived from Cryptoapi, by Alexander Kjeldaas <astor@fast.no>
 * and Nettle, by Niels Möller.
 */

#include <linux/err.h>
#include <linux/errno.h>
#include <linux/jump_label.h>
#include <linux/kernel.h>
#include <linux/kmod.h>
#include <linux/module.h>
#include <linux/param.h>
#include <linux/sched/signal.h>
#include <linux/slab.h>
#include <linux/string.h>
#include <linux/completion.h>
#include "internal.h"

LIST_HEAD(crypto_alg_list);
EXPORT_SYMBOL_GPL(crypto_alg_list);
DECLARE_RWSEM(crypto_alg_sem);
EXPORT_SYMBOL_GPL(crypto_alg_sem);

BLOCKING_NOTIFIER_HEAD(crypto_chain);
EXPORT_SYMBOL_GPL(crypto_chain);

#if IS_BUILTIN(CONFIG_CRYPTO_ALGAPI) && \
    !IS_ENABLED(CONFIG_CRYPTO_MANAGER_DISABLE_TESTS)
DEFINE_STATIC_KEY_FALSE(__crypto_boot_test_finished);
#endif

static struct crypto_alg *crypto_larval_wait(struct crypto_alg *alg);
static struct crypto_alg *crypto_alg_lookup(const char *name, u32 type,
                                            u32 mask);

struct crypto_alg *crypto_mod_get(struct crypto_alg *alg)
{
        return try_module_get(alg->cra_module) ? crypto_alg_get(alg) : NULL;
}
EXPORT_SYMBOL_GPL(crypto_mod_get);

void crypto_mod_put(struct crypto_alg *alg)
{
        struct module *module = alg->cra_module;

        crypto_alg_put(alg);
        module_put(module);
}
EXPORT_SYMBOL_GPL(crypto_mod_put);

static struct crypto_alg *__crypto_alg_lookup(const char *name, u32 type,
                                              u32 mask)
{
        struct crypto_alg *q, *alg = NULL;
        int best = -2;

        list_for_each_entry(q, &crypto_alg_list, cra_list) {
                int exact, fuzzy;

                if (crypto_is_moribund(q))
                        continue;

                if ((q->cra_flags ^ type) & mask)
                        continue;

                exact = !strcmp(q->cra_driver_name, name);
                fuzzy = !strcmp(q->cra_name, name);
                if (!exact && !(fuzzy && q->cra_priority > best))
                        continue;

                if (unlikely(!crypto_mod_get(q)))
                        continue;

                best = q->cra_priority;
                if (alg)
                        crypto_mod_put(alg);
                alg = q;

                if (exact)
                        break;
        }

        return alg;
}

static void crypto_larval_destroy(struct crypto_alg *alg)
{
        struct crypto_larval *larval = (void *)alg;

        BUG_ON(!crypto_is_larval(alg));
        if (!IS_ERR_OR_NULL(larval->adult))
                crypto_mod_put(larval->adult);
        kfree(larval);
}

struct crypto_larval *crypto_larval_alloc(const char *name, u32 type, u32 mask)
{
        struct crypto_larval *larval;

        larval = kzalloc(sizeof(*larval), GFP_KERNEL);
        if (!larval)
                return ERR_PTR(-ENOMEM);

        type &= ~CRYPTO_ALG_TYPE_MASK | (mask ?: CRYPTO_ALG_TYPE_MASK);

        larval->mask = mask;
        larval->alg.cra_flags = CRYPTO_ALG_LARVAL | type;
        larval->alg.cra_priority = -1;
        larval->alg.cra_destroy = crypto_larval_destroy;

        strscpy(larval->alg.cra_name, name, CRYPTO_MAX_ALG_NAME);
        init_completion(&larval->completion);

        return larval;
}
EXPORT_SYMBOL_GPL(crypto_larval_alloc);

static struct crypto_alg *crypto_larval_add(const char *name, u32 type,
                                            u32 mask)
{
        struct crypto_alg *alg;
        struct crypto_larval *larval;

        larval = crypto_larval_alloc(name, type, mask);
        if (IS_ERR(larval))
                return ERR_CAST(larval);

        refcount_set(&larval->alg.cra_refcnt, 2);

        down_write(&crypto_alg_sem);
        alg = __crypto_alg_lookup(name, type, mask);
        if (!alg) {
                alg = &larval->alg;
                list_add(&alg->cra_list, &crypto_alg_list);
        }
        up_write(&crypto_alg_sem);

        if (alg != &larval->alg) {
                kfree(larval);
                if (crypto_is_larval(alg))
                        alg = crypto_larval_wait(alg);
        }

        return alg;
}

static void crypto_larval_kill(struct crypto_larval *larval)
{
        bool unlinked;

        down_write(&crypto_alg_sem);
        unlinked = list_empty(&larval->alg.cra_list);
        if (!unlinked)
                list_del_init(&larval->alg.cra_list);
        up_write(&crypto_alg_sem);

        if (unlinked)
                return;

        complete_all(&larval->completion);
        crypto_alg_put(&larval->alg);
}

void crypto_schedule_test(struct crypto_larval *larval)
{
        int err;

        err = crypto_probing_notify(CRYPTO_MSG_ALG_REGISTER, larval->adult);
        WARN_ON_ONCE(err != NOTIFY_STOP);
}
EXPORT_SYMBOL_GPL(crypto_schedule_test);

static void crypto_start_test(struct crypto_larval *larval)
{
        if (!crypto_is_test_larval(larval))
                return;

        if (larval->test_started)
                return;

        down_write(&crypto_alg_sem);
        if (larval->test_started) {
                up_write(&crypto_alg_sem);
                return;
        }

        larval->test_started = true;
        up_write(&crypto_alg_sem);

        crypto_schedule_test(larval);
}

static struct crypto_alg *crypto_larval_wait(struct crypto_alg *alg)
{
        struct crypto_larval *larval;
        long time_left;

again:
        larval = container_of(alg, struct crypto_larval, alg);

        if (!crypto_boot_test_finished())
                crypto_start_test(larval);

        time_left = wait_for_completion_killable_timeout(
                &larval->completion, 60 * HZ);

        alg = larval->adult;
        if (time_left < 0)
                alg = ERR_PTR(-EINTR);
        else if (!time_left) {
                if (crypto_is_test_larval(larval))
                        crypto_larval_kill(larval);
                alg = ERR_PTR(-ETIMEDOUT);
        } else if (!alg) {
                u32 type;
                u32 mask;

                alg = &larval->alg;
                type = alg->cra_flags & ~(CRYPTO_ALG_LARVAL | CRYPTO_ALG_DEAD);
                mask = larval->mask;
                alg = crypto_alg_lookup(alg->cra_name, type, mask) ?:
                      ERR_PTR(-EAGAIN);
        } else if (IS_ERR(alg))
                ;
        else if (crypto_is_test_larval(larval) &&
                 !(alg->cra_flags & CRYPTO_ALG_TESTED))
                alg = ERR_PTR(-EAGAIN);
        else if (alg->cra_flags & CRYPTO_ALG_FIPS_INTERNAL)
                alg = ERR_PTR(-EAGAIN);
        else if (!crypto_mod_get(alg))
                alg = ERR_PTR(-EAGAIN);
        crypto_mod_put(&larval->alg);

        if (!IS_ERR(alg) && crypto_is_larval(alg))
                goto again;

        return alg;
}

static struct crypto_alg *crypto_alg_lookup(const char *name, u32 type,
                                            u32 mask)
{
        const u32 fips = CRYPTO_ALG_FIPS_INTERNAL;
        struct crypto_alg *alg;
        u32 test = 0;

        if (!((type | mask) & CRYPTO_ALG_TESTED))
                test |= CRYPTO_ALG_TESTED;

        down_read(&crypto_alg_sem);
        alg = __crypto_alg_lookup(name, (type | test) & ~fips,
                                  (mask | test) & ~fips);
        if (alg) {
                if (((type | mask) ^ fips) & fips)
                        mask |= fips;
                mask &= fips;

                if (!crypto_is_larval(alg) &&
                    ((type ^ alg->cra_flags) & mask)) {
                        /* Algorithm is disallowed in FIPS mode. */
                        crypto_mod_put(alg);
                        alg = ERR_PTR(-ENOENT);
                }
        } else if (test) {
                alg = __crypto_alg_lookup(name, type, mask);
                if (alg && !crypto_is_larval(alg)) {
                        /* Test failed */
                        crypto_mod_put(alg);
                        alg = ERR_PTR(-ELIBBAD);
                }
        }
        up_read(&crypto_alg_sem);

        return alg;
}

static struct crypto_alg *crypto_larval_lookup(const char *name, u32 type,
                                               u32 mask)
{
        struct crypto_alg *alg;

        if (!name)
                return ERR_PTR(-ENOENT);

        type &= ~(CRYPTO_ALG_LARVAL | CRYPTO_ALG_DEAD);
        mask &= ~(CRYPTO_ALG_LARVAL | CRYPTO_ALG_DEAD);

        alg = crypto_alg_lookup(name, type, mask);
        if (!alg && !(mask & CRYPTO_NOLOAD)) {
                request_module("crypto-%s", name);

                if (!((type ^ CRYPTO_ALG_NEED_FALLBACK) & mask &
                      CRYPTO_ALG_NEED_FALLBACK))
                        request_module("crypto-%s-all", name);

                alg = crypto_alg_lookup(name, type, mask);
        }

        if (!IS_ERR_OR_NULL(alg) && crypto_is_larval(alg))
                alg = crypto_larval_wait(alg);
        else if (alg)
                ;
        else if (!(mask & CRYPTO_ALG_TESTED))
                alg = crypto_larval_add(name, type, mask);
        else
                alg = ERR_PTR(-ENOENT);

        return alg;
}

int crypto_probing_notify(unsigned long val, void *v)
{
        int ok;

        ok = blocking_notifier_call_chain(&crypto_chain, val, v);
        if (ok == NOTIFY_DONE) {
                request_module("cryptomgr");
                ok = blocking_notifier_call_chain(&crypto_chain, val, v);
        }

        return ok;
}
EXPORT_SYMBOL_GPL(crypto_probing_notify);

struct crypto_alg *crypto_alg_mod_lookup(const char *name, u32 type, u32 mask)
{
        struct crypto_alg *alg;
        struct crypto_alg *larval;
        int ok;

        /*
         * If the internal flag is set for a cipher, require a caller to
         * invoke the cipher with the internal flag to use that cipher.
         * Also, if a caller wants to allocate a cipher that may or may
         * not be an internal cipher, use type | CRYPTO_ALG_INTERNAL and
         * !(mask & CRYPTO_ALG_INTERNAL).
         */
        if (!((type | mask) & CRYPTO_ALG_INTERNAL))
                mask |= CRYPTO_ALG_INTERNAL;

        larval = crypto_larval_lookup(name, type, mask);
        if (IS_ERR(larval) || !crypto_is_larval(larval))
                return larval;

        ok = crypto_probing_notify(CRYPTO_MSG_ALG_REQUEST, larval);

        if (ok == NOTIFY_STOP)
                alg = crypto_larval_wait(larval);
        else {
                crypto_mod_put(larval);
                alg = ERR_PTR(-ENOENT);
        }
        crypto_larval_kill(container_of(larval, struct crypto_larval, alg));
        return alg;
}
EXPORT_SYMBOL_GPL(crypto_alg_mod_lookup);

static void crypto_exit_ops(struct crypto_tfm *tfm)
{
        const struct crypto_type *type = tfm->__crt_alg->cra_type;

        if (type && tfm->exit)
                tfm->exit(tfm);
}

static unsigned int crypto_ctxsize(struct crypto_alg *alg, u32 type, u32 mask)
{
        const struct crypto_type *type_obj = alg->cra_type;
        unsigned int len;

        len = alg->cra_alignmask & ~(crypto_tfm_ctx_alignment() - 1);
        if (type_obj)
                return len + type_obj->ctxsize(alg, type, mask);

        switch (alg->cra_flags & CRYPTO_ALG_TYPE_MASK) {
        default:
                BUG();

        case CRYPTO_ALG_TYPE_CIPHER:
                len += crypto_cipher_ctxsize(alg);
                break;

        case CRYPTO_ALG_TYPE_COMPRESS:
                len += crypto_compress_ctxsize(alg);
                break;
        }

        return len;
}

void crypto_shoot_alg(struct crypto_alg *alg)
{
        down_write(&crypto_alg_sem);
        alg->cra_flags |= CRYPTO_ALG_DYING;
        up_write(&crypto_alg_sem);
}
EXPORT_SYMBOL_GPL(crypto_shoot_alg);

struct crypto_tfm *__crypto_alloc_tfmgfp(struct crypto_alg *alg, u32 type,
                                         u32 mask, gfp_t gfp)
{
        struct crypto_tfm *tfm;
        unsigned int tfm_size;
        int err = -ENOMEM;

        tfm_size = sizeof(*tfm) + crypto_ctxsize(alg, type, mask);
        tfm = kzalloc(tfm_size, gfp);
        if (tfm == NULL)
                goto out_err;

        tfm->__crt_alg = alg;
        refcount_set(&tfm->refcnt, 1);

        if (!tfm->exit && alg->cra_init && (err = alg->cra_init(tfm)))
                goto cra_init_failed;

        goto out;

cra_init_failed:
        crypto_exit_ops(tfm);
        if (err == -EAGAIN)
                crypto_shoot_alg(alg);
        kfree(tfm);
out_err:
        tfm = ERR_PTR(err);
out:
        return tfm;
}
EXPORT_SYMBOL_GPL(__crypto_alloc_tfmgfp);

struct crypto_tfm *__crypto_alloc_tfm(struct crypto_alg *alg, u32 type,
                                      u32 mask)
{
        return __crypto_alloc_tfmgfp(alg, type, mask, GFP_KERNEL);
}
EXPORT_SYMBOL_GPL(__crypto_alloc_tfm);

/*
 *        crypto_alloc_base - Locate algorithm and allocate transform
 *        @alg_name: Name of algorithm
 *        @type: Type of algorithm
 *        @mask: Mask for type comparison
 *
 *        This function should not be used by new algorithm types.
 *        Please use crypto_alloc_tfm instead.
 *
 *        crypto_alloc_base() will first attempt to locate an already loaded
 *        algorithm.  If that fails and the kernel supports dynamically loadable
 *        modules, it will then attempt to load a module of the same name or
 *        alias.  If that fails it will send a query to any loaded crypto manager
 *        to construct an algorithm on the fly.  A refcount is grabbed on the
 *        algorithm which is then associated with the new transform.
 *
 *        The returned transform is of a non-determinate type.  Most people
 *        should use one of the more specific allocation functions such as
 *        crypto_alloc_skcipher().
 *
 *        In case of error the return value is an error pointer.
 */
struct crypto_tfm *crypto_alloc_base(const char *alg_name, u32 type, u32 mask)
{
        struct crypto_tfm *tfm;
        int err;

        for (;;) {
                struct crypto_alg *alg;

                alg = crypto_alg_mod_lookup(alg_name, type, mask);
                if (IS_ERR(alg)) {
                        err = PTR_ERR(alg);
                        goto err;
                }

                tfm = __crypto_alloc_tfm(alg, type, mask);
                if (!IS_ERR(tfm))
                        return tfm;

                crypto_mod_put(alg);
                err = PTR_ERR(tfm);

err:
                if (err != -EAGAIN)
                        break;
                if (fatal_signal_pending(current)) {
                        err = -EINTR;
                        break;
                }
        }

        return ERR_PTR(err);
}
EXPORT_SYMBOL_GPL(crypto_alloc_base);

static void *crypto_alloc_tfmmem(struct crypto_alg *alg,
                                 const struct crypto_type *frontend, int node,
                                 gfp_t gfp)
{
        struct crypto_tfm *tfm;
        unsigned int tfmsize;
        unsigned int total;
        char *mem;

        tfmsize = frontend->tfmsize;
        total = tfmsize + sizeof(*tfm) + frontend->extsize(alg);

        mem = kzalloc_node(total, gfp, node);
        if (mem == NULL)
                return ERR_PTR(-ENOMEM);

        tfm = (struct crypto_tfm *)(mem + tfmsize);
        tfm->__crt_alg = alg;
        tfm->node = node;
        refcount_set(&tfm->refcnt, 1);

        return mem;
}

void *crypto_create_tfm_node(struct crypto_alg *alg,
                             const struct crypto_type *frontend,
                             int node)
{
        struct crypto_tfm *tfm;
        char *mem;
        int err;

        mem = crypto_alloc_tfmmem(alg, frontend, node, GFP_KERNEL);
        if (IS_ERR(mem))
                goto out;

        tfm = (struct crypto_tfm *)(mem + frontend->tfmsize);

        err = frontend->init_tfm(tfm);
        if (err)
                goto out_free_tfm;

        if (!tfm->exit && alg->cra_init && (err = alg->cra_init(tfm)))
                goto cra_init_failed;

        goto out;

cra_init_failed:
        crypto_exit_ops(tfm);
out_free_tfm:
        if (err == -EAGAIN)
                crypto_shoot_alg(alg);
        kfree(mem);
        mem = ERR_PTR(err);
out:
        return mem;
}
EXPORT_SYMBOL_GPL(crypto_create_tfm_node);

void *crypto_clone_tfm(const struct crypto_type *frontend,
                       struct crypto_tfm *otfm)
{
        struct crypto_alg *alg = otfm->__crt_alg;
        struct crypto_tfm *tfm;
        char *mem;

        mem = ERR_PTR(-ESTALE);
        if (unlikely(!crypto_mod_get(alg)))
                goto out;

        mem = crypto_alloc_tfmmem(alg, frontend, otfm->node, GFP_ATOMIC);
        if (IS_ERR(mem)) {
                crypto_mod_put(alg);
                goto out;
        }

        tfm = (struct crypto_tfm *)(mem + frontend->tfmsize);
        tfm->crt_flags = otfm->crt_flags;
        tfm->exit = otfm->exit;

out:
        return mem;
}
EXPORT_SYMBOL_GPL(crypto_clone_tfm);

struct crypto_alg *crypto_find_alg(const char *alg_name,
                                   const struct crypto_type *frontend,
                                   u32 type, u32 mask)
{
        if (frontend) {
                type &= frontend->maskclear;
                mask &= frontend->maskclear;
                type |= frontend->type;
                mask |= frontend->maskset;
        }

        return crypto_alg_mod_lookup(alg_name, type, mask);
}
EXPORT_SYMBOL_GPL(crypto_find_alg);

/*
 *        crypto_alloc_tfm_node - Locate algorithm and allocate transform
 *        @alg_name: Name of algorithm
 *        @frontend: Frontend algorithm type
 *        @type: Type of algorithm
 *        @mask: Mask for type comparison
 *        @node: NUMA node in which users desire to put requests, if node is
 *                NUMA_NO_NODE, it means users have no special requirement.
 *
 *        crypto_alloc_tfm() will first attempt to locate an already loaded
 *        algorithm.  If that fails and the kernel supports dynamically loadable
 *        modules, it will then attempt to load a module of the same name or
 *        alias.  If that fails it will send a query to any loaded crypto manager
 *        to construct an algorithm on the fly.  A refcount is grabbed on the
 *        algorithm which is then associated with the new transform.
 *
 *        The returned transform is of a non-determinate type.  Most people
 *        should use one of the more specific allocation functions such as
 *        crypto_alloc_skcipher().
 *
 *        In case of error the return value is an error pointer.
 */

void *crypto_alloc_tfm_node(const char *alg_name,
                       const struct crypto_type *frontend, u32 type, u32 mask,
                       int node)
{
        void *tfm;
        int err;

        for (;;) {
                struct crypto_alg *alg;

                alg = crypto_find_alg(alg_name, frontend, type, mask);
                if (IS_ERR(alg)) {
                        err = PTR_ERR(alg);
                        goto err;
                }

                tfm = crypto_create_tfm_node(alg, frontend, node);
                if (!IS_ERR(tfm))
                        return tfm;

                crypto_mod_put(alg);
                err = PTR_ERR(tfm);

err:
                if (err != -EAGAIN)
                        break;
                if (fatal_signal_pending(current)) {
                        err = -EINTR;
                        break;
                }
        }

        return ERR_PTR(err);
}
EXPORT_SYMBOL_GPL(crypto_alloc_tfm_node);

/*
 *        crypto_destroy_tfm - Free crypto transform
 *        @mem: Start of tfm slab
 *        @tfm: Transform to free
 *
 *        This function frees up the transform and any associated resources,
 *        then drops the refcount on the associated algorithm.
 */
void crypto_destroy_tfm(void *mem, struct crypto_tfm *tfm)
{
        struct crypto_alg *alg;

        if (IS_ERR_OR_NULL(mem))
                return;

        if (!refcount_dec_and_test(&tfm->refcnt))
                return;
        alg = tfm->__crt_alg;

        if (!tfm->exit && alg->cra_exit)
                alg->cra_exit(tfm);
        crypto_exit_ops(tfm);
        crypto_mod_put(alg);
        kfree_sensitive(mem);
}
EXPORT_SYMBOL_GPL(crypto_destroy_tfm);

int crypto_has_alg(const char *name, u32 type, u32 mask)
{
        int ret = 0;
        struct crypto_alg *alg = crypto_alg_mod_lookup(name, type, mask);

        if (!IS_ERR(alg)) {
                crypto_mod_put(alg);
                ret = 1;
        }

        return ret;
}
EXPORT_SYMBOL_GPL(crypto_has_alg);

void crypto_req_done(void *data, int err)
{
        struct crypto_wait *wait = data;

        if (err == -EINPROGRESS)
                return;

        wait->err = err;
        complete(&wait->completion);
}
EXPORT_SYMBOL_GPL(crypto_req_done);

MODULE_DESCRIPTION("Cryptographic core API");
MODULE_LICENSE("GPL");

























    1 


    1 



    1 




















   10 
















   13 

   15 












   12 

























    1 



    1 






























    2 





    2 





















































    2 

    1 






















































   12 

   12 



    4 
   12 












   15 







   15 

   15 



   13 






   13 




   14 
    1 









   14 



    1 






   14 







    1 











   14 










    9 



    5 







    4 









   11 
    3 
    8 

    4 









   12 


   13 



   10 
    6 






    1 





    4 









    1 



    3 

    2 






















    4 


   18 
    2 






   15 


    1 






    1 


   17 





    1 


    1 


    2 




    1 


    2 





    2 





    2 




    1 


    2 




    1 



    1 

    4 

    3 








    4 



























    1 






































   42 

















   42 



   17 












    1 






   15 


    5 


    2 


















    4 









   16 




















    1 













    2 
   15 





























































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
// SPDX-License-Identifier: GPL-2.0-or-later
/* AF_RXRPC sendmsg() implementation.
 *
 * Copyright (C) 2007, 2016 Red Hat, Inc. All Rights Reserved.
 * Written by David Howells (dhowells@redhat.com)
 */

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#include <linux/net.h>
#include <linux/gfp.h>
#include <linux/skbuff.h>
#include <linux/export.h>
#include <linux/sched/signal.h>

#include <net/sock.h>
#include <net/af_rxrpc.h>
#include "ar-internal.h"

/*
 * Propose an abort to be made in the I/O thread.
 */
bool rxrpc_propose_abort(struct rxrpc_call *call, s32 abort_code, int error,
                         enum rxrpc_abort_reason why)
{
        _enter("{%d},%d,%d,%u", call->debug_id, abort_code, error, why);

        if (!call->send_abort && !rxrpc_call_is_complete(call)) {
                call->send_abort_why = why;
                call->send_abort_err = error;
                call->send_abort_seq = 0;
                trace_rxrpc_abort_call(call, abort_code);
                /* Request abort locklessly vs rxrpc_input_call_event(). */
                smp_store_release(&call->send_abort, abort_code);
                rxrpc_poke_call(call, rxrpc_call_poke_abort);
                return true;
        }

        return false;
}

/*
 * Wait for a call to become connected.  Interruption here doesn't cause the
 * call to be aborted.
 */
static int rxrpc_wait_to_be_connected(struct rxrpc_call *call, long *timeo)
{
        DECLARE_WAITQUEUE(myself, current);
        int ret = 0;

        _enter("%d", call->debug_id);

        if (rxrpc_call_state(call) != RXRPC_CALL_CLIENT_AWAIT_CONN)
                goto no_wait;

        add_wait_queue_exclusive(&call->waitq, &myself);

        for (;;) {
                switch (call->interruptibility) {
                case RXRPC_INTERRUPTIBLE:
                case RXRPC_PREINTERRUPTIBLE:
                        set_current_state(TASK_INTERRUPTIBLE);
                        break;
                case RXRPC_UNINTERRUPTIBLE:
                default:
                        set_current_state(TASK_UNINTERRUPTIBLE);
                        break;
                }

                if (rxrpc_call_state(call) != RXRPC_CALL_CLIENT_AWAIT_CONN)
                        break;
                if ((call->interruptibility == RXRPC_INTERRUPTIBLE ||
                     call->interruptibility == RXRPC_PREINTERRUPTIBLE) &&
                    signal_pending(current)) {
                        ret = sock_intr_errno(*timeo);
                        break;
                }
                *timeo = schedule_timeout(*timeo);
        }

        remove_wait_queue(&call->waitq, &myself);
        __set_current_state(TASK_RUNNING);

no_wait:
        if (ret == 0 && rxrpc_call_is_complete(call))
                ret = call->error;

        _leave(" = %d", ret);
        return ret;
}

/*
 * Return true if there's sufficient Tx queue space.
 */
static bool rxrpc_check_tx_space(struct rxrpc_call *call, rxrpc_seq_t *_tx_win)
{
        if (_tx_win)
                *_tx_win = call->tx_bottom;
        return call->tx_prepared - call->tx_bottom < 256;
}

/*
 * Wait for space to appear in the Tx queue or a signal to occur.
 */
static int rxrpc_wait_for_tx_window_intr(struct rxrpc_sock *rx,
                                         struct rxrpc_call *call,
                                         long *timeo)
{
        for (;;) {
                set_current_state(TASK_INTERRUPTIBLE);
                if (rxrpc_check_tx_space(call, NULL))
                        return 0;

                if (rxrpc_call_is_complete(call))
                        return call->error;

                if (signal_pending(current))
                        return sock_intr_errno(*timeo);

                trace_rxrpc_txqueue(call, rxrpc_txqueue_wait);
                *timeo = schedule_timeout(*timeo);
        }
}

/*
 * Wait for space to appear in the Tx queue uninterruptibly, but with
 * a timeout of 2*RTT if no progress was made and a signal occurred.
 */
static int rxrpc_wait_for_tx_window_waitall(struct rxrpc_sock *rx,
                                            struct rxrpc_call *call)
{
        rxrpc_seq_t tx_start, tx_win;
        signed long rtt, timeout;

        rtt = READ_ONCE(call->peer->srtt_us) >> 3;
        rtt = usecs_to_jiffies(rtt) * 2;
        if (rtt < 2)
                rtt = 2;

        timeout = rtt;
        tx_start = smp_load_acquire(&call->acks_hard_ack);

        for (;;) {
                set_current_state(TASK_UNINTERRUPTIBLE);

                if (rxrpc_check_tx_space(call, &tx_win))
                        return 0;

                if (rxrpc_call_is_complete(call))
                        return call->error;

                if (timeout == 0 &&
                    tx_win == tx_start && signal_pending(current))
                        return -EINTR;

                if (tx_win != tx_start) {
                        timeout = rtt;
                        tx_start = tx_win;
                }

                trace_rxrpc_txqueue(call, rxrpc_txqueue_wait);
                timeout = schedule_timeout(timeout);
        }
}

/*
 * Wait for space to appear in the Tx queue uninterruptibly.
 */
static int rxrpc_wait_for_tx_window_nonintr(struct rxrpc_sock *rx,
                                            struct rxrpc_call *call,
                                            long *timeo)
{
        for (;;) {
                set_current_state(TASK_UNINTERRUPTIBLE);
                if (rxrpc_check_tx_space(call, NULL))
                        return 0;

                if (rxrpc_call_is_complete(call))
                        return call->error;

                trace_rxrpc_txqueue(call, rxrpc_txqueue_wait);
                *timeo = schedule_timeout(*timeo);
        }
}

/*
 * wait for space to appear in the transmit/ACK window
 * - caller holds the socket locked
 */
static int rxrpc_wait_for_tx_window(struct rxrpc_sock *rx,
                                    struct rxrpc_call *call,
                                    long *timeo,
                                    bool waitall)
{
        DECLARE_WAITQUEUE(myself, current);
        int ret;

        _enter(",{%u,%u,%u,%u}",
               call->tx_bottom, call->acks_hard_ack, call->tx_top, call->tx_winsize);

        add_wait_queue(&call->waitq, &myself);

        switch (call->interruptibility) {
        case RXRPC_INTERRUPTIBLE:
                if (waitall)
                        ret = rxrpc_wait_for_tx_window_waitall(rx, call);
                else
                        ret = rxrpc_wait_for_tx_window_intr(rx, call, timeo);
                break;
        case RXRPC_PREINTERRUPTIBLE:
        case RXRPC_UNINTERRUPTIBLE:
        default:
                ret = rxrpc_wait_for_tx_window_nonintr(rx, call, timeo);
                break;
        }

        remove_wait_queue(&call->waitq, &myself);
        set_current_state(TASK_RUNNING);
        _leave(" = %d", ret);
        return ret;
}

/*
 * Notify the owner of the call that the transmit phase is ended and the last
 * packet has been queued.
 */
static void rxrpc_notify_end_tx(struct rxrpc_sock *rx, struct rxrpc_call *call,
                                rxrpc_notify_end_tx_t notify_end_tx)
{
        if (notify_end_tx)
                notify_end_tx(&rx->sk, call, call->user_call_ID);
}

/*
 * Queue a DATA packet for transmission, set the resend timeout and send
 * the packet immediately.  Returns the error from rxrpc_send_data_packet()
 * in case the caller wants to do something with it.
 */
static void rxrpc_queue_packet(struct rxrpc_sock *rx, struct rxrpc_call *call,
                               struct rxrpc_txbuf *txb,
                               rxrpc_notify_end_tx_t notify_end_tx)
{
        rxrpc_seq_t seq = txb->seq;
        bool poke, last = txb->flags & RXRPC_LAST_PACKET;

        rxrpc_inc_stat(call->rxnet, stat_tx_data);

        ASSERTCMP(txb->seq, ==, call->tx_prepared + 1);

        /* We have to set the timestamp before queueing as the retransmit
         * algorithm can see the packet as soon as we queue it.
         */
        txb->last_sent = ktime_get_real();

        if (last)
                trace_rxrpc_txqueue(call, rxrpc_txqueue_queue_last);
        else
                trace_rxrpc_txqueue(call, rxrpc_txqueue_queue);

        /* Add the packet to the call's output buffer */
        spin_lock(&call->tx_lock);
        poke = list_empty(&call->tx_sendmsg);
        list_add_tail(&txb->call_link, &call->tx_sendmsg);
        call->tx_prepared = seq;
        if (last)
                rxrpc_notify_end_tx(rx, call, notify_end_tx);
        spin_unlock(&call->tx_lock);

        if (poke)
                rxrpc_poke_call(call, rxrpc_call_poke_start);
}

/*
 * send data through a socket
 * - must be called in process context
 * - The caller holds the call user access mutex, but not the socket lock.
 */
static int rxrpc_send_data(struct rxrpc_sock *rx,
                           struct rxrpc_call *call,
                           struct msghdr *msg, size_t len,
                           rxrpc_notify_end_tx_t notify_end_tx,
                           bool *_dropped_lock)
{
        struct rxrpc_txbuf *txb;
        struct sock *sk = &rx->sk;
        enum rxrpc_call_state state;
        long timeo;
        bool more = msg->msg_flags & MSG_MORE;
        int ret, copied = 0;

        timeo = sock_sndtimeo(sk, msg->msg_flags & MSG_DONTWAIT);

        ret = rxrpc_wait_to_be_connected(call, &timeo);
        if (ret < 0)
                return ret;

        if (call->conn->state == RXRPC_CONN_CLIENT_UNSECURED) {
                ret = rxrpc_init_client_conn_security(call->conn);
                if (ret < 0)
                        return ret;
        }

        /* this should be in poll */
        sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk);

reload:
        txb = call->tx_pending;
        call->tx_pending = NULL;
        if (txb)
                rxrpc_see_txbuf(txb, rxrpc_txbuf_see_send_more);

        ret = -EPIPE;
        if (sk->sk_shutdown & SEND_SHUTDOWN)
                goto maybe_error;
        state = rxrpc_call_state(call);
        ret = -ESHUTDOWN;
        if (state >= RXRPC_CALL_COMPLETE)
                goto maybe_error;
        ret = -EPROTO;
        if (state != RXRPC_CALL_CLIENT_SEND_REQUEST &&
            state != RXRPC_CALL_SERVER_ACK_REQUEST &&
            state != RXRPC_CALL_SERVER_SEND_REPLY) {
                /* Request phase complete for this client call */
                trace_rxrpc_abort(call->debug_id, rxrpc_sendmsg_late_send,
                                  call->cid, call->call_id, call->rx_consumed,
                                  0, -EPROTO);
                goto maybe_error;
        }

        ret = -EMSGSIZE;
        if (call->tx_total_len != -1) {
                if (len - copied > call->tx_total_len)
                        goto maybe_error;
                if (!more && len - copied != call->tx_total_len)
                        goto maybe_error;
        }

        do {
                if (!txb) {
                        size_t remain;

                        _debug("alloc");

                        if (!rxrpc_check_tx_space(call, NULL))
                                goto wait_for_space;

                        /* Work out the maximum size of a packet.  Assume that
                         * the security header is going to be in the padded
                         * region (enc blocksize), but the trailer is not.
                         */
                        remain = more ? INT_MAX : msg_data_left(msg);
                        txb = call->conn->security->alloc_txbuf(call, remain, sk->sk_allocation);
                        if (!txb) {
                                ret = -ENOMEM;
                                goto maybe_error;
                        }
                }

                _debug("append");

                /* append next segment of data to the current buffer */
                if (msg_data_left(msg) > 0) {
                        size_t copy = min_t(size_t, txb->space, msg_data_left(msg));

                        _debug("add %zu", copy);
                        if (!copy_from_iter_full(txb->kvec[0].iov_base + txb->offset,
                                                 copy, &msg->msg_iter))
                                goto efault;
                        _debug("added");
                        txb->space -= copy;
                        txb->len += copy;
                        txb->offset += copy;
                        copied += copy;
                        if (call->tx_total_len != -1)
                                call->tx_total_len -= copy;
                }

                /* check for the far side aborting the call or a network error
                 * occurring */
                if (rxrpc_call_is_complete(call))
                        goto call_terminated;

                /* add the packet to the send queue if it's now full */
                if (!txb->space ||
                    (msg_data_left(msg) == 0 && !more)) {
                        if (msg_data_left(msg) == 0 && !more)
                                txb->flags |= RXRPC_LAST_PACKET;
                        else if (call->tx_top - call->acks_hard_ack <
                                 call->tx_winsize)
                                txb->flags |= RXRPC_MORE_PACKETS;

                        ret = call->security->secure_packet(call, txb);
                        if (ret < 0)
                                goto out;

                        txb->kvec[0].iov_len += txb->len;
                        txb->len = txb->kvec[0].iov_len;
                        rxrpc_queue_packet(rx, call, txb, notify_end_tx);
                        txb = NULL;
                }
        } while (msg_data_left(msg) > 0);

success:
        ret = copied;
        if (rxrpc_call_is_complete(call) &&
            call->error < 0)
                ret = call->error;
out:
        call->tx_pending = txb;
        _leave(" = %d", ret);
        return ret;

call_terminated:
        rxrpc_put_txbuf(txb, rxrpc_txbuf_put_send_aborted);
        _leave(" = %d", call->error);
        return call->error;

maybe_error:
        if (copied)
                goto success;
        goto out;

efault:
        ret = -EFAULT;
        goto out;

wait_for_space:
        ret = -EAGAIN;
        if (msg->msg_flags & MSG_DONTWAIT)
                goto maybe_error;
        mutex_unlock(&call->user_mutex);
        *_dropped_lock = true;
        ret = rxrpc_wait_for_tx_window(rx, call, &timeo,
                                       msg->msg_flags & MSG_WAITALL);
        if (ret < 0)
                goto maybe_error;
        if (call->interruptibility == RXRPC_INTERRUPTIBLE) {
                if (mutex_lock_interruptible(&call->user_mutex) < 0) {
                        ret = sock_intr_errno(timeo);
                        goto maybe_error;
                }
        } else {
                mutex_lock(&call->user_mutex);
        }
        *_dropped_lock = false;
        goto reload;
}

/*
 * extract control messages from the sendmsg() control buffer
 */
static int rxrpc_sendmsg_cmsg(struct msghdr *msg, struct rxrpc_send_params *p)
{
        struct cmsghdr *cmsg;
        bool got_user_ID = false;
        int len;

        if (msg->msg_controllen == 0)
                return -EINVAL;

        for_each_cmsghdr(cmsg, msg) {
                if (!CMSG_OK(msg, cmsg))
                        return -EINVAL;

                len = cmsg->cmsg_len - sizeof(struct cmsghdr);
                _debug("CMSG %d, %d, %d",
                       cmsg->cmsg_level, cmsg->cmsg_type, len);

                if (cmsg->cmsg_level != SOL_RXRPC)
                        continue;

                switch (cmsg->cmsg_type) {
                case RXRPC_USER_CALL_ID:
                        if (msg->msg_flags & MSG_CMSG_COMPAT) {
                                if (len != sizeof(u32))
                                        return -EINVAL;
                                p->call.user_call_ID = *(u32 *)CMSG_DATA(cmsg);
                        } else {
                                if (len != sizeof(unsigned long))
                                        return -EINVAL;
                                p->call.user_call_ID = *(unsigned long *)
                                        CMSG_DATA(cmsg);
                        }
                        got_user_ID = true;
                        break;

                case RXRPC_ABORT:
                        if (p->command != RXRPC_CMD_SEND_DATA)
                                return -EINVAL;
                        p->command = RXRPC_CMD_SEND_ABORT;
                        if (len != sizeof(p->abort_code))
                                return -EINVAL;
                        p->abort_code = *(unsigned int *)CMSG_DATA(cmsg);
                        if (p->abort_code == 0)
                                return -EINVAL;
                        break;

                case RXRPC_CHARGE_ACCEPT:
                        if (p->command != RXRPC_CMD_SEND_DATA)
                                return -EINVAL;
                        p->command = RXRPC_CMD_CHARGE_ACCEPT;
                        if (len != 0)
                                return -EINVAL;
                        break;

                case RXRPC_EXCLUSIVE_CALL:
                        p->exclusive = true;
                        if (len != 0)
                                return -EINVAL;
                        break;

                case RXRPC_UPGRADE_SERVICE:
                        p->upgrade = true;
                        if (len != 0)
                                return -EINVAL;
                        break;

                case RXRPC_TX_LENGTH:
                        if (p->call.tx_total_len != -1 || len != sizeof(__s64))
                                return -EINVAL;
                        p->call.tx_total_len = *(__s64 *)CMSG_DATA(cmsg);
                        if (p->call.tx_total_len < 0)
                                return -EINVAL;
                        break;

                case RXRPC_SET_CALL_TIMEOUT:
                        if (len & 3 || len < 4 || len > 12)
                                return -EINVAL;
                        memcpy(&p->call.timeouts, CMSG_DATA(cmsg), len);
                        p->call.nr_timeouts = len / 4;
                        if (p->call.timeouts.hard > INT_MAX / HZ)
                                return -ERANGE;
                        if (p->call.nr_timeouts >= 2 && p->call.timeouts.idle > 60 * 60 * 1000)
                                return -ERANGE;
                        if (p->call.nr_timeouts >= 3 && p->call.timeouts.normal > 60 * 60 * 1000)
                                return -ERANGE;
                        break;

                default:
                        return -EINVAL;
                }
        }

        if (!got_user_ID)
                return -EINVAL;
        if (p->call.tx_total_len != -1 && p->command != RXRPC_CMD_SEND_DATA)
                return -EINVAL;
        _leave(" = 0");
        return 0;
}

/*
 * Create a new client call for sendmsg().
 * - Called with the socket lock held, which it must release.
 * - If it returns a call, the call's lock will need releasing by the caller.
 */
static struct rxrpc_call *
rxrpc_new_client_call_for_sendmsg(struct rxrpc_sock *rx, struct msghdr *msg,
                                  struct rxrpc_send_params *p)
        __releases(&rx->sk.sk_lock.slock)
        __acquires(&call->user_mutex)
{
        struct rxrpc_conn_parameters cp;
        struct rxrpc_peer *peer;
        struct rxrpc_call *call;
        struct key *key;

        DECLARE_SOCKADDR(struct sockaddr_rxrpc *, srx, msg->msg_name);

        _enter("");

        if (!msg->msg_name) {
                release_sock(&rx->sk);
                return ERR_PTR(-EDESTADDRREQ);
        }

        peer = rxrpc_lookup_peer(rx->local, srx, GFP_KERNEL);
        if (!peer) {
                release_sock(&rx->sk);
                return ERR_PTR(-ENOMEM);
        }

        key = rx->key;
        if (key && !rx->key->payload.data[0])
                key = NULL;

        memset(&cp, 0, sizeof(cp));
        cp.local                = rx->local;
        cp.peer                        = peer;
        cp.key                        = rx->key;
        cp.security_level        = rx->min_sec_level;
        cp.exclusive                = rx->exclusive | p->exclusive;
        cp.upgrade                = p->upgrade;
        cp.service_id                = srx->srx_service;
        call = rxrpc_new_client_call(rx, &cp, &p->call, GFP_KERNEL,
                                     atomic_inc_return(&rxrpc_debug_id));
        /* The socket is now unlocked */

        rxrpc_put_peer(peer, rxrpc_peer_put_application);
        _leave(" = %p\n", call);
        return call;
}

/*
 * send a message forming part of a client call through an RxRPC socket
 * - caller holds the socket locked
 * - the socket may be either a client socket or a server socket
 */
int rxrpc_do_sendmsg(struct rxrpc_sock *rx, struct msghdr *msg, size_t len)
        __releases(&rx->sk.sk_lock.slock)
{
        struct rxrpc_call *call;
        bool dropped_lock = false;
        int ret;

        struct rxrpc_send_params p = {
                .call.tx_total_len        = -1,
                .call.user_call_ID        = 0,
                .call.nr_timeouts        = 0,
                .call.interruptibility        = RXRPC_INTERRUPTIBLE,
                .abort_code                = 0,
                .command                = RXRPC_CMD_SEND_DATA,
                .exclusive                = false,
                .upgrade                = false,
        };

        _enter("");

        ret = rxrpc_sendmsg_cmsg(msg, &p);
        if (ret < 0)
                goto error_release_sock;

        if (p.command == RXRPC_CMD_CHARGE_ACCEPT) {
                ret = -EINVAL;
                if (rx->sk.sk_state != RXRPC_SERVER_LISTENING)
                        goto error_release_sock;
                ret = rxrpc_user_charge_accept(rx, p.call.user_call_ID);
                goto error_release_sock;
        }

        call = rxrpc_find_call_by_user_ID(rx, p.call.user_call_ID);
        if (!call) {
                ret = -EBADSLT;
                if (p.command != RXRPC_CMD_SEND_DATA)
                        goto error_release_sock;
                call = rxrpc_new_client_call_for_sendmsg(rx, msg, &p);
                /* The socket is now unlocked... */
                if (IS_ERR(call))
                        return PTR_ERR(call);
                /* ... and we have the call lock. */
                p.call.nr_timeouts = 0;
                ret = 0;
                if (rxrpc_call_is_complete(call))
                        goto out_put_unlock;
        } else {
                switch (rxrpc_call_state(call)) {
                case RXRPC_CALL_CLIENT_AWAIT_CONN:
                case RXRPC_CALL_SERVER_SECURING:
                        if (p.command == RXRPC_CMD_SEND_ABORT)
                                break;
                        fallthrough;
                case RXRPC_CALL_UNINITIALISED:
                case RXRPC_CALL_SERVER_PREALLOC:
                        rxrpc_put_call(call, rxrpc_call_put_sendmsg);
                        ret = -EBUSY;
                        goto error_release_sock;
                default:
                        break;
                }

                ret = mutex_lock_interruptible(&call->user_mutex);
                release_sock(&rx->sk);
                if (ret < 0) {
                        ret = -ERESTARTSYS;
                        goto error_put;
                }

                if (p.call.tx_total_len != -1) {
                        ret = -EINVAL;
                        if (call->tx_total_len != -1 ||
                            call->tx_pending ||
                            call->tx_top != 0)
                                goto out_put_unlock;
                        call->tx_total_len = p.call.tx_total_len;
                }
        }

        switch (p.call.nr_timeouts) {
        case 3:
                WRITE_ONCE(call->next_rx_timo, p.call.timeouts.normal);
                fallthrough;
        case 2:
                WRITE_ONCE(call->next_req_timo, p.call.timeouts.idle);
                fallthrough;
        case 1:
                if (p.call.timeouts.hard > 0) {
                        ktime_t delay = ms_to_ktime(p.call.timeouts.hard * MSEC_PER_SEC);

                        WRITE_ONCE(call->expect_term_by,
                                   ktime_add(p.call.timeouts.hard,
                                             ktime_get_real()));
                        trace_rxrpc_timer_set(call, delay, rxrpc_timer_trace_hard);
                        rxrpc_poke_call(call, rxrpc_call_poke_set_timeout);

                }
                break;
        }

        if (rxrpc_call_is_complete(call)) {
                /* it's too late for this call */
                ret = -ESHUTDOWN;
        } else if (p.command == RXRPC_CMD_SEND_ABORT) {
                rxrpc_propose_abort(call, p.abort_code, -ECONNABORTED,
                                    rxrpc_abort_call_sendmsg);
                ret = 0;
        } else if (p.command != RXRPC_CMD_SEND_DATA) {
                ret = -EINVAL;
        } else {
                ret = rxrpc_send_data(rx, call, msg, len, NULL, &dropped_lock);
        }

out_put_unlock:
        if (!dropped_lock)
                mutex_unlock(&call->user_mutex);
error_put:
        rxrpc_put_call(call, rxrpc_call_put_sendmsg);
        _leave(" = %d", ret);
        return ret;

error_release_sock:
        release_sock(&rx->sk);
        return ret;
}

/**
 * rxrpc_kernel_send_data - Allow a kernel service to send data on a call
 * @sock: The socket the call is on
 * @call: The call to send data through
 * @msg: The data to send
 * @len: The amount of data to send
 * @notify_end_tx: Notification that the last packet is queued.
 *
 * Allow a kernel service to send data on a call.  The call must be in an state
 * appropriate to sending data.  No control data should be supplied in @msg,
 * nor should an address be supplied.  MSG_MORE should be flagged if there's
 * more data to come, otherwise this data will end the transmission phase.
 */
int rxrpc_kernel_send_data(struct socket *sock, struct rxrpc_call *call,
                           struct msghdr *msg, size_t len,
                           rxrpc_notify_end_tx_t notify_end_tx)
{
        bool dropped_lock = false;
        int ret;

        _enter("{%d},", call->debug_id);

        ASSERTCMP(msg->msg_name, ==, NULL);
        ASSERTCMP(msg->msg_control, ==, NULL);

        mutex_lock(&call->user_mutex);

        ret = rxrpc_send_data(rxrpc_sk(sock->sk), call, msg, len,
                              notify_end_tx, &dropped_lock);
        if (ret == -ESHUTDOWN)
                ret = call->error;

        if (!dropped_lock)
                mutex_unlock(&call->user_mutex);
        _leave(" = %d", ret);
        return ret;
}
EXPORT_SYMBOL(rxrpc_kernel_send_data);

/**
 * rxrpc_kernel_abort_call - Allow a kernel service to abort a call
 * @sock: The socket the call is on
 * @call: The call to be aborted
 * @abort_code: The abort code to stick into the ABORT packet
 * @error: Local error value
 * @why: Indication as to why.
 *
 * Allow a kernel service to abort a call, if it's still in an abortable state
 * and return true if the call was aborted, false if it was already complete.
 */
bool rxrpc_kernel_abort_call(struct socket *sock, struct rxrpc_call *call,
                             u32 abort_code, int error, enum rxrpc_abort_reason why)
{
        bool aborted;

        _enter("{%d},%d,%d,%u", call->debug_id, abort_code, error, why);

        mutex_lock(&call->user_mutex);
        aborted = rxrpc_propose_abort(call, abort_code, error, why);
        mutex_unlock(&call->user_mutex);
        return aborted;
}
EXPORT_SYMBOL(rxrpc_kernel_abort_call);

/**
 * rxrpc_kernel_set_tx_length - Set the total Tx length on a call
 * @sock: The socket the call is on
 * @call: The call to be informed
 * @tx_total_len: The amount of data to be transmitted for this call
 *
 * Allow a kernel service to set the total transmit length on a call.  This
 * allows buffer-to-packet encrypt-and-copy to be performed.
 *
 * This function is primarily for use for setting the reply length since the
 * request length can be set when beginning the call.
 */
void rxrpc_kernel_set_tx_length(struct socket *sock, struct rxrpc_call *call,
                                s64 tx_total_len)
{
        WARN_ON(call->tx_total_len != -1);
        call->tx_total_len = tx_total_len;
}
EXPORT_SYMBOL(rxrpc_kernel_set_tx_length);








































































































































































































































































































































































































































 10597 
 10590 


























































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
// SPDX-License-Identifier: GPL-2.0-only
/*
 * x86 APERF/MPERF KHz calculation for
 * /sys/.../cpufreq/scaling_cur_freq
 *
 * Copyright (C) 2017 Intel Corp.
 * Author: Len Brown <len.brown@intel.com>
 */
#include <linux/cpufreq.h>
#include <linux/delay.h>
#include <linux/ktime.h>
#include <linux/math64.h>
#include <linux/percpu.h>
#include <linux/rcupdate.h>
#include <linux/sched/isolation.h>
#include <linux/sched/topology.h>
#include <linux/smp.h>
#include <linux/syscore_ops.h>

#include <asm/cpu.h>
#include <asm/cpu_device_id.h>
#include <asm/intel-family.h>

#include "cpu.h"

struct aperfmperf {
        seqcount_t        seq;
        unsigned long        last_update;
        u64                acnt;
        u64                mcnt;
        u64                aperf;
        u64                mperf;
};

static DEFINE_PER_CPU_SHARED_ALIGNED(struct aperfmperf, cpu_samples) = {
        .seq = SEQCNT_ZERO(cpu_samples.seq)
};

static void init_counter_refs(void)
{
        u64 aperf, mperf;

        rdmsrl(MSR_IA32_APERF, aperf);
        rdmsrl(MSR_IA32_MPERF, mperf);

        this_cpu_write(cpu_samples.aperf, aperf);
        this_cpu_write(cpu_samples.mperf, mperf);
}

#if defined(CONFIG_X86_64) && defined(CONFIG_SMP)
/*
 * APERF/MPERF frequency ratio computation.
 *
 * The scheduler wants to do frequency invariant accounting and needs a <1
 * ratio to account for the 'current' frequency, corresponding to
 * freq_curr / freq_max.
 *
 * Since the frequency freq_curr on x86 is controlled by micro-controller and
 * our P-state setting is little more than a request/hint, we need to observe
 * the effective frequency 'BusyMHz', i.e. the average frequency over a time
 * interval after discarding idle time. This is given by:
 *
 *   BusyMHz = delta_APERF / delta_MPERF * freq_base
 *
 * where freq_base is the max non-turbo P-state.
 *
 * The freq_max term has to be set to a somewhat arbitrary value, because we
 * can't know which turbo states will be available at a given point in time:
 * it all depends on the thermal headroom of the entire package. We set it to
 * the turbo level with 4 cores active.
 *
 * Benchmarks show that's a good compromise between the 1C turbo ratio
 * (freq_curr/freq_max would rarely reach 1) and something close to freq_base,
 * which would ignore the entire turbo range (a conspicuous part, making
 * freq_curr/freq_max always maxed out).
 *
 * An exception to the heuristic above is the Atom uarch, where we choose the
 * highest turbo level for freq_max since Atom's are generally oriented towards
 * power efficiency.
 *
 * Setting freq_max to anything less than the 1C turbo ratio makes the ratio
 * freq_curr / freq_max to eventually grow >1, in which case we clip it to 1.
 */

DEFINE_STATIC_KEY_FALSE(arch_scale_freq_key);

static u64 arch_turbo_freq_ratio = SCHED_CAPACITY_SCALE;
static u64 arch_max_freq_ratio = SCHED_CAPACITY_SCALE;

void arch_set_max_freq_ratio(bool turbo_disabled)
{
        arch_max_freq_ratio = turbo_disabled ? SCHED_CAPACITY_SCALE :
                                        arch_turbo_freq_ratio;
}
EXPORT_SYMBOL_GPL(arch_set_max_freq_ratio);

static bool __init turbo_disabled(void)
{
        u64 misc_en;
        int err;

        err = rdmsrl_safe(MSR_IA32_MISC_ENABLE, &misc_en);
        if (err)
                return false;

        return (misc_en & MSR_IA32_MISC_ENABLE_TURBO_DISABLE);
}

static bool __init slv_set_max_freq_ratio(u64 *base_freq, u64 *turbo_freq)
{
        int err;

        err = rdmsrl_safe(MSR_ATOM_CORE_RATIOS, base_freq);
        if (err)
                return false;

        err = rdmsrl_safe(MSR_ATOM_CORE_TURBO_RATIOS, turbo_freq);
        if (err)
                return false;

        *base_freq = (*base_freq >> 16) & 0x3F;     /* max P state */
        *turbo_freq = *turbo_freq & 0x3F;           /* 1C turbo    */

        return true;
}

#define X86_MATCH(vfm)                                                \
        X86_MATCH_VFM_FEATURE(vfm, X86_FEATURE_APERFMPERF, NULL)

static const struct x86_cpu_id has_knl_turbo_ratio_limits[] __initconst = {
        X86_MATCH(INTEL_XEON_PHI_KNL),
        X86_MATCH(INTEL_XEON_PHI_KNM),
        {}
};

static const struct x86_cpu_id has_skx_turbo_ratio_limits[] __initconst = {
        X86_MATCH(INTEL_SKYLAKE_X),
        {}
};

static const struct x86_cpu_id has_glm_turbo_ratio_limits[] __initconst = {
        X86_MATCH(INTEL_ATOM_GOLDMONT),
        X86_MATCH(INTEL_ATOM_GOLDMONT_D),
        X86_MATCH(INTEL_ATOM_GOLDMONT_PLUS),
        {}
};

static bool __init knl_set_max_freq_ratio(u64 *base_freq, u64 *turbo_freq,
                                          int num_delta_fratio)
{
        int fratio, delta_fratio, found;
        int err, i;
        u64 msr;

        err = rdmsrl_safe(MSR_PLATFORM_INFO, base_freq);
        if (err)
                return false;

        *base_freq = (*base_freq >> 8) & 0xFF;            /* max P state */

        err = rdmsrl_safe(MSR_TURBO_RATIO_LIMIT, &msr);
        if (err)
                return false;

        fratio = (msr >> 8) & 0xFF;
        i = 16;
        found = 0;
        do {
                if (found >= num_delta_fratio) {
                        *turbo_freq = fratio;
                        return true;
                }

                delta_fratio = (msr >> (i + 5)) & 0x7;

                if (delta_fratio) {
                        found += 1;
                        fratio -= delta_fratio;
                }

                i += 8;
        } while (i < 64);

        return true;
}

static bool __init skx_set_max_freq_ratio(u64 *base_freq, u64 *turbo_freq, int size)
{
        u64 ratios, counts;
        u32 group_size;
        int err, i;

        err = rdmsrl_safe(MSR_PLATFORM_INFO, base_freq);
        if (err)
                return false;

        *base_freq = (*base_freq >> 8) & 0xFF;      /* max P state */

        err = rdmsrl_safe(MSR_TURBO_RATIO_LIMIT, &ratios);
        if (err)
                return false;

        err = rdmsrl_safe(MSR_TURBO_RATIO_LIMIT1, &counts);
        if (err)
                return false;

        for (i = 0; i < 64; i += 8) {
                group_size = (counts >> i) & 0xFF;
                if (group_size >= size) {
                        *turbo_freq = (ratios >> i) & 0xFF;
                        return true;
                }
        }

        return false;
}

static bool __init core_set_max_freq_ratio(u64 *base_freq, u64 *turbo_freq)
{
        u64 msr;
        int err;

        err = rdmsrl_safe(MSR_PLATFORM_INFO, base_freq);
        if (err)
                return false;

        err = rdmsrl_safe(MSR_TURBO_RATIO_LIMIT, &msr);
        if (err)
                return false;

        *base_freq = (*base_freq >> 8) & 0xFF;    /* max P state */
        *turbo_freq = (msr >> 24) & 0xFF;         /* 4C turbo    */

        /* The CPU may have less than 4 cores */
        if (!*turbo_freq)
                *turbo_freq = msr & 0xFF;         /* 1C turbo    */

        return true;
}

static bool __init intel_set_max_freq_ratio(void)
{
        u64 base_freq, turbo_freq;
        u64 turbo_ratio;

        if (slv_set_max_freq_ratio(&base_freq, &turbo_freq))
                goto out;

        if (x86_match_cpu(has_glm_turbo_ratio_limits) &&
            skx_set_max_freq_ratio(&base_freq, &turbo_freq, 1))
                goto out;

        if (x86_match_cpu(has_knl_turbo_ratio_limits) &&
            knl_set_max_freq_ratio(&base_freq, &turbo_freq, 1))
                goto out;

        if (x86_match_cpu(has_skx_turbo_ratio_limits) &&
            skx_set_max_freq_ratio(&base_freq, &turbo_freq, 4))
                goto out;

        if (core_set_max_freq_ratio(&base_freq, &turbo_freq))
                goto out;

        return false;

out:
        /*
         * Some hypervisors advertise X86_FEATURE_APERFMPERF
         * but then fill all MSR's with zeroes.
         * Some CPUs have turbo boost but don't declare any turbo ratio
         * in MSR_TURBO_RATIO_LIMIT.
         */
        if (!base_freq || !turbo_freq) {
                pr_debug("Couldn't determine cpu base or turbo frequency, necessary for scale-invariant accounting.\n");
                return false;
        }

        turbo_ratio = div_u64(turbo_freq * SCHED_CAPACITY_SCALE, base_freq);
        if (!turbo_ratio) {
                pr_debug("Non-zero turbo and base frequencies led to a 0 ratio.\n");
                return false;
        }

        arch_turbo_freq_ratio = turbo_ratio;
        arch_set_max_freq_ratio(turbo_disabled());

        return true;
}

#ifdef CONFIG_PM_SLEEP
static struct syscore_ops freq_invariance_syscore_ops = {
        .resume = init_counter_refs,
};

static void register_freq_invariance_syscore_ops(void)
{
        register_syscore_ops(&freq_invariance_syscore_ops);
}
#else
static inline void register_freq_invariance_syscore_ops(void) {}
#endif

static void freq_invariance_enable(void)
{
        if (static_branch_unlikely(&arch_scale_freq_key)) {
                WARN_ON_ONCE(1);
                return;
        }
        static_branch_enable_cpuslocked(&arch_scale_freq_key);
        register_freq_invariance_syscore_ops();
        pr_info("Estimated ratio of average max frequency by base frequency (times 1024): %llu\n", arch_max_freq_ratio);
}

void freq_invariance_set_perf_ratio(u64 ratio, bool turbo_disabled)
{
        arch_turbo_freq_ratio = ratio;
        arch_set_max_freq_ratio(turbo_disabled);
        freq_invariance_enable();
}

static void __init bp_init_freq_invariance(void)
{
        if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL)
                return;

        if (intel_set_max_freq_ratio()) {
                guard(cpus_read_lock)();
                freq_invariance_enable();
        }
}

static void disable_freq_invariance_workfn(struct work_struct *work)
{
        int cpu;

        static_branch_disable(&arch_scale_freq_key);

        /*
         * Set arch_freq_scale to a default value on all cpus
         * This negates the effect of scaling
         */
        for_each_possible_cpu(cpu)
                per_cpu(arch_freq_scale, cpu) = SCHED_CAPACITY_SCALE;
}

static DECLARE_WORK(disable_freq_invariance_work,
                    disable_freq_invariance_workfn);

DEFINE_PER_CPU(unsigned long, arch_freq_scale) = SCHED_CAPACITY_SCALE;
EXPORT_PER_CPU_SYMBOL_GPL(arch_freq_scale);

static DEFINE_STATIC_KEY_FALSE(arch_hybrid_cap_scale_key);

struct arch_hybrid_cpu_scale {
        unsigned long capacity;
        unsigned long freq_ratio;
};

static struct arch_hybrid_cpu_scale __percpu *arch_cpu_scale;

/**
 * arch_enable_hybrid_capacity_scale() - Enable hybrid CPU capacity scaling
 *
 * Allocate memory for per-CPU data used by hybrid CPU capacity scaling,
 * initialize it and set the static key controlling its code paths.
 *
 * Must be called before arch_set_cpu_capacity().
 */
bool arch_enable_hybrid_capacity_scale(void)
{
        int cpu;

        if (static_branch_unlikely(&arch_hybrid_cap_scale_key)) {
                WARN_ONCE(1, "Hybrid CPU capacity scaling already enabled");
                return true;
        }

        arch_cpu_scale = alloc_percpu(struct arch_hybrid_cpu_scale);
        if (!arch_cpu_scale)
                return false;

        for_each_possible_cpu(cpu) {
                per_cpu_ptr(arch_cpu_scale, cpu)->capacity = SCHED_CAPACITY_SCALE;
                per_cpu_ptr(arch_cpu_scale, cpu)->freq_ratio = arch_max_freq_ratio;
        }

        static_branch_enable(&arch_hybrid_cap_scale_key);

        pr_info("Hybrid CPU capacity scaling enabled\n");

        return true;
}

/**
 * arch_set_cpu_capacity() - Set scale-invariance parameters for a CPU
 * @cpu: Target CPU.
 * @cap: Capacity of @cpu at its maximum frequency, relative to @max_cap.
 * @max_cap: System-wide maximum CPU capacity.
 * @cap_freq: Frequency of @cpu corresponding to @cap.
 * @base_freq: Frequency of @cpu at which MPERF counts.
 *
 * The units in which @cap and @max_cap are expressed do not matter, so long
 * as they are consistent, because the former is effectively divided by the
 * latter.  Analogously for @cap_freq and @base_freq.
 *
 * After calling this function for all CPUs, call arch_rebuild_sched_domains()
 * to let the scheduler know that capacity-aware scheduling can be used going
 * forward.
 */
void arch_set_cpu_capacity(int cpu, unsigned long cap, unsigned long max_cap,
                           unsigned long cap_freq, unsigned long base_freq)
{
        if (static_branch_likely(&arch_hybrid_cap_scale_key)) {
                WRITE_ONCE(per_cpu_ptr(arch_cpu_scale, cpu)->capacity,
                           div_u64(cap << SCHED_CAPACITY_SHIFT, max_cap));
                WRITE_ONCE(per_cpu_ptr(arch_cpu_scale, cpu)->freq_ratio,
                           div_u64(cap_freq << SCHED_CAPACITY_SHIFT, base_freq));
        } else {
                WARN_ONCE(1, "Hybrid CPU capacity scaling not enabled");
        }
}

unsigned long arch_scale_cpu_capacity(int cpu)
{
        if (static_branch_unlikely(&arch_hybrid_cap_scale_key))
                return READ_ONCE(per_cpu_ptr(arch_cpu_scale, cpu)->capacity);

        return SCHED_CAPACITY_SCALE;
}
EXPORT_SYMBOL_GPL(arch_scale_cpu_capacity);

static void scale_freq_tick(u64 acnt, u64 mcnt)
{
        u64 freq_scale, freq_ratio;

        if (!arch_scale_freq_invariant())
                return;

        if (check_shl_overflow(acnt, 2*SCHED_CAPACITY_SHIFT, &acnt))
                goto error;

        if (static_branch_unlikely(&arch_hybrid_cap_scale_key))
                freq_ratio = READ_ONCE(this_cpu_ptr(arch_cpu_scale)->freq_ratio);
        else
                freq_ratio = arch_max_freq_ratio;

        if (check_mul_overflow(mcnt, freq_ratio, &mcnt) || !mcnt)
                goto error;

        freq_scale = div64_u64(acnt, mcnt);
        if (!freq_scale)
                goto error;

        if (freq_scale > SCHED_CAPACITY_SCALE)
                freq_scale = SCHED_CAPACITY_SCALE;

        this_cpu_write(arch_freq_scale, freq_scale);
        return;

error:
        pr_warn("Scheduler frequency invariance went wobbly, disabling!\n");
        schedule_work(&disable_freq_invariance_work);
}
#else
static inline void bp_init_freq_invariance(void) { }
static inline void scale_freq_tick(u64 acnt, u64 mcnt) { }
#endif /* CONFIG_X86_64 && CONFIG_SMP */

void arch_scale_freq_tick(void)
{
        struct aperfmperf *s = this_cpu_ptr(&cpu_samples);
        u64 acnt, mcnt, aperf, mperf;

        if (!cpu_feature_enabled(X86_FEATURE_APERFMPERF))
                return;

        rdmsrl(MSR_IA32_APERF, aperf);
        rdmsrl(MSR_IA32_MPERF, mperf);
        acnt = aperf - s->aperf;
        mcnt = mperf - s->mperf;

        s->aperf = aperf;
        s->mperf = mperf;

        raw_write_seqcount_begin(&s->seq);
        s->last_update = jiffies;
        s->acnt = acnt;
        s->mcnt = mcnt;
        raw_write_seqcount_end(&s->seq);

        scale_freq_tick(acnt, mcnt);
}

/*
 * Discard samples older than the define maximum sample age of 20ms. There
 * is no point in sending IPIs in such a case. If the scheduler tick was
 * not running then the CPU is either idle or isolated.
 */
#define MAX_SAMPLE_AGE        ((unsigned long)HZ / 50)

unsigned int arch_freq_get_on_cpu(int cpu)
{
        struct aperfmperf *s = per_cpu_ptr(&cpu_samples, cpu);
        unsigned int seq, freq;
        unsigned long last;
        u64 acnt, mcnt;

        if (!cpu_feature_enabled(X86_FEATURE_APERFMPERF))
                goto fallback;

        do {
                seq = raw_read_seqcount_begin(&s->seq);
                last = s->last_update;
                acnt = s->acnt;
                mcnt = s->mcnt;
        } while (read_seqcount_retry(&s->seq, seq));

        /*
         * Bail on invalid count and when the last update was too long ago,
         * which covers idle and NOHZ full CPUs.
         */
        if (!mcnt || (jiffies - last) > MAX_SAMPLE_AGE)
                goto fallback;

        return div64_u64((cpu_khz * acnt), mcnt);

fallback:
        freq = cpufreq_quick_get(cpu);
        return freq ? freq : cpu_khz;
}

static int __init bp_init_aperfmperf(void)
{
        if (!cpu_feature_enabled(X86_FEATURE_APERFMPERF))
                return 0;

        init_counter_refs();
        bp_init_freq_invariance();
        return 0;
}
early_initcall(bp_init_aperfmperf);

void ap_init_aperfmperf(void)
{
        if (cpu_feature_enabled(X86_FEATURE_APERFMPERF))
                init_counter_refs();
}






























    2 




    2 










    2 



    2 
    2 
    2 

    2 

















    4 








    4 

    1 



    1 



    1 











    1 
















    2 



    2 

    1 








    1 








    2 







    2 









   12 









   12 

    1 



    1 


    8 

    2 












    6 






    8 






    1 

    7 








    3 




    4 
















    5 




    5 





    1 












    5 






    5 

    1 



    1 




    1 










    1 




    1 





    1 


















    1 




    1 


    1 








    2 



1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
// SPDX-License-Identifier: GPL-2.0-only
/*
 * Netlink interface for IEEE 802.15.4 stack
 *
 * Copyright 2007, 2008 Siemens AG
 *
 * Written by:
 * Sergey Lapin <slapin@ossfans.org>
 * Dmitry Eremin-Solenikov <dbaryshkov@gmail.com>
 * Maxim Osipov <maxim.osipov@siemens.com>
 */

#include <linux/kernel.h>
#include <linux/slab.h>
#include <linux/if_arp.h>
#include <net/netlink.h>
#include <net/genetlink.h>
#include <net/cfg802154.h>
#include <net/af_ieee802154.h>
#include <net/ieee802154_netdev.h>
#include <net/rtnetlink.h> /* for rtnl_{un,}lock */
#include <linux/nl802154.h>

#include "ieee802154.h"
#include "rdev-ops.h"
#include "core.h"

static int ieee802154_nl_fill_phy(struct sk_buff *msg, u32 portid,
                                  u32 seq, int flags, struct wpan_phy *phy)
{
        void *hdr;
        int i, pages = 0;
        u32 *buf = kcalloc(IEEE802154_MAX_PAGE + 1, sizeof(u32), GFP_KERNEL);

        pr_debug("%s\n", __func__);

        if (!buf)
                return -EMSGSIZE;

        hdr = genlmsg_put(msg, 0, seq, &nl802154_family, flags,
                          IEEE802154_LIST_PHY);
        if (!hdr)
                goto out;

        rtnl_lock();
        if (nla_put_string(msg, IEEE802154_ATTR_PHY_NAME, wpan_phy_name(phy)) ||
            nla_put_u8(msg, IEEE802154_ATTR_PAGE, phy->current_page) ||
            nla_put_u8(msg, IEEE802154_ATTR_CHANNEL, phy->current_channel))
                goto nla_put_failure;
        for (i = 0; i <= IEEE802154_MAX_PAGE; i++) {
                if (phy->supported.channels[i])
                        buf[pages++] = phy->supported.channels[i] | (i << 27);
        }
        if (pages &&
            nla_put(msg, IEEE802154_ATTR_CHANNEL_PAGE_LIST,
                    pages * sizeof(uint32_t), buf))
                goto nla_put_failure;
        rtnl_unlock();
        kfree(buf);
        genlmsg_end(msg, hdr);
        return 0;

nla_put_failure:
        rtnl_unlock();
        genlmsg_cancel(msg, hdr);
out:
        kfree(buf);
        return -EMSGSIZE;
}

int ieee802154_list_phy(struct sk_buff *skb, struct genl_info *info)
{
        /* Request for interface name, index, type, IEEE address,
         * PAN Id, short address
         */
        struct sk_buff *msg;
        struct wpan_phy *phy;
        const char *name;
        int rc = -ENOBUFS;

        pr_debug("%s\n", __func__);

        if (!info->attrs[IEEE802154_ATTR_PHY_NAME])
                return -EINVAL;

        name = nla_data(info->attrs[IEEE802154_ATTR_PHY_NAME]);
        if (name[nla_len(info->attrs[IEEE802154_ATTR_PHY_NAME]) - 1] != '\0')
                return -EINVAL; /* phy name should be null-terminated */

        phy = wpan_phy_find(name);
        if (!phy)
                return -ENODEV;

        msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
        if (!msg)
                goto out_dev;

        rc = ieee802154_nl_fill_phy(msg, info->snd_portid, info->snd_seq,
                                    0, phy);
        if (rc < 0)
                goto out_free;

        wpan_phy_put(phy);

        return genlmsg_reply(msg, info);
out_free:
        nlmsg_free(msg);
out_dev:
        wpan_phy_put(phy);
        return rc;
}

struct dump_phy_data {
        struct sk_buff *skb;
        struct netlink_callback *cb;
        int idx, s_idx;
};

static int ieee802154_dump_phy_iter(struct wpan_phy *phy, void *_data)
{
        int rc;
        struct dump_phy_data *data = _data;

        pr_debug("%s\n", __func__);

        if (data->idx++ < data->s_idx)
                return 0;

        rc = ieee802154_nl_fill_phy(data->skb,
                                    NETLINK_CB(data->cb->skb).portid,
                                    data->cb->nlh->nlmsg_seq,
                                    NLM_F_MULTI,
                                    phy);

        if (rc < 0) {
                data->idx--;
                return rc;
        }

        return 0;
}

int ieee802154_dump_phy(struct sk_buff *skb, struct netlink_callback *cb)
{
        struct dump_phy_data data = {
                .cb = cb,
                .skb = skb,
                .s_idx = cb->args[0],
                .idx = 0,
        };

        pr_debug("%s\n", __func__);

        wpan_phy_for_each(ieee802154_dump_phy_iter, &data);

        cb->args[0] = data.idx;

        return skb->len;
}

int ieee802154_add_iface(struct sk_buff *skb, struct genl_info *info)
{
        struct sk_buff *msg;
        struct wpan_phy *phy;
        const char *name;
        const char *devname;
        int rc = -ENOBUFS;
        struct net_device *dev;
        int type = __IEEE802154_DEV_INVALID;
        unsigned char name_assign_type;

        pr_debug("%s\n", __func__);

        if (!info->attrs[IEEE802154_ATTR_PHY_NAME])
                return -EINVAL;

        name = nla_data(info->attrs[IEEE802154_ATTR_PHY_NAME]);
        if (name[nla_len(info->attrs[IEEE802154_ATTR_PHY_NAME]) - 1] != '\0')
                return -EINVAL; /* phy name should be null-terminated */

        if (info->attrs[IEEE802154_ATTR_DEV_NAME]) {
                devname = nla_data(info->attrs[IEEE802154_ATTR_DEV_NAME]);
                if (devname[nla_len(info->attrs[IEEE802154_ATTR_DEV_NAME]) - 1]
                                != '\0')
                        return -EINVAL; /* phy name should be null-terminated */
                name_assign_type = NET_NAME_USER;
        } else  {
                devname = "wpan%d";
                name_assign_type = NET_NAME_ENUM;
        }

        if (strlen(devname) >= IFNAMSIZ)
                return -ENAMETOOLONG;

        phy = wpan_phy_find(name);
        if (!phy)
                return -ENODEV;

        msg = ieee802154_nl_new_reply(info, 0, IEEE802154_ADD_IFACE);
        if (!msg)
                goto out_dev;

        if (info->attrs[IEEE802154_ATTR_HW_ADDR] &&
            nla_len(info->attrs[IEEE802154_ATTR_HW_ADDR]) !=
                        IEEE802154_ADDR_LEN) {
                rc = -EINVAL;
                goto nla_put_failure;
        }

        if (info->attrs[IEEE802154_ATTR_DEV_TYPE]) {
                type = nla_get_u8(info->attrs[IEEE802154_ATTR_DEV_TYPE]);
                if (type >= __IEEE802154_DEV_MAX) {
                        rc = -EINVAL;
                        goto nla_put_failure;
                }
        }

        dev = rdev_add_virtual_intf_deprecated(wpan_phy_to_rdev(phy), devname,
                                               name_assign_type, type);
        if (IS_ERR(dev)) {
                rc = PTR_ERR(dev);
                goto nla_put_failure;
        }
        dev_hold(dev);

        if (info->attrs[IEEE802154_ATTR_HW_ADDR]) {
                struct sockaddr addr;

                addr.sa_family = ARPHRD_IEEE802154;
                nla_memcpy(&addr.sa_data, info->attrs[IEEE802154_ATTR_HW_ADDR],
                           IEEE802154_ADDR_LEN);

                /* strangely enough, some callbacks (inetdev_event) from
                 * dev_set_mac_address require RTNL_LOCK
                 */
                rtnl_lock();
                rc = dev_set_mac_address(dev, &addr, NULL);
                rtnl_unlock();
                if (rc)
                        goto dev_unregister;
        }

        if (nla_put_string(msg, IEEE802154_ATTR_PHY_NAME, wpan_phy_name(phy)) ||
            nla_put_string(msg, IEEE802154_ATTR_DEV_NAME, dev->name)) {
                rc = -EMSGSIZE;
                goto nla_put_failure;
        }
        dev_put(dev);

        wpan_phy_put(phy);

        return ieee802154_nl_reply(msg, info);

dev_unregister:
        rtnl_lock(); /* del_iface must be called with RTNL lock */
        rdev_del_virtual_intf_deprecated(wpan_phy_to_rdev(phy), dev);
        dev_put(dev);
        rtnl_unlock();
nla_put_failure:
        nlmsg_free(msg);
out_dev:
        wpan_phy_put(phy);
        return rc;
}

int ieee802154_del_iface(struct sk_buff *skb, struct genl_info *info)
{
        struct sk_buff *msg;
        struct wpan_phy *phy;
        const char *name;
        int rc;
        struct net_device *dev;

        pr_debug("%s\n", __func__);

        if (!info->attrs[IEEE802154_ATTR_DEV_NAME])
                return -EINVAL;

        name = nla_data(info->attrs[IEEE802154_ATTR_DEV_NAME]);
        if (name[nla_len(info->attrs[IEEE802154_ATTR_DEV_NAME]) - 1] != '\0')
                return -EINVAL; /* name should be null-terminated */

        rc = -ENODEV;
        dev = dev_get_by_name(genl_info_net(info), name);
        if (!dev)
                return rc;
        if (dev->type != ARPHRD_IEEE802154)
                goto out;

        phy = dev->ieee802154_ptr->wpan_phy;
        BUG_ON(!phy);
        get_device(&phy->dev);

        rc = -EINVAL;
        /* phy name is optional, but should be checked if it's given */
        if (info->attrs[IEEE802154_ATTR_PHY_NAME]) {
                struct wpan_phy *phy2;

                const char *pname =
                        nla_data(info->attrs[IEEE802154_ATTR_PHY_NAME]);
                if (pname[nla_len(info->attrs[IEEE802154_ATTR_PHY_NAME]) - 1]
                                != '\0')
                        /* name should be null-terminated */
                        goto out_dev;

                phy2 = wpan_phy_find(pname);
                if (!phy2)
                        goto out_dev;

                if (phy != phy2) {
                        wpan_phy_put(phy2);
                        goto out_dev;
                }
        }

        rc = -ENOBUFS;

        msg = ieee802154_nl_new_reply(info, 0, IEEE802154_DEL_IFACE);
        if (!msg)
                goto out_dev;

        rtnl_lock();
        rdev_del_virtual_intf_deprecated(wpan_phy_to_rdev(phy), dev);

        /* We don't have device anymore */
        dev_put(dev);
        dev = NULL;

        rtnl_unlock();

        if (nla_put_string(msg, IEEE802154_ATTR_PHY_NAME, wpan_phy_name(phy)) ||
            nla_put_string(msg, IEEE802154_ATTR_DEV_NAME, name))
                goto nla_put_failure;
        wpan_phy_put(phy);

        return ieee802154_nl_reply(msg, info);

nla_put_failure:
        nlmsg_free(msg);
out_dev:
        wpan_phy_put(phy);
out:
        dev_put(dev);

        return rc;
}











   11 




    2 




    1 


    1 
































    5 
    2 





    4 









   10 


    4 







   10 

    1 
    1 
    1 
    1 
    6 





   13 



    2 




    5 
    6 




   12 


    5 






    1 

    1 

    4 





    2 


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
// SPDX-License-Identifier: GPL-2.0-only
/* Copyright (c) 2017 Facebook
 */
#include <linux/slab.h>
#include <linux/bpf.h>
#include <linux/btf.h>

#include "map_in_map.h"

struct bpf_map *bpf_map_meta_alloc(int inner_map_ufd)
{
        struct bpf_map *inner_map, *inner_map_meta;
        u32 inner_map_meta_size;
        CLASS(fd, f)(inner_map_ufd);

        inner_map = __bpf_map_get(f);
        if (IS_ERR(inner_map))
                return inner_map;

        /* Does not support >1 level map-in-map */
        if (inner_map->inner_map_meta)
                return ERR_PTR(-EINVAL);

        if (!inner_map->ops->map_meta_equal)
                return ERR_PTR(-ENOTSUPP);

        inner_map_meta_size = sizeof(*inner_map_meta);
        /* In some cases verifier needs to access beyond just base map. */
        if (inner_map->ops == &array_map_ops || inner_map->ops == &percpu_array_map_ops)
                inner_map_meta_size = sizeof(struct bpf_array);

        inner_map_meta = kzalloc(inner_map_meta_size, GFP_USER);
        if (!inner_map_meta)
                return ERR_PTR(-ENOMEM);

        inner_map_meta->map_type = inner_map->map_type;
        inner_map_meta->key_size = inner_map->key_size;
        inner_map_meta->value_size = inner_map->value_size;
        inner_map_meta->map_flags = inner_map->map_flags;
        inner_map_meta->max_entries = inner_map->max_entries;

        inner_map_meta->record = btf_record_dup(inner_map->record);
        if (IS_ERR(inner_map_meta->record)) {
                /* btf_record_dup returns NULL or valid pointer in case of
                 * invalid/empty/valid, but ERR_PTR in case of errors. During
                 * equality NULL or IS_ERR is equivalent.
                 */
                struct bpf_map *ret = ERR_CAST(inner_map_meta->record);
                kfree(inner_map_meta);
                return ret;
        }
        /* Note: We must use the same BTF, as we also used btf_record_dup above
         * which relies on BTF being same for both maps, as some members like
         * record->fields.list_head have pointers like value_rec pointing into
         * inner_map->btf.
         */
        if (inner_map->btf) {
                btf_get(inner_map->btf);
                inner_map_meta->btf = inner_map->btf;
        }

        /* Misc members not needed in bpf_map_meta_equal() check. */
        inner_map_meta->ops = inner_map->ops;
        if (inner_map->ops == &array_map_ops || inner_map->ops == &percpu_array_map_ops) {
                struct bpf_array *inner_array_meta =
                        container_of(inner_map_meta, struct bpf_array, map);
                struct bpf_array *inner_array = container_of(inner_map, struct bpf_array, map);

                inner_array_meta->index_mask = inner_array->index_mask;
                inner_array_meta->elem_size = inner_array->elem_size;
                inner_map_meta->bypass_spec_v1 = inner_map->bypass_spec_v1;
        }
        return inner_map_meta;
}

void bpf_map_meta_free(struct bpf_map *map_meta)
{
        bpf_map_free_record(map_meta);
        btf_put(map_meta->btf);
        kfree(map_meta);
}

bool bpf_map_meta_equal(const struct bpf_map *meta0,
                        const struct bpf_map *meta1)
{
        /* No need to compare ops because it is covered by map_type */
        return meta0->map_type == meta1->map_type &&
                meta0->key_size == meta1->key_size &&
                meta0->value_size == meta1->value_size &&
                meta0->map_flags == meta1->map_flags &&
                btf_record_equal(meta0->record, meta1->record);
}

void *bpf_map_fd_get_ptr(struct bpf_map *map,
                         struct file *map_file /* not used */,
                         int ufd)
{
        struct bpf_map *inner_map, *inner_map_meta;
        CLASS(fd, f)(ufd);

        inner_map = __bpf_map_get(f);
        if (IS_ERR(inner_map))
                return inner_map;

        inner_map_meta = map->inner_map_meta;
        if (inner_map_meta->ops->map_meta_equal(inner_map_meta, inner_map))
                bpf_map_inc(inner_map);
        else
                inner_map = ERR_PTR(-EINVAL);

        return inner_map;
}

void bpf_map_fd_put_ptr(struct bpf_map *map, void *ptr, bool need_defer)
{
        struct bpf_map *inner_map = ptr;

        /* Defer the freeing of inner map according to the sleepable attribute
         * of bpf program which owns the outer map, so unnecessary waiting for
         * RCU tasks trace grace period can be avoided.
         */
        if (need_defer) {
                if (atomic64_read(&map->sleepable_refcnt))
                        WRITE_ONCE(inner_map->free_after_mult_rcu_gp, true);
                else
                        WRITE_ONCE(inner_map->free_after_rcu_gp, true);
        }
        bpf_map_put(inner_map);
}

u32 bpf_map_fd_sys_lookup_elem(void *ptr)
{
        return ((struct bpf_map *)ptr)->id;
}






































































































































































































































































































































































































































































































































  190 









  189 




































































































































































































































































































































































































































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
// SPDX-License-Identifier: GPL-2.0
/*
 * drivers/base/devres.c - device resource management
 *
 * Copyright (c) 2006  SUSE Linux Products GmbH
 * Copyright (c) 2006  Tejun Heo <teheo@suse.de>
 */

#include <linux/device.h>
#include <linux/module.h>
#include <linux/slab.h>
#include <linux/percpu.h>

#include <asm/sections.h>

#include "base.h"
#include "trace.h"

struct devres_node {
        struct list_head                entry;
        dr_release_t                        release;
        const char                        *name;
        size_t                                size;
};

struct devres {
        struct devres_node                node;
        /*
         * Some archs want to perform DMA into kmalloc caches
         * and need a guaranteed alignment larger than
         * the alignment of a 64-bit integer.
         * Thus we use ARCH_DMA_MINALIGN for data[] which will force the same
         * alignment for struct devres when allocated by kmalloc().
         */
        u8 __aligned(ARCH_DMA_MINALIGN) data[];
};

struct devres_group {
        struct devres_node                node[2];
        void                                *id;
        int                                color;
        /* -- 8 pointers */
};

static void set_node_dbginfo(struct devres_node *node, const char *name,
                             size_t size)
{
        node->name = name;
        node->size = size;
}

#ifdef CONFIG_DEBUG_DEVRES
static int log_devres = 0;
module_param_named(log, log_devres, int, S_IRUGO | S_IWUSR);

static void devres_dbg(struct device *dev, struct devres_node *node,
                       const char *op)
{
        if (unlikely(log_devres))
                dev_err(dev, "DEVRES %3s %p %s (%zu bytes)\n",
                        op, node, node->name, node->size);
}
#else /* CONFIG_DEBUG_DEVRES */
#define devres_dbg(dev, node, op)        do {} while (0)
#endif /* CONFIG_DEBUG_DEVRES */

static void devres_log(struct device *dev, struct devres_node *node,
                       const char *op)
{
        trace_devres_log(dev, op, node, node->name, node->size);
        devres_dbg(dev, node, op);
}

/*
 * Release functions for devres group.  These callbacks are used only
 * for identification.
 */
static void group_open_release(struct device *dev, void *res)
{
        /* noop */
}

static void group_close_release(struct device *dev, void *res)
{
        /* noop */
}

static struct devres_group *node_to_group(struct devres_node *node)
{
        if (node->release == &group_open_release)
                return container_of(node, struct devres_group, node[0]);
        if (node->release == &group_close_release)
                return container_of(node, struct devres_group, node[1]);
        return NULL;
}

static bool check_dr_size(size_t size, size_t *tot_size)
{
        /* We must catch any near-SIZE_MAX cases that could overflow. */
        if (unlikely(check_add_overflow(sizeof(struct devres),
                                        size, tot_size)))
                return false;

        /* Actually allocate the full kmalloc bucket size. */
        *tot_size = kmalloc_size_roundup(*tot_size);

        return true;
}

static __always_inline struct devres *alloc_dr(dr_release_t release,
                                               size_t size, gfp_t gfp, int nid)
{
        size_t tot_size;
        struct devres *dr;

        if (!check_dr_size(size, &tot_size))
                return NULL;

        dr = kmalloc_node_track_caller(tot_size, gfp, nid);
        if (unlikely(!dr))
                return NULL;

        /* No need to clear memory twice */
        if (!(gfp & __GFP_ZERO))
                memset(dr, 0, offsetof(struct devres, data));

        INIT_LIST_HEAD(&dr->node.entry);
        dr->node.release = release;
        return dr;
}

static void add_dr(struct device *dev, struct devres_node *node)
{
        devres_log(dev, node, "ADD");
        BUG_ON(!list_empty(&node->entry));
        list_add_tail(&node->entry, &dev->devres_head);
}

static void replace_dr(struct device *dev,
                       struct devres_node *old, struct devres_node *new)
{
        devres_log(dev, old, "REPLACE");
        BUG_ON(!list_empty(&new->entry));
        list_replace(&old->entry, &new->entry);
}

/**
 * __devres_alloc_node - Allocate device resource data
 * @release: Release function devres will be associated with
 * @size: Allocation size
 * @gfp: Allocation flags
 * @nid: NUMA node
 * @name: Name of the resource
 *
 * Allocate devres of @size bytes.  The allocated area is zeroed, then
 * associated with @release.  The returned pointer can be passed to
 * other devres_*() functions.
 *
 * RETURNS:
 * Pointer to allocated devres on success, NULL on failure.
 */
void *__devres_alloc_node(dr_release_t release, size_t size, gfp_t gfp, int nid,
                          const char *name)
{
        struct devres *dr;

        dr = alloc_dr(release, size, gfp | __GFP_ZERO, nid);
        if (unlikely(!dr))
                return NULL;
        set_node_dbginfo(&dr->node, name, size);
        return dr->data;
}
EXPORT_SYMBOL_GPL(__devres_alloc_node);

/**
 * devres_for_each_res - Resource iterator
 * @dev: Device to iterate resource from
 * @release: Look for resources associated with this release function
 * @match: Match function (optional)
 * @match_data: Data for the match function
 * @fn: Function to be called for each matched resource.
 * @data: Data for @fn, the 3rd parameter of @fn
 *
 * Call @fn for each devres of @dev which is associated with @release
 * and for which @match returns 1.
 *
 * RETURNS:
 *         void
 */
void devres_for_each_res(struct device *dev, dr_release_t release,
                        dr_match_t match, void *match_data,
                        void (*fn)(struct device *, void *, void *),
                        void *data)
{
        struct devres_node *node;
        struct devres_node *tmp;
        unsigned long flags;

        if (!fn)
                return;

        spin_lock_irqsave(&dev->devres_lock, flags);
        list_for_each_entry_safe_reverse(node, tmp,
                        &dev->devres_head, entry) {
                struct devres *dr = container_of(node, struct devres, node);

                if (node->release != release)
                        continue;
                if (match && !match(dev, dr->data, match_data))
                        continue;
                fn(dev, dr->data, data);
        }
        spin_unlock_irqrestore(&dev->devres_lock, flags);
}
EXPORT_SYMBOL_GPL(devres_for_each_res);

/**
 * devres_free - Free device resource data
 * @res: Pointer to devres data to free
 *
 * Free devres created with devres_alloc().
 */
void devres_free(void *res)
{
        if (res) {
                struct devres *dr = container_of(res, struct devres, data);

                BUG_ON(!list_empty(&dr->node.entry));
                kfree(dr);
        }
}
EXPORT_SYMBOL_GPL(devres_free);

/**
 * devres_add - Register device resource
 * @dev: Device to add resource to
 * @res: Resource to register
 *
 * Register devres @res to @dev.  @res should have been allocated
 * using devres_alloc().  On driver detach, the associated release
 * function will be invoked and devres will be freed automatically.
 */
void devres_add(struct device *dev, void *res)
{
        struct devres *dr = container_of(res, struct devres, data);
        unsigned long flags;

        spin_lock_irqsave(&dev->devres_lock, flags);
        add_dr(dev, &dr->node);
        spin_unlock_irqrestore(&dev->devres_lock, flags);
}
EXPORT_SYMBOL_GPL(devres_add);

static struct devres *find_dr(struct device *dev, dr_release_t release,
                              dr_match_t match, void *match_data)
{
        struct devres_node *node;

        list_for_each_entry_reverse(node, &dev->devres_head, entry) {
                struct devres *dr = container_of(node, struct devres, node);

                if (node->release != release)
                        continue;
                if (match && !match(dev, dr->data, match_data))
                        continue;
                return dr;
        }

        return NULL;
}

/**
 * devres_find - Find device resource
 * @dev: Device to lookup resource from
 * @release: Look for resources associated with this release function
 * @match: Match function (optional)
 * @match_data: Data for the match function
 *
 * Find the latest devres of @dev which is associated with @release
 * and for which @match returns 1.  If @match is NULL, it's considered
 * to match all.
 *
 * RETURNS:
 * Pointer to found devres, NULL if not found.
 */
void *devres_find(struct device *dev, dr_release_t release,
                  dr_match_t match, void *match_data)
{
        struct devres *dr;
        unsigned long flags;

        spin_lock_irqsave(&dev->devres_lock, flags);
        dr = find_dr(dev, release, match, match_data);
        spin_unlock_irqrestore(&dev->devres_lock, flags);

        if (dr)
                return dr->data;
        return NULL;
}
EXPORT_SYMBOL_GPL(devres_find);

/**
 * devres_get - Find devres, if non-existent, add one atomically
 * @dev: Device to lookup or add devres for
 * @new_res: Pointer to new initialized devres to add if not found
 * @match: Match function (optional)
 * @match_data: Data for the match function
 *
 * Find the latest devres of @dev which has the same release function
 * as @new_res and for which @match return 1.  If found, @new_res is
 * freed; otherwise, @new_res is added atomically.
 *
 * RETURNS:
 * Pointer to found or added devres.
 */
void *devres_get(struct device *dev, void *new_res,
                 dr_match_t match, void *match_data)
{
        struct devres *new_dr = container_of(new_res, struct devres, data);
        struct devres *dr;
        unsigned long flags;

        spin_lock_irqsave(&dev->devres_lock, flags);
        dr = find_dr(dev, new_dr->node.release, match, match_data);
        if (!dr) {
                add_dr(dev, &new_dr->node);
                dr = new_dr;
                new_res = NULL;
        }
        spin_unlock_irqrestore(&dev->devres_lock, flags);
        devres_free(new_res);

        return dr->data;
}
EXPORT_SYMBOL_GPL(devres_get);

/**
 * devres_remove - Find a device resource and remove it
 * @dev: Device to find resource from
 * @release: Look for resources associated with this release function
 * @match: Match function (optional)
 * @match_data: Data for the match function
 *
 * Find the latest devres of @dev associated with @release and for
 * which @match returns 1.  If @match is NULL, it's considered to
 * match all.  If found, the resource is removed atomically and
 * returned.
 *
 * RETURNS:
 * Pointer to removed devres on success, NULL if not found.
 */
void *devres_remove(struct device *dev, dr_release_t release,
                    dr_match_t match, void *match_data)
{
        struct devres *dr;
        unsigned long flags;

        spin_lock_irqsave(&dev->devres_lock, flags);
        dr = find_dr(dev, release, match, match_data);
        if (dr) {
                list_del_init(&dr->node.entry);
                devres_log(dev, &dr->node, "REM");
        }
        spin_unlock_irqrestore(&dev->devres_lock, flags);

        if (dr)
                return dr->data;
        return NULL;
}
EXPORT_SYMBOL_GPL(devres_remove);

/**
 * devres_destroy - Find a device resource and destroy it
 * @dev: Device to find resource from
 * @release: Look for resources associated with this release function
 * @match: Match function (optional)
 * @match_data: Data for the match function
 *
 * Find the latest devres of @dev associated with @release and for
 * which @match returns 1.  If @match is NULL, it's considered to
 * match all.  If found, the resource is removed atomically and freed.
 *
 * Note that the release function for the resource will not be called,
 * only the devres-allocated data will be freed.  The caller becomes
 * responsible for freeing any other data.
 *
 * RETURNS:
 * 0 if devres is found and freed, -ENOENT if not found.
 */
int devres_destroy(struct device *dev, dr_release_t release,
                   dr_match_t match, void *match_data)
{
        void *res;

        res = devres_remove(dev, release, match, match_data);
        if (unlikely(!res))
                return -ENOENT;

        devres_free(res);
        return 0;
}
EXPORT_SYMBOL_GPL(devres_destroy);


/**
 * devres_release - Find a device resource and destroy it, calling release
 * @dev: Device to find resource from
 * @release: Look for resources associated with this release function
 * @match: Match function (optional)
 * @match_data: Data for the match function
 *
 * Find the latest devres of @dev associated with @release and for
 * which @match returns 1.  If @match is NULL, it's considered to
 * match all.  If found, the resource is removed atomically, the
 * release function called and the resource freed.
 *
 * RETURNS:
 * 0 if devres is found and freed, -ENOENT if not found.
 */
int devres_release(struct device *dev, dr_release_t release,
                   dr_match_t match, void *match_data)
{
        void *res;

        res = devres_remove(dev, release, match, match_data);
        if (unlikely(!res))
                return -ENOENT;

        (*release)(dev, res);
        devres_free(res);
        return 0;
}
EXPORT_SYMBOL_GPL(devres_release);

static int remove_nodes(struct device *dev,
                        struct list_head *first, struct list_head *end,
                        struct list_head *todo)
{
        struct devres_node *node, *n;
        int cnt = 0, nr_groups = 0;

        /* First pass - move normal devres entries to @todo and clear
         * devres_group colors.
         */
        node = list_entry(first, struct devres_node, entry);
        list_for_each_entry_safe_from(node, n, end, entry) {
                struct devres_group *grp;

                grp = node_to_group(node);
                if (grp) {
                        /* clear color of group markers in the first pass */
                        grp->color = 0;
                        nr_groups++;
                } else {
                        /* regular devres entry */
                        if (&node->entry == first)
                                first = first->next;
                        list_move_tail(&node->entry, todo);
                        cnt++;
                }
        }

        if (!nr_groups)
                return cnt;

        /* Second pass - Scan groups and color them.  A group gets
         * color value of two iff the group is wholly contained in
         * [current node, end). That is, for a closed group, both opening
         * and closing markers should be in the range, while just the
         * opening marker is enough for an open group.
         */
        node = list_entry(first, struct devres_node, entry);
        list_for_each_entry_safe_from(node, n, end, entry) {
                struct devres_group *grp;

                grp = node_to_group(node);
                BUG_ON(!grp || list_empty(&grp->node[0].entry));

                grp->color++;
                if (list_empty(&grp->node[1].entry))
                        grp->color++;

                BUG_ON(grp->color <= 0 || grp->color > 2);
                if (grp->color == 2) {
                        /* No need to update current node or end. The removed
                         * nodes are always before both.
                         */
                        list_move_tail(&grp->node[0].entry, todo);
                        list_del_init(&grp->node[1].entry);
                }
        }

        return cnt;
}

static void release_nodes(struct device *dev, struct list_head *todo)
{
        struct devres *dr, *tmp;

        /* Release.  Note that both devres and devres_group are
         * handled as devres in the following loop.  This is safe.
         */
        list_for_each_entry_safe_reverse(dr, tmp, todo, node.entry) {
                devres_log(dev, &dr->node, "REL");
                dr->node.release(dev, dr->data);
                kfree(dr);
        }
}

/**
 * devres_release_all - Release all managed resources
 * @dev: Device to release resources for
 *
 * Release all resources associated with @dev.  This function is
 * called on driver detach.
 */
int devres_release_all(struct device *dev)
{
        unsigned long flags;
        LIST_HEAD(todo);
        int cnt;

        /* Looks like an uninitialized device structure */
        if (WARN_ON(dev->devres_head.next == NULL))
                return -ENODEV;

        /* Nothing to release if list is empty */
        if (list_empty(&dev->devres_head))
                return 0;

        spin_lock_irqsave(&dev->devres_lock, flags);
        cnt = remove_nodes(dev, dev->devres_head.next, &dev->devres_head, &todo);
        spin_unlock_irqrestore(&dev->devres_lock, flags);

        release_nodes(dev, &todo);
        return cnt;
}

/**
 * devres_open_group - Open a new devres group
 * @dev: Device to open devres group for
 * @id: Separator ID
 * @gfp: Allocation flags
 *
 * Open a new devres group for @dev with @id.  For @id, using a
 * pointer to an object which won't be used for another group is
 * recommended.  If @id is NULL, address-wise unique ID is created.
 *
 * RETURNS:
 * ID of the new group, NULL on failure.
 */
void *devres_open_group(struct device *dev, void *id, gfp_t gfp)
{
        struct devres_group *grp;
        unsigned long flags;

        grp = kmalloc(sizeof(*grp), gfp);
        if (unlikely(!grp))
                return NULL;

        grp->node[0].release = &group_open_release;
        grp->node[1].release = &group_close_release;
        INIT_LIST_HEAD(&grp->node[0].entry);
        INIT_LIST_HEAD(&grp->node[1].entry);
        set_node_dbginfo(&grp->node[0], "grp<", 0);
        set_node_dbginfo(&grp->node[1], "grp>", 0);
        grp->id = grp;
        if (id)
                grp->id = id;
        grp->color = 0;

        spin_lock_irqsave(&dev->devres_lock, flags);
        add_dr(dev, &grp->node[0]);
        spin_unlock_irqrestore(&dev->devres_lock, flags);
        return grp->id;
}
EXPORT_SYMBOL_GPL(devres_open_group);

/* Find devres group with ID @id.  If @id is NULL, look for the latest. */
static struct devres_group *find_group(struct device *dev, void *id)
{
        struct devres_node *node;

        list_for_each_entry_reverse(node, &dev->devres_head, entry) {
                struct devres_group *grp;

                if (node->release != &group_open_release)
                        continue;

                grp = container_of(node, struct devres_group, node[0]);

                if (id) {
                        if (grp->id == id)
                                return grp;
                } else if (list_empty(&grp->node[1].entry))
                        return grp;
        }

        return NULL;
}

/**
 * devres_close_group - Close a devres group
 * @dev: Device to close devres group for
 * @id: ID of target group, can be NULL
 *
 * Close the group identified by @id.  If @id is NULL, the latest open
 * group is selected.
 */
void devres_close_group(struct device *dev, void *id)
{
        struct devres_group *grp;
        unsigned long flags;

        spin_lock_irqsave(&dev->devres_lock, flags);

        grp = find_group(dev, id);
        if (grp)
                add_dr(dev, &grp->node[1]);
        else
                WARN_ON(1);

        spin_unlock_irqrestore(&dev->devres_lock, flags);
}
EXPORT_SYMBOL_GPL(devres_close_group);

/**
 * devres_remove_group - Remove a devres group
 * @dev: Device to remove group for
 * @id: ID of target group, can be NULL
 *
 * Remove the group identified by @id.  If @id is NULL, the latest
 * open group is selected.  Note that removing a group doesn't affect
 * any other resources.
 */
void devres_remove_group(struct device *dev, void *id)
{
        struct devres_group *grp;
        unsigned long flags;

        spin_lock_irqsave(&dev->devres_lock, flags);

        grp = find_group(dev, id);
        if (grp) {
                list_del_init(&grp->node[0].entry);
                list_del_init(&grp->node[1].entry);
                devres_log(dev, &grp->node[0], "REM");
        } else
                WARN_ON(1);

        spin_unlock_irqrestore(&dev->devres_lock, flags);

        kfree(grp);
}
EXPORT_SYMBOL_GPL(devres_remove_group);

/**
 * devres_release_group - Release resources in a devres group
 * @dev: Device to release group for
 * @id: ID of target group, can be NULL
 *
 * Release all resources in the group identified by @id.  If @id is
 * NULL, the latest open group is selected.  The selected group and
 * groups properly nested inside the selected group are removed.
 *
 * RETURNS:
 * The number of released non-group resources.
 */
int devres_release_group(struct device *dev, void *id)
{
        struct devres_group *grp;
        unsigned long flags;
        LIST_HEAD(todo);
        int cnt = 0;

        spin_lock_irqsave(&dev->devres_lock, flags);

        grp = find_group(dev, id);
        if (grp) {
                struct list_head *first = &grp->node[0].entry;
                struct list_head *end = &dev->devres_head;

                if (!list_empty(&grp->node[1].entry))
                        end = grp->node[1].entry.next;

                cnt = remove_nodes(dev, first, end, &todo);
                spin_unlock_irqrestore(&dev->devres_lock, flags);

                release_nodes(dev, &todo);
        } else {
                WARN_ON(1);
                spin_unlock_irqrestore(&dev->devres_lock, flags);
        }

        return cnt;
}
EXPORT_SYMBOL_GPL(devres_release_group);

/*
 * Custom devres actions allow inserting a simple function call
 * into the teardown sequence.
 */

struct action_devres {
        void *data;
        void (*action)(void *);
};

static int devm_action_match(struct device *dev, void *res, void *p)
{
        struct action_devres *devres = res;
        struct action_devres *target = p;

        return devres->action == target->action &&
               devres->data == target->data;
}

static void devm_action_release(struct device *dev, void *res)
{
        struct action_devres *devres = res;

        devres->action(devres->data);
}

/**
 * __devm_add_action() - add a custom action to list of managed resources
 * @dev: Device that owns the action
 * @action: Function that should be called
 * @data: Pointer to data passed to @action implementation
 * @name: Name of the resource (for debugging purposes)
 *
 * This adds a custom action to the list of managed resources so that
 * it gets executed as part of standard resource unwinding.
 */
int __devm_add_action(struct device *dev, void (*action)(void *), void *data, const char *name)
{
        struct action_devres *devres;

        devres = __devres_alloc_node(devm_action_release, sizeof(struct action_devres),
                                     GFP_KERNEL, NUMA_NO_NODE, name);
        if (!devres)
                return -ENOMEM;

        devres->data = data;
        devres->action = action;

        devres_add(dev, devres);
        return 0;
}
EXPORT_SYMBOL_GPL(__devm_add_action);

/**
 * devm_remove_action() - removes previously added custom action
 * @dev: Device that owns the action
 * @action: Function implementing the action
 * @data: Pointer to data passed to @action implementation
 *
 * Removes instance of @action previously added by devm_add_action().
 * Both action and data should match one of the existing entries.
 */
void devm_remove_action(struct device *dev, void (*action)(void *), void *data)
{
        struct action_devres devres = {
                .data = data,
                .action = action,
        };

        WARN_ON(devres_destroy(dev, devm_action_release, devm_action_match,
                               &devres));
}
EXPORT_SYMBOL_GPL(devm_remove_action);

/**
 * devm_release_action() - release previously added custom action
 * @dev: Device that owns the action
 * @action: Function implementing the action
 * @data: Pointer to data passed to @action implementation
 *
 * Releases and removes instance of @action previously added by
 * devm_add_action().  Both action and data should match one of the
 * existing entries.
 */
void devm_release_action(struct device *dev, void (*action)(void *), void *data)
{
        struct action_devres devres = {
                .data = data,
                .action = action,
        };

        WARN_ON(devres_release(dev, devm_action_release, devm_action_match,
                               &devres));

}
EXPORT_SYMBOL_GPL(devm_release_action);

/*
 * Managed kmalloc/kfree
 */
static void devm_kmalloc_release(struct device *dev, void *res)
{
        /* noop */
}

static int devm_kmalloc_match(struct device *dev, void *res, void *data)
{
        return res == data;
}

/**
 * devm_kmalloc - Resource-managed kmalloc
 * @dev: Device to allocate memory for
 * @size: Allocation size
 * @gfp: Allocation gfp flags
 *
 * Managed kmalloc.  Memory allocated with this function is
 * automatically freed on driver detach.  Like all other devres
 * resources, guaranteed alignment is unsigned long long.
 *
 * RETURNS:
 * Pointer to allocated memory on success, NULL on failure.
 */
void *devm_kmalloc(struct device *dev, size_t size, gfp_t gfp)
{
        struct devres *dr;

        if (unlikely(!size))
                return ZERO_SIZE_PTR;

        /* use raw alloc_dr for kmalloc caller tracing */
        dr = alloc_dr(devm_kmalloc_release, size, gfp, dev_to_node(dev));
        if (unlikely(!dr))
                return NULL;

        /*
         * This is named devm_kzalloc_release for historical reasons
         * The initial implementation did not support kmalloc, only kzalloc
         */
        set_node_dbginfo(&dr->node, "devm_kzalloc_release", size);
        devres_add(dev, dr->data);
        return dr->data;
}
EXPORT_SYMBOL_GPL(devm_kmalloc);

/**
 * devm_krealloc - Resource-managed krealloc()
 * @dev: Device to re-allocate memory for
 * @ptr: Pointer to the memory chunk to re-allocate
 * @new_size: New allocation size
 * @gfp: Allocation gfp flags
 *
 * Managed krealloc(). Resizes the memory chunk allocated with devm_kmalloc().
 * Behaves similarly to regular krealloc(): if @ptr is NULL or ZERO_SIZE_PTR,
 * it's the equivalent of devm_kmalloc(). If new_size is zero, it frees the
 * previously allocated memory and returns ZERO_SIZE_PTR. This function doesn't
 * change the order in which the release callback for the re-alloc'ed devres
 * will be called (except when falling back to devm_kmalloc() or when freeing
 * resources when new_size is zero). The contents of the memory are preserved
 * up to the lesser of new and old sizes.
 */
void *devm_krealloc(struct device *dev, void *ptr, size_t new_size, gfp_t gfp)
{
        size_t total_new_size, total_old_size;
        struct devres *old_dr, *new_dr;
        unsigned long flags;

        if (unlikely(!new_size)) {
                devm_kfree(dev, ptr);
                return ZERO_SIZE_PTR;
        }

        if (unlikely(ZERO_OR_NULL_PTR(ptr)))
                return devm_kmalloc(dev, new_size, gfp);

        if (WARN_ON(is_kernel_rodata((unsigned long)ptr)))
                /*
                 * We cannot reliably realloc a const string returned by
                 * devm_kstrdup_const().
                 */
                return NULL;

        if (!check_dr_size(new_size, &total_new_size))
                return NULL;

        total_old_size = ksize(container_of(ptr, struct devres, data));
        if (total_old_size == 0) {
                WARN(1, "Pointer doesn't point to dynamically allocated memory.");
                return NULL;
        }

        /*
         * If new size is smaller or equal to the actual number of bytes
         * allocated previously - just return the same pointer.
         */
        if (total_new_size <= total_old_size)
                return ptr;

        /*
         * Otherwise: allocate new, larger chunk. We need to allocate before
         * taking the lock as most probably the caller uses GFP_KERNEL.
         * alloc_dr() will call check_dr_size() to reserve extra memory
         * for struct devres automatically, so size @new_size user request
         * is delivered to it directly as devm_kmalloc() does.
         */
        new_dr = alloc_dr(devm_kmalloc_release,
                          new_size, gfp, dev_to_node(dev));
        if (!new_dr)
                return NULL;

        /*
         * The spinlock protects the linked list against concurrent
         * modifications but not the resource itself.
         */
        spin_lock_irqsave(&dev->devres_lock, flags);

        old_dr = find_dr(dev, devm_kmalloc_release, devm_kmalloc_match, ptr);
        if (!old_dr) {
                spin_unlock_irqrestore(&dev->devres_lock, flags);
                kfree(new_dr);
                WARN(1, "Memory chunk not managed or managed by a different device.");
                return NULL;
        }

        replace_dr(dev, &old_dr->node, &new_dr->node);

        spin_unlock_irqrestore(&dev->devres_lock, flags);

        /*
         * We can copy the memory contents after releasing the lock as we're
         * no longer modifying the list links.
         */
        memcpy(new_dr->data, old_dr->data,
               total_old_size - offsetof(struct devres, data));
        /*
         * Same for releasing the old devres - it's now been removed from the
         * list. This is also the reason why we must not use devm_kfree() - the
         * links are no longer valid.
         */
        kfree(old_dr);

        return new_dr->data;
}
EXPORT_SYMBOL_GPL(devm_krealloc);

/**
 * devm_kstrdup - Allocate resource managed space and
 *                copy an existing string into that.
 * @dev: Device to allocate memory for
 * @s: the string to duplicate
 * @gfp: the GFP mask used in the devm_kmalloc() call when
 *       allocating memory
 * RETURNS:
 * Pointer to allocated string on success, NULL on failure.
 */
char *devm_kstrdup(struct device *dev, const char *s, gfp_t gfp)
{
        size_t size;
        char *buf;

        if (!s)
                return NULL;

        size = strlen(s) + 1;
        buf = devm_kmalloc(dev, size, gfp);
        if (buf)
                memcpy(buf, s, size);
        return buf;
}
EXPORT_SYMBOL_GPL(devm_kstrdup);

/**
 * devm_kstrdup_const - resource managed conditional string duplication
 * @dev: device for which to duplicate the string
 * @s: the string to duplicate
 * @gfp: the GFP mask used in the kmalloc() call when allocating memory
 *
 * Strings allocated by devm_kstrdup_const will be automatically freed when
 * the associated device is detached.
 *
 * RETURNS:
 * Source string if it is in .rodata section otherwise it falls back to
 * devm_kstrdup.
 */
const char *devm_kstrdup_const(struct device *dev, const char *s, gfp_t gfp)
{
        if (is_kernel_rodata((unsigned long)s))
                return s;

        return devm_kstrdup(dev, s, gfp);
}
EXPORT_SYMBOL_GPL(devm_kstrdup_const);

/**
 * devm_kvasprintf - Allocate resource managed space and format a string
 *                     into that.
 * @dev: Device to allocate memory for
 * @gfp: the GFP mask used in the devm_kmalloc() call when
 *       allocating memory
 * @fmt: The printf()-style format string
 * @ap: Arguments for the format string
 * RETURNS:
 * Pointer to allocated string on success, NULL on failure.
 */
char *devm_kvasprintf(struct device *dev, gfp_t gfp, const char *fmt,
                      va_list ap)
{
        unsigned int len;
        char *p;
        va_list aq;

        va_copy(aq, ap);
        len = vsnprintf(NULL, 0, fmt, aq);
        va_end(aq);

        p = devm_kmalloc(dev, len+1, gfp);
        if (!p)
                return NULL;

        vsnprintf(p, len+1, fmt, ap);

        return p;
}
EXPORT_SYMBOL(devm_kvasprintf);

/**
 * devm_kasprintf - Allocate resource managed space and format a string
 *                    into that.
 * @dev: Device to allocate memory for
 * @gfp: the GFP mask used in the devm_kmalloc() call when
 *       allocating memory
 * @fmt: The printf()-style format string
 * @...: Arguments for the format string
 * RETURNS:
 * Pointer to allocated string on success, NULL on failure.
 */
char *devm_kasprintf(struct device *dev, gfp_t gfp, const char *fmt, ...)
{
        va_list ap;
        char *p;

        va_start(ap, fmt);
        p = devm_kvasprintf(dev, gfp, fmt, ap);
        va_end(ap);

        return p;
}
EXPORT_SYMBOL_GPL(devm_kasprintf);

/**
 * devm_kfree - Resource-managed kfree
 * @dev: Device this memory belongs to
 * @p: Memory to free
 *
 * Free memory allocated with devm_kmalloc().
 */
void devm_kfree(struct device *dev, const void *p)
{
        int rc;

        /*
         * Special cases: pointer to a string in .rodata returned by
         * devm_kstrdup_const() or NULL/ZERO ptr.
         */
        if (unlikely(is_kernel_rodata((unsigned long)p) || ZERO_OR_NULL_PTR(p)))
                return;

        rc = devres_destroy(dev, devm_kmalloc_release,
                            devm_kmalloc_match, (void *)p);
        WARN_ON(rc);
}
EXPORT_SYMBOL_GPL(devm_kfree);

/**
 * devm_kmemdup - Resource-managed kmemdup
 * @dev: Device this memory belongs to
 * @src: Memory region to duplicate
 * @len: Memory region length
 * @gfp: GFP mask to use
 *
 * Duplicate region of a memory using resource managed kmalloc
 */
void *devm_kmemdup(struct device *dev, const void *src, size_t len, gfp_t gfp)
{
        void *p;

        p = devm_kmalloc(dev, len, gfp);
        if (p)
                memcpy(p, src, len);

        return p;
}
EXPORT_SYMBOL_GPL(devm_kmemdup);

struct pages_devres {
        unsigned long addr;
        unsigned int order;
};

static int devm_pages_match(struct device *dev, void *res, void *p)
{
        struct pages_devres *devres = res;
        struct pages_devres *target = p;

        return devres->addr == target->addr;
}

static void devm_pages_release(struct device *dev, void *res)
{
        struct pages_devres *devres = res;

        free_pages(devres->addr, devres->order);
}

/**
 * devm_get_free_pages - Resource-managed __get_free_pages
 * @dev: Device to allocate memory for
 * @gfp_mask: Allocation gfp flags
 * @order: Allocation size is (1 << order) pages
 *
 * Managed get_free_pages.  Memory allocated with this function is
 * automatically freed on driver detach.
 *
 * RETURNS:
 * Address of allocated memory on success, 0 on failure.
 */

unsigned long devm_get_free_pages(struct device *dev,
                                  gfp_t gfp_mask, unsigned int order)
{
        struct pages_devres *devres;
        unsigned long addr;

        addr = __get_free_pages(gfp_mask, order);

        if (unlikely(!addr))
                return 0;

        devres = devres_alloc(devm_pages_release,
                              sizeof(struct pages_devres), GFP_KERNEL);
        if (unlikely(!devres)) {
                free_pages(addr, order);
                return 0;
        }

        devres->addr = addr;
        devres->order = order;

        devres_add(dev, devres);
        return addr;
}
EXPORT_SYMBOL_GPL(devm_get_free_pages);

/**
 * devm_free_pages - Resource-managed free_pages
 * @dev: Device this memory belongs to
 * @addr: Memory to free
 *
 * Free memory allocated with devm_get_free_pages(). Unlike free_pages,
 * there is no need to supply the @order.
 */
void devm_free_pages(struct device *dev, unsigned long addr)
{
        struct pages_devres devres = { .addr = addr };

        WARN_ON(devres_release(dev, devm_pages_release, devm_pages_match,
                               &devres));
}
EXPORT_SYMBOL_GPL(devm_free_pages);

static void devm_percpu_release(struct device *dev, void *pdata)
{
        void __percpu *p;

        p = *(void __percpu **)pdata;
        free_percpu(p);
}

static int devm_percpu_match(struct device *dev, void *data, void *p)
{
        struct devres *devr = container_of(data, struct devres, data);

        return *(void **)devr->data == p;
}

/**
 * __devm_alloc_percpu - Resource-managed alloc_percpu
 * @dev: Device to allocate per-cpu memory for
 * @size: Size of per-cpu memory to allocate
 * @align: Alignment of per-cpu memory to allocate
 *
 * Managed alloc_percpu. Per-cpu memory allocated with this function is
 * automatically freed on driver detach.
 *
 * RETURNS:
 * Pointer to allocated memory on success, NULL on failure.
 */
void __percpu *__devm_alloc_percpu(struct device *dev, size_t size,
                size_t align)
{
        void *p;
        void __percpu *pcpu;

        pcpu = __alloc_percpu(size, align);
        if (!pcpu)
                return NULL;

        p = devres_alloc(devm_percpu_release, sizeof(void *), GFP_KERNEL);
        if (!p) {
                free_percpu(pcpu);
                return NULL;
        }

        *(void __percpu **)p = pcpu;

        devres_add(dev, p);

        return pcpu;
}
EXPORT_SYMBOL_GPL(__devm_alloc_percpu);

/**
 * devm_free_percpu - Resource-managed free_percpu
 * @dev: Device this memory belongs to
 * @pdata: Per-cpu memory to free
 *
 * Free memory allocated with devm_alloc_percpu().
 */
void devm_free_percpu(struct device *dev, void __percpu *pdata)
{
        /*
         * Use devres_release() to prevent memory leakage as
         * devm_free_pages() does.
         */
        WARN_ON(devres_release(dev, devm_percpu_release, devm_percpu_match,
                               (void *)(__force unsigned long)pdata));
}
EXPORT_SYMBOL_GPL(devm_free_percpu);

























































































































  214 




































 20549 

  651 
 20777 
















1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
/* SPDX-License-Identifier: GPL-2.0-only */
#ifndef _LINUX_FILE_REF_H
#define _LINUX_FILE_REF_H

#include <linux/atomic.h>
#include <linux/preempt.h>
#include <linux/types.h>

/*
 * file_ref is a reference count implementation specifically for use by
 * files. It takes inspiration from rcuref but differs in key aspects
 * such as support for SLAB_TYPESAFE_BY_RCU type caches.
 *
 * FILE_REF_ONEREF                FILE_REF_MAXREF
 * 0x0000000000000000UL      0x7FFFFFFFFFFFFFFFUL
 * <-------------------valid ------------------->
 *
 *                       FILE_REF_SATURATED
 * 0x8000000000000000UL 0xA000000000000000UL 0xBFFFFFFFFFFFFFFFUL
 * <-----------------------saturation zone---------------------->
 *
 * FILE_REF_RELEASED                   FILE_REF_DEAD
 * 0xC000000000000000UL         0xE000000000000000UL
 * <-------------------dead zone------------------->
 *
 * FILE_REF_NOREF
 * 0xFFFFFFFFFFFFFFFFUL
 */

#ifdef CONFIG_64BIT
#define FILE_REF_ONEREF                0x0000000000000000UL
#define FILE_REF_MAXREF                0x7FFFFFFFFFFFFFFFUL
#define FILE_REF_SATURATED        0xA000000000000000UL
#define FILE_REF_RELEASED        0xC000000000000000UL
#define FILE_REF_DEAD                0xE000000000000000UL
#define FILE_REF_NOREF                0xFFFFFFFFFFFFFFFFUL
#else
#define FILE_REF_ONEREF                0x00000000U
#define FILE_REF_MAXREF                0x7FFFFFFFU
#define FILE_REF_SATURATED        0xA0000000U
#define FILE_REF_RELEASED        0xC0000000U
#define FILE_REF_DEAD                0xE0000000U
#define FILE_REF_NOREF                0xFFFFFFFFU
#endif

typedef struct {
#ifdef CONFIG_64BIT
        atomic64_t refcnt;
#else
        atomic_t refcnt;
#endif
} file_ref_t;

/**
 * file_ref_init - Initialize a file reference count
 * @ref: Pointer to the reference count
 * @cnt: The initial reference count typically '1'
 */
static inline void file_ref_init(file_ref_t *ref, unsigned long cnt)
{
        atomic_long_set(&ref->refcnt, cnt - 1);
}

bool __file_ref_put(file_ref_t *ref, unsigned long cnt);

/**
 * file_ref_get - Acquire one reference on a file
 * @ref: Pointer to the reference count
 *
 * Similar to atomic_inc_not_zero() but saturates at FILE_REF_MAXREF.
 *
 * Provides full memory ordering.
 *
 * Return: False if the attempt to acquire a reference failed. This happens
 *         when the last reference has been put already. True if a reference
 *         was successfully acquired
 */
static __always_inline __must_check bool file_ref_get(file_ref_t *ref)
{
        /*
         * Unconditionally increase the reference count with full
         * ordering. The saturation and dead zones provide enough
         * tolerance for this.
         *
         * If this indicates negative the file in question the fail can
         * be freed and immediately reused due to SLAB_TYPSAFE_BY_RCU.
         * Hence, unconditionally altering the file reference count to
         * e.g., reset the file reference count back to the middle of
         * the deadzone risk end up marking someone else's file as dead
         * behind their back.
         *
         * It would be possible to do a careful:
         *
         * cnt = atomic_long_inc_return();
         * if (likely(cnt >= 0))
         *        return true;
         *
         * and then something like:
         *
         * if (cnt >= FILE_REF_RELEASE)
         *        atomic_long_try_cmpxchg(&ref->refcnt, &cnt, FILE_REF_DEAD),
         *
         * to set the value back to the middle of the deadzone. But it's
         * practically impossible to go from FILE_REF_DEAD to
         * FILE_REF_ONEREF. It would need 2305843009213693952/2^61
         * file_ref_get()s to resurrect such a dead file.
         */
        return !atomic_long_add_negative(1, &ref->refcnt);
}

/**
 * file_ref_inc - Acquire one reference on a file
 * @ref: Pointer to the reference count
 *
 * Acquire an additional reference on a file. Warns if the caller didn't
 * already hold a reference.
 */
static __always_inline void file_ref_inc(file_ref_t *ref)
{
        long prior = atomic_long_fetch_inc_relaxed(&ref->refcnt);
        WARN_ONCE(prior < 0, "file_ref_inc() on a released file reference");
}

/**
 * file_ref_put -- Release a file reference
 * @ref:        Pointer to the reference count
 *
 * Provides release memory ordering, such that prior loads and stores
 * are done before, and provides an acquire ordering on success such
 * that free() must come after.
 *
 * Return: True if this was the last reference with no future references
 *         possible. This signals the caller that it can safely release
 *         the object which is protected by the reference counter.
 *         False if there are still active references or the put() raced
 *         with a concurrent get()/put() pair. Caller is not allowed to
 *         release the protected object.
 */
static __always_inline __must_check bool file_ref_put(file_ref_t *ref)
{
        long cnt;

        /*
         * While files are SLAB_TYPESAFE_BY_RCU and thus file_ref_put()
         * calls don't risk UAFs when a file is recyclyed, it is still
         * vulnerable to UAFs caused by freeing the whole slab page once
         * it becomes unused. Prevent file_ref_put() from being
         * preempted protects against this.
         */
        guard(preempt)();
        /*
         * Unconditionally decrease the reference count. The saturation
         * and dead zones provide enough tolerance for this. If this
         * fails then we need to handle the last reference drop and
         * cases inside the saturation and dead zones.
         */
        cnt = atomic_long_dec_return(&ref->refcnt);
        if (cnt >= 0)
                return false;
        return __file_ref_put(ref, cnt);
}

/**
 * file_ref_read - Read the number of file references
 * @ref: Pointer to the reference count
 *
 * Return: The number of held references (0 ... N)
 */
static inline unsigned long file_ref_read(file_ref_t *ref)
{
        unsigned long c = atomic_long_read(&ref->refcnt);

        /* Return 0 if within the DEAD zone. */
        return c >= FILE_REF_RELEASED ? 0 : c + 1;
}

#endif










































    1 
















    1 



    1 




    1 


































    1 









    1 

















































    1 












































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
// SPDX-License-Identifier: GPL-2.0-only
/*
 *
 *   YeAH TCP
 *
 * For further details look at:
 *   https://web.archive.org/web/20080316215752/http://wil.cs.caltech.edu/pfldnet2007/paper/YeAH_TCP.pdf
 *
 */
#include <linux/mm.h>
#include <linux/module.h>
#include <linux/skbuff.h>
#include <linux/inet_diag.h>

#include <net/tcp.h>

#include "tcp_vegas.h"

#define TCP_YEAH_ALPHA       80 /* number of packets queued at the bottleneck */
#define TCP_YEAH_GAMMA        1 /* fraction of queue to be removed per rtt */
#define TCP_YEAH_DELTA        3 /* log minimum fraction of cwnd to be removed on loss */
#define TCP_YEAH_EPSILON      1 /* log maximum fraction to be removed on early decongestion */
#define TCP_YEAH_PHY          8 /* maximum delta from base */
#define TCP_YEAH_RHO         16 /* minimum number of consecutive rtt to consider competition on loss */
#define TCP_YEAH_ZETA        50 /* minimum number of state switches to reset reno_count */

#define TCP_SCALABLE_AI_CNT         100U

/* YeAH variables */
struct yeah {
        struct vegas vegas;        /* must be first */

        /* YeAH */
        u32 lastQ;
        u32 doing_reno_now;

        u32 reno_count;
        u32 fast_count;
};

static void tcp_yeah_init(struct sock *sk)
{
        struct tcp_sock *tp = tcp_sk(sk);
        struct yeah *yeah = inet_csk_ca(sk);

        tcp_vegas_init(sk);

        yeah->doing_reno_now = 0;
        yeah->lastQ = 0;

        yeah->reno_count = 2;

        /* Ensure the MD arithmetic works.  This is somewhat pedantic,
         * since I don't think we will see a cwnd this large. :) */
        tp->snd_cwnd_clamp = min_t(u32, tp->snd_cwnd_clamp, 0xffffffff/128);
}

static void tcp_yeah_cong_avoid(struct sock *sk, u32 ack, u32 acked)
{
        struct tcp_sock *tp = tcp_sk(sk);
        struct yeah *yeah = inet_csk_ca(sk);

        if (!tcp_is_cwnd_limited(sk))
                return;

        if (tcp_in_slow_start(tp)) {
                acked = tcp_slow_start(tp, acked);
                if (!acked)
                        goto do_vegas;
        }

        if (!yeah->doing_reno_now) {
                /* Scalable */
                tcp_cong_avoid_ai(tp, min(tcp_snd_cwnd(tp), TCP_SCALABLE_AI_CNT),
                                  acked);
        } else {
                /* Reno */
                tcp_cong_avoid_ai(tp, tcp_snd_cwnd(tp), acked);
        }

        /* The key players are v_vegas.beg_snd_una and v_beg_snd_nxt.
         *
         * These are so named because they represent the approximate values
         * of snd_una and snd_nxt at the beginning of the current RTT. More
         * precisely, they represent the amount of data sent during the RTT.
         * At the end of the RTT, when we receive an ACK for v_beg_snd_nxt,
         * we will calculate that (v_beg_snd_nxt - v_vegas.beg_snd_una) outstanding
         * bytes of data have been ACKed during the course of the RTT, giving
         * an "actual" rate of:
         *
         *     (v_beg_snd_nxt - v_vegas.beg_snd_una) / (rtt duration)
         *
         * Unfortunately, v_vegas.beg_snd_una is not exactly equal to snd_una,
         * because delayed ACKs can cover more than one segment, so they
         * don't line up yeahly with the boundaries of RTTs.
         *
         * Another unfortunate fact of life is that delayed ACKs delay the
         * advance of the left edge of our send window, so that the number
         * of bytes we send in an RTT is often less than our cwnd will allow.
         * So we keep track of our cwnd separately, in v_beg_snd_cwnd.
         */
do_vegas:
        if (after(ack, yeah->vegas.beg_snd_nxt)) {
                /* We do the Vegas calculations only if we got enough RTT
                 * samples that we can be reasonably sure that we got
                 * at least one RTT sample that wasn't from a delayed ACK.
                 * If we only had 2 samples total,
                 * then that means we're getting only 1 ACK per RTT, which
                 * means they're almost certainly delayed ACKs.
                 * If  we have 3 samples, we should be OK.
                 */

                if (yeah->vegas.cntRTT > 2) {
                        u32 rtt, queue;
                        u64 bw;

                        /* We have enough RTT samples, so, using the Vegas
                         * algorithm, we determine if we should increase or
                         * decrease cwnd, and by how much.
                         */

                        /* Pluck out the RTT we are using for the Vegas
                         * calculations. This is the min RTT seen during the
                         * last RTT. Taking the min filters out the effects
                         * of delayed ACKs, at the cost of noticing congestion
                         * a bit later.
                         */
                        rtt = yeah->vegas.minRTT;

                        /* Compute excess number of packets above bandwidth
                         * Avoid doing full 64 bit divide.
                         */
                        bw = tcp_snd_cwnd(tp);
                        bw *= rtt - yeah->vegas.baseRTT;
                        do_div(bw, rtt);
                        queue = bw;

                        if (queue > TCP_YEAH_ALPHA ||
                            rtt - yeah->vegas.baseRTT > (yeah->vegas.baseRTT / TCP_YEAH_PHY)) {
                                if (queue > TCP_YEAH_ALPHA &&
                                    tcp_snd_cwnd(tp) > yeah->reno_count) {
                                        u32 reduction = min(queue / TCP_YEAH_GAMMA ,
                                                            tcp_snd_cwnd(tp) >> TCP_YEAH_EPSILON);

                                        tcp_snd_cwnd_set(tp, tcp_snd_cwnd(tp) - reduction);

                                        tcp_snd_cwnd_set(tp, max(tcp_snd_cwnd(tp),
                                                                 yeah->reno_count));

                                        tp->snd_ssthresh = tcp_snd_cwnd(tp);
                                }

                                if (yeah->reno_count <= 2)
                                        yeah->reno_count = max(tcp_snd_cwnd(tp)>>1, 2U);
                                else
                                        yeah->reno_count++;

                                yeah->doing_reno_now = min(yeah->doing_reno_now + 1,
                                                           0xffffffU);
                        } else {
                                yeah->fast_count++;

                                if (yeah->fast_count > TCP_YEAH_ZETA) {
                                        yeah->reno_count = 2;
                                        yeah->fast_count = 0;
                                }

                                yeah->doing_reno_now = 0;
                        }

                        yeah->lastQ = queue;
                }

                /* Save the extent of the current window so we can use this
                 * at the end of the next RTT.
                 */
                yeah->vegas.beg_snd_una  = yeah->vegas.beg_snd_nxt;
                yeah->vegas.beg_snd_nxt  = tp->snd_nxt;
                yeah->vegas.beg_snd_cwnd = tcp_snd_cwnd(tp);

                /* Wipe the slate clean for the next RTT. */
                yeah->vegas.cntRTT = 0;
                yeah->vegas.minRTT = 0x7fffffff;
        }
}

static u32 tcp_yeah_ssthresh(struct sock *sk)
{
        const struct tcp_sock *tp = tcp_sk(sk);
        struct yeah *yeah = inet_csk_ca(sk);
        u32 reduction;

        if (yeah->doing_reno_now < TCP_YEAH_RHO) {
                reduction = yeah->lastQ;

                reduction = min(reduction, max(tcp_snd_cwnd(tp)>>1, 2U));

                reduction = max(reduction, tcp_snd_cwnd(tp) >> TCP_YEAH_DELTA);
        } else
                reduction = max(tcp_snd_cwnd(tp)>>1, 2U);

        yeah->fast_count = 0;
        yeah->reno_count = max(yeah->reno_count>>1, 2U);

        return max_t(int, tcp_snd_cwnd(tp) - reduction, 2);
}

static struct tcp_congestion_ops tcp_yeah __read_mostly = {
        .init                = tcp_yeah_init,
        .ssthresh        = tcp_yeah_ssthresh,
        .undo_cwnd      = tcp_reno_undo_cwnd,
        .cong_avoid        = tcp_yeah_cong_avoid,
        .set_state        = tcp_vegas_state,
        .cwnd_event        = tcp_vegas_cwnd_event,
        .get_info        = tcp_vegas_get_info,
        .pkts_acked        = tcp_vegas_pkts_acked,

        .owner                = THIS_MODULE,
        .name                = "yeah",
};

static int __init tcp_yeah_register(void)
{
        BUILD_BUG_ON(sizeof(struct yeah) > ICSK_CA_PRIV_SIZE);
        tcp_register_congestion_control(&tcp_yeah);
        return 0;
}

static void __exit tcp_yeah_unregister(void)
{
        tcp_unregister_congestion_control(&tcp_yeah);
}

module_init(tcp_yeah_register);
module_exit(tcp_yeah_unregister);

MODULE_AUTHOR("Angelo P. Castellani");
MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("YeAH TCP");












































    1 



    1 






















    7 






    1 


    1 

    1 


    1 





    1 



    2 























































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
// SPDX-License-Identifier: GPL-2.0-only
/*
 * Authors:
 * (C) 2020 Alexander Aring <alex.aring@gmail.com>
 */

#include <linux/rpl_iptunnel.h>

#include <net/dst_cache.h>
#include <net/ip6_route.h>
#include <net/lwtunnel.h>
#include <net/ipv6.h>
#include <net/rpl.h>

struct rpl_iptunnel_encap {
        DECLARE_FLEX_ARRAY(struct ipv6_rpl_sr_hdr, srh);
};

struct rpl_lwt {
        struct dst_cache cache;
        struct rpl_iptunnel_encap tuninfo;
};

static inline struct rpl_lwt *rpl_lwt_lwtunnel(struct lwtunnel_state *lwt)
{
        return (struct rpl_lwt *)lwt->data;
}

static inline struct rpl_iptunnel_encap *
rpl_encap_lwtunnel(struct lwtunnel_state *lwt)
{
        return &rpl_lwt_lwtunnel(lwt)->tuninfo;
}

static const struct nla_policy rpl_iptunnel_policy[RPL_IPTUNNEL_MAX + 1] = {
        [RPL_IPTUNNEL_SRH]        = { .type = NLA_BINARY },
};

static bool rpl_validate_srh(struct net *net, struct ipv6_rpl_sr_hdr *srh,
                             size_t seglen)
{
        int err;

        if ((srh->hdrlen << 3) != seglen)
                return false;

        /* check at least one segment and seglen fit with segments_left */
        if (!srh->segments_left ||
            (srh->segments_left * sizeof(struct in6_addr)) != seglen)
                return false;

        if (srh->cmpri || srh->cmpre)
                return false;

        err = ipv6_chk_rpl_srh_loop(net, srh->rpl_segaddr,
                                    srh->segments_left);
        if (err)
                return false;

        if (ipv6_addr_type(&srh->rpl_segaddr[srh->segments_left - 1]) &
            IPV6_ADDR_MULTICAST)
                return false;

        return true;
}

static int rpl_build_state(struct net *net, struct nlattr *nla,
                           unsigned int family, const void *cfg,
                           struct lwtunnel_state **ts,
                           struct netlink_ext_ack *extack)
{
        struct nlattr *tb[RPL_IPTUNNEL_MAX + 1];
        struct lwtunnel_state *newts;
        struct ipv6_rpl_sr_hdr *srh;
        struct rpl_lwt *rlwt;
        int err, srh_len;

        if (family != AF_INET6)
                return -EINVAL;

        err = nla_parse_nested(tb, RPL_IPTUNNEL_MAX, nla,
                               rpl_iptunnel_policy, extack);
        if (err < 0)
                return err;

        if (!tb[RPL_IPTUNNEL_SRH])
                return -EINVAL;

        srh = nla_data(tb[RPL_IPTUNNEL_SRH]);
        srh_len = nla_len(tb[RPL_IPTUNNEL_SRH]);

        if (srh_len < sizeof(*srh))
                return -EINVAL;

        /* verify that SRH is consistent */
        if (!rpl_validate_srh(net, srh, srh_len - sizeof(*srh)))
                return -EINVAL;

        newts = lwtunnel_state_alloc(srh_len + sizeof(*rlwt));
        if (!newts)
                return -ENOMEM;

        rlwt = rpl_lwt_lwtunnel(newts);

        err = dst_cache_init(&rlwt->cache, GFP_ATOMIC);
        if (err) {
                kfree(newts);
                return err;
        }

        memcpy(&rlwt->tuninfo.srh, srh, srh_len);

        newts->type = LWTUNNEL_ENCAP_RPL;
        newts->flags |= LWTUNNEL_STATE_INPUT_REDIRECT;
        newts->flags |= LWTUNNEL_STATE_OUTPUT_REDIRECT;

        *ts = newts;

        return 0;
}

static void rpl_destroy_state(struct lwtunnel_state *lwt)
{
        dst_cache_destroy(&rpl_lwt_lwtunnel(lwt)->cache);
}

static int rpl_do_srh_inline(struct sk_buff *skb, const struct rpl_lwt *rlwt,
                             const struct ipv6_rpl_sr_hdr *srh)
{
        struct ipv6_rpl_sr_hdr *isrh, *csrh;
        const struct ipv6hdr *oldhdr;
        struct ipv6hdr *hdr;
        unsigned char *buf;
        size_t hdrlen;
        int err;

        oldhdr = ipv6_hdr(skb);

        buf = kcalloc(struct_size(srh, segments.addr, srh->segments_left), 2, GFP_ATOMIC);
        if (!buf)
                return -ENOMEM;

        isrh = (struct ipv6_rpl_sr_hdr *)buf;
        csrh = (struct ipv6_rpl_sr_hdr *)(buf + ((srh->hdrlen + 1) << 3));

        memcpy(isrh, srh, sizeof(*isrh));
        memcpy(isrh->rpl_segaddr, &srh->rpl_segaddr[1],
               (srh->segments_left - 1) * 16);
        isrh->rpl_segaddr[srh->segments_left - 1] = oldhdr->daddr;

        ipv6_rpl_srh_compress(csrh, isrh, &srh->rpl_segaddr[0],
                              isrh->segments_left - 1);

        hdrlen = ((csrh->hdrlen + 1) << 3);

        err = skb_cow_head(skb, hdrlen + skb->mac_len);
        if (unlikely(err)) {
                kfree(buf);
                return err;
        }

        skb_pull(skb, sizeof(struct ipv6hdr));
        skb_postpull_rcsum(skb, skb_network_header(skb),
                           sizeof(struct ipv6hdr));

        skb_push(skb, sizeof(struct ipv6hdr) + hdrlen);
        skb_reset_network_header(skb);
        skb_mac_header_rebuild(skb);

        hdr = ipv6_hdr(skb);
        memmove(hdr, oldhdr, sizeof(*hdr));
        isrh = (void *)hdr + sizeof(*hdr);
        memcpy(isrh, csrh, hdrlen);

        isrh->nexthdr = hdr->nexthdr;
        hdr->nexthdr = NEXTHDR_ROUTING;
        hdr->daddr = srh->rpl_segaddr[0];

        ipv6_hdr(skb)->payload_len = htons(skb->len - sizeof(struct ipv6hdr));
        skb_set_transport_header(skb, sizeof(struct ipv6hdr));

        skb_postpush_rcsum(skb, hdr, sizeof(struct ipv6hdr) + hdrlen);

        kfree(buf);

        return 0;
}

static int rpl_do_srh(struct sk_buff *skb, const struct rpl_lwt *rlwt)
{
        struct dst_entry *dst = skb_dst(skb);
        struct rpl_iptunnel_encap *tinfo;

        if (skb->protocol != htons(ETH_P_IPV6))
                return -EINVAL;

        tinfo = rpl_encap_lwtunnel(dst->lwtstate);

        return rpl_do_srh_inline(skb, rlwt, tinfo->srh);
}

static int rpl_output(struct net *net, struct sock *sk, struct sk_buff *skb)
{
        struct dst_entry *orig_dst = skb_dst(skb);
        struct dst_entry *dst = NULL;
        struct rpl_lwt *rlwt;
        int err;

        rlwt = rpl_lwt_lwtunnel(orig_dst->lwtstate);

        err = rpl_do_srh(skb, rlwt);
        if (unlikely(err))
                goto drop;

        local_bh_disable();
        dst = dst_cache_get(&rlwt->cache);
        local_bh_enable();

        if (unlikely(!dst)) {
                struct ipv6hdr *hdr = ipv6_hdr(skb);
                struct flowi6 fl6;

                memset(&fl6, 0, sizeof(fl6));
                fl6.daddr = hdr->daddr;
                fl6.saddr = hdr->saddr;
                fl6.flowlabel = ip6_flowinfo(hdr);
                fl6.flowi6_mark = skb->mark;
                fl6.flowi6_proto = hdr->nexthdr;

                dst = ip6_route_output(net, NULL, &fl6);
                if (dst->error) {
                        err = dst->error;
                        dst_release(dst);
                        goto drop;
                }

                local_bh_disable();
                dst_cache_set_ip6(&rlwt->cache, dst, &fl6.saddr);
                local_bh_enable();
        }

        skb_dst_drop(skb);
        skb_dst_set(skb, dst);

        err = skb_cow_head(skb, LL_RESERVED_SPACE(dst->dev));
        if (unlikely(err))
                goto drop;

        return dst_output(net, sk, skb);

drop:
        kfree_skb(skb);
        return err;
}

static int rpl_input(struct sk_buff *skb)
{
        struct dst_entry *orig_dst = skb_dst(skb);
        struct dst_entry *dst = NULL;
        struct rpl_lwt *rlwt;
        int err;

        rlwt = rpl_lwt_lwtunnel(orig_dst->lwtstate);

        err = rpl_do_srh(skb, rlwt);
        if (unlikely(err))
                goto drop;

        local_bh_disable();
        dst = dst_cache_get(&rlwt->cache);

        if (!dst) {
                ip6_route_input(skb);
                dst = skb_dst(skb);
                if (!dst->error) {
                        dst_cache_set_ip6(&rlwt->cache, dst,
                                          &ipv6_hdr(skb)->saddr);
                }
        } else {
                skb_dst_drop(skb);
                skb_dst_set(skb, dst);
        }
        local_bh_enable();

        err = skb_cow_head(skb, LL_RESERVED_SPACE(dst->dev));
        if (unlikely(err))
                goto drop;

        return dst_input(skb);

drop:
        kfree_skb(skb);
        return err;
}

static int nla_put_rpl_srh(struct sk_buff *skb, int attrtype,
                           struct rpl_iptunnel_encap *tuninfo)
{
        struct rpl_iptunnel_encap *data;
        struct nlattr *nla;
        int len;

        len = RPL_IPTUNNEL_SRH_SIZE(tuninfo->srh);

        nla = nla_reserve(skb, attrtype, len);
        if (!nla)
                return -EMSGSIZE;

        data = nla_data(nla);
        memcpy(data, tuninfo->srh, len);

        return 0;
}

static int rpl_fill_encap_info(struct sk_buff *skb,
                               struct lwtunnel_state *lwtstate)
{
        struct rpl_iptunnel_encap *tuninfo = rpl_encap_lwtunnel(lwtstate);

        if (nla_put_rpl_srh(skb, RPL_IPTUNNEL_SRH, tuninfo))
                return -EMSGSIZE;

        return 0;
}

static int rpl_encap_nlsize(struct lwtunnel_state *lwtstate)
{
        struct rpl_iptunnel_encap *tuninfo = rpl_encap_lwtunnel(lwtstate);

        return nla_total_size(RPL_IPTUNNEL_SRH_SIZE(tuninfo->srh));
}

static int rpl_encap_cmp(struct lwtunnel_state *a, struct lwtunnel_state *b)
{
        struct rpl_iptunnel_encap *a_hdr = rpl_encap_lwtunnel(a);
        struct rpl_iptunnel_encap *b_hdr = rpl_encap_lwtunnel(b);
        int len = RPL_IPTUNNEL_SRH_SIZE(a_hdr->srh);

        if (len != RPL_IPTUNNEL_SRH_SIZE(b_hdr->srh))
                return 1;

        return memcmp(a_hdr, b_hdr, len);
}

static const struct lwtunnel_encap_ops rpl_ops = {
        .build_state        = rpl_build_state,
        .destroy_state        = rpl_destroy_state,
        .output                = rpl_output,
        .input                = rpl_input,
        .fill_encap        = rpl_fill_encap_info,
        .get_encap_size        = rpl_encap_nlsize,
        .cmp_encap        = rpl_encap_cmp,
        .owner                = THIS_MODULE,
};

int __init rpl_init(void)
{
        int err;

        err = lwtunnel_encap_add_ops(&rpl_ops, LWTUNNEL_ENCAP_RPL);
        if (err)
                goto out;

        pr_info("RPL Segment Routing with IPv6\n");

        return 0;

out:
        return err;
}

void rpl_exit(void)
{
        lwtunnel_encap_del_ops(&rpl_ops, LWTUNNEL_ENCAP_RPL);
}



































    3 



    3 




   15 
   15 




    4 
    4 





    4 
    4 






































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * Cryptographic API.
 *
 * SHA1 Secure Hash Algorithm.
 *
 * Derived from cryptoapi implementation, adapted for in-place
 * scatterlist interface.
 *
 * Copyright (c) Alan Smithee.
 * Copyright (c) Andrew McDonald <andrew@mcdonald.org.uk>
 * Copyright (c) Jean-Francois Dive <jef@linuxbe.org>
 */
#include <crypto/internal/hash.h>
#include <linux/init.h>
#include <linux/module.h>
#include <linux/mm.h>
#include <linux/types.h>
#include <crypto/sha1.h>
#include <crypto/sha1_base.h>
#include <asm/byteorder.h>

const u8 sha1_zero_message_hash[SHA1_DIGEST_SIZE] = {
        0xda, 0x39, 0xa3, 0xee, 0x5e, 0x6b, 0x4b, 0x0d,
        0x32, 0x55, 0xbf, 0xef, 0x95, 0x60, 0x18, 0x90,
        0xaf, 0xd8, 0x07, 0x09
};
EXPORT_SYMBOL_GPL(sha1_zero_message_hash);

static void sha1_generic_block_fn(struct sha1_state *sst, u8 const *src,
                                  int blocks)
{
        u32 temp[SHA1_WORKSPACE_WORDS];

        while (blocks--) {
                sha1_transform(sst->state, src, temp);
                src += SHA1_BLOCK_SIZE;
        }
        memzero_explicit(temp, sizeof(temp));
}

int crypto_sha1_update(struct shash_desc *desc, const u8 *data,
                       unsigned int len)
{
        return sha1_base_do_update(desc, data, len, sha1_generic_block_fn);
}
EXPORT_SYMBOL(crypto_sha1_update);

static int sha1_final(struct shash_desc *desc, u8 *out)
{
        sha1_base_do_finalize(desc, sha1_generic_block_fn);
        return sha1_base_finish(desc, out);
}

int crypto_sha1_finup(struct shash_desc *desc, const u8 *data,
                      unsigned int len, u8 *out)
{
        sha1_base_do_update(desc, data, len, sha1_generic_block_fn);
        return sha1_final(desc, out);
}
EXPORT_SYMBOL(crypto_sha1_finup);

static struct shash_alg alg = {
        .digestsize        =        SHA1_DIGEST_SIZE,
        .init                =        sha1_base_init,
        .update                =        crypto_sha1_update,
        .final                =        sha1_final,
        .finup                =        crypto_sha1_finup,
        .descsize        =        sizeof(struct sha1_state),
        .base                =        {
                .cra_name        =        "sha1",
                .cra_driver_name=        "sha1-generic",
                .cra_priority        =        100,
                .cra_blocksize        =        SHA1_BLOCK_SIZE,
                .cra_module        =        THIS_MODULE,
        }
};

static int __init sha1_generic_mod_init(void)
{
        return crypto_register_shash(&alg);
}

static void __exit sha1_generic_mod_fini(void)
{
        crypto_unregister_shash(&alg);
}

subsys_initcall(sha1_generic_mod_init);
module_exit(sha1_generic_mod_fini);

MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("SHA1 Secure Hash Algorithm");

MODULE_ALIAS_CRYPTO("sha1");
MODULE_ALIAS_CRYPTO("sha1-generic");





























   19 


   18 



   19 







1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
// SPDX-License-Identifier: GPL-2.0-only
/*
 * ksyms_common.c: A split of kernel/kallsyms.c
 * Contains a few generic function definations independent of config KALLSYMS.
 */
#include <linux/kallsyms.h>
#include <linux/security.h>

static inline int kallsyms_for_perf(void)
{
#ifdef CONFIG_PERF_EVENTS
        extern int sysctl_perf_event_paranoid;

        if (sysctl_perf_event_paranoid <= 1)
                return 1;
#endif
        return 0;
}

/*
 * We show kallsyms information even to normal users if we've enabled
 * kernel profiling and are explicitly not paranoid (so kptr_restrict
 * is clear, and sysctl_perf_event_paranoid isn't set).
 *
 * Otherwise, require CAP_SYSLOG (assuming kptr_restrict isn't set to
 * block even that).
 */
bool kallsyms_show_value(const struct cred *cred)
{
        switch (kptr_restrict) {
        case 0:
                if (kallsyms_for_perf())
                        return true;
                fallthrough;
        case 1:
                if (security_capable(cred, &init_user_ns, CAP_SYSLOG,
                                     CAP_OPT_NOAUDIT) == 0)
                        return true;
                fallthrough;
        default:
                return false;
        }
}






















































































    2 



    2 
    1 

    2 








    4 














    4 

    2 

    1 
    1 
    1 

    1 












    2 




    2 

    2 



















    2 
































  431 
    9 






























  172 
    4 
    1 


    1 























































    1 

    1 



    6 
    1 














    3 


    3 










































    1 
    1 
















    2 
    1 








    1 


    2 







    1 





















    3 









    3 



























    3 
    3 






    3 


























    1 


    8 


    4 

    2 








    1 


    1 


    2 

    2 


    3 


















    1 



    1 



    1 



    1 








    9 


    9 



    3 
    2 

    1 






    1 














    1 









    2 











    2 
    2 




    1 

    1 










    1 












    2 




    1 
















    1 





    2 








































































































































    4 




























  858 




  859 


  173 



  431 



  183 


  191 

















    5 









    4 

































  322 


  323 

   90 





































    6 





















    1 


























































    6 








    6 




    6 
    5 


    1 























    6 
    6 














    6 





















    6 







    6 








    6 








    6 































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
// SPDX-License-Identifier: GPL-2.0
/*
 *  Shared Memory Communications over RDMA (SMC-R) and RoCE
 *
 *  Generic netlink support functions to configure an SMC-R PNET table
 *
 *  Copyright IBM Corp. 2016
 *
 *  Author(s):  Thomas Richter <tmricht@linux.vnet.ibm.com>
 */

#include <linux/module.h>
#include <linux/list.h>
#include <linux/ctype.h>
#include <linux/mutex.h>
#include <net/netlink.h>
#include <net/genetlink.h>

#include <uapi/linux/if.h>
#include <uapi/linux/smc.h>

#include <rdma/ib_verbs.h>

#include <net/netns/generic.h>
#include "smc_netns.h"

#include "smc_pnet.h"
#include "smc_ib.h"
#include "smc_ism.h"
#include "smc_core.h"

static struct net_device *__pnet_find_base_ndev(struct net_device *ndev);
static struct net_device *pnet_find_base_ndev(struct net_device *ndev);

static const struct nla_policy smc_pnet_policy[SMC_PNETID_MAX + 1] = {
        [SMC_PNETID_NAME] = {
                .type = NLA_NUL_STRING,
                .len = SMC_MAX_PNETID_LEN
        },
        [SMC_PNETID_ETHNAME] = {
                .type = NLA_NUL_STRING,
                .len = IFNAMSIZ - 1
        },
        [SMC_PNETID_IBNAME] = {
                .type = NLA_NUL_STRING,
                .len = IB_DEVICE_NAME_MAX - 1
        },
        [SMC_PNETID_IBPORT] = { .type = NLA_U8 }
};

static struct genl_family smc_pnet_nl_family;

enum smc_pnet_nametype {
        SMC_PNET_ETH        = 1,
        SMC_PNET_IB        = 2,
};

/* pnet entry stored in pnet table */
struct smc_pnetentry {
        struct list_head list;
        char pnet_name[SMC_MAX_PNETID_LEN + 1];
        enum smc_pnet_nametype type;
        union {
                struct {
                        char eth_name[IFNAMSIZ + 1];
                        struct net_device *ndev;
                        netdevice_tracker dev_tracker;
                };
                struct {
                        char ib_name[IB_DEVICE_NAME_MAX + 1];
                        u8 ib_port;
                };
        };
};

/* Check if the pnetid is set */
bool smc_pnet_is_pnetid_set(u8 *pnetid)
{
        if (pnetid[0] == 0 || pnetid[0] == _S)
                return false;
        return true;
}

/* Check if two given pnetids match */
static bool smc_pnet_match(u8 *pnetid1, u8 *pnetid2)
{
        int i;

        for (i = 0; i < SMC_MAX_PNETID_LEN; i++) {
                if ((pnetid1[i] == 0 || pnetid1[i] == _S) &&
                    (pnetid2[i] == 0 || pnetid2[i] == _S))
                        break;
                if (pnetid1[i] != pnetid2[i])
                        return false;
        }
        return true;
}

/* Remove a pnetid from the pnet table.
 */
static int smc_pnet_remove_by_pnetid(struct net *net, char *pnet_name)
{
        struct smc_pnetentry *pnetelem, *tmp_pe;
        struct smc_pnettable *pnettable;
        struct smc_ib_device *ibdev;
        struct smcd_dev *smcd;
        struct smc_net *sn;
        int rc = -ENOENT;
        int ibport;

        /* get pnettable for namespace */
        sn = net_generic(net, smc_net_id);
        pnettable = &sn->pnettable;

        /* remove table entry */
        mutex_lock(&pnettable->lock);
        list_for_each_entry_safe(pnetelem, tmp_pe, &pnettable->pnetlist,
                                 list) {
                if (!pnet_name ||
                    smc_pnet_match(pnetelem->pnet_name, pnet_name)) {
                        list_del(&pnetelem->list);
                        if (pnetelem->type == SMC_PNET_ETH && pnetelem->ndev) {
                                netdev_put(pnetelem->ndev,
                                           &pnetelem->dev_tracker);
                                pr_warn_ratelimited("smc: net device %s "
                                                    "erased user defined "
                                                    "pnetid %.16s\n",
                                                    pnetelem->eth_name,
                                                    pnetelem->pnet_name);
                        }
                        kfree(pnetelem);
                        rc = 0;
                }
        }
        mutex_unlock(&pnettable->lock);

        /* if this is not the initial namespace, stop here */
        if (net != &init_net)
                return rc;

        /* remove ib devices */
        mutex_lock(&smc_ib_devices.mutex);
        list_for_each_entry(ibdev, &smc_ib_devices.list, list) {
                for (ibport = 0; ibport < SMC_MAX_PORTS; ibport++) {
                        if (ibdev->pnetid_by_user[ibport] &&
                            (!pnet_name ||
                             smc_pnet_match(pnet_name,
                                            ibdev->pnetid[ibport]))) {
                                pr_warn_ratelimited("smc: ib device %s ibport "
                                                    "%d erased user defined "
                                                    "pnetid %.16s\n",
                                                    ibdev->ibdev->name,
                                                    ibport + 1,
                                                    ibdev->pnetid[ibport]);
                                memset(ibdev->pnetid[ibport], 0,
                                       SMC_MAX_PNETID_LEN);
                                ibdev->pnetid_by_user[ibport] = false;
                                rc = 0;
                        }
                }
        }
        mutex_unlock(&smc_ib_devices.mutex);
        /* remove smcd devices */
        mutex_lock(&smcd_dev_list.mutex);
        list_for_each_entry(smcd, &smcd_dev_list.list, list) {
                if (smcd->pnetid_by_user &&
                    (!pnet_name ||
                     smc_pnet_match(pnet_name, smcd->pnetid))) {
                        pr_warn_ratelimited("smc: smcd device %s "
                                            "erased user defined pnetid "
                                            "%.16s\n",
                                            dev_name(smcd->ops->get_dev(smcd)),
                                            smcd->pnetid);
                        memset(smcd->pnetid, 0, SMC_MAX_PNETID_LEN);
                        smcd->pnetid_by_user = false;
                        rc = 0;
                }
        }
        mutex_unlock(&smcd_dev_list.mutex);
        return rc;
}

/* Add the reference to a given network device to the pnet table.
 */
static int smc_pnet_add_by_ndev(struct net_device *ndev)
{
        struct smc_pnetentry *pnetelem, *tmp_pe;
        struct smc_pnettable *pnettable;
        struct net *net = dev_net(ndev);
        struct smc_net *sn;
        int rc = -ENOENT;

        /* get pnettable for namespace */
        sn = net_generic(net, smc_net_id);
        pnettable = &sn->pnettable;

        mutex_lock(&pnettable->lock);
        list_for_each_entry_safe(pnetelem, tmp_pe, &pnettable->pnetlist, list) {
                if (pnetelem->type == SMC_PNET_ETH && !pnetelem->ndev &&
                    !strncmp(pnetelem->eth_name, ndev->name, IFNAMSIZ)) {
                        netdev_hold(ndev, &pnetelem->dev_tracker, GFP_ATOMIC);
                        pnetelem->ndev = ndev;
                        rc = 0;
                        pr_warn_ratelimited("smc: adding net device %s with "
                                            "user defined pnetid %.16s\n",
                                            pnetelem->eth_name,
                                            pnetelem->pnet_name);
                        break;
                }
        }
        mutex_unlock(&pnettable->lock);
        return rc;
}

/* Remove the reference to a given network device from the pnet table.
 */
static int smc_pnet_remove_by_ndev(struct net_device *ndev)
{
        struct smc_pnetentry *pnetelem, *tmp_pe;
        struct smc_pnettable *pnettable;
        struct net *net = dev_net(ndev);
        struct smc_net *sn;
        int rc = -ENOENT;

        /* get pnettable for namespace */
        sn = net_generic(net, smc_net_id);
        pnettable = &sn->pnettable;

        mutex_lock(&pnettable->lock);
        list_for_each_entry_safe(pnetelem, tmp_pe, &pnettable->pnetlist, list) {
                if (pnetelem->type == SMC_PNET_ETH && pnetelem->ndev == ndev) {
                        netdev_put(pnetelem->ndev, &pnetelem->dev_tracker);
                        pnetelem->ndev = NULL;
                        rc = 0;
                        pr_warn_ratelimited("smc: removing net device %s with "
                                            "user defined pnetid %.16s\n",
                                            pnetelem->eth_name,
                                            pnetelem->pnet_name);
                        break;
                }
        }
        mutex_unlock(&pnettable->lock);
        return rc;
}

/* Apply pnetid to ib device when no pnetid is set.
 */
static bool smc_pnet_apply_ib(struct smc_ib_device *ib_dev, u8 ib_port,
                              char *pnet_name)
{
        bool applied = false;

        mutex_lock(&smc_ib_devices.mutex);
        if (!smc_pnet_is_pnetid_set(ib_dev->pnetid[ib_port - 1])) {
                memcpy(ib_dev->pnetid[ib_port - 1], pnet_name,
                       SMC_MAX_PNETID_LEN);
                ib_dev->pnetid_by_user[ib_port - 1] = true;
                applied = true;
        }
        mutex_unlock(&smc_ib_devices.mutex);
        return applied;
}

/* Apply pnetid to smcd device when no pnetid is set.
 */
static bool smc_pnet_apply_smcd(struct smcd_dev *smcd_dev, char *pnet_name)
{
        bool applied = false;

        mutex_lock(&smcd_dev_list.mutex);
        if (!smc_pnet_is_pnetid_set(smcd_dev->pnetid)) {
                memcpy(smcd_dev->pnetid, pnet_name, SMC_MAX_PNETID_LEN);
                smcd_dev->pnetid_by_user = true;
                applied = true;
        }
        mutex_unlock(&smcd_dev_list.mutex);
        return applied;
}

/* The limit for pnetid is 16 characters.
 * Valid characters should be (single-byte character set) a-z, A-Z, 0-9.
 * Lower case letters are converted to upper case.
 * Interior blanks should not be used.
 */
static bool smc_pnetid_valid(const char *pnet_name, char *pnetid)
{
        char *bf = skip_spaces(pnet_name);
        size_t len = strlen(bf);
        char *end = bf + len;

        if (!len)
                return false;
        while (--end >= bf && isspace(*end))
                ;
        if (end - bf >= SMC_MAX_PNETID_LEN)
                return false;
        while (bf <= end) {
                if (!isalnum(*bf))
                        return false;
                *pnetid++ = islower(*bf) ? toupper(*bf) : *bf;
                bf++;
        }
        *pnetid = '\0';
        return true;
}

/* Find an infiniband device by a given name. The device might not exist. */
static struct smc_ib_device *smc_pnet_find_ib(char *ib_name)
{
        struct smc_ib_device *ibdev;

        mutex_lock(&smc_ib_devices.mutex);
        list_for_each_entry(ibdev, &smc_ib_devices.list, list) {
                if (!strncmp(ibdev->ibdev->name, ib_name,
                             sizeof(ibdev->ibdev->name)) ||
                    (ibdev->ibdev->dev.parent &&
                     !strncmp(dev_name(ibdev->ibdev->dev.parent), ib_name,
                             IB_DEVICE_NAME_MAX - 1))) {
                        goto out;
                }
        }
        ibdev = NULL;
out:
        mutex_unlock(&smc_ib_devices.mutex);
        return ibdev;
}

/* Find an smcd device by a given name. The device might not exist. */
static struct smcd_dev *smc_pnet_find_smcd(char *smcd_name)
{
        struct smcd_dev *smcd_dev;

        mutex_lock(&smcd_dev_list.mutex);
        list_for_each_entry(smcd_dev, &smcd_dev_list.list, list) {
                if (!strncmp(dev_name(smcd_dev->ops->get_dev(smcd_dev)),
                             smcd_name, IB_DEVICE_NAME_MAX - 1))
                        goto out;
        }
        smcd_dev = NULL;
out:
        mutex_unlock(&smcd_dev_list.mutex);
        return smcd_dev;
}

static int smc_pnet_add_eth(struct smc_pnettable *pnettable, struct net *net,
                            char *eth_name, char *pnet_name)
{
        struct smc_pnetentry *tmp_pe, *new_pe;
        struct net_device *ndev, *base_ndev;
        u8 ndev_pnetid[SMC_MAX_PNETID_LEN];
        bool new_netdev;
        int rc;

        /* check if (base) netdev already has a pnetid. If there is one, we do
         * not want to add a pnet table entry
         */
        rc = -EEXIST;
        ndev = dev_get_by_name(net, eth_name);        /* dev_hold() */
        if (ndev) {
                base_ndev = pnet_find_base_ndev(ndev);
                if (!smc_pnetid_by_dev_port(base_ndev->dev.parent,
                                            base_ndev->dev_port, ndev_pnetid))
                        goto out_put;
        }

        /* add a new netdev entry to the pnet table if there isn't one */
        rc = -ENOMEM;
        new_pe = kzalloc(sizeof(*new_pe), GFP_KERNEL);
        if (!new_pe)
                goto out_put;
        new_pe->type = SMC_PNET_ETH;
        memcpy(new_pe->pnet_name, pnet_name, SMC_MAX_PNETID_LEN);
        strncpy(new_pe->eth_name, eth_name, IFNAMSIZ);
        rc = -EEXIST;
        new_netdev = true;
        mutex_lock(&pnettable->lock);
        list_for_each_entry(tmp_pe, &pnettable->pnetlist, list) {
                if (tmp_pe->type == SMC_PNET_ETH &&
                    !strncmp(tmp_pe->eth_name, eth_name, IFNAMSIZ)) {
                        new_netdev = false;
                        break;
                }
        }
        if (new_netdev) {
                if (ndev) {
                        new_pe->ndev = ndev;
                        netdev_tracker_alloc(ndev, &new_pe->dev_tracker,
                                             GFP_ATOMIC);
                }
                list_add_tail(&new_pe->list, &pnettable->pnetlist);
                mutex_unlock(&pnettable->lock);
        } else {
                mutex_unlock(&pnettable->lock);
                kfree(new_pe);
                goto out_put;
        }
        if (ndev)
                pr_warn_ratelimited("smc: net device %s "
                                    "applied user defined pnetid %.16s\n",
                                    new_pe->eth_name, new_pe->pnet_name);
        return 0;

out_put:
        dev_put(ndev);
        return rc;
}

static int smc_pnet_add_ib(struct smc_pnettable *pnettable, char *ib_name,
                           u8 ib_port, char *pnet_name)
{
        struct smc_pnetentry *tmp_pe, *new_pe;
        struct smc_ib_device *ib_dev;
        bool smcddev_applied = true;
        bool ibdev_applied = true;
        struct smcd_dev *smcd;
        struct device *dev;
        bool new_ibdev;

        /* try to apply the pnetid to active devices */
        ib_dev = smc_pnet_find_ib(ib_name);
        if (ib_dev) {
                ibdev_applied = smc_pnet_apply_ib(ib_dev, ib_port, pnet_name);
                if (ibdev_applied)
                        pr_warn_ratelimited("smc: ib device %s ibport %d "
                                            "applied user defined pnetid "
                                            "%.16s\n", ib_dev->ibdev->name,
                                            ib_port,
                                            ib_dev->pnetid[ib_port - 1]);
        }
        smcd = smc_pnet_find_smcd(ib_name);
        if (smcd) {
                smcddev_applied = smc_pnet_apply_smcd(smcd, pnet_name);
                if (smcddev_applied) {
                        dev = smcd->ops->get_dev(smcd);
                        pr_warn_ratelimited("smc: smcd device %s "
                                            "applied user defined pnetid "
                                            "%.16s\n", dev_name(dev),
                                            smcd->pnetid);
                }
        }
        /* Apply fails when a device has a hardware-defined pnetid set, do not
         * add a pnet table entry in that case.
         */
        if (!ibdev_applied || !smcddev_applied)
                return -EEXIST;

        /* add a new ib entry to the pnet table if there isn't one */
        new_pe = kzalloc(sizeof(*new_pe), GFP_KERNEL);
        if (!new_pe)
                return -ENOMEM;
        new_pe->type = SMC_PNET_IB;
        memcpy(new_pe->pnet_name, pnet_name, SMC_MAX_PNETID_LEN);
        strncpy(new_pe->ib_name, ib_name, IB_DEVICE_NAME_MAX);
        new_pe->ib_port = ib_port;

        new_ibdev = true;
        mutex_lock(&pnettable->lock);
        list_for_each_entry(tmp_pe, &pnettable->pnetlist, list) {
                if (tmp_pe->type == SMC_PNET_IB &&
                    !strncmp(tmp_pe->ib_name, ib_name, IB_DEVICE_NAME_MAX)) {
                        new_ibdev = false;
                        break;
                }
        }
        if (new_ibdev) {
                list_add_tail(&new_pe->list, &pnettable->pnetlist);
                mutex_unlock(&pnettable->lock);
        } else {
                mutex_unlock(&pnettable->lock);
                kfree(new_pe);
        }
        return (new_ibdev) ? 0 : -EEXIST;
}

/* Append a pnetid to the end of the pnet table if not already on this list.
 */
static int smc_pnet_enter(struct net *net, struct nlattr *tb[])
{
        char pnet_name[SMC_MAX_PNETID_LEN + 1];
        struct smc_pnettable *pnettable;
        bool new_netdev = false;
        bool new_ibdev = false;
        struct smc_net *sn;
        u8 ibport = 1;
        char *string;
        int rc;

        /* get pnettable for namespace */
        sn = net_generic(net, smc_net_id);
        pnettable = &sn->pnettable;

        rc = -EINVAL;
        if (!tb[SMC_PNETID_NAME])
                goto error;
        string = (char *)nla_data(tb[SMC_PNETID_NAME]);
        if (!smc_pnetid_valid(string, pnet_name))
                goto error;

        if (tb[SMC_PNETID_ETHNAME]) {
                string = (char *)nla_data(tb[SMC_PNETID_ETHNAME]);
                rc = smc_pnet_add_eth(pnettable, net, string, pnet_name);
                if (!rc)
                        new_netdev = true;
                else if (rc != -EEXIST)
                        goto error;
        }

        /* if this is not the initial namespace, stop here */
        if (net != &init_net)
                return new_netdev ? 0 : -EEXIST;

        rc = -EINVAL;
        if (tb[SMC_PNETID_IBNAME]) {
                string = (char *)nla_data(tb[SMC_PNETID_IBNAME]);
                string = strim(string);
                if (tb[SMC_PNETID_IBPORT]) {
                        ibport = nla_get_u8(tb[SMC_PNETID_IBPORT]);
                        if (ibport < 1 || ibport > SMC_MAX_PORTS)
                                goto error;
                }
                rc = smc_pnet_add_ib(pnettable, string, ibport, pnet_name);
                if (!rc)
                        new_ibdev = true;
                else if (rc != -EEXIST)
                        goto error;
        }
        return (new_netdev || new_ibdev) ? 0 : -EEXIST;

error:
        return rc;
}

/* Convert an smc_pnetentry to a netlink attribute sequence */
static int smc_pnet_set_nla(struct sk_buff *msg,
                            struct smc_pnetentry *pnetelem)
{
        if (nla_put_string(msg, SMC_PNETID_NAME, pnetelem->pnet_name))
                return -1;
        if (pnetelem->type == SMC_PNET_ETH) {
                if (nla_put_string(msg, SMC_PNETID_ETHNAME,
                                   pnetelem->eth_name))
                        return -1;
        } else {
                if (nla_put_string(msg, SMC_PNETID_ETHNAME, "n/a"))
                        return -1;
        }
        if (pnetelem->type == SMC_PNET_IB) {
                if (nla_put_string(msg, SMC_PNETID_IBNAME, pnetelem->ib_name) ||
                    nla_put_u8(msg, SMC_PNETID_IBPORT, pnetelem->ib_port))
                        return -1;
        } else {
                if (nla_put_string(msg, SMC_PNETID_IBNAME, "n/a") ||
                    nla_put_u8(msg, SMC_PNETID_IBPORT, 0xff))
                        return -1;
        }

        return 0;
}

static int smc_pnet_add(struct sk_buff *skb, struct genl_info *info)
{
        struct net *net = genl_info_net(info);

        return smc_pnet_enter(net, info->attrs);
}

static int smc_pnet_del(struct sk_buff *skb, struct genl_info *info)
{
        struct net *net = genl_info_net(info);

        if (!info->attrs[SMC_PNETID_NAME])
                return -EINVAL;
        return smc_pnet_remove_by_pnetid(net,
                                (char *)nla_data(info->attrs[SMC_PNETID_NAME]));
}

static int smc_pnet_dump_start(struct netlink_callback *cb)
{
        cb->args[0] = 0;
        return 0;
}

static int smc_pnet_dumpinfo(struct sk_buff *skb,
                             u32 portid, u32 seq, u32 flags,
                             struct smc_pnetentry *pnetelem)
{
        void *hdr;

        hdr = genlmsg_put(skb, portid, seq, &smc_pnet_nl_family,
                          flags, SMC_PNETID_GET);
        if (!hdr)
                return -ENOMEM;
        if (smc_pnet_set_nla(skb, pnetelem) < 0) {
                genlmsg_cancel(skb, hdr);
                return -EMSGSIZE;
        }
        genlmsg_end(skb, hdr);
        return 0;
}

static int _smc_pnet_dump(struct net *net, struct sk_buff *skb, u32 portid,
                          u32 seq, u8 *pnetid, int start_idx)
{
        struct smc_pnettable *pnettable;
        struct smc_pnetentry *pnetelem;
        struct smc_net *sn;
        int idx = 0;

        /* get pnettable for namespace */
        sn = net_generic(net, smc_net_id);
        pnettable = &sn->pnettable;

        /* dump pnettable entries */
        mutex_lock(&pnettable->lock);
        list_for_each_entry(pnetelem, &pnettable->pnetlist, list) {
                if (pnetid && !smc_pnet_match(pnetelem->pnet_name, pnetid))
                        continue;
                if (idx++ < start_idx)
                        continue;
                /* if this is not the initial namespace, dump only netdev */
                if (net != &init_net && pnetelem->type != SMC_PNET_ETH)
                        continue;
                if (smc_pnet_dumpinfo(skb, portid, seq, NLM_F_MULTI,
                                      pnetelem)) {
                        --idx;
                        break;
                }
        }
        mutex_unlock(&pnettable->lock);
        return idx;
}

static int smc_pnet_dump(struct sk_buff *skb, struct netlink_callback *cb)
{
        struct net *net = sock_net(skb->sk);
        int idx;

        idx = _smc_pnet_dump(net, skb, NETLINK_CB(cb->skb).portid,
                             cb->nlh->nlmsg_seq, NULL, cb->args[0]);

        cb->args[0] = idx;
        return skb->len;
}

/* Retrieve one PNETID entry */
static int smc_pnet_get(struct sk_buff *skb, struct genl_info *info)
{
        struct net *net = genl_info_net(info);
        struct sk_buff *msg;
        void *hdr;

        if (!info->attrs[SMC_PNETID_NAME])
                return -EINVAL;

        msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
        if (!msg)
                return -ENOMEM;

        _smc_pnet_dump(net, msg, info->snd_portid, info->snd_seq,
                       nla_data(info->attrs[SMC_PNETID_NAME]), 0);

        /* finish multi part message and send it */
        hdr = nlmsg_put(msg, info->snd_portid, info->snd_seq, NLMSG_DONE, 0,
                        NLM_F_MULTI);
        if (!hdr) {
                nlmsg_free(msg);
                return -EMSGSIZE;
        }
        return genlmsg_reply(msg, info);
}

/* Remove and delete all pnetids from pnet table.
 */
static int smc_pnet_flush(struct sk_buff *skb, struct genl_info *info)
{
        struct net *net = genl_info_net(info);

        smc_pnet_remove_by_pnetid(net, NULL);
        return 0;
}

/* SMC_PNETID generic netlink operation definition */
static const struct genl_ops smc_pnet_ops[] = {
        {
                .cmd = SMC_PNETID_GET,
                .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
                /* can be retrieved by unprivileged users */
                .doit = smc_pnet_get,
                .dumpit = smc_pnet_dump,
                .start = smc_pnet_dump_start
        },
        {
                .cmd = SMC_PNETID_ADD,
                .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
                .flags = GENL_ADMIN_PERM,
                .doit = smc_pnet_add
        },
        {
                .cmd = SMC_PNETID_DEL,
                .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
                .flags = GENL_ADMIN_PERM,
                .doit = smc_pnet_del
        },
        {
                .cmd = SMC_PNETID_FLUSH,
                .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
                .flags = GENL_ADMIN_PERM,
                .doit = smc_pnet_flush
        }
};

/* SMC_PNETID family definition */
static struct genl_family smc_pnet_nl_family __ro_after_init = {
        .hdrsize = 0,
        .name = SMCR_GENL_FAMILY_NAME,
        .version = SMCR_GENL_FAMILY_VERSION,
        .maxattr = SMC_PNETID_MAX,
        .policy = smc_pnet_policy,
        .netnsok = true,
        .module = THIS_MODULE,
        .ops = smc_pnet_ops,
        .n_ops =  ARRAY_SIZE(smc_pnet_ops),
        .resv_start_op = SMC_PNETID_FLUSH + 1,
};

bool smc_pnet_is_ndev_pnetid(struct net *net, u8 *pnetid)
{
        struct smc_net *sn = net_generic(net, smc_net_id);
        struct smc_pnetids_ndev_entry *pe;
        bool rc = false;

        read_lock(&sn->pnetids_ndev.lock);
        list_for_each_entry(pe, &sn->pnetids_ndev.list, list) {
                if (smc_pnet_match(pnetid, pe->pnetid)) {
                        rc = true;
                        goto unlock;
                }
        }

unlock:
        read_unlock(&sn->pnetids_ndev.lock);
        return rc;
}

static int smc_pnet_add_pnetid(struct net *net, u8 *pnetid)
{
        struct smc_net *sn = net_generic(net, smc_net_id);
        struct smc_pnetids_ndev_entry *pe, *pi;

        pe = kzalloc(sizeof(*pe), GFP_KERNEL);
        if (!pe)
                return -ENOMEM;

        write_lock(&sn->pnetids_ndev.lock);
        list_for_each_entry(pi, &sn->pnetids_ndev.list, list) {
                if (smc_pnet_match(pnetid, pi->pnetid)) {
                        refcount_inc(&pi->refcnt);
                        kfree(pe);
                        goto unlock;
                }
        }
        refcount_set(&pe->refcnt, 1);
        memcpy(pe->pnetid, pnetid, SMC_MAX_PNETID_LEN);
        list_add_tail(&pe->list, &sn->pnetids_ndev.list);

unlock:
        write_unlock(&sn->pnetids_ndev.lock);
        return 0;
}

static void smc_pnet_remove_pnetid(struct net *net, u8 *pnetid)
{
        struct smc_net *sn = net_generic(net, smc_net_id);
        struct smc_pnetids_ndev_entry *pe, *pe2;

        write_lock(&sn->pnetids_ndev.lock);
        list_for_each_entry_safe(pe, pe2, &sn->pnetids_ndev.list, list) {
                if (smc_pnet_match(pnetid, pe->pnetid)) {
                        if (refcount_dec_and_test(&pe->refcnt)) {
                                list_del(&pe->list);
                                kfree(pe);
                        }
                        break;
                }
        }
        write_unlock(&sn->pnetids_ndev.lock);
}

static void smc_pnet_add_base_pnetid(struct net *net, struct net_device *dev,
                                     u8 *ndev_pnetid)
{
        struct net_device *base_dev;

        base_dev = __pnet_find_base_ndev(dev);
        if (base_dev->flags & IFF_UP &&
            !smc_pnetid_by_dev_port(base_dev->dev.parent, base_dev->dev_port,
                                    ndev_pnetid)) {
                /* add to PNETIDs list */
                smc_pnet_add_pnetid(net, ndev_pnetid);
        }
}

/* create initial list of netdevice pnetids */
static void smc_pnet_create_pnetids_list(struct net *net)
{
        u8 ndev_pnetid[SMC_MAX_PNETID_LEN];
        struct net_device *dev;

        /* Newly created netns do not have devices.
         * Do not even acquire rtnl.
         */
        if (list_empty(&net->dev_base_head))
                return;

        /* Note: This might not be needed, because smc_pnet_netdev_event()
         * is also calling smc_pnet_add_base_pnetid() when handling
         * NETDEV_UP event.
         */
        rtnl_lock();
        for_each_netdev(net, dev)
                smc_pnet_add_base_pnetid(net, dev, ndev_pnetid);
        rtnl_unlock();
}

/* clean up list of netdevice pnetids */
static void smc_pnet_destroy_pnetids_list(struct net *net)
{
        struct smc_net *sn = net_generic(net, smc_net_id);
        struct smc_pnetids_ndev_entry *pe, *temp_pe;

        write_lock(&sn->pnetids_ndev.lock);
        list_for_each_entry_safe(pe, temp_pe, &sn->pnetids_ndev.list, list) {
                list_del(&pe->list);
                kfree(pe);
        }
        write_unlock(&sn->pnetids_ndev.lock);
}

static int smc_pnet_netdev_event(struct notifier_block *this,
                                 unsigned long event, void *ptr)
{
        struct net_device *event_dev = netdev_notifier_info_to_dev(ptr);
        struct net *net = dev_net(event_dev);
        u8 ndev_pnetid[SMC_MAX_PNETID_LEN];

        switch (event) {
        case NETDEV_REBOOT:
        case NETDEV_UNREGISTER:
                smc_pnet_remove_by_ndev(event_dev);
                smc_ib_ndev_change(event_dev, event);
                return NOTIFY_OK;
        case NETDEV_REGISTER:
                smc_pnet_add_by_ndev(event_dev);
                smc_ib_ndev_change(event_dev, event);
                return NOTIFY_OK;
        case NETDEV_UP:
                smc_pnet_add_base_pnetid(net, event_dev, ndev_pnetid);
                return NOTIFY_OK;
        case NETDEV_DOWN:
                event_dev = __pnet_find_base_ndev(event_dev);
                if (!smc_pnetid_by_dev_port(event_dev->dev.parent,
                                            event_dev->dev_port, ndev_pnetid)) {
                        /* remove from PNETIDs list */
                        smc_pnet_remove_pnetid(net, ndev_pnetid);
                }
                return NOTIFY_OK;
        default:
                return NOTIFY_DONE;
        }
}

static struct notifier_block smc_netdev_notifier = {
        .notifier_call = smc_pnet_netdev_event
};

/* init network namespace */
int smc_pnet_net_init(struct net *net)
{
        struct smc_net *sn = net_generic(net, smc_net_id);
        struct smc_pnettable *pnettable = &sn->pnettable;
        struct smc_pnetids_ndev *pnetids_ndev = &sn->pnetids_ndev;

        INIT_LIST_HEAD(&pnettable->pnetlist);
        mutex_init(&pnettable->lock);
        INIT_LIST_HEAD(&pnetids_ndev->list);
        rwlock_init(&pnetids_ndev->lock);

        smc_pnet_create_pnetids_list(net);

        return 0;
}

int __init smc_pnet_init(void)
{
        int rc;

        rc = genl_register_family(&smc_pnet_nl_family);
        if (rc)
                return rc;
        rc = register_netdevice_notifier(&smc_netdev_notifier);
        if (rc)
                genl_unregister_family(&smc_pnet_nl_family);

        return rc;
}

/* exit network namespace */
void smc_pnet_net_exit(struct net *net)
{
        /* flush pnet table */
        smc_pnet_remove_by_pnetid(net, NULL);
        smc_pnet_destroy_pnetids_list(net);
}

void smc_pnet_exit(void)
{
        unregister_netdevice_notifier(&smc_netdev_notifier);
        genl_unregister_family(&smc_pnet_nl_family);
}

static struct net_device *__pnet_find_base_ndev(struct net_device *ndev)
{
        int i, nest_lvl;

        ASSERT_RTNL();
        nest_lvl = ndev->lower_level;
        for (i = 0; i < nest_lvl; i++) {
                struct list_head *lower = &ndev->adj_list.lower;

                if (list_empty(lower))
                        break;
                lower = lower->next;
                ndev = netdev_lower_get_next(ndev, &lower);
        }
        return ndev;
}

/* Determine one base device for stacked net devices.
 * If the lower device level contains more than one devices
 * (for instance with bonding slaves), just the first device
 * is used to reach a base device.
 */
static struct net_device *pnet_find_base_ndev(struct net_device *ndev)
{
        rtnl_lock();
        ndev = __pnet_find_base_ndev(ndev);
        rtnl_unlock();
        return ndev;
}

static int smc_pnet_find_ndev_pnetid_by_table(struct net_device *ndev,
                                              u8 *pnetid)
{
        struct smc_pnettable *pnettable;
        struct net *net = dev_net(ndev);
        struct smc_pnetentry *pnetelem;
        struct smc_net *sn;
        int rc = -ENOENT;

        /* get pnettable for namespace */
        sn = net_generic(net, smc_net_id);
        pnettable = &sn->pnettable;

        mutex_lock(&pnettable->lock);
        list_for_each_entry(pnetelem, &pnettable->pnetlist, list) {
                if (pnetelem->type == SMC_PNET_ETH && ndev == pnetelem->ndev) {
                        /* get pnetid of netdev device */
                        memcpy(pnetid, pnetelem->pnet_name, SMC_MAX_PNETID_LEN);
                        rc = 0;
                        break;
                }
        }
        mutex_unlock(&pnettable->lock);
        return rc;
}

static int smc_pnet_determine_gid(struct smc_ib_device *ibdev, int i,
                                  struct smc_init_info *ini)
{
        if (!ini->check_smcrv2 &&
            !smc_ib_determine_gid(ibdev, i, ini->vlan_id, ini->ib_gid, NULL,
                                  NULL)) {
                ini->ib_dev = ibdev;
                ini->ib_port = i;
                return 0;
        }
        if (ini->check_smcrv2 &&
            !smc_ib_determine_gid(ibdev, i, ini->vlan_id, ini->smcrv2.ib_gid_v2,
                                  NULL, &ini->smcrv2)) {
                ini->smcrv2.ib_dev_v2 = ibdev;
                ini->smcrv2.ib_port_v2 = i;
                return 0;
        }
        return -ENODEV;
}

/* find a roce device for the given pnetid */
static void _smc_pnet_find_roce_by_pnetid(u8 *pnet_id,
                                          struct smc_init_info *ini,
                                          struct smc_ib_device *known_dev,
                                          struct net *net)
{
        struct smc_ib_device *ibdev;
        int i;

        mutex_lock(&smc_ib_devices.mutex);
        list_for_each_entry(ibdev, &smc_ib_devices.list, list) {
                if (ibdev == known_dev ||
                    !rdma_dev_access_netns(ibdev->ibdev, net))
                        continue;
                for (i = 1; i <= SMC_MAX_PORTS; i++) {
                        if (!rdma_is_port_valid(ibdev->ibdev, i))
                                continue;
                        if (smc_pnet_match(ibdev->pnetid[i - 1], pnet_id) &&
                            smc_ib_port_active(ibdev, i) &&
                            !test_bit(i - 1, ibdev->ports_going_away)) {
                                if (!smc_pnet_determine_gid(ibdev, i, ini))
                                        goto out;
                        }
                }
        }
out:
        mutex_unlock(&smc_ib_devices.mutex);
}

/* find alternate roce device with same pnet_id, vlan_id and net namespace */
void smc_pnet_find_alt_roce(struct smc_link_group *lgr,
                            struct smc_init_info *ini,
                            struct smc_ib_device *known_dev)
{
        struct net *net = lgr->net;

        _smc_pnet_find_roce_by_pnetid(lgr->pnet_id, ini, known_dev, net);
}

/* if handshake network device belongs to a roce device, return its
 * IB device and port
 */
static void smc_pnet_find_rdma_dev(struct net_device *netdev,
                                   struct smc_init_info *ini)
{
        struct net *net = dev_net(netdev);
        struct smc_ib_device *ibdev;

        mutex_lock(&smc_ib_devices.mutex);
        list_for_each_entry(ibdev, &smc_ib_devices.list, list) {
                struct net_device *ndev;
                int i;

                /* check rdma net namespace */
                if (!rdma_dev_access_netns(ibdev->ibdev, net))
                        continue;

                for (i = 1; i <= SMC_MAX_PORTS; i++) {
                        if (!rdma_is_port_valid(ibdev->ibdev, i))
                                continue;
                        ndev = ib_device_get_netdev(ibdev->ibdev, i);
                        if (!ndev)
                                continue;
                        dev_put(ndev);
                        if (netdev == ndev &&
                            smc_ib_port_active(ibdev, i) &&
                            !test_bit(i - 1, ibdev->ports_going_away)) {
                                if (!smc_pnet_determine_gid(ibdev, i, ini))
                                        break;
                        }
                }
        }
        mutex_unlock(&smc_ib_devices.mutex);
}

/* Determine the corresponding IB device port based on the hardware PNETID.
 * Searching stops at the first matching active IB device port with vlan_id
 * configured.
 * If nothing found, check pnetid table.
 * If nothing found, try to use handshake device
 */
static void smc_pnet_find_roce_by_pnetid(struct net_device *ndev,
                                         struct smc_init_info *ini)
{
        u8 ndev_pnetid[SMC_MAX_PNETID_LEN];
        struct net *net;

        ndev = pnet_find_base_ndev(ndev);
        net = dev_net(ndev);
        if (smc_pnetid_by_dev_port(ndev->dev.parent, ndev->dev_port,
                                   ndev_pnetid) &&
            smc_pnet_find_ndev_pnetid_by_table(ndev, ndev_pnetid)) {
                smc_pnet_find_rdma_dev(ndev, ini);
                return; /* pnetid could not be determined */
        }
        _smc_pnet_find_roce_by_pnetid(ndev_pnetid, ini, NULL, net);
}

static void smc_pnet_find_ism_by_pnetid(struct net_device *ndev,
                                        struct smc_init_info *ini)
{
        u8 ndev_pnetid[SMC_MAX_PNETID_LEN];
        struct smcd_dev *ismdev;

        ndev = pnet_find_base_ndev(ndev);
        if (smc_pnetid_by_dev_port(ndev->dev.parent, ndev->dev_port,
                                   ndev_pnetid) &&
            smc_pnet_find_ndev_pnetid_by_table(ndev, ndev_pnetid))
                return; /* pnetid could not be determined */

        mutex_lock(&smcd_dev_list.mutex);
        list_for_each_entry(ismdev, &smcd_dev_list.list, list) {
                if (smc_pnet_match(ismdev->pnetid, ndev_pnetid) &&
                    !ismdev->going_away &&
                    (!ini->ism_peer_gid[0].gid ||
                     !smc_ism_cantalk(&ini->ism_peer_gid[0], ini->vlan_id,
                                      ismdev))) {
                        ini->ism_dev[0] = ismdev;
                        break;
                }
        }
        mutex_unlock(&smcd_dev_list.mutex);
}

/* PNET table analysis for a given sock:
 * determine ib_device and port belonging to used internal TCP socket
 * ethernet interface.
 */
void smc_pnet_find_roce_resource(struct sock *sk, struct smc_init_info *ini)
{
        struct dst_entry *dst = sk_dst_get(sk);

        if (!dst)
                goto out;
        if (!dst->dev)
                goto out_rel;

        smc_pnet_find_roce_by_pnetid(dst->dev, ini);

out_rel:
        dst_release(dst);
out:
        return;
}

void smc_pnet_find_ism_resource(struct sock *sk, struct smc_init_info *ini)
{
        struct dst_entry *dst = sk_dst_get(sk);

        ini->ism_dev[0] = NULL;
        if (!dst)
                goto out;
        if (!dst->dev)
                goto out_rel;

        smc_pnet_find_ism_by_pnetid(dst->dev, ini);

out_rel:
        dst_release(dst);
out:
        return;
}

/* Lookup and apply a pnet table entry to the given ib device.
 */
int smc_pnetid_by_table_ib(struct smc_ib_device *smcibdev, u8 ib_port)
{
        char *ib_name = smcibdev->ibdev->name;
        struct smc_pnettable *pnettable;
        struct smc_pnetentry *tmp_pe;
        struct smc_net *sn;
        int rc = -ENOENT;

        /* get pnettable for init namespace */
        sn = net_generic(&init_net, smc_net_id);
        pnettable = &sn->pnettable;

        mutex_lock(&pnettable->lock);
        list_for_each_entry(tmp_pe, &pnettable->pnetlist, list) {
                if (tmp_pe->type == SMC_PNET_IB &&
                    !strncmp(tmp_pe->ib_name, ib_name, IB_DEVICE_NAME_MAX) &&
                    tmp_pe->ib_port == ib_port) {
                        smc_pnet_apply_ib(smcibdev, ib_port, tmp_pe->pnet_name);
                        rc = 0;
                        break;
                }
        }
        mutex_unlock(&pnettable->lock);

        return rc;
}

/* Lookup and apply a pnet table entry to the given smcd device.
 */
int smc_pnetid_by_table_smcd(struct smcd_dev *smcddev)
{
        const char *ib_name = dev_name(smcddev->ops->get_dev(smcddev));
        struct smc_pnettable *pnettable;
        struct smc_pnetentry *tmp_pe;
        struct smc_net *sn;
        int rc = -ENOENT;

        /* get pnettable for init namespace */
        sn = net_generic(&init_net, smc_net_id);
        pnettable = &sn->pnettable;

        mutex_lock(&pnettable->lock);
        list_for_each_entry(tmp_pe, &pnettable->pnetlist, list) {
                if (tmp_pe->type == SMC_PNET_IB &&
                    !strncmp(tmp_pe->ib_name, ib_name, IB_DEVICE_NAME_MAX)) {
                        smc_pnet_apply_smcd(smcddev, tmp_pe->pnet_name);
                        rc = 0;
                        break;
                }
        }
        mutex_unlock(&pnettable->lock);

        return rc;
}





































































































 2447 
































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_X86_ATOMIC64_64_H
#define _ASM_X86_ATOMIC64_64_H

#include <linux/types.h>
#include <asm/alternative.h>
#include <asm/cmpxchg.h>

/* The 64-bit atomic type */

#define ATOMIC64_INIT(i)        { (i) }

static __always_inline s64 arch_atomic64_read(const atomic64_t *v)
{
        return __READ_ONCE((v)->counter);
}

static __always_inline void arch_atomic64_set(atomic64_t *v, s64 i)
{
        __WRITE_ONCE(v->counter, i);
}

static __always_inline void arch_atomic64_add(s64 i, atomic64_t *v)
{
        asm volatile(LOCK_PREFIX "addq %1,%0"
                     : "=m" (v->counter)
                     : "er" (i), "m" (v->counter) : "memory");
}

static __always_inline void arch_atomic64_sub(s64 i, atomic64_t *v)
{
        asm volatile(LOCK_PREFIX "subq %1,%0"
                     : "=m" (v->counter)
                     : "er" (i), "m" (v->counter) : "memory");
}

static __always_inline bool arch_atomic64_sub_and_test(s64 i, atomic64_t *v)
{
        return GEN_BINARY_RMWcc(LOCK_PREFIX "subq", v->counter, e, "er", i);
}
#define arch_atomic64_sub_and_test arch_atomic64_sub_and_test

static __always_inline void arch_atomic64_inc(atomic64_t *v)
{
        asm volatile(LOCK_PREFIX "incq %0"
                     : "=m" (v->counter)
                     : "m" (v->counter) : "memory");
}
#define arch_atomic64_inc arch_atomic64_inc

static __always_inline void arch_atomic64_dec(atomic64_t *v)
{
        asm volatile(LOCK_PREFIX "decq %0"
                     : "=m" (v->counter)
                     : "m" (v->counter) : "memory");
}
#define arch_atomic64_dec arch_atomic64_dec

static __always_inline bool arch_atomic64_dec_and_test(atomic64_t *v)
{
        return GEN_UNARY_RMWcc(LOCK_PREFIX "decq", v->counter, e);
}
#define arch_atomic64_dec_and_test arch_atomic64_dec_and_test

static __always_inline bool arch_atomic64_inc_and_test(atomic64_t *v)
{
        return GEN_UNARY_RMWcc(LOCK_PREFIX "incq", v->counter, e);
}
#define arch_atomic64_inc_and_test arch_atomic64_inc_and_test

static __always_inline bool arch_atomic64_add_negative(s64 i, atomic64_t *v)
{
        return GEN_BINARY_RMWcc(LOCK_PREFIX "addq", v->counter, s, "er", i);
}
#define arch_atomic64_add_negative arch_atomic64_add_negative

static __always_inline s64 arch_atomic64_add_return(s64 i, atomic64_t *v)
{
        return i + xadd(&v->counter, i);
}
#define arch_atomic64_add_return arch_atomic64_add_return

#define arch_atomic64_sub_return(i, v) arch_atomic64_add_return(-(i), v)

static __always_inline s64 arch_atomic64_fetch_add(s64 i, atomic64_t *v)
{
        return xadd(&v->counter, i);
}
#define arch_atomic64_fetch_add arch_atomic64_fetch_add

#define arch_atomic64_fetch_sub(i, v) arch_atomic64_fetch_add(-(i), v)

static __always_inline s64 arch_atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new)
{
        return arch_cmpxchg(&v->counter, old, new);
}
#define arch_atomic64_cmpxchg arch_atomic64_cmpxchg

static __always_inline bool arch_atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new)
{
        return arch_try_cmpxchg(&v->counter, old, new);
}
#define arch_atomic64_try_cmpxchg arch_atomic64_try_cmpxchg

static __always_inline s64 arch_atomic64_xchg(atomic64_t *v, s64 new)
{
        return arch_xchg(&v->counter, new);
}
#define arch_atomic64_xchg arch_atomic64_xchg

static __always_inline void arch_atomic64_and(s64 i, atomic64_t *v)
{
        asm volatile(LOCK_PREFIX "andq %1,%0"
                        : "+m" (v->counter)
                        : "er" (i)
                        : "memory");
}

static __always_inline s64 arch_atomic64_fetch_and(s64 i, atomic64_t *v)
{
        s64 val = arch_atomic64_read(v);

        do {
        } while (!arch_atomic64_try_cmpxchg(v, &val, val & i));
        return val;
}
#define arch_atomic64_fetch_and arch_atomic64_fetch_and

static __always_inline void arch_atomic64_or(s64 i, atomic64_t *v)
{
        asm volatile(LOCK_PREFIX "orq %1,%0"
                        : "+m" (v->counter)
                        : "er" (i)
                        : "memory");
}

static __always_inline s64 arch_atomic64_fetch_or(s64 i, atomic64_t *v)
{
        s64 val = arch_atomic64_read(v);

        do {
        } while (!arch_atomic64_try_cmpxchg(v, &val, val | i));
        return val;
}
#define arch_atomic64_fetch_or arch_atomic64_fetch_or

static __always_inline void arch_atomic64_xor(s64 i, atomic64_t *v)
{
        asm volatile(LOCK_PREFIX "xorq %1,%0"
                        : "+m" (v->counter)
                        : "er" (i)
                        : "memory");
}

static __always_inline s64 arch_atomic64_fetch_xor(s64 i, atomic64_t *v)
{
        s64 val = arch_atomic64_read(v);

        do {
        } while (!arch_atomic64_try_cmpxchg(v, &val, val ^ i));
        return val;
}
#define arch_atomic64_fetch_xor arch_atomic64_fetch_xor

#endif /* _ASM_X86_ATOMIC64_64_H */
















 18719 






  898 




 18905 







 13967 





 13978 





 15421 






  721 





 15527 



 15540 



































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
// SPDX-License-Identifier: GPL-2.0
#include <linux/memblock.h>
#include <linux/mmdebug.h>
#include <linux/export.h>
#include <linux/mm.h>

#include <asm/page.h>
#include <linux/vmalloc.h>

#include "physaddr.h"

#ifdef CONFIG_X86_64

#ifdef CONFIG_DEBUG_VIRTUAL
unsigned long __phys_addr(unsigned long x)
{
        unsigned long y = x - __START_KERNEL_map;

        /* use the carry flag to determine if x was < __START_KERNEL_map */
        if (unlikely(x > y)) {
                x = y + phys_base;

                VIRTUAL_BUG_ON(y >= KERNEL_IMAGE_SIZE);
        } else {
                x = y + (__START_KERNEL_map - PAGE_OFFSET);

                /* carry flag will be set if starting x was >= PAGE_OFFSET */
                VIRTUAL_BUG_ON((x > y) || !phys_addr_valid(x));
        }

        return x;
}
EXPORT_SYMBOL(__phys_addr);

unsigned long __phys_addr_symbol(unsigned long x)
{
        unsigned long y = x - __START_KERNEL_map;

        /* only check upper bounds since lower bounds will trigger carry */
        VIRTUAL_BUG_ON(y >= KERNEL_IMAGE_SIZE);

        return y + phys_base;
}
EXPORT_SYMBOL(__phys_addr_symbol);
#endif

bool __virt_addr_valid(unsigned long x)
{
        unsigned long y = x - __START_KERNEL_map;

        /* use the carry flag to determine if x was < __START_KERNEL_map */
        if (unlikely(x > y)) {
                x = y + phys_base;

                if (y >= KERNEL_IMAGE_SIZE)
                        return false;
        } else {
                x = y + (__START_KERNEL_map - PAGE_OFFSET);

                /* carry flag will be set if starting x was >= PAGE_OFFSET */
                if ((x > y) || !phys_addr_valid(x))
                        return false;
        }

        return pfn_valid(x >> PAGE_SHIFT);
}
EXPORT_SYMBOL(__virt_addr_valid);

#else

#ifdef CONFIG_DEBUG_VIRTUAL
unsigned long __phys_addr(unsigned long x)
{
        unsigned long phys_addr = x - PAGE_OFFSET;
        /* VMALLOC_* aren't constants  */
        VIRTUAL_BUG_ON(x < PAGE_OFFSET);
        VIRTUAL_BUG_ON(__vmalloc_start_set && is_vmalloc_addr((void *) x));
        /* max_low_pfn is set early, but not _that_ early */
        if (max_low_pfn) {
                VIRTUAL_BUG_ON((phys_addr >> PAGE_SHIFT) > max_low_pfn);
                BUG_ON(slow_virt_to_phys((void *)x) != phys_addr);
        }
        return phys_addr;
}
EXPORT_SYMBOL(__phys_addr);
#endif

bool __virt_addr_valid(unsigned long x)
{
        if (x < PAGE_OFFSET)
                return false;
        if (__vmalloc_start_set && is_vmalloc_addr((void *) x))
                return false;
        if (x >= FIXADDR_START)
                return false;
        return pfn_valid((x - PAGE_OFFSET) >> PAGE_SHIFT);
}
EXPORT_SYMBOL(__virt_addr_valid);

#endif        /* CONFIG_X86_64 */





















































































































































































































































































































































































































































  383 








   38 













   38 
   38 














   30 













   30 

   31 














   16 
   31 




   30 




























































































































































































































































































































































































































































































 1966 










































































































































































































































































































   49 















































































































































































































































































































































































































































































































































































































  112 










































































































































  136 






















































































































































































































































































   53 

   18 

































































































 2005 




 1979 

































































































































































































  182 
   34 































    2 



    8 




















































































































































































































































































































































































    3 
    5 








   65 


























































































































































































































































































































































































































































































 2191 
















1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965
2966
2967
2968
2969
2970
2971
2972
2973
2974
2975
2976
2977
2978
2979
2980
2981
2982
2983
2984
2985
2986
2987
2988
2989
2990
2991
2992
2993
2994
2995
2996
2997
2998
2999
3000
3001
3002
3003
3004
3005
3006
3007
3008
3009
3010
3011
3012
3013
3014
3015
3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029
3030
3031
3032
3033
3034
3035
3036
3037
3038
3039
3040
3041
3042
3043
3044
3045
3046
3047
3048
3049
3050
3051
3052
3053
3054
3055
3056
3057
3058
3059
3060
3061
3062
3063
3064
3065
3066
3067
3068
3069
3070
3071
3072
3073
3074
3075
3076
3077
3078
3079
3080
3081
3082
3083
3084
3085
3086
3087
3088
3089
3090
3091
3092
3093
3094
3095
3096
3097
3098
3099
3100
3101
3102
3103
3104
3105
3106
3107
3108
3109
3110
3111
3112
3113
3114
3115
3116
3117
3118
3119
3120
3121
3122
3123
3124
3125
3126
3127
3128
3129
3130
3131
3132
3133
3134
3135
3136
3137
3138
3139
3140
3141
3142
3143
3144
3145
3146
3147
3148
3149
3150
3151
3152
3153
3154
3155
3156
3157
3158
3159
3160
3161
3162
3163
3164
3165
3166
3167
3168
3169
3170
3171
3172
3173
3174
3175
3176
3177
3178
3179
3180
3181
3182
3183
3184
3185
3186
3187
3188
3189
3190
3191
3192
3193
3194
3195
3196
3197
3198
3199
3200
3201
3202
3203
3204
3205
3206
3207
3208
3209
3210
3211
3212
3213
3214
3215
3216
3217
3218
3219
3220
3221
3222
3223
3224
3225
3226
3227
3228
3229
3230
3231
3232
3233
3234
3235
3236
3237
3238
3239
3240
3241
3242
3243
3244
3245
3246
3247
3248
3249
3250
3251
3252
3253
3254
3255
3256
3257
3258
3259
3260
3261
3262
3263
3264
3265
3266
3267
3268
3269
3270
3271
3272
3273
3274
3275
3276
3277
3278
3279
3280
3281
3282
3283
3284
3285
3286
3287
3288
3289
3290
3291
3292
3293
3294
3295
3296
3297
3298
3299
3300
3301
3302
3303
3304
3305
3306
3307
3308
3309
3310
3311
3312
3313
3314
3315
3316
3317
3318
3319
3320
3321
3322
3323
3324
3325
3326
3327
3328
3329
3330
3331
3332
3333
3334
3335
3336
3337
3338
3339
3340
3341
3342
3343
3344
3345
3346
3347
3348
3349
3350
3351
3352
3353
3354
3355
3356
3357
3358
3359
3360
3361
3362
3363
3364
3365
3366
3367
3368
3369
3370
3371
3372
3373
3374
3375
3376
3377
3378
3379
3380
3381
3382
3383
3384
3385
3386
3387
3388
3389
3390
3391
3392
3393
3394
3395
3396
3397
3398
3399
3400
3401
3402
3403
3404
3405
3406
3407
3408
3409
3410
3411
3412
3413
3414
3415
3416
3417
3418
3419
3420
3421
3422
3423
3424
3425
3426
3427
3428
3429
3430
3431
3432
3433
3434
3435
3436
3437
3438
3439
3440
3441
3442
3443
3444
3445
3446
3447
3448
3449
3450
3451
3452
3453
3454
3455
3456
3457
3458
3459
3460
3461
3462
3463
3464
3465
3466
3467
3468
3469
3470
3471
3472
3473
3474
3475
3476
3477
3478
3479
3480
3481
3482
3483
3484
3485
3486
3487
3488
3489
3490
3491
3492
3493
3494
3495
3496
3497
3498
3499
3500
3501
3502
3503
3504
3505
3506
3507
3508
3509
/* SPDX-License-Identifier: GPL-2.0-only */
/* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com
 */
#ifndef _LINUX_BPF_H
#define _LINUX_BPF_H 1

#include <uapi/linux/bpf.h>
#include <uapi/linux/filter.h>

#include <linux/workqueue.h>
#include <linux/file.h>
#include <linux/percpu.h>
#include <linux/err.h>
#include <linux/rbtree_latch.h>
#include <linux/numa.h>
#include <linux/mm_types.h>
#include <linux/wait.h>
#include <linux/refcount.h>
#include <linux/mutex.h>
#include <linux/module.h>
#include <linux/kallsyms.h>
#include <linux/capability.h>
#include <linux/sched/mm.h>
#include <linux/slab.h>
#include <linux/percpu-refcount.h>
#include <linux/stddef.h>
#include <linux/bpfptr.h>
#include <linux/btf.h>
#include <linux/rcupdate_trace.h>
#include <linux/static_call.h>
#include <linux/memcontrol.h>
#include <linux/cfi.h>

struct bpf_verifier_env;
struct bpf_verifier_log;
struct perf_event;
struct bpf_prog;
struct bpf_prog_aux;
struct bpf_map;
struct bpf_arena;
struct sock;
struct seq_file;
struct btf;
struct btf_type;
struct exception_table_entry;
struct seq_operations;
struct bpf_iter_aux_info;
struct bpf_local_storage;
struct bpf_local_storage_map;
struct kobject;
struct mem_cgroup;
struct module;
struct bpf_func_state;
struct ftrace_ops;
struct cgroup;
struct bpf_token;
struct user_namespace;
struct super_block;
struct inode;

extern struct idr btf_idr;
extern spinlock_t btf_idr_lock;
extern struct kobject *btf_kobj;
extern struct bpf_mem_alloc bpf_global_ma, bpf_global_percpu_ma;
extern bool bpf_global_ma_set;

typedef u64 (*bpf_callback_t)(u64, u64, u64, u64, u64);
typedef int (*bpf_iter_init_seq_priv_t)(void *private_data,
                                        struct bpf_iter_aux_info *aux);
typedef void (*bpf_iter_fini_seq_priv_t)(void *private_data);
typedef unsigned int (*bpf_func_t)(const void *,
                                   const struct bpf_insn *);
struct bpf_iter_seq_info {
        const struct seq_operations *seq_ops;
        bpf_iter_init_seq_priv_t init_seq_private;
        bpf_iter_fini_seq_priv_t fini_seq_private;
        u32 seq_priv_size;
};

/* map is generic key/value storage optionally accessible by eBPF programs */
struct bpf_map_ops {
        /* funcs callable from userspace (via syscall) */
        int (*map_alloc_check)(union bpf_attr *attr);
        struct bpf_map *(*map_alloc)(union bpf_attr *attr);
        void (*map_release)(struct bpf_map *map, struct file *map_file);
        void (*map_free)(struct bpf_map *map);
        int (*map_get_next_key)(struct bpf_map *map, void *key, void *next_key);
        void (*map_release_uref)(struct bpf_map *map);
        void *(*map_lookup_elem_sys_only)(struct bpf_map *map, void *key);
        int (*map_lookup_batch)(struct bpf_map *map, const union bpf_attr *attr,
                                union bpf_attr __user *uattr);
        int (*map_lookup_and_delete_elem)(struct bpf_map *map, void *key,
                                          void *value, u64 flags);
        int (*map_lookup_and_delete_batch)(struct bpf_map *map,
                                           const union bpf_attr *attr,
                                           union bpf_attr __user *uattr);
        int (*map_update_batch)(struct bpf_map *map, struct file *map_file,
                                const union bpf_attr *attr,
                                union bpf_attr __user *uattr);
        int (*map_delete_batch)(struct bpf_map *map, const union bpf_attr *attr,
                                union bpf_attr __user *uattr);

        /* funcs callable from userspace and from eBPF programs */
        void *(*map_lookup_elem)(struct bpf_map *map, void *key);
        long (*map_update_elem)(struct bpf_map *map, void *key, void *value, u64 flags);
        long (*map_delete_elem)(struct bpf_map *map, void *key);
        long (*map_push_elem)(struct bpf_map *map, void *value, u64 flags);
        long (*map_pop_elem)(struct bpf_map *map, void *value);
        long (*map_peek_elem)(struct bpf_map *map, void *value);
        void *(*map_lookup_percpu_elem)(struct bpf_map *map, void *key, u32 cpu);

        /* funcs called by prog_array and perf_event_array map */
        void *(*map_fd_get_ptr)(struct bpf_map *map, struct file *map_file,
                                int fd);
        /* If need_defer is true, the implementation should guarantee that
         * the to-be-put element is still alive before the bpf program, which
         * may manipulate it, exists.
         */
        void (*map_fd_put_ptr)(struct bpf_map *map, void *ptr, bool need_defer);
        int (*map_gen_lookup)(struct bpf_map *map, struct bpf_insn *insn_buf);
        u32 (*map_fd_sys_lookup_elem)(void *ptr);
        void (*map_seq_show_elem)(struct bpf_map *map, void *key,
                                  struct seq_file *m);
        int (*map_check_btf)(const struct bpf_map *map,
                             const struct btf *btf,
                             const struct btf_type *key_type,
                             const struct btf_type *value_type);

        /* Prog poke tracking helpers. */
        int (*map_poke_track)(struct bpf_map *map, struct bpf_prog_aux *aux);
        void (*map_poke_untrack)(struct bpf_map *map, struct bpf_prog_aux *aux);
        void (*map_poke_run)(struct bpf_map *map, u32 key, struct bpf_prog *old,
                             struct bpf_prog *new);

        /* Direct value access helpers. */
        int (*map_direct_value_addr)(const struct bpf_map *map,
                                     u64 *imm, u32 off);
        int (*map_direct_value_meta)(const struct bpf_map *map,
                                     u64 imm, u32 *off);
        int (*map_mmap)(struct bpf_map *map, struct vm_area_struct *vma);
        __poll_t (*map_poll)(struct bpf_map *map, struct file *filp,
                             struct poll_table_struct *pts);
        unsigned long (*map_get_unmapped_area)(struct file *filep, unsigned long addr,
                                               unsigned long len, unsigned long pgoff,
                                               unsigned long flags);

        /* Functions called by bpf_local_storage maps */
        int (*map_local_storage_charge)(struct bpf_local_storage_map *smap,
                                        void *owner, u32 size);
        void (*map_local_storage_uncharge)(struct bpf_local_storage_map *smap,
                                           void *owner, u32 size);
        struct bpf_local_storage __rcu ** (*map_owner_storage_ptr)(void *owner);

        /* Misc helpers.*/
        long (*map_redirect)(struct bpf_map *map, u64 key, u64 flags);

        /* map_meta_equal must be implemented for maps that can be
         * used as an inner map.  It is a runtime check to ensure
         * an inner map can be inserted to an outer map.
         *
         * Some properties of the inner map has been used during the
         * verification time.  When inserting an inner map at the runtime,
         * map_meta_equal has to ensure the inserting map has the same
         * properties that the verifier has used earlier.
         */
        bool (*map_meta_equal)(const struct bpf_map *meta0,
                               const struct bpf_map *meta1);


        int (*map_set_for_each_callback_args)(struct bpf_verifier_env *env,
                                              struct bpf_func_state *caller,
                                              struct bpf_func_state *callee);
        long (*map_for_each_callback)(struct bpf_map *map,
                                     bpf_callback_t callback_fn,
                                     void *callback_ctx, u64 flags);

        u64 (*map_mem_usage)(const struct bpf_map *map);

        /* BTF id of struct allocated by map_alloc */
        int *map_btf_id;

        /* bpf_iter info used to open a seq_file */
        const struct bpf_iter_seq_info *iter_seq_info;
};

enum {
        /* Support at most 11 fields in a BTF type */
        BTF_FIELDS_MAX           = 11,
};

enum btf_field_type {
        BPF_SPIN_LOCK  = (1 << 0),
        BPF_TIMER      = (1 << 1),
        BPF_KPTR_UNREF = (1 << 2),
        BPF_KPTR_REF   = (1 << 3),
        BPF_KPTR_PERCPU = (1 << 4),
        BPF_KPTR       = BPF_KPTR_UNREF | BPF_KPTR_REF | BPF_KPTR_PERCPU,
        BPF_LIST_HEAD  = (1 << 5),
        BPF_LIST_NODE  = (1 << 6),
        BPF_RB_ROOT    = (1 << 7),
        BPF_RB_NODE    = (1 << 8),
        BPF_GRAPH_NODE = BPF_RB_NODE | BPF_LIST_NODE,
        BPF_GRAPH_ROOT = BPF_RB_ROOT | BPF_LIST_HEAD,
        BPF_REFCOUNT   = (1 << 9),
        BPF_WORKQUEUE  = (1 << 10),
        BPF_UPTR       = (1 << 11),
};

typedef void (*btf_dtor_kfunc_t)(void *);

struct btf_field_kptr {
        struct btf *btf;
        struct module *module;
        /* dtor used if btf_is_kernel(btf), otherwise the type is
         * program-allocated, dtor is NULL,  and __bpf_obj_drop_impl is used
         */
        btf_dtor_kfunc_t dtor;
        u32 btf_id;
};

struct btf_field_graph_root {
        struct btf *btf;
        u32 value_btf_id;
        u32 node_offset;
        struct btf_record *value_rec;
};

struct btf_field {
        u32 offset;
        u32 size;
        enum btf_field_type type;
        union {
                struct btf_field_kptr kptr;
                struct btf_field_graph_root graph_root;
        };
};

struct btf_record {
        u32 cnt;
        u32 field_mask;
        int spin_lock_off;
        int timer_off;
        int wq_off;
        int refcount_off;
        struct btf_field fields[];
};

/* Non-opaque version of bpf_rb_node in uapi/linux/bpf.h */
struct bpf_rb_node_kern {
        struct rb_node rb_node;
        void *owner;
} __attribute__((aligned(8)));

/* Non-opaque version of bpf_list_node in uapi/linux/bpf.h */
struct bpf_list_node_kern {
        struct list_head list_head;
        void *owner;
} __attribute__((aligned(8)));

struct bpf_map {
        const struct bpf_map_ops *ops;
        struct bpf_map *inner_map_meta;
#ifdef CONFIG_SECURITY
        void *security;
#endif
        enum bpf_map_type map_type;
        u32 key_size;
        u32 value_size;
        u32 max_entries;
        u64 map_extra; /* any per-map-type extra fields */
        u32 map_flags;
        u32 id;
        struct btf_record *record;
        int numa_node;
        u32 btf_key_type_id;
        u32 btf_value_type_id;
        u32 btf_vmlinux_value_type_id;
        struct btf *btf;
#ifdef CONFIG_MEMCG
        struct obj_cgroup *objcg;
#endif
        char name[BPF_OBJ_NAME_LEN];
        struct mutex freeze_mutex;
        atomic64_t refcnt;
        atomic64_t usercnt;
        /* rcu is used before freeing and work is only used during freeing */
        union {
                struct work_struct work;
                struct rcu_head rcu;
        };
        atomic64_t writecnt;
        /* 'Ownership' of program-containing map is claimed by the first program
         * that is going to use this map or by the first program which FD is
         * stored in the map to make sure that all callers and callees have the
         * same prog type, JITed flag and xdp_has_frags flag.
         */
        struct {
                const struct btf_type *attach_func_proto;
                spinlock_t lock;
                enum bpf_prog_type type;
                bool jited;
                bool xdp_has_frags;
        } owner;
        bool bypass_spec_v1;
        bool frozen; /* write-once; write-protected by freeze_mutex */
        bool free_after_mult_rcu_gp;
        bool free_after_rcu_gp;
        atomic64_t sleepable_refcnt;
        s64 __percpu *elem_count;
};

static inline const char *btf_field_type_name(enum btf_field_type type)
{
        switch (type) {
        case BPF_SPIN_LOCK:
                return "bpf_spin_lock";
        case BPF_TIMER:
                return "bpf_timer";
        case BPF_WORKQUEUE:
                return "bpf_wq";
        case BPF_KPTR_UNREF:
        case BPF_KPTR_REF:
                return "kptr";
        case BPF_KPTR_PERCPU:
                return "percpu_kptr";
        case BPF_UPTR:
                return "uptr";
        case BPF_LIST_HEAD:
                return "bpf_list_head";
        case BPF_LIST_NODE:
                return "bpf_list_node";
        case BPF_RB_ROOT:
                return "bpf_rb_root";
        case BPF_RB_NODE:
                return "bpf_rb_node";
        case BPF_REFCOUNT:
                return "bpf_refcount";
        default:
                WARN_ON_ONCE(1);
                return "unknown";
        }
}

static inline u32 btf_field_type_size(enum btf_field_type type)
{
        switch (type) {
        case BPF_SPIN_LOCK:
                return sizeof(struct bpf_spin_lock);
        case BPF_TIMER:
                return sizeof(struct bpf_timer);
        case BPF_WORKQUEUE:
                return sizeof(struct bpf_wq);
        case BPF_KPTR_UNREF:
        case BPF_KPTR_REF:
        case BPF_KPTR_PERCPU:
        case BPF_UPTR:
                return sizeof(u64);
        case BPF_LIST_HEAD:
                return sizeof(struct bpf_list_head);
        case BPF_LIST_NODE:
                return sizeof(struct bpf_list_node);
        case BPF_RB_ROOT:
                return sizeof(struct bpf_rb_root);
        case BPF_RB_NODE:
                return sizeof(struct bpf_rb_node);
        case BPF_REFCOUNT:
                return sizeof(struct bpf_refcount);
        default:
                WARN_ON_ONCE(1);
                return 0;
        }
}

static inline u32 btf_field_type_align(enum btf_field_type type)
{
        switch (type) {
        case BPF_SPIN_LOCK:
                return __alignof__(struct bpf_spin_lock);
        case BPF_TIMER:
                return __alignof__(struct bpf_timer);
        case BPF_WORKQUEUE:
                return __alignof__(struct bpf_wq);
        case BPF_KPTR_UNREF:
        case BPF_KPTR_REF:
        case BPF_KPTR_PERCPU:
        case BPF_UPTR:
                return __alignof__(u64);
        case BPF_LIST_HEAD:
                return __alignof__(struct bpf_list_head);
        case BPF_LIST_NODE:
                return __alignof__(struct bpf_list_node);
        case BPF_RB_ROOT:
                return __alignof__(struct bpf_rb_root);
        case BPF_RB_NODE:
                return __alignof__(struct bpf_rb_node);
        case BPF_REFCOUNT:
                return __alignof__(struct bpf_refcount);
        default:
                WARN_ON_ONCE(1);
                return 0;
        }
}

static inline void bpf_obj_init_field(const struct btf_field *field, void *addr)
{
        memset(addr, 0, field->size);

        switch (field->type) {
        case BPF_REFCOUNT:
                refcount_set((refcount_t *)addr, 1);
                break;
        case BPF_RB_NODE:
                RB_CLEAR_NODE((struct rb_node *)addr);
                break;
        case BPF_LIST_HEAD:
        case BPF_LIST_NODE:
                INIT_LIST_HEAD((struct list_head *)addr);
                break;
        case BPF_RB_ROOT:
                /* RB_ROOT_CACHED 0-inits, no need to do anything after memset */
        case BPF_SPIN_LOCK:
        case BPF_TIMER:
        case BPF_WORKQUEUE:
        case BPF_KPTR_UNREF:
        case BPF_KPTR_REF:
        case BPF_KPTR_PERCPU:
        case BPF_UPTR:
                break;
        default:
                WARN_ON_ONCE(1);
                return;
        }
}

static inline bool btf_record_has_field(const struct btf_record *rec, enum btf_field_type type)
{
        if (IS_ERR_OR_NULL(rec))
                return false;
        return rec->field_mask & type;
}

static inline void bpf_obj_init(const struct btf_record *rec, void *obj)
{
        int i;

        if (IS_ERR_OR_NULL(rec))
                return;
        for (i = 0; i < rec->cnt; i++)
                bpf_obj_init_field(&rec->fields[i], obj + rec->fields[i].offset);
}

/* 'dst' must be a temporary buffer and should not point to memory that is being
 * used in parallel by a bpf program or bpf syscall, otherwise the access from
 * the bpf program or bpf syscall may be corrupted by the reinitialization,
 * leading to weird problems. Even 'dst' is newly-allocated from bpf memory
 * allocator, it is still possible for 'dst' to be used in parallel by a bpf
 * program or bpf syscall.
 */
static inline void check_and_init_map_value(struct bpf_map *map, void *dst)
{
        bpf_obj_init(map->record, dst);
}

/* memcpy that is used with 8-byte aligned pointers, power-of-8 size and
 * forced to use 'long' read/writes to try to atomically copy long counters.
 * Best-effort only.  No barriers here, since it _will_ race with concurrent
 * updates from BPF programs. Called from bpf syscall and mostly used with
 * size 8 or 16 bytes, so ask compiler to inline it.
 */
static inline void bpf_long_memcpy(void *dst, const void *src, u32 size)
{
        const long *lsrc = src;
        long *ldst = dst;

        size /= sizeof(long);
        while (size--)
                data_race(*ldst++ = *lsrc++);
}

/* copy everything but bpf_spin_lock, bpf_timer, and kptrs. There could be one of each. */
static inline void bpf_obj_memcpy(struct btf_record *rec,
                                  void *dst, void *src, u32 size,
                                  bool long_memcpy)
{
        u32 curr_off = 0;
        int i;

        if (IS_ERR_OR_NULL(rec)) {
                if (long_memcpy)
                        bpf_long_memcpy(dst, src, round_up(size, 8));
                else
                        memcpy(dst, src, size);
                return;
        }

        for (i = 0; i < rec->cnt; i++) {
                u32 next_off = rec->fields[i].offset;
                u32 sz = next_off - curr_off;

                memcpy(dst + curr_off, src + curr_off, sz);
                curr_off += rec->fields[i].size + sz;
        }
        memcpy(dst + curr_off, src + curr_off, size - curr_off);
}

static inline void copy_map_value(struct bpf_map *map, void *dst, void *src)
{
        bpf_obj_memcpy(map->record, dst, src, map->value_size, false);
}

static inline void copy_map_value_long(struct bpf_map *map, void *dst, void *src)
{
        bpf_obj_memcpy(map->record, dst, src, map->value_size, true);
}

static inline void bpf_obj_swap_uptrs(const struct btf_record *rec, void *dst, void *src)
{
        unsigned long *src_uptr, *dst_uptr;
        const struct btf_field *field;
        int i;

        if (!btf_record_has_field(rec, BPF_UPTR))
                return;

        for (i = 0, field = rec->fields; i < rec->cnt; i++, field++) {
                if (field->type != BPF_UPTR)
                        continue;

                src_uptr = src + field->offset;
                dst_uptr = dst + field->offset;
                swap(*src_uptr, *dst_uptr);
        }
}

static inline void bpf_obj_memzero(struct btf_record *rec, void *dst, u32 size)
{
        u32 curr_off = 0;
        int i;

        if (IS_ERR_OR_NULL(rec)) {
                memset(dst, 0, size);
                return;
        }

        for (i = 0; i < rec->cnt; i++) {
                u32 next_off = rec->fields[i].offset;
                u32 sz = next_off - curr_off;

                memset(dst + curr_off, 0, sz);
                curr_off += rec->fields[i].size + sz;
        }
        memset(dst + curr_off, 0, size - curr_off);
}

static inline void zero_map_value(struct bpf_map *map, void *dst)
{
        bpf_obj_memzero(map->record, dst, map->value_size);
}

void copy_map_value_locked(struct bpf_map *map, void *dst, void *src,
                           bool lock_src);
void bpf_timer_cancel_and_free(void *timer);
void bpf_wq_cancel_and_free(void *timer);
void bpf_list_head_free(const struct btf_field *field, void *list_head,
                        struct bpf_spin_lock *spin_lock);
void bpf_rb_root_free(const struct btf_field *field, void *rb_root,
                      struct bpf_spin_lock *spin_lock);
u64 bpf_arena_get_kern_vm_start(struct bpf_arena *arena);
u64 bpf_arena_get_user_vm_start(struct bpf_arena *arena);
int bpf_obj_name_cpy(char *dst, const char *src, unsigned int size);

struct bpf_offload_dev;
struct bpf_offloaded_map;

struct bpf_map_dev_ops {
        int (*map_get_next_key)(struct bpf_offloaded_map *map,
                                void *key, void *next_key);
        int (*map_lookup_elem)(struct bpf_offloaded_map *map,
                               void *key, void *value);
        int (*map_update_elem)(struct bpf_offloaded_map *map,
                               void *key, void *value, u64 flags);
        int (*map_delete_elem)(struct bpf_offloaded_map *map, void *key);
};

struct bpf_offloaded_map {
        struct bpf_map map;
        struct net_device *netdev;
        const struct bpf_map_dev_ops *dev_ops;
        void *dev_priv;
        struct list_head offloads;
};

static inline struct bpf_offloaded_map *map_to_offmap(struct bpf_map *map)
{
        return container_of(map, struct bpf_offloaded_map, map);
}

static inline bool bpf_map_offload_neutral(const struct bpf_map *map)
{
        return map->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY;
}

static inline bool bpf_map_support_seq_show(const struct bpf_map *map)
{
        return (map->btf_value_type_id || map->btf_vmlinux_value_type_id) &&
                map->ops->map_seq_show_elem;
}

int map_check_no_btf(const struct bpf_map *map,
                     const struct btf *btf,
                     const struct btf_type *key_type,
                     const struct btf_type *value_type);

bool bpf_map_meta_equal(const struct bpf_map *meta0,
                        const struct bpf_map *meta1);

extern const struct bpf_map_ops bpf_map_offload_ops;

/* bpf_type_flag contains a set of flags that are applicable to the values of
 * arg_type, ret_type and reg_type. For example, a pointer value may be null,
 * or a memory is read-only. We classify types into two categories: base types
 * and extended types. Extended types are base types combined with a type flag.
 *
 * Currently there are no more than 32 base types in arg_type, ret_type and
 * reg_types.
 */
#define BPF_BASE_TYPE_BITS        8

enum bpf_type_flag {
        /* PTR may be NULL. */
        PTR_MAYBE_NULL                = BIT(0 + BPF_BASE_TYPE_BITS),

        /* MEM is read-only. When applied on bpf_arg, it indicates the arg is
         * compatible with both mutable and immutable memory.
         */
        MEM_RDONLY                = BIT(1 + BPF_BASE_TYPE_BITS),

        /* MEM points to BPF ring buffer reservation. */
        MEM_RINGBUF                = BIT(2 + BPF_BASE_TYPE_BITS),

        /* MEM is in user address space. */
        MEM_USER                = BIT(3 + BPF_BASE_TYPE_BITS),

        /* MEM is a percpu memory. MEM_PERCPU tags PTR_TO_BTF_ID. When tagged
         * with MEM_PERCPU, PTR_TO_BTF_ID _cannot_ be directly accessed. In
         * order to drop this tag, it must be passed into bpf_per_cpu_ptr()
         * or bpf_this_cpu_ptr(), which will return the pointer corresponding
         * to the specified cpu.
         */
        MEM_PERCPU                = BIT(4 + BPF_BASE_TYPE_BITS),

        /* Indicates that the argument will be released. */
        OBJ_RELEASE                = BIT(5 + BPF_BASE_TYPE_BITS),

        /* PTR is not trusted. This is only used with PTR_TO_BTF_ID, to mark
         * unreferenced and referenced kptr loaded from map value using a load
         * instruction, so that they can only be dereferenced but not escape the
         * BPF program into the kernel (i.e. cannot be passed as arguments to
         * kfunc or bpf helpers).
         */
        PTR_UNTRUSTED                = BIT(6 + BPF_BASE_TYPE_BITS),

        /* MEM can be uninitialized. */
        MEM_UNINIT                = BIT(7 + BPF_BASE_TYPE_BITS),

        /* DYNPTR points to memory local to the bpf program. */
        DYNPTR_TYPE_LOCAL        = BIT(8 + BPF_BASE_TYPE_BITS),

        /* DYNPTR points to a kernel-produced ringbuf record. */
        DYNPTR_TYPE_RINGBUF        = BIT(9 + BPF_BASE_TYPE_BITS),

        /* Size is known at compile time. */
        MEM_FIXED_SIZE                = BIT(10 + BPF_BASE_TYPE_BITS),

        /* MEM is of an allocated object of type in program BTF. This is used to
         * tag PTR_TO_BTF_ID allocated using bpf_obj_new.
         */
        MEM_ALLOC                = BIT(11 + BPF_BASE_TYPE_BITS),

        /* PTR was passed from the kernel in a trusted context, and may be
         * passed to KF_TRUSTED_ARGS kfuncs or BPF helper functions.
         * Confusingly, this is _not_ the opposite of PTR_UNTRUSTED above.
         * PTR_UNTRUSTED refers to a kptr that was read directly from a map
         * without invoking bpf_kptr_xchg(). What we really need to know is
         * whether a pointer is safe to pass to a kfunc or BPF helper function.
         * While PTR_UNTRUSTED pointers are unsafe to pass to kfuncs and BPF
         * helpers, they do not cover all possible instances of unsafe
         * pointers. For example, a pointer that was obtained from walking a
         * struct will _not_ get the PTR_UNTRUSTED type modifier, despite the
         * fact that it may be NULL, invalid, etc. This is due to backwards
         * compatibility requirements, as this was the behavior that was first
         * introduced when kptrs were added. The behavior is now considered
         * deprecated, and PTR_UNTRUSTED will eventually be removed.
         *
         * PTR_TRUSTED, on the other hand, is a pointer that the kernel
         * guarantees to be valid and safe to pass to kfuncs and BPF helpers.
         * For example, pointers passed to tracepoint arguments are considered
         * PTR_TRUSTED, as are pointers that are passed to struct_ops
         * callbacks. As alluded to above, pointers that are obtained from
         * walking PTR_TRUSTED pointers are _not_ trusted. For example, if a
         * struct task_struct *task is PTR_TRUSTED, then accessing
         * task->last_wakee will lose the PTR_TRUSTED modifier when it's stored
         * in a BPF register. Similarly, pointers passed to certain programs
         * types such as kretprobes are not guaranteed to be valid, as they may
         * for example contain an object that was recently freed.
         */
        PTR_TRUSTED                = BIT(12 + BPF_BASE_TYPE_BITS),

        /* MEM is tagged with rcu and memory access needs rcu_read_lock protection. */
        MEM_RCU                        = BIT(13 + BPF_BASE_TYPE_BITS),

        /* Used to tag PTR_TO_BTF_ID | MEM_ALLOC references which are non-owning.
         * Currently only valid for linked-list and rbtree nodes. If the nodes
         * have a bpf_refcount_field, they must be tagged MEM_RCU as well.
         */
        NON_OWN_REF                = BIT(14 + BPF_BASE_TYPE_BITS),

        /* DYNPTR points to sk_buff */
        DYNPTR_TYPE_SKB                = BIT(15 + BPF_BASE_TYPE_BITS),

        /* DYNPTR points to xdp_buff */
        DYNPTR_TYPE_XDP                = BIT(16 + BPF_BASE_TYPE_BITS),

        /* Memory must be aligned on some architectures, used in combination with
         * MEM_FIXED_SIZE.
         */
        MEM_ALIGNED                = BIT(17 + BPF_BASE_TYPE_BITS),

        /* MEM is being written to, often combined with MEM_UNINIT. Non-presence
         * of MEM_WRITE means that MEM is only being read. MEM_WRITE without the
         * MEM_UNINIT means that memory needs to be initialized since it is also
         * read.
         */
        MEM_WRITE                = BIT(18 + BPF_BASE_TYPE_BITS),

        __BPF_TYPE_FLAG_MAX,
        __BPF_TYPE_LAST_FLAG        = __BPF_TYPE_FLAG_MAX - 1,
};

#define DYNPTR_TYPE_FLAG_MASK        (DYNPTR_TYPE_LOCAL | DYNPTR_TYPE_RINGBUF | DYNPTR_TYPE_SKB \
                                 | DYNPTR_TYPE_XDP)

/* Max number of base types. */
#define BPF_BASE_TYPE_LIMIT        (1UL << BPF_BASE_TYPE_BITS)

/* Max number of all types. */
#define BPF_TYPE_LIMIT                (__BPF_TYPE_LAST_FLAG | (__BPF_TYPE_LAST_FLAG - 1))

/* function argument constraints */
enum bpf_arg_type {
        ARG_DONTCARE = 0,        /* unused argument in helper function */

        /* the following constraints used to prototype
         * bpf_map_lookup/update/delete_elem() functions
         */
        ARG_CONST_MAP_PTR,        /* const argument used as pointer to bpf_map */
        ARG_PTR_TO_MAP_KEY,        /* pointer to stack used as map key */
        ARG_PTR_TO_MAP_VALUE,        /* pointer to stack used as map value */

        /* Used to prototype bpf_memcmp() and other functions that access data
         * on eBPF program stack
         */
        ARG_PTR_TO_MEM,                /* pointer to valid memory (stack, packet, map value) */
        ARG_PTR_TO_ARENA,

        ARG_CONST_SIZE,                /* number of bytes accessed from memory */
        ARG_CONST_SIZE_OR_ZERO,        /* number of bytes accessed from memory or 0 */

        ARG_PTR_TO_CTX,                /* pointer to context */
        ARG_ANYTHING,                /* any (initialized) argument is ok */
        ARG_PTR_TO_SPIN_LOCK,        /* pointer to bpf_spin_lock */
        ARG_PTR_TO_SOCK_COMMON,        /* pointer to sock_common */
        ARG_PTR_TO_SOCKET,        /* pointer to bpf_sock (fullsock) */
        ARG_PTR_TO_BTF_ID,        /* pointer to in-kernel struct */
        ARG_PTR_TO_RINGBUF_MEM,        /* pointer to dynamically reserved ringbuf memory */
        ARG_CONST_ALLOC_SIZE_OR_ZERO,        /* number of allocated bytes requested */
        ARG_PTR_TO_BTF_ID_SOCK_COMMON,        /* pointer to in-kernel sock_common or bpf-mirrored bpf_sock */
        ARG_PTR_TO_PERCPU_BTF_ID,        /* pointer to in-kernel percpu type */
        ARG_PTR_TO_FUNC,        /* pointer to a bpf program function */
        ARG_PTR_TO_STACK,        /* pointer to stack */
        ARG_PTR_TO_CONST_STR,        /* pointer to a null terminated read-only string */
        ARG_PTR_TO_TIMER,        /* pointer to bpf_timer */
        ARG_KPTR_XCHG_DEST,        /* pointer to destination that kptrs are bpf_kptr_xchg'd into */
        ARG_PTR_TO_DYNPTR,      /* pointer to bpf_dynptr. See bpf_type_flag for dynptr type */
        __BPF_ARG_TYPE_MAX,

        /* Extended arg_types. */
        ARG_PTR_TO_MAP_VALUE_OR_NULL        = PTR_MAYBE_NULL | ARG_PTR_TO_MAP_VALUE,
        ARG_PTR_TO_MEM_OR_NULL                = PTR_MAYBE_NULL | ARG_PTR_TO_MEM,
        ARG_PTR_TO_CTX_OR_NULL                = PTR_MAYBE_NULL | ARG_PTR_TO_CTX,
        ARG_PTR_TO_SOCKET_OR_NULL        = PTR_MAYBE_NULL | ARG_PTR_TO_SOCKET,
        ARG_PTR_TO_STACK_OR_NULL        = PTR_MAYBE_NULL | ARG_PTR_TO_STACK,
        ARG_PTR_TO_BTF_ID_OR_NULL        = PTR_MAYBE_NULL | ARG_PTR_TO_BTF_ID,
        /* Pointer to memory does not need to be initialized, since helper function
         * fills all bytes or clears them in error case.
         */
        ARG_PTR_TO_UNINIT_MEM                = MEM_UNINIT | MEM_WRITE | ARG_PTR_TO_MEM,
        /* Pointer to valid memory of size known at compile time. */
        ARG_PTR_TO_FIXED_SIZE_MEM        = MEM_FIXED_SIZE | ARG_PTR_TO_MEM,

        /* This must be the last entry. Its purpose is to ensure the enum is
         * wide enough to hold the higher bits reserved for bpf_type_flag.
         */
        __BPF_ARG_TYPE_LIMIT        = BPF_TYPE_LIMIT,
};
static_assert(__BPF_ARG_TYPE_MAX <= BPF_BASE_TYPE_LIMIT);

/* type of values returned from helper functions */
enum bpf_return_type {
        RET_INTEGER,                        /* function returns integer */
        RET_VOID,                        /* function doesn't return anything */
        RET_PTR_TO_MAP_VALUE,                /* returns a pointer to map elem value */
        RET_PTR_TO_SOCKET,                /* returns a pointer to a socket */
        RET_PTR_TO_TCP_SOCK,                /* returns a pointer to a tcp_sock */
        RET_PTR_TO_SOCK_COMMON,                /* returns a pointer to a sock_common */
        RET_PTR_TO_MEM,                        /* returns a pointer to memory */
        RET_PTR_TO_MEM_OR_BTF_ID,        /* returns a pointer to a valid memory or a btf_id */
        RET_PTR_TO_BTF_ID,                /* returns a pointer to a btf_id */
        __BPF_RET_TYPE_MAX,

        /* Extended ret_types. */
        RET_PTR_TO_MAP_VALUE_OR_NULL        = PTR_MAYBE_NULL | RET_PTR_TO_MAP_VALUE,
        RET_PTR_TO_SOCKET_OR_NULL        = PTR_MAYBE_NULL | RET_PTR_TO_SOCKET,
        RET_PTR_TO_TCP_SOCK_OR_NULL        = PTR_MAYBE_NULL | RET_PTR_TO_TCP_SOCK,
        RET_PTR_TO_SOCK_COMMON_OR_NULL        = PTR_MAYBE_NULL | RET_PTR_TO_SOCK_COMMON,
        RET_PTR_TO_RINGBUF_MEM_OR_NULL        = PTR_MAYBE_NULL | MEM_RINGBUF | RET_PTR_TO_MEM,
        RET_PTR_TO_DYNPTR_MEM_OR_NULL        = PTR_MAYBE_NULL | RET_PTR_TO_MEM,
        RET_PTR_TO_BTF_ID_OR_NULL        = PTR_MAYBE_NULL | RET_PTR_TO_BTF_ID,
        RET_PTR_TO_BTF_ID_TRUSTED        = PTR_TRUSTED         | RET_PTR_TO_BTF_ID,

        /* This must be the last entry. Its purpose is to ensure the enum is
         * wide enough to hold the higher bits reserved for bpf_type_flag.
         */
        __BPF_RET_TYPE_LIMIT        = BPF_TYPE_LIMIT,
};
static_assert(__BPF_RET_TYPE_MAX <= BPF_BASE_TYPE_LIMIT);

/* eBPF function prototype used by verifier to allow BPF_CALLs from eBPF programs
 * to in-kernel helper functions and for adjusting imm32 field in BPF_CALL
 * instructions after verifying
 */
struct bpf_func_proto {
        u64 (*func)(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5);
        bool gpl_only;
        bool pkt_access;
        bool might_sleep;
        /* set to true if helper follows contract for llvm
         * attribute bpf_fastcall:
         * - void functions do not scratch r0
         * - functions taking N arguments scratch only registers r1-rN
         */
        bool allow_fastcall;
        enum bpf_return_type ret_type;
        union {
                struct {
                        enum bpf_arg_type arg1_type;
                        enum bpf_arg_type arg2_type;
                        enum bpf_arg_type arg3_type;
                        enum bpf_arg_type arg4_type;
                        enum bpf_arg_type arg5_type;
                };
                enum bpf_arg_type arg_type[5];
        };
        union {
                struct {
                        u32 *arg1_btf_id;
                        u32 *arg2_btf_id;
                        u32 *arg3_btf_id;
                        u32 *arg4_btf_id;
                        u32 *arg5_btf_id;
                };
                u32 *arg_btf_id[5];
                struct {
                        size_t arg1_size;
                        size_t arg2_size;
                        size_t arg3_size;
                        size_t arg4_size;
                        size_t arg5_size;
                };
                size_t arg_size[5];
        };
        int *ret_btf_id; /* return value btf_id */
        bool (*allowed)(const struct bpf_prog *prog);
};

/* bpf_context is intentionally undefined structure. Pointer to bpf_context is
 * the first argument to eBPF programs.
 * For socket filters: 'struct bpf_context *' == 'struct sk_buff *'
 */
struct bpf_context;

enum bpf_access_type {
        BPF_READ = 1,
        BPF_WRITE = 2
};

/* types of values stored in eBPF registers */
/* Pointer types represent:
 * pointer
 * pointer + imm
 * pointer + (u16) var
 * pointer + (u16) var + imm
 * if (range > 0) then [ptr, ptr + range - off) is safe to access
 * if (id > 0) means that some 'var' was added
 * if (off > 0) means that 'imm' was added
 */
enum bpf_reg_type {
        NOT_INIT = 0,                 /* nothing was written into register */
        SCALAR_VALUE,                 /* reg doesn't contain a valid pointer */
        PTR_TO_CTX,                 /* reg points to bpf_context */
        CONST_PTR_TO_MAP,         /* reg points to struct bpf_map */
        PTR_TO_MAP_VALUE,         /* reg points to map element value */
        PTR_TO_MAP_KEY,                 /* reg points to a map element key */
        PTR_TO_STACK,                 /* reg == frame_pointer + offset */
        PTR_TO_PACKET_META,         /* skb->data - meta_len */
        PTR_TO_PACKET,                 /* reg points to skb->data */
        PTR_TO_PACKET_END,         /* skb->data + headlen */
        PTR_TO_FLOW_KEYS,         /* reg points to bpf_flow_keys */
        PTR_TO_SOCKET,                 /* reg points to struct bpf_sock */
        PTR_TO_SOCK_COMMON,         /* reg points to sock_common */
        PTR_TO_TCP_SOCK,         /* reg points to struct tcp_sock */
        PTR_TO_TP_BUFFER,         /* reg points to a writable raw tp's buffer */
        PTR_TO_XDP_SOCK,         /* reg points to struct xdp_sock */
        /* PTR_TO_BTF_ID points to a kernel struct that does not need
         * to be null checked by the BPF program. This does not imply the
         * pointer is _not_ null and in practice this can easily be a null
         * pointer when reading pointer chains. The assumption is program
         * context will handle null pointer dereference typically via fault
         * handling. The verifier must keep this in mind and can make no
         * assumptions about null or non-null when doing branch analysis.
         * Further, when passed into helpers the helpers can not, without
         * additional context, assume the value is non-null.
         */
        PTR_TO_BTF_ID,
        PTR_TO_MEM,                 /* reg points to valid memory region */
        PTR_TO_ARENA,
        PTR_TO_BUF,                 /* reg points to a read/write buffer */
        PTR_TO_FUNC,                 /* reg points to a bpf program function */
        CONST_PTR_TO_DYNPTR,         /* reg points to a const struct bpf_dynptr */
        __BPF_REG_TYPE_MAX,

        /* Extended reg_types. */
        PTR_TO_MAP_VALUE_OR_NULL        = PTR_MAYBE_NULL | PTR_TO_MAP_VALUE,
        PTR_TO_SOCKET_OR_NULL                = PTR_MAYBE_NULL | PTR_TO_SOCKET,
        PTR_TO_SOCK_COMMON_OR_NULL        = PTR_MAYBE_NULL | PTR_TO_SOCK_COMMON,
        PTR_TO_TCP_SOCK_OR_NULL                = PTR_MAYBE_NULL | PTR_TO_TCP_SOCK,
        /* PTR_TO_BTF_ID_OR_NULL points to a kernel struct that has not
         * been checked for null. Used primarily to inform the verifier
         * an explicit null check is required for this struct.
         */
        PTR_TO_BTF_ID_OR_NULL                = PTR_MAYBE_NULL | PTR_TO_BTF_ID,

        /* This must be the last entry. Its purpose is to ensure the enum is
         * wide enough to hold the higher bits reserved for bpf_type_flag.
         */
        __BPF_REG_TYPE_LIMIT        = BPF_TYPE_LIMIT,
};
static_assert(__BPF_REG_TYPE_MAX <= BPF_BASE_TYPE_LIMIT);

/* The information passed from prog-specific *_is_valid_access
 * back to the verifier.
 */
struct bpf_insn_access_aux {
        enum bpf_reg_type reg_type;
        bool is_ldsx;
        union {
                int ctx_field_size;
                struct {
                        struct btf *btf;
                        u32 btf_id;
                };
        };
        struct bpf_verifier_log *log; /* for verbose logs */
        bool is_retval; /* is accessing function return value ? */
};

static inline void
bpf_ctx_record_field_size(struct bpf_insn_access_aux *aux, u32 size)
{
        aux->ctx_field_size = size;
}

static bool bpf_is_ldimm64(const struct bpf_insn *insn)
{
        return insn->code == (BPF_LD | BPF_IMM | BPF_DW);
}

static inline bool bpf_pseudo_func(const struct bpf_insn *insn)
{
        return bpf_is_ldimm64(insn) && insn->src_reg == BPF_PSEUDO_FUNC;
}

struct bpf_prog_ops {
        int (*test_run)(struct bpf_prog *prog, const union bpf_attr *kattr,
                        union bpf_attr __user *uattr);
};

struct bpf_reg_state;
struct bpf_verifier_ops {
        /* return eBPF function prototype for verification */
        const struct bpf_func_proto *
        (*get_func_proto)(enum bpf_func_id func_id,
                          const struct bpf_prog *prog);

        /* return true if 'size' wide access at offset 'off' within bpf_context
         * with 'type' (read or write) is allowed
         */
        bool (*is_valid_access)(int off, int size, enum bpf_access_type type,
                                const struct bpf_prog *prog,
                                struct bpf_insn_access_aux *info);
        int (*gen_prologue)(struct bpf_insn *insn, bool direct_write,
                            const struct bpf_prog *prog);
        int (*gen_epilogue)(struct bpf_insn *insn, const struct bpf_prog *prog,
                            s16 ctx_stack_off);
        int (*gen_ld_abs)(const struct bpf_insn *orig,
                          struct bpf_insn *insn_buf);
        u32 (*convert_ctx_access)(enum bpf_access_type type,
                                  const struct bpf_insn *src,
                                  struct bpf_insn *dst,
                                  struct bpf_prog *prog, u32 *target_size);
        int (*btf_struct_access)(struct bpf_verifier_log *log,
                                 const struct bpf_reg_state *reg,
                                 int off, int size);
};

struct bpf_prog_offload_ops {
        /* verifier basic callbacks */
        int (*insn_hook)(struct bpf_verifier_env *env,
                         int insn_idx, int prev_insn_idx);
        int (*finalize)(struct bpf_verifier_env *env);
        /* verifier optimization callbacks (called after .finalize) */
        int (*replace_insn)(struct bpf_verifier_env *env, u32 off,
                            struct bpf_insn *insn);
        int (*remove_insns)(struct bpf_verifier_env *env, u32 off, u32 cnt);
        /* program management callbacks */
        int (*prepare)(struct bpf_prog *prog);
        int (*translate)(struct bpf_prog *prog);
        void (*destroy)(struct bpf_prog *prog);
};

struct bpf_prog_offload {
        struct bpf_prog                *prog;
        struct net_device        *netdev;
        struct bpf_offload_dev        *offdev;
        void                        *dev_priv;
        struct list_head        offloads;
        bool                        dev_state;
        bool                        opt_failed;
        void                        *jited_image;
        u32                        jited_len;
};

enum bpf_cgroup_storage_type {
        BPF_CGROUP_STORAGE_SHARED,
        BPF_CGROUP_STORAGE_PERCPU,
        __BPF_CGROUP_STORAGE_MAX
};

#define MAX_BPF_CGROUP_STORAGE_TYPE __BPF_CGROUP_STORAGE_MAX

/* The longest tracepoint has 12 args.
 * See include/trace/bpf_probe.h
 */
#define MAX_BPF_FUNC_ARGS 12

/* The maximum number of arguments passed through registers
 * a single function may have.
 */
#define MAX_BPF_FUNC_REG_ARGS 5

/* The argument is a structure. */
#define BTF_FMODEL_STRUCT_ARG                BIT(0)

/* The argument is signed. */
#define BTF_FMODEL_SIGNED_ARG                BIT(1)

struct btf_func_model {
        u8 ret_size;
        u8 ret_flags;
        u8 nr_args;
        u8 arg_size[MAX_BPF_FUNC_ARGS];
        u8 arg_flags[MAX_BPF_FUNC_ARGS];
};

/* Restore arguments before returning from trampoline to let original function
 * continue executing. This flag is used for fentry progs when there are no
 * fexit progs.
 */
#define BPF_TRAMP_F_RESTORE_REGS        BIT(0)
/* Call original function after fentry progs, but before fexit progs.
 * Makes sense for fentry/fexit, normal calls and indirect calls.
 */
#define BPF_TRAMP_F_CALL_ORIG                BIT(1)
/* Skip current frame and return to parent.  Makes sense for fentry/fexit
 * programs only. Should not be used with normal calls and indirect calls.
 */
#define BPF_TRAMP_F_SKIP_FRAME                BIT(2)
/* Store IP address of the caller on the trampoline stack,
 * so it's available for trampoline's programs.
 */
#define BPF_TRAMP_F_IP_ARG                BIT(3)
/* Return the return value of fentry prog. Only used by bpf_struct_ops. */
#define BPF_TRAMP_F_RET_FENTRY_RET        BIT(4)

/* Get original function from stack instead of from provided direct address.
 * Makes sense for trampolines with fexit or fmod_ret programs.
 */
#define BPF_TRAMP_F_ORIG_STACK                BIT(5)

/* This trampoline is on a function with another ftrace_ops with IPMODIFY,
 * e.g., a live patch. This flag is set and cleared by ftrace call backs,
 */
#define BPF_TRAMP_F_SHARE_IPMODIFY        BIT(6)

/* Indicate that current trampoline is in a tail call context. Then, it has to
 * cache and restore tail_call_cnt to avoid infinite tail call loop.
 */
#define BPF_TRAMP_F_TAIL_CALL_CTX        BIT(7)

/*
 * Indicate the trampoline should be suitable to receive indirect calls;
 * without this indirectly calling the generated code can result in #UD/#CP,
 * depending on the CFI options.
 *
 * Used by bpf_struct_ops.
 *
 * Incompatible with FENTRY usage, overloads @func_addr argument.
 */
#define BPF_TRAMP_F_INDIRECT                BIT(8)

/* Each call __bpf_prog_enter + call bpf_func + call __bpf_prog_exit is ~50
 * bytes on x86.
 */
enum {
#if defined(__s390x__)
        BPF_MAX_TRAMP_LINKS = 27,
#else
        BPF_MAX_TRAMP_LINKS = 38,
#endif
};

struct bpf_tramp_links {
        struct bpf_tramp_link *links[BPF_MAX_TRAMP_LINKS];
        int nr_links;
};

struct bpf_tramp_run_ctx;

/* Different use cases for BPF trampoline:
 * 1. replace nop at the function entry (kprobe equivalent)
 *    flags = BPF_TRAMP_F_RESTORE_REGS
 *    fentry = a set of programs to run before returning from trampoline
 *
 * 2. replace nop at the function entry (kprobe + kretprobe equivalent)
 *    flags = BPF_TRAMP_F_CALL_ORIG | BPF_TRAMP_F_SKIP_FRAME
 *    orig_call = fentry_ip + MCOUNT_INSN_SIZE
 *    fentry = a set of program to run before calling original function
 *    fexit = a set of program to run after original function
 *
 * 3. replace direct call instruction anywhere in the function body
 *    or assign a function pointer for indirect call (like tcp_congestion_ops->cong_avoid)
 *    With flags = 0
 *      fentry = a set of programs to run before returning from trampoline
 *    With flags = BPF_TRAMP_F_CALL_ORIG
 *      orig_call = original callback addr or direct function addr
 *      fentry = a set of program to run before calling original function
 *      fexit = a set of program to run after original function
 */
struct bpf_tramp_image;
int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *image_end,
                                const struct btf_func_model *m, u32 flags,
                                struct bpf_tramp_links *tlinks,
                                void *func_addr);
void *arch_alloc_bpf_trampoline(unsigned int size);
void arch_free_bpf_trampoline(void *image, unsigned int size);
int __must_check arch_protect_bpf_trampoline(void *image, unsigned int size);
int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags,
                             struct bpf_tramp_links *tlinks, void *func_addr);

u64 notrace __bpf_prog_enter_sleepable_recur(struct bpf_prog *prog,
                                             struct bpf_tramp_run_ctx *run_ctx);
void notrace __bpf_prog_exit_sleepable_recur(struct bpf_prog *prog, u64 start,
                                             struct bpf_tramp_run_ctx *run_ctx);
void notrace __bpf_tramp_enter(struct bpf_tramp_image *tr);
void notrace __bpf_tramp_exit(struct bpf_tramp_image *tr);
typedef u64 (*bpf_trampoline_enter_t)(struct bpf_prog *prog,
                                      struct bpf_tramp_run_ctx *run_ctx);
typedef void (*bpf_trampoline_exit_t)(struct bpf_prog *prog, u64 start,
                                      struct bpf_tramp_run_ctx *run_ctx);
bpf_trampoline_enter_t bpf_trampoline_enter(const struct bpf_prog *prog);
bpf_trampoline_exit_t bpf_trampoline_exit(const struct bpf_prog *prog);

struct bpf_ksym {
        unsigned long                 start;
        unsigned long                 end;
        char                         name[KSYM_NAME_LEN];
        struct list_head         lnode;
        struct latch_tree_node         tnode;
        bool                         prog;
};

enum bpf_tramp_prog_type {
        BPF_TRAMP_FENTRY,
        BPF_TRAMP_FEXIT,
        BPF_TRAMP_MODIFY_RETURN,
        BPF_TRAMP_MAX,
        BPF_TRAMP_REPLACE, /* more than MAX */
};

struct bpf_tramp_image {
        void *image;
        int size;
        struct bpf_ksym ksym;
        struct percpu_ref pcref;
        void *ip_after_call;
        void *ip_epilogue;
        union {
                struct rcu_head rcu;
                struct work_struct work;
        };
};

struct bpf_trampoline {
        /* hlist for trampoline_table */
        struct hlist_node hlist;
        struct ftrace_ops *fops;
        /* serializes access to fields of this trampoline */
        struct mutex mutex;
        refcount_t refcnt;
        u32 flags;
        u64 key;
        struct {
                struct btf_func_model model;
                void *addr;
                bool ftrace_managed;
        } func;
        /* if !NULL this is BPF_PROG_TYPE_EXT program that extends another BPF
         * program by replacing one of its functions. func.addr is the address
         * of the function it replaced.
         */
        struct bpf_prog *extension_prog;
        /* list of BPF programs using this trampoline */
        struct hlist_head progs_hlist[BPF_TRAMP_MAX];
        /* Number of attached programs. A counter per kind. */
        int progs_cnt[BPF_TRAMP_MAX];
        /* Executable image of trampoline */
        struct bpf_tramp_image *cur_image;
};

struct bpf_attach_target_info {
        struct btf_func_model fmodel;
        long tgt_addr;
        struct module *tgt_mod;
        const char *tgt_name;
        const struct btf_type *tgt_type;
};

#define BPF_DISPATCHER_MAX 48 /* Fits in 2048B */

struct bpf_dispatcher_prog {
        struct bpf_prog *prog;
        refcount_t users;
};

struct bpf_dispatcher {
        /* dispatcher mutex */
        struct mutex mutex;
        void *func;
        struct bpf_dispatcher_prog progs[BPF_DISPATCHER_MAX];
        int num_progs;
        void *image;
        void *rw_image;
        u32 image_off;
        struct bpf_ksym ksym;
#ifdef CONFIG_HAVE_STATIC_CALL
        struct static_call_key *sc_key;
        void *sc_tramp;
#endif
};

#ifndef __bpfcall
#define __bpfcall __nocfi
#endif

static __always_inline __bpfcall unsigned int bpf_dispatcher_nop_func(
        const void *ctx,
        const struct bpf_insn *insnsi,
        bpf_func_t bpf_func)
{
        return bpf_func(ctx, insnsi);
}

/* the implementation of the opaque uapi struct bpf_dynptr */
struct bpf_dynptr_kern {
        void *data;
        /* Size represents the number of usable bytes of dynptr data.
         * If for example the offset is at 4 for a local dynptr whose data is
         * of type u64, the number of usable bytes is 4.
         *
         * The upper 8 bits are reserved. It is as follows:
         * Bits 0 - 23 = size
         * Bits 24 - 30 = dynptr type
         * Bit 31 = whether dynptr is read-only
         */
        u32 size;
        u32 offset;
} __aligned(8);

enum bpf_dynptr_type {
        BPF_DYNPTR_TYPE_INVALID,
        /* Points to memory that is local to the bpf program */
        BPF_DYNPTR_TYPE_LOCAL,
        /* Underlying data is a ringbuf record */
        BPF_DYNPTR_TYPE_RINGBUF,
        /* Underlying data is a sk_buff */
        BPF_DYNPTR_TYPE_SKB,
        /* Underlying data is a xdp_buff */
        BPF_DYNPTR_TYPE_XDP,
};

int bpf_dynptr_check_size(u32 size);
u32 __bpf_dynptr_size(const struct bpf_dynptr_kern *ptr);
const void *__bpf_dynptr_data(const struct bpf_dynptr_kern *ptr, u32 len);
void *__bpf_dynptr_data_rw(const struct bpf_dynptr_kern *ptr, u32 len);
bool __bpf_dynptr_is_rdonly(const struct bpf_dynptr_kern *ptr);

#ifdef CONFIG_BPF_JIT
int bpf_trampoline_link_prog(struct bpf_tramp_link *link,
                             struct bpf_trampoline *tr,
                             struct bpf_prog *tgt_prog);
int bpf_trampoline_unlink_prog(struct bpf_tramp_link *link,
                               struct bpf_trampoline *tr,
                               struct bpf_prog *tgt_prog);
struct bpf_trampoline *bpf_trampoline_get(u64 key,
                                          struct bpf_attach_target_info *tgt_info);
void bpf_trampoline_put(struct bpf_trampoline *tr);
int arch_prepare_bpf_dispatcher(void *image, void *buf, s64 *funcs, int num_funcs);

/*
 * When the architecture supports STATIC_CALL replace the bpf_dispatcher_fn
 * indirection with a direct call to the bpf program. If the architecture does
 * not have STATIC_CALL, avoid a double-indirection.
 */
#ifdef CONFIG_HAVE_STATIC_CALL

#define __BPF_DISPATCHER_SC_INIT(_name)                                \
        .sc_key = &STATIC_CALL_KEY(_name),                        \
        .sc_tramp = STATIC_CALL_TRAMP_ADDR(_name),

#define __BPF_DISPATCHER_SC(name)                                \
        DEFINE_STATIC_CALL(bpf_dispatcher_##name##_call, bpf_dispatcher_nop_func)

#define __BPF_DISPATCHER_CALL(name)                                \
        static_call(bpf_dispatcher_##name##_call)(ctx, insnsi, bpf_func)

#define __BPF_DISPATCHER_UPDATE(_d, _new)                        \
        __static_call_update((_d)->sc_key, (_d)->sc_tramp, (_new))

#else
#define __BPF_DISPATCHER_SC_INIT(name)
#define __BPF_DISPATCHER_SC(name)
#define __BPF_DISPATCHER_CALL(name)                bpf_func(ctx, insnsi)
#define __BPF_DISPATCHER_UPDATE(_d, _new)
#endif

#define BPF_DISPATCHER_INIT(_name) {                                \
        .mutex = __MUTEX_INITIALIZER(_name.mutex),                \
        .func = &_name##_func,                                        \
        .progs = {},                                                \
        .num_progs = 0,                                                \
        .image = NULL,                                                \
        .image_off = 0,                                                \
        .ksym = {                                                \
                .name  = #_name,                                \
                .lnode = LIST_HEAD_INIT(_name.ksym.lnode),        \
        },                                                        \
        __BPF_DISPATCHER_SC_INIT(_name##_call)                        \
}

#define DEFINE_BPF_DISPATCHER(name)                                        \
        __BPF_DISPATCHER_SC(name);                                        \
        noinline __bpfcall unsigned int bpf_dispatcher_##name##_func(        \
                const void *ctx,                                        \
                const struct bpf_insn *insnsi,                                \
                bpf_func_t bpf_func)                                        \
        {                                                                \
                return __BPF_DISPATCHER_CALL(name);                        \
        }                                                                \
        EXPORT_SYMBOL(bpf_dispatcher_##name##_func);                        \
        struct bpf_dispatcher bpf_dispatcher_##name =                        \
                BPF_DISPATCHER_INIT(bpf_dispatcher_##name);

#define DECLARE_BPF_DISPATCHER(name)                                        \
        unsigned int bpf_dispatcher_##name##_func(                        \
                const void *ctx,                                        \
                const struct bpf_insn *insnsi,                                \
                bpf_func_t bpf_func);                                        \
        extern struct bpf_dispatcher bpf_dispatcher_##name;

#define BPF_DISPATCHER_FUNC(name) bpf_dispatcher_##name##_func
#define BPF_DISPATCHER_PTR(name) (&bpf_dispatcher_##name)
void bpf_dispatcher_change_prog(struct bpf_dispatcher *d, struct bpf_prog *from,
                                struct bpf_prog *to);
/* Called only from JIT-enabled code, so there's no need for stubs. */
void bpf_image_ksym_init(void *data, unsigned int size, struct bpf_ksym *ksym);
void bpf_image_ksym_add(struct bpf_ksym *ksym);
void bpf_image_ksym_del(struct bpf_ksym *ksym);
void bpf_ksym_add(struct bpf_ksym *ksym);
void bpf_ksym_del(struct bpf_ksym *ksym);
int bpf_jit_charge_modmem(u32 size);
void bpf_jit_uncharge_modmem(u32 size);
bool bpf_prog_has_trampoline(const struct bpf_prog *prog);
#else
static inline int bpf_trampoline_link_prog(struct bpf_tramp_link *link,
                                           struct bpf_trampoline *tr,
                                           struct bpf_prog *tgt_prog)
{
        return -ENOTSUPP;
}
static inline int bpf_trampoline_unlink_prog(struct bpf_tramp_link *link,
                                             struct bpf_trampoline *tr,
                                             struct bpf_prog *tgt_prog)
{
        return -ENOTSUPP;
}
static inline struct bpf_trampoline *bpf_trampoline_get(u64 key,
                                                        struct bpf_attach_target_info *tgt_info)
{
        return NULL;
}
static inline void bpf_trampoline_put(struct bpf_trampoline *tr) {}
#define DEFINE_BPF_DISPATCHER(name)
#define DECLARE_BPF_DISPATCHER(name)
#define BPF_DISPATCHER_FUNC(name) bpf_dispatcher_nop_func
#define BPF_DISPATCHER_PTR(name) NULL
static inline void bpf_dispatcher_change_prog(struct bpf_dispatcher *d,
                                              struct bpf_prog *from,
                                              struct bpf_prog *to) {}
static inline bool is_bpf_image_address(unsigned long address)
{
        return false;
}
static inline bool bpf_prog_has_trampoline(const struct bpf_prog *prog)
{
        return false;
}
#endif

struct bpf_func_info_aux {
        u16 linkage;
        bool unreliable;
        bool called : 1;
        bool verified : 1;
};

enum bpf_jit_poke_reason {
        BPF_POKE_REASON_TAIL_CALL,
};

/* Descriptor of pokes pointing /into/ the JITed image. */
struct bpf_jit_poke_descriptor {
        void *tailcall_target;
        void *tailcall_bypass;
        void *bypass_addr;
        void *aux;
        union {
                struct {
                        struct bpf_map *map;
                        u32 key;
                } tail_call;
        };
        bool tailcall_target_stable;
        u8 adj_off;
        u16 reason;
        u32 insn_idx;
};

/* reg_type info for ctx arguments */
struct bpf_ctx_arg_aux {
        u32 offset;
        enum bpf_reg_type reg_type;
        struct btf *btf;
        u32 btf_id;
};

struct btf_mod_pair {
        struct btf *btf;
        struct module *module;
};

struct bpf_kfunc_desc_tab;

struct bpf_prog_aux {
        atomic64_t refcnt;
        u32 used_map_cnt;
        u32 used_btf_cnt;
        u32 max_ctx_offset;
        u32 max_pkt_offset;
        u32 max_tp_access;
        u32 stack_depth;
        u32 id;
        u32 func_cnt; /* used by non-func prog as the number of func progs */
        u32 real_func_cnt; /* includes hidden progs, only used for JIT and freeing progs */
        u32 func_idx; /* 0 for non-func prog, the index in func array for func prog */
        u32 attach_btf_id; /* in-kernel BTF type id to attach to */
        u32 ctx_arg_info_size;
        u32 max_rdonly_access;
        u32 max_rdwr_access;
        struct btf *attach_btf;
        const struct bpf_ctx_arg_aux *ctx_arg_info;
        void __percpu *priv_stack_ptr;
        struct mutex dst_mutex; /* protects dst_* pointers below, *after* prog becomes visible */
        struct bpf_prog *dst_prog;
        struct bpf_trampoline *dst_trampoline;
        enum bpf_prog_type saved_dst_prog_type;
        enum bpf_attach_type saved_dst_attach_type;
        bool verifier_zext; /* Zero extensions has been inserted by verifier. */
        bool dev_bound; /* Program is bound to the netdev. */
        bool offload_requested; /* Program is bound and offloaded to the netdev. */
        bool attach_btf_trace; /* true if attaching to BTF-enabled raw tp */
        bool attach_tracing_prog; /* true if tracing another tracing program */
        bool func_proto_unreliable;
        bool tail_call_reachable;
        bool xdp_has_frags;
        bool exception_cb;
        bool exception_boundary;
        bool is_extended; /* true if extended by freplace program */
        bool jits_use_priv_stack;
        bool priv_stack_requested;
        u64 prog_array_member_cnt; /* counts how many times as member of prog_array */
        struct mutex ext_mutex; /* mutex for is_extended and prog_array_member_cnt */
        struct bpf_arena *arena;
        void (*recursion_detected)(struct bpf_prog *prog); /* callback if recursion is detected */
        /* BTF_KIND_FUNC_PROTO for valid attach_btf_id */
        const struct btf_type *attach_func_proto;
        /* function name for valid attach_btf_id */
        const char *attach_func_name;
        struct bpf_prog **func;
        void *jit_data; /* JIT specific data. arch dependent */
        struct bpf_jit_poke_descriptor *poke_tab;
        struct bpf_kfunc_desc_tab *kfunc_tab;
        struct bpf_kfunc_btf_tab *kfunc_btf_tab;
        u32 size_poke_tab;
#ifdef CONFIG_FINEIBT
        struct bpf_ksym ksym_prefix;
#endif
        struct bpf_ksym ksym;
        const struct bpf_prog_ops *ops;
        struct bpf_map **used_maps;
        struct mutex used_maps_mutex; /* mutex for used_maps and used_map_cnt */
        struct btf_mod_pair *used_btfs;
        struct bpf_prog *prog;
        struct user_struct *user;
        u64 load_time; /* ns since boottime */
        u32 verified_insns;
        int cgroup_atype; /* enum cgroup_bpf_attach_type */
        struct bpf_map *cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE];
        char name[BPF_OBJ_NAME_LEN];
        u64 (*bpf_exception_cb)(u64 cookie, u64 sp, u64 bp, u64, u64);
#ifdef CONFIG_SECURITY
        void *security;
#endif
        struct bpf_token *token;
        struct bpf_prog_offload *offload;
        struct btf *btf;
        struct bpf_func_info *func_info;
        struct bpf_func_info_aux *func_info_aux;
        /* bpf_line_info loaded from userspace.  linfo->insn_off
         * has the xlated insn offset.
         * Both the main and sub prog share the same linfo.
         * The subprog can access its first linfo by
         * using the linfo_idx.
         */
        struct bpf_line_info *linfo;
        /* jited_linfo is the jited addr of the linfo.  It has a
         * one to one mapping to linfo:
         * jited_linfo[i] is the jited addr for the linfo[i]->insn_off.
         * Both the main and sub prog share the same jited_linfo.
         * The subprog can access its first jited_linfo by
         * using the linfo_idx.
         */
        void **jited_linfo;
        u32 func_info_cnt;
        u32 nr_linfo;
        /* subprog can use linfo_idx to access its first linfo and
         * jited_linfo.
         * main prog always has linfo_idx == 0
         */
        u32 linfo_idx;
        struct module *mod;
        u32 num_exentries;
        struct exception_table_entry *extable;
        union {
                struct work_struct work;
                struct rcu_head        rcu;
        };
};

struct bpf_prog {
        u16                        pages;                /* Number of allocated pages */
        u16                        jited:1,        /* Is our filter JIT'ed? */
                                jit_requested:1,/* archs need to JIT the prog */
                                gpl_compatible:1, /* Is filter GPL compatible? */
                                cb_access:1,        /* Is control block accessed? */
                                dst_needed:1,        /* Do we need dst entry? */
                                blinding_requested:1, /* needs constant blinding */
                                blinded:1,        /* Was blinded */
                                is_func:1,        /* program is a bpf function */
                                kprobe_override:1, /* Do we override a kprobe? */
                                has_callchain_buf:1, /* callchain buffer allocated? */
                                enforce_expected_attach_type:1, /* Enforce expected_attach_type checking at attach time */
                                call_get_stack:1, /* Do we call bpf_get_stack() or bpf_get_stackid() */
                                call_get_func_ip:1, /* Do we call get_func_ip() */
                                tstamp_type_access:1, /* Accessed __sk_buff->tstamp_type */
                                sleepable:1;        /* BPF program is sleepable */
        enum bpf_prog_type        type;                /* Type of BPF program */
        enum bpf_attach_type        expected_attach_type; /* For some prog types */
        u32                        len;                /* Number of filter blocks */
        u32                        jited_len;        /* Size of jited insns in bytes */
        u8                        tag[BPF_TAG_SIZE];
        struct bpf_prog_stats __percpu *stats;
        int __percpu                *active;
        unsigned int                (*bpf_func)(const void *ctx,
                                            const struct bpf_insn *insn);
        struct bpf_prog_aux        *aux;                /* Auxiliary fields */
        struct sock_fprog_kern        *orig_prog;        /* Original BPF program */
        /* Instructions for interpreter */
        union {
                DECLARE_FLEX_ARRAY(struct sock_filter, insns);
                DECLARE_FLEX_ARRAY(struct bpf_insn, insnsi);
        };
};

struct bpf_array_aux {
        /* Programs with direct jumps into programs part of this array. */
        struct list_head poke_progs;
        struct bpf_map *map;
        struct mutex poke_mutex;
        struct work_struct work;
};

struct bpf_link {
        atomic64_t refcnt;
        u32 id;
        enum bpf_link_type type;
        const struct bpf_link_ops *ops;
        struct bpf_prog *prog;
        /* rcu is used before freeing, work can be used to schedule that
         * RCU-based freeing before that, so they never overlap
         */
        union {
                struct rcu_head rcu;
                struct work_struct work;
        };
};

struct bpf_link_ops {
        void (*release)(struct bpf_link *link);
        /* deallocate link resources callback, called without RCU grace period
         * waiting
         */
        void (*dealloc)(struct bpf_link *link);
        /* deallocate link resources callback, called after RCU grace period;
         * if underlying BPF program is sleepable we go through tasks trace
         * RCU GP and then "classic" RCU GP
         */
        void (*dealloc_deferred)(struct bpf_link *link);
        int (*detach)(struct bpf_link *link);
        int (*update_prog)(struct bpf_link *link, struct bpf_prog *new_prog,
                           struct bpf_prog *old_prog);
        void (*show_fdinfo)(const struct bpf_link *link, struct seq_file *seq);
        int (*fill_link_info)(const struct bpf_link *link,
                              struct bpf_link_info *info);
        int (*update_map)(struct bpf_link *link, struct bpf_map *new_map,
                          struct bpf_map *old_map);
        __poll_t (*poll)(struct file *file, struct poll_table_struct *pts);
};

struct bpf_tramp_link {
        struct bpf_link link;
        struct hlist_node tramp_hlist;
        u64 cookie;
};

struct bpf_shim_tramp_link {
        struct bpf_tramp_link link;
        struct bpf_trampoline *trampoline;
};

struct bpf_tracing_link {
        struct bpf_tramp_link link;
        enum bpf_attach_type attach_type;
        struct bpf_trampoline *trampoline;
        struct bpf_prog *tgt_prog;
};

struct bpf_raw_tp_link {
        struct bpf_link link;
        struct bpf_raw_event_map *btp;
        u64 cookie;
};

struct bpf_link_primer {
        struct bpf_link *link;
        struct file *file;
        int fd;
        u32 id;
};

struct bpf_mount_opts {
        kuid_t uid;
        kgid_t gid;
        umode_t mode;

        /* BPF token-related delegation options */
        u64 delegate_cmds;
        u64 delegate_maps;
        u64 delegate_progs;
        u64 delegate_attachs;
};

struct bpf_token {
        struct work_struct work;
        atomic64_t refcnt;
        struct user_namespace *userns;
        u64 allowed_cmds;
        u64 allowed_maps;
        u64 allowed_progs;
        u64 allowed_attachs;
#ifdef CONFIG_SECURITY
        void *security;
#endif
};

struct bpf_struct_ops_value;
struct btf_member;

#define BPF_STRUCT_OPS_MAX_NR_MEMBERS 64
/**
 * struct bpf_struct_ops - A structure of callbacks allowing a subsystem to
 *                           define a BPF_MAP_TYPE_STRUCT_OPS map type composed
 *                           of BPF_PROG_TYPE_STRUCT_OPS progs.
 * @verifier_ops: A structure of callbacks that are invoked by the verifier
 *                  when determining whether the struct_ops progs in the
 *                  struct_ops map are valid.
 * @init: A callback that is invoked a single time, and before any other
 *          callback, to initialize the structure. A nonzero return value means
 *          the subsystem could not be initialized.
 * @check_member: When defined, a callback invoked by the verifier to allow
 *                  the subsystem to determine if an entry in the struct_ops map
 *                  is valid. A nonzero return value means that the map is
 *                  invalid and should be rejected by the verifier.
 * @init_member: A callback that is invoked for each member of the struct_ops
 *                 map to allow the subsystem to initialize the member. A nonzero
 *                 value means the member could not be initialized. This callback
 *                 is exclusive with the @type, @type_id, @value_type, and
 *                 @value_id fields.
 * @reg: A callback that is invoked when the struct_ops map has been
 *         initialized and is being attached to. Zero means the struct_ops map
 *         has been successfully registered and is live. A nonzero return value
 *         means the struct_ops map could not be registered.
 * @unreg: A callback that is invoked when the struct_ops map should be
 *           unregistered.
 * @update: A callback that is invoked when the live struct_ops map is being
 *            updated to contain new values. This callback is only invoked when
 *            the struct_ops map is loaded with BPF_F_LINK. If not defined, the
 *            it is assumed that the struct_ops map cannot be updated.
 * @validate: A callback that is invoked after all of the members have been
 *              initialized. This callback should perform static checks on the
 *              map, meaning that it should either fail or succeed
 *              deterministically. A struct_ops map that has been validated may
 *              not necessarily succeed in being registered if the call to @reg
 *              fails. For example, a valid struct_ops map may be loaded, but
 *              then fail to be registered due to there being another active
 *              struct_ops map on the system in the subsystem already. For this
 *              reason, if this callback is not defined, the check is skipped as
 *              the struct_ops map will have final verification performed in
 *              @reg.
 * @type: BTF type.
 * @value_type: Value type.
 * @name: The name of the struct bpf_struct_ops object.
 * @func_models: Func models
 * @type_id: BTF type id.
 * @value_id: BTF value id.
 */
struct bpf_struct_ops {
        const struct bpf_verifier_ops *verifier_ops;
        int (*init)(struct btf *btf);
        int (*check_member)(const struct btf_type *t,
                            const struct btf_member *member,
                            const struct bpf_prog *prog);
        int (*init_member)(const struct btf_type *t,
                           const struct btf_member *member,
                           void *kdata, const void *udata);
        int (*reg)(void *kdata, struct bpf_link *link);
        void (*unreg)(void *kdata, struct bpf_link *link);
        int (*update)(void *kdata, void *old_kdata, struct bpf_link *link);
        int (*validate)(void *kdata);
        void *cfi_stubs;
        struct module *owner;
        const char *name;
        struct btf_func_model func_models[BPF_STRUCT_OPS_MAX_NR_MEMBERS];
};

/* Every member of a struct_ops type has an instance even a member is not
 * an operator (function pointer). The "info" field will be assigned to
 * prog->aux->ctx_arg_info of BPF struct_ops programs to provide the
 * argument information required by the verifier to verify the program.
 *
 * btf_ctx_access() will lookup prog->aux->ctx_arg_info to find the
 * corresponding entry for an given argument.
 */
struct bpf_struct_ops_arg_info {
        struct bpf_ctx_arg_aux *info;
        u32 cnt;
};

struct bpf_struct_ops_desc {
        struct bpf_struct_ops *st_ops;

        const struct btf_type *type;
        const struct btf_type *value_type;
        u32 type_id;
        u32 value_id;

        /* Collection of argument information for each member */
        struct bpf_struct_ops_arg_info *arg_info;
};

enum bpf_struct_ops_state {
        BPF_STRUCT_OPS_STATE_INIT,
        BPF_STRUCT_OPS_STATE_INUSE,
        BPF_STRUCT_OPS_STATE_TOBEFREE,
        BPF_STRUCT_OPS_STATE_READY,
};

struct bpf_struct_ops_common_value {
        refcount_t refcnt;
        enum bpf_struct_ops_state state;
};

#if defined(CONFIG_BPF_JIT) && defined(CONFIG_BPF_SYSCALL)
/* This macro helps developer to register a struct_ops type and generate
 * type information correctly. Developers should use this macro to register
 * a struct_ops type instead of calling __register_bpf_struct_ops() directly.
 */
#define register_bpf_struct_ops(st_ops, type)                                \
        ({                                                                \
                struct bpf_struct_ops_##type {                                \
                        struct bpf_struct_ops_common_value common;        \
                        struct type data ____cacheline_aligned_in_smp;        \
                };                                                        \
                BTF_TYPE_EMIT(struct bpf_struct_ops_##type);                \
                __register_bpf_struct_ops(st_ops);                        \
        })
#define BPF_MODULE_OWNER ((void *)((0xeB9FUL << 2) + POISON_POINTER_DELTA))
bool bpf_struct_ops_get(const void *kdata);
void bpf_struct_ops_put(const void *kdata);
int bpf_struct_ops_supported(const struct bpf_struct_ops *st_ops, u32 moff);
int bpf_struct_ops_map_sys_lookup_elem(struct bpf_map *map, void *key,
                                       void *value);
int bpf_struct_ops_prepare_trampoline(struct bpf_tramp_links *tlinks,
                                      struct bpf_tramp_link *link,
                                      const struct btf_func_model *model,
                                      void *stub_func,
                                      void **image, u32 *image_off,
                                      bool allow_alloc);
void bpf_struct_ops_image_free(void *image);
static inline bool bpf_try_module_get(const void *data, struct module *owner)
{
        if (owner == BPF_MODULE_OWNER)
                return bpf_struct_ops_get(data);
        else
                return try_module_get(owner);
}
static inline void bpf_module_put(const void *data, struct module *owner)
{
        if (owner == BPF_MODULE_OWNER)
                bpf_struct_ops_put(data);
        else
                module_put(owner);
}
int bpf_struct_ops_link_create(union bpf_attr *attr);

#ifdef CONFIG_NET
/* Define it here to avoid the use of forward declaration */
struct bpf_dummy_ops_state {
        int val;
};

struct bpf_dummy_ops {
        int (*test_1)(struct bpf_dummy_ops_state *cb);
        int (*test_2)(struct bpf_dummy_ops_state *cb, int a1, unsigned short a2,
                      char a3, unsigned long a4);
        int (*test_sleepable)(struct bpf_dummy_ops_state *cb);
};

int bpf_struct_ops_test_run(struct bpf_prog *prog, const union bpf_attr *kattr,
                            union bpf_attr __user *uattr);
#endif
int bpf_struct_ops_desc_init(struct bpf_struct_ops_desc *st_ops_desc,
                             struct btf *btf,
                             struct bpf_verifier_log *log);
void bpf_map_struct_ops_info_fill(struct bpf_map_info *info, struct bpf_map *map);
void bpf_struct_ops_desc_release(struct bpf_struct_ops_desc *st_ops_desc);
#else
#define register_bpf_struct_ops(st_ops, type) ({ (void *)(st_ops); 0; })
static inline bool bpf_try_module_get(const void *data, struct module *owner)
{
        return try_module_get(owner);
}
static inline void bpf_module_put(const void *data, struct module *owner)
{
        module_put(owner);
}
static inline int bpf_struct_ops_supported(const struct bpf_struct_ops *st_ops, u32 moff)
{
        return -ENOTSUPP;
}
static inline int bpf_struct_ops_map_sys_lookup_elem(struct bpf_map *map,
                                                     void *key,
                                                     void *value)
{
        return -EINVAL;
}
static inline int bpf_struct_ops_link_create(union bpf_attr *attr)
{
        return -EOPNOTSUPP;
}
static inline void bpf_map_struct_ops_info_fill(struct bpf_map_info *info, struct bpf_map *map)
{
}

static inline void bpf_struct_ops_desc_release(struct bpf_struct_ops_desc *st_ops_desc)
{
}

#endif

#if defined(CONFIG_CGROUP_BPF) && defined(CONFIG_BPF_LSM)
int bpf_trampoline_link_cgroup_shim(struct bpf_prog *prog,
                                    int cgroup_atype);
void bpf_trampoline_unlink_cgroup_shim(struct bpf_prog *prog);
#else
static inline int bpf_trampoline_link_cgroup_shim(struct bpf_prog *prog,
                                                  int cgroup_atype)
{
        return -EOPNOTSUPP;
}
static inline void bpf_trampoline_unlink_cgroup_shim(struct bpf_prog *prog)
{
}
#endif

struct bpf_array {
        struct bpf_map map;
        u32 elem_size;
        u32 index_mask;
        struct bpf_array_aux *aux;
        union {
                DECLARE_FLEX_ARRAY(char, value) __aligned(8);
                DECLARE_FLEX_ARRAY(void *, ptrs) __aligned(8);
                DECLARE_FLEX_ARRAY(void __percpu *, pptrs) __aligned(8);
        };
};

#define BPF_COMPLEXITY_LIMIT_INSNS      1000000 /* yes. 1M insns */
#define MAX_TAIL_CALL_CNT 33

/* Maximum number of loops for bpf_loop and bpf_iter_num.
 * It's enum to expose it (and thus make it discoverable) through BTF.
 */
enum {
        BPF_MAX_LOOPS = 8 * 1024 * 1024,
};

#define BPF_F_ACCESS_MASK        (BPF_F_RDONLY |                \
                                 BPF_F_RDONLY_PROG |        \
                                 BPF_F_WRONLY |                \
                                 BPF_F_WRONLY_PROG)

#define BPF_MAP_CAN_READ        BIT(0)
#define BPF_MAP_CAN_WRITE        BIT(1)

/* Maximum number of user-producer ring buffer samples that can be drained in
 * a call to bpf_user_ringbuf_drain().
 */
#define BPF_MAX_USER_RINGBUF_SAMPLES (128 * 1024)

static inline u32 bpf_map_flags_to_cap(struct bpf_map *map)
{
        u32 access_flags = map->map_flags & (BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG);

        /* Combination of BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG is
         * not possible.
         */
        if (access_flags & BPF_F_RDONLY_PROG)
                return BPF_MAP_CAN_READ;
        else if (access_flags & BPF_F_WRONLY_PROG)
                return BPF_MAP_CAN_WRITE;
        else
                return BPF_MAP_CAN_READ | BPF_MAP_CAN_WRITE;
}

static inline bool bpf_map_flags_access_ok(u32 access_flags)
{
        return (access_flags & (BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG)) !=
               (BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG);
}

struct bpf_event_entry {
        struct perf_event *event;
        struct file *perf_file;
        struct file *map_file;
        struct rcu_head rcu;
};

static inline bool map_type_contains_progs(struct bpf_map *map)
{
        return map->map_type == BPF_MAP_TYPE_PROG_ARRAY ||
               map->map_type == BPF_MAP_TYPE_DEVMAP ||
               map->map_type == BPF_MAP_TYPE_CPUMAP;
}

bool bpf_prog_map_compatible(struct bpf_map *map, const struct bpf_prog *fp);
int bpf_prog_calc_tag(struct bpf_prog *fp);

const struct bpf_func_proto *bpf_get_trace_printk_proto(void);
const struct bpf_func_proto *bpf_get_trace_vprintk_proto(void);

typedef unsigned long (*bpf_ctx_copy_t)(void *dst, const void *src,
                                        unsigned long off, unsigned long len);
typedef u32 (*bpf_convert_ctx_access_t)(enum bpf_access_type type,
                                        const struct bpf_insn *src,
                                        struct bpf_insn *dst,
                                        struct bpf_prog *prog,
                                        u32 *target_size);

u64 bpf_event_output(struct bpf_map *map, u64 flags, void *meta, u64 meta_size,
                     void *ctx, u64 ctx_size, bpf_ctx_copy_t ctx_copy);

/* an array of programs to be executed under rcu_lock.
 *
 * Typical usage:
 * ret = bpf_prog_run_array(rcu_dereference(&bpf_prog_array), ctx, bpf_prog_run);
 *
 * the structure returned by bpf_prog_array_alloc() should be populated
 * with program pointers and the last pointer must be NULL.
 * The user has to keep refcnt on the program and make sure the program
 * is removed from the array before bpf_prog_put().
 * The 'struct bpf_prog_array *' should only be replaced with xchg()
 * since other cpus are walking the array of pointers in parallel.
 */
struct bpf_prog_array_item {
        struct bpf_prog *prog;
        union {
                struct bpf_cgroup_storage *cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE];
                u64 bpf_cookie;
        };
};

struct bpf_prog_array {
        struct rcu_head rcu;
        struct bpf_prog_array_item items[];
};

struct bpf_empty_prog_array {
        struct bpf_prog_array hdr;
        struct bpf_prog *null_prog;
};

/* to avoid allocating empty bpf_prog_array for cgroups that
 * don't have bpf program attached use one global 'bpf_empty_prog_array'
 * It will not be modified the caller of bpf_prog_array_alloc()
 * (since caller requested prog_cnt == 0)
 * that pointer should be 'freed' by bpf_prog_array_free()
 */
extern struct bpf_empty_prog_array bpf_empty_prog_array;

struct bpf_prog_array *bpf_prog_array_alloc(u32 prog_cnt, gfp_t flags);
void bpf_prog_array_free(struct bpf_prog_array *progs);
/* Use when traversal over the bpf_prog_array uses tasks_trace rcu */
void bpf_prog_array_free_sleepable(struct bpf_prog_array *progs);
int bpf_prog_array_length(struct bpf_prog_array *progs);
bool bpf_prog_array_is_empty(struct bpf_prog_array *array);
int bpf_prog_array_copy_to_user(struct bpf_prog_array *progs,
                                __u32 __user *prog_ids, u32 cnt);

void bpf_prog_array_delete_safe(struct bpf_prog_array *progs,
                                struct bpf_prog *old_prog);
int bpf_prog_array_delete_safe_at(struct bpf_prog_array *array, int index);
int bpf_prog_array_update_at(struct bpf_prog_array *array, int index,
                             struct bpf_prog *prog);
int bpf_prog_array_copy_info(struct bpf_prog_array *array,
                             u32 *prog_ids, u32 request_cnt,
                             u32 *prog_cnt);
int bpf_prog_array_copy(struct bpf_prog_array *old_array,
                        struct bpf_prog *exclude_prog,
                        struct bpf_prog *include_prog,
                        u64 bpf_cookie,
                        struct bpf_prog_array **new_array);

struct bpf_run_ctx {};

struct bpf_cg_run_ctx {
        struct bpf_run_ctx run_ctx;
        const struct bpf_prog_array_item *prog_item;
        int retval;
};

struct bpf_trace_run_ctx {
        struct bpf_run_ctx run_ctx;
        u64 bpf_cookie;
        bool is_uprobe;
};

struct bpf_tramp_run_ctx {
        struct bpf_run_ctx run_ctx;
        u64 bpf_cookie;
        struct bpf_run_ctx *saved_run_ctx;
};

static inline struct bpf_run_ctx *bpf_set_run_ctx(struct bpf_run_ctx *new_ctx)
{
        struct bpf_run_ctx *old_ctx = NULL;

#ifdef CONFIG_BPF_SYSCALL
        old_ctx = current->bpf_ctx;
        current->bpf_ctx = new_ctx;
#endif
        return old_ctx;
}

static inline void bpf_reset_run_ctx(struct bpf_run_ctx *old_ctx)
{
#ifdef CONFIG_BPF_SYSCALL
        current->bpf_ctx = old_ctx;
#endif
}

/* BPF program asks to bypass CAP_NET_BIND_SERVICE in bind. */
#define BPF_RET_BIND_NO_CAP_NET_BIND_SERVICE                        (1 << 0)
/* BPF program asks to set CN on the packet. */
#define BPF_RET_SET_CN                                                (1 << 0)

typedef u32 (*bpf_prog_run_fn)(const struct bpf_prog *prog, const void *ctx);

static __always_inline u32
bpf_prog_run_array(const struct bpf_prog_array *array,
                   const void *ctx, bpf_prog_run_fn run_prog)
{
        const struct bpf_prog_array_item *item;
        const struct bpf_prog *prog;
        struct bpf_run_ctx *old_run_ctx;
        struct bpf_trace_run_ctx run_ctx;
        u32 ret = 1;

        RCU_LOCKDEP_WARN(!rcu_read_lock_held(), "no rcu lock held");

        if (unlikely(!array))
                return ret;

        run_ctx.is_uprobe = false;

        migrate_disable();
        old_run_ctx = bpf_set_run_ctx(&run_ctx.run_ctx);
        item = &array->items[0];
        while ((prog = READ_ONCE(item->prog))) {
                run_ctx.bpf_cookie = item->bpf_cookie;
                ret &= run_prog(prog, ctx);
                item++;
        }
        bpf_reset_run_ctx(old_run_ctx);
        migrate_enable();
        return ret;
}

/* Notes on RCU design for bpf_prog_arrays containing sleepable programs:
 *
 * We use the tasks_trace rcu flavor read section to protect the bpf_prog_array
 * overall. As a result, we must use the bpf_prog_array_free_sleepable
 * in order to use the tasks_trace rcu grace period.
 *
 * When a non-sleepable program is inside the array, we take the rcu read
 * section and disable preemption for that program alone, so it can access
 * rcu-protected dynamically sized maps.
 */
static __always_inline u32
bpf_prog_run_array_uprobe(const struct bpf_prog_array __rcu *array_rcu,
                          const void *ctx, bpf_prog_run_fn run_prog)
{
        const struct bpf_prog_array_item *item;
        const struct bpf_prog *prog;
        const struct bpf_prog_array *array;
        struct bpf_run_ctx *old_run_ctx;
        struct bpf_trace_run_ctx run_ctx;
        u32 ret = 1;

        might_fault();

        rcu_read_lock_trace();
        migrate_disable();

        run_ctx.is_uprobe = true;

        array = rcu_dereference_check(array_rcu, rcu_read_lock_trace_held());
        if (unlikely(!array))
                goto out;
        old_run_ctx = bpf_set_run_ctx(&run_ctx.run_ctx);
        item = &array->items[0];
        while ((prog = READ_ONCE(item->prog))) {
                if (!prog->sleepable)
                        rcu_read_lock();

                run_ctx.bpf_cookie = item->bpf_cookie;
                ret &= run_prog(prog, ctx);
                item++;

                if (!prog->sleepable)
                        rcu_read_unlock();
        }
        bpf_reset_run_ctx(old_run_ctx);
out:
        migrate_enable();
        rcu_read_unlock_trace();
        return ret;
}

#ifdef CONFIG_BPF_SYSCALL
DECLARE_PER_CPU(int, bpf_prog_active);
extern struct mutex bpf_stats_enabled_mutex;

/*
 * Block execution of BPF programs attached to instrumentation (perf,
 * kprobes, tracepoints) to prevent deadlocks on map operations as any of
 * these events can happen inside a region which holds a map bucket lock
 * and can deadlock on it.
 */
static inline void bpf_disable_instrumentation(void)
{
        migrate_disable();
        this_cpu_inc(bpf_prog_active);
}

static inline void bpf_enable_instrumentation(void)
{
        this_cpu_dec(bpf_prog_active);
        migrate_enable();
}

extern const struct super_operations bpf_super_ops;
extern const struct file_operations bpf_map_fops;
extern const struct file_operations bpf_prog_fops;
extern const struct file_operations bpf_iter_fops;

#define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type) \
        extern const struct bpf_prog_ops _name ## _prog_ops; \
        extern const struct bpf_verifier_ops _name ## _verifier_ops;
#define BPF_MAP_TYPE(_id, _ops) \
        extern const struct bpf_map_ops _ops;
#define BPF_LINK_TYPE(_id, _name)
#include <linux/bpf_types.h>
#undef BPF_PROG_TYPE
#undef BPF_MAP_TYPE
#undef BPF_LINK_TYPE

extern const struct bpf_prog_ops bpf_offload_prog_ops;
extern const struct bpf_verifier_ops tc_cls_act_analyzer_ops;
extern const struct bpf_verifier_ops xdp_analyzer_ops;

struct bpf_prog *bpf_prog_get(u32 ufd);
struct bpf_prog *bpf_prog_get_type_dev(u32 ufd, enum bpf_prog_type type,
                                       bool attach_drv);
void bpf_prog_add(struct bpf_prog *prog, int i);
void bpf_prog_sub(struct bpf_prog *prog, int i);
void bpf_prog_inc(struct bpf_prog *prog);
struct bpf_prog * __must_check bpf_prog_inc_not_zero(struct bpf_prog *prog);
void bpf_prog_put(struct bpf_prog *prog);

void bpf_prog_free_id(struct bpf_prog *prog);
void bpf_map_free_id(struct bpf_map *map);

struct btf_field *btf_record_find(const struct btf_record *rec,
                                  u32 offset, u32 field_mask);
void btf_record_free(struct btf_record *rec);
void bpf_map_free_record(struct bpf_map *map);
struct btf_record *btf_record_dup(const struct btf_record *rec);
bool btf_record_equal(const struct btf_record *rec_a, const struct btf_record *rec_b);
void bpf_obj_free_timer(const struct btf_record *rec, void *obj);
void bpf_obj_free_workqueue(const struct btf_record *rec, void *obj);
void bpf_obj_free_fields(const struct btf_record *rec, void *obj);
void __bpf_obj_drop_impl(void *p, const struct btf_record *rec, bool percpu);

struct bpf_map *bpf_map_get(u32 ufd);
struct bpf_map *bpf_map_get_with_uref(u32 ufd);

static inline struct bpf_map *__bpf_map_get(struct fd f)
{
        if (fd_empty(f))
                return ERR_PTR(-EBADF);
        if (unlikely(fd_file(f)->f_op != &bpf_map_fops))
                return ERR_PTR(-EINVAL);
        return fd_file(f)->private_data;
}

void bpf_map_inc(struct bpf_map *map);
void bpf_map_inc_with_uref(struct bpf_map *map);
struct bpf_map *__bpf_map_inc_not_zero(struct bpf_map *map, bool uref);
struct bpf_map * __must_check bpf_map_inc_not_zero(struct bpf_map *map);
void bpf_map_put_with_uref(struct bpf_map *map);
void bpf_map_put(struct bpf_map *map);
void *bpf_map_area_alloc(u64 size, int numa_node);
void *bpf_map_area_mmapable_alloc(u64 size, int numa_node);
void bpf_map_area_free(void *base);
bool bpf_map_write_active(const struct bpf_map *map);
void bpf_map_init_from_attr(struct bpf_map *map, union bpf_attr *attr);
int  generic_map_lookup_batch(struct bpf_map *map,
                              const union bpf_attr *attr,
                              union bpf_attr __user *uattr);
int  generic_map_update_batch(struct bpf_map *map, struct file *map_file,
                              const union bpf_attr *attr,
                              union bpf_attr __user *uattr);
int  generic_map_delete_batch(struct bpf_map *map,
                              const union bpf_attr *attr,
                              union bpf_attr __user *uattr);
struct bpf_map *bpf_map_get_curr_or_next(u32 *id);
struct bpf_prog *bpf_prog_get_curr_or_next(u32 *id);

int bpf_map_alloc_pages(const struct bpf_map *map, gfp_t gfp, int nid,
                        unsigned long nr_pages, struct page **page_array);
#ifdef CONFIG_MEMCG
void *bpf_map_kmalloc_node(const struct bpf_map *map, size_t size, gfp_t flags,
                           int node);
void *bpf_map_kzalloc(const struct bpf_map *map, size_t size, gfp_t flags);
void *bpf_map_kvcalloc(struct bpf_map *map, size_t n, size_t size,
                       gfp_t flags);
void __percpu *bpf_map_alloc_percpu(const struct bpf_map *map, size_t size,
                                    size_t align, gfp_t flags);
#else
/*
 * These specialized allocators have to be macros for their allocations to be
 * accounted separately (to have separate alloc_tag).
 */
#define bpf_map_kmalloc_node(_map, _size, _flags, _node)        \
                kmalloc_node(_size, _flags, _node)
#define bpf_map_kzalloc(_map, _size, _flags)                        \
                kzalloc(_size, _flags)
#define bpf_map_kvcalloc(_map, _n, _size, _flags)                \
                kvcalloc(_n, _size, _flags)
#define bpf_map_alloc_percpu(_map, _size, _align, _flags)        \
                __alloc_percpu_gfp(_size, _align, _flags)
#endif

static inline int
bpf_map_init_elem_count(struct bpf_map *map)
{
        size_t size = sizeof(*map->elem_count), align = size;
        gfp_t flags = GFP_USER | __GFP_NOWARN;

        map->elem_count = bpf_map_alloc_percpu(map, size, align, flags);
        if (!map->elem_count)
                return -ENOMEM;

        return 0;
}

static inline void
bpf_map_free_elem_count(struct bpf_map *map)
{
        free_percpu(map->elem_count);
}

static inline void bpf_map_inc_elem_count(struct bpf_map *map)
{
        this_cpu_inc(*map->elem_count);
}

static inline void bpf_map_dec_elem_count(struct bpf_map *map)
{
        this_cpu_dec(*map->elem_count);
}

extern int sysctl_unprivileged_bpf_disabled;

bool bpf_token_capable(const struct bpf_token *token, int cap);

static inline bool bpf_allow_ptr_leaks(const struct bpf_token *token)
{
        return bpf_token_capable(token, CAP_PERFMON);
}

static inline bool bpf_allow_uninit_stack(const struct bpf_token *token)
{
        return bpf_token_capable(token, CAP_PERFMON);
}

static inline bool bpf_bypass_spec_v1(const struct bpf_token *token)
{
        return cpu_mitigations_off() || bpf_token_capable(token, CAP_PERFMON);
}

static inline bool bpf_bypass_spec_v4(const struct bpf_token *token)
{
        return cpu_mitigations_off() || bpf_token_capable(token, CAP_PERFMON);
}

int bpf_map_new_fd(struct bpf_map *map, int flags);
int bpf_prog_new_fd(struct bpf_prog *prog);

void bpf_link_init(struct bpf_link *link, enum bpf_link_type type,
                   const struct bpf_link_ops *ops, struct bpf_prog *prog);
int bpf_link_prime(struct bpf_link *link, struct bpf_link_primer *primer);
int bpf_link_settle(struct bpf_link_primer *primer);
void bpf_link_cleanup(struct bpf_link_primer *primer);
void bpf_link_inc(struct bpf_link *link);
struct bpf_link *bpf_link_inc_not_zero(struct bpf_link *link);
void bpf_link_put(struct bpf_link *link);
int bpf_link_new_fd(struct bpf_link *link);
struct bpf_link *bpf_link_get_from_fd(u32 ufd);
struct bpf_link *bpf_link_get_curr_or_next(u32 *id);

void bpf_token_inc(struct bpf_token *token);
void bpf_token_put(struct bpf_token *token);
int bpf_token_create(union bpf_attr *attr);
struct bpf_token *bpf_token_get_from_fd(u32 ufd);

bool bpf_token_allow_cmd(const struct bpf_token *token, enum bpf_cmd cmd);
bool bpf_token_allow_map_type(const struct bpf_token *token, enum bpf_map_type type);
bool bpf_token_allow_prog_type(const struct bpf_token *token,
                               enum bpf_prog_type prog_type,
                               enum bpf_attach_type attach_type);

int bpf_obj_pin_user(u32 ufd, int path_fd, const char __user *pathname);
int bpf_obj_get_user(int path_fd, const char __user *pathname, int flags);
struct inode *bpf_get_inode(struct super_block *sb, const struct inode *dir,
                            umode_t mode);

#define BPF_ITER_FUNC_PREFIX "bpf_iter_"
#define DEFINE_BPF_ITER_FUNC(target, args...)                        \
        extern int bpf_iter_ ## target(args);                        \
        int __init bpf_iter_ ## target(args) { return 0; }

/*
 * The task type of iterators.
 *
 * For BPF task iterators, they can be parameterized with various
 * parameters to visit only some of tasks.
 *
 * BPF_TASK_ITER_ALL (default)
 *        Iterate over resources of every task.
 *
 * BPF_TASK_ITER_TID
 *        Iterate over resources of a task/tid.
 *
 * BPF_TASK_ITER_TGID
 *        Iterate over resources of every task of a process / task group.
 */
enum bpf_iter_task_type {
        BPF_TASK_ITER_ALL = 0,
        BPF_TASK_ITER_TID,
        BPF_TASK_ITER_TGID,
};

struct bpf_iter_aux_info {
        /* for map_elem iter */
        struct bpf_map *map;

        /* for cgroup iter */
        struct {
                struct cgroup *start; /* starting cgroup */
                enum bpf_cgroup_iter_order order;
        } cgroup;
        struct {
                enum bpf_iter_task_type        type;
                u32 pid;
        } task;
};

typedef int (*bpf_iter_attach_target_t)(struct bpf_prog *prog,
                                        union bpf_iter_link_info *linfo,
                                        struct bpf_iter_aux_info *aux);
typedef void (*bpf_iter_detach_target_t)(struct bpf_iter_aux_info *aux);
typedef void (*bpf_iter_show_fdinfo_t) (const struct bpf_iter_aux_info *aux,
                                        struct seq_file *seq);
typedef int (*bpf_iter_fill_link_info_t)(const struct bpf_iter_aux_info *aux,
                                         struct bpf_link_info *info);
typedef const struct bpf_func_proto *
(*bpf_iter_get_func_proto_t)(enum bpf_func_id func_id,
                             const struct bpf_prog *prog);

enum bpf_iter_feature {
        BPF_ITER_RESCHED        = BIT(0),
};

#define BPF_ITER_CTX_ARG_MAX 2
struct bpf_iter_reg {
        const char *target;
        bpf_iter_attach_target_t attach_target;
        bpf_iter_detach_target_t detach_target;
        bpf_iter_show_fdinfo_t show_fdinfo;
        bpf_iter_fill_link_info_t fill_link_info;
        bpf_iter_get_func_proto_t get_func_proto;
        u32 ctx_arg_info_size;
        u32 feature;
        struct bpf_ctx_arg_aux ctx_arg_info[BPF_ITER_CTX_ARG_MAX];
        const struct bpf_iter_seq_info *seq_info;
};

struct bpf_iter_meta {
        __bpf_md_ptr(struct seq_file *, seq);
        u64 session_id;
        u64 seq_num;
};

struct bpf_iter__bpf_map_elem {
        __bpf_md_ptr(struct bpf_iter_meta *, meta);
        __bpf_md_ptr(struct bpf_map *, map);
        __bpf_md_ptr(void *, key);
        __bpf_md_ptr(void *, value);
};

int bpf_iter_reg_target(const struct bpf_iter_reg *reg_info);
void bpf_iter_unreg_target(const struct bpf_iter_reg *reg_info);
bool bpf_iter_prog_supported(struct bpf_prog *prog);
const struct bpf_func_proto *
bpf_iter_get_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog);
int bpf_iter_link_attach(const union bpf_attr *attr, bpfptr_t uattr, struct bpf_prog *prog);
int bpf_iter_new_fd(struct bpf_link *link);
bool bpf_link_is_iter(struct bpf_link *link);
struct bpf_prog *bpf_iter_get_info(struct bpf_iter_meta *meta, bool in_stop);
int bpf_iter_run_prog(struct bpf_prog *prog, void *ctx);
void bpf_iter_map_show_fdinfo(const struct bpf_iter_aux_info *aux,
                              struct seq_file *seq);
int bpf_iter_map_fill_link_info(const struct bpf_iter_aux_info *aux,
                                struct bpf_link_info *info);

int map_set_for_each_callback_args(struct bpf_verifier_env *env,
                                   struct bpf_func_state *caller,
                                   struct bpf_func_state *callee);

int bpf_percpu_hash_copy(struct bpf_map *map, void *key, void *value);
int bpf_percpu_array_copy(struct bpf_map *map, void *key, void *value);
int bpf_percpu_hash_update(struct bpf_map *map, void *key, void *value,
                           u64 flags);
int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value,
                            u64 flags);

int bpf_stackmap_copy(struct bpf_map *map, void *key, void *value);

int bpf_fd_array_map_update_elem(struct bpf_map *map, struct file *map_file,
                                 void *key, void *value, u64 map_flags);
int bpf_fd_array_map_lookup_elem(struct bpf_map *map, void *key, u32 *value);
int bpf_fd_htab_map_update_elem(struct bpf_map *map, struct file *map_file,
                                void *key, void *value, u64 map_flags);
int bpf_fd_htab_map_lookup_elem(struct bpf_map *map, void *key, u32 *value);

int bpf_get_file_flag(int flags);
int bpf_check_uarg_tail_zero(bpfptr_t uaddr, size_t expected_size,
                             size_t actual_size);

/* verify correctness of eBPF program */
int bpf_check(struct bpf_prog **fp, union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size);

#ifndef CONFIG_BPF_JIT_ALWAYS_ON
void bpf_patch_call_args(struct bpf_insn *insn, u32 stack_depth);
#endif

struct btf *bpf_get_btf_vmlinux(void);

/* Map specifics */
struct xdp_frame;
struct sk_buff;
struct bpf_dtab_netdev;
struct bpf_cpu_map_entry;

void __dev_flush(struct list_head *flush_list);
int dev_xdp_enqueue(struct net_device *dev, struct xdp_frame *xdpf,
                    struct net_device *dev_rx);
int dev_map_enqueue(struct bpf_dtab_netdev *dst, struct xdp_frame *xdpf,
                    struct net_device *dev_rx);
int dev_map_enqueue_multi(struct xdp_frame *xdpf, struct net_device *dev_rx,
                          struct bpf_map *map, bool exclude_ingress);
int dev_map_generic_redirect(struct bpf_dtab_netdev *dst, struct sk_buff *skb,
                             struct bpf_prog *xdp_prog);
int dev_map_redirect_multi(struct net_device *dev, struct sk_buff *skb,
                           struct bpf_prog *xdp_prog, struct bpf_map *map,
                           bool exclude_ingress);

void __cpu_map_flush(struct list_head *flush_list);
int cpu_map_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_frame *xdpf,
                    struct net_device *dev_rx);
int cpu_map_generic_redirect(struct bpf_cpu_map_entry *rcpu,
                             struct sk_buff *skb);

/* Return map's numa specified by userspace */
static inline int bpf_map_attr_numa_node(const union bpf_attr *attr)
{
        return (attr->map_flags & BPF_F_NUMA_NODE) ?
                attr->numa_node : NUMA_NO_NODE;
}

struct bpf_prog *bpf_prog_get_type_path(const char *name, enum bpf_prog_type type);
int array_map_alloc_check(union bpf_attr *attr);

int bpf_prog_test_run_xdp(struct bpf_prog *prog, const union bpf_attr *kattr,
                          union bpf_attr __user *uattr);
int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr,
                          union bpf_attr __user *uattr);
int bpf_prog_test_run_tracing(struct bpf_prog *prog,
                              const union bpf_attr *kattr,
                              union bpf_attr __user *uattr);
int bpf_prog_test_run_flow_dissector(struct bpf_prog *prog,
                                     const union bpf_attr *kattr,
                                     union bpf_attr __user *uattr);
int bpf_prog_test_run_raw_tp(struct bpf_prog *prog,
                             const union bpf_attr *kattr,
                             union bpf_attr __user *uattr);
int bpf_prog_test_run_sk_lookup(struct bpf_prog *prog,
                                const union bpf_attr *kattr,
                                union bpf_attr __user *uattr);
int bpf_prog_test_run_nf(struct bpf_prog *prog,
                         const union bpf_attr *kattr,
                         union bpf_attr __user *uattr);
bool btf_ctx_access(int off, int size, enum bpf_access_type type,
                    const struct bpf_prog *prog,
                    struct bpf_insn_access_aux *info);

static inline bool bpf_tracing_ctx_access(int off, int size,
                                          enum bpf_access_type type)
{
        if (off < 0 || off >= sizeof(__u64) * MAX_BPF_FUNC_ARGS)
                return false;
        if (type != BPF_READ)
                return false;
        if (off % size != 0)
                return false;
        return true;
}

static inline bool bpf_tracing_btf_ctx_access(int off, int size,
                                              enum bpf_access_type type,
                                              const struct bpf_prog *prog,
                                              struct bpf_insn_access_aux *info)
{
        if (!bpf_tracing_ctx_access(off, size, type))
                return false;
        return btf_ctx_access(off, size, type, prog, info);
}

int btf_struct_access(struct bpf_verifier_log *log,
                      const struct bpf_reg_state *reg,
                      int off, int size, enum bpf_access_type atype,
                      u32 *next_btf_id, enum bpf_type_flag *flag, const char **field_name);
bool btf_struct_ids_match(struct bpf_verifier_log *log,
                          const struct btf *btf, u32 id, int off,
                          const struct btf *need_btf, u32 need_type_id,
                          bool strict);

int btf_distill_func_proto(struct bpf_verifier_log *log,
                           struct btf *btf,
                           const struct btf_type *func_proto,
                           const char *func_name,
                           struct btf_func_model *m);

struct bpf_reg_state;
int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog);
int btf_check_type_match(struct bpf_verifier_log *log, const struct bpf_prog *prog,
                         struct btf *btf, const struct btf_type *t);
const char *btf_find_decl_tag_value(const struct btf *btf, const struct btf_type *pt,
                                    int comp_idx, const char *tag_key);
int btf_find_next_decl_tag(const struct btf *btf, const struct btf_type *pt,
                           int comp_idx, const char *tag_key, int last_id);

struct bpf_prog *bpf_prog_by_id(u32 id);
struct bpf_link *bpf_link_by_id(u32 id);

const struct bpf_func_proto *bpf_base_func_proto(enum bpf_func_id func_id,
                                                 const struct bpf_prog *prog);
void bpf_task_storage_free(struct task_struct *task);
void bpf_cgrp_storage_free(struct cgroup *cgroup);
bool bpf_prog_has_kfunc_call(const struct bpf_prog *prog);
const struct btf_func_model *
bpf_jit_find_kfunc_model(const struct bpf_prog *prog,
                         const struct bpf_insn *insn);
int bpf_get_kfunc_addr(const struct bpf_prog *prog, u32 func_id,
                       u16 btf_fd_idx, u8 **func_addr);

struct bpf_core_ctx {
        struct bpf_verifier_log *log;
        const struct btf *btf;
};

bool btf_nested_type_is_trusted(struct bpf_verifier_log *log,
                                const struct bpf_reg_state *reg,
                                const char *field_name, u32 btf_id, const char *suffix);

bool btf_type_ids_nocast_alias(struct bpf_verifier_log *log,
                               const struct btf *reg_btf, u32 reg_id,
                               const struct btf *arg_btf, u32 arg_id);

int bpf_core_apply(struct bpf_core_ctx *ctx, const struct bpf_core_relo *relo,
                   int relo_idx, void *insn);

static inline bool unprivileged_ebpf_enabled(void)
{
        return !sysctl_unprivileged_bpf_disabled;
}

/* Not all bpf prog type has the bpf_ctx.
 * For the bpf prog type that has initialized the bpf_ctx,
 * this function can be used to decide if a kernel function
 * is called by a bpf program.
 */
static inline bool has_current_bpf_ctx(void)
{
        return !!current->bpf_ctx;
}

void notrace bpf_prog_inc_misses_counter(struct bpf_prog *prog);

void bpf_dynptr_init(struct bpf_dynptr_kern *ptr, void *data,
                     enum bpf_dynptr_type type, u32 offset, u32 size);
void bpf_dynptr_set_null(struct bpf_dynptr_kern *ptr);
void bpf_dynptr_set_rdonly(struct bpf_dynptr_kern *ptr);

#else /* !CONFIG_BPF_SYSCALL */
static inline struct bpf_prog *bpf_prog_get(u32 ufd)
{
        return ERR_PTR(-EOPNOTSUPP);
}

static inline struct bpf_prog *bpf_prog_get_type_dev(u32 ufd,
                                                     enum bpf_prog_type type,
                                                     bool attach_drv)
{
        return ERR_PTR(-EOPNOTSUPP);
}

static inline void bpf_prog_add(struct bpf_prog *prog, int i)
{
}

static inline void bpf_prog_sub(struct bpf_prog *prog, int i)
{
}

static inline void bpf_prog_put(struct bpf_prog *prog)
{
}

static inline void bpf_prog_inc(struct bpf_prog *prog)
{
}

static inline struct bpf_prog *__must_check
bpf_prog_inc_not_zero(struct bpf_prog *prog)
{
        return ERR_PTR(-EOPNOTSUPP);
}

static inline void bpf_link_init(struct bpf_link *link, enum bpf_link_type type,
                                 const struct bpf_link_ops *ops,
                                 struct bpf_prog *prog)
{
}

static inline int bpf_link_prime(struct bpf_link *link,
                                 struct bpf_link_primer *primer)
{
        return -EOPNOTSUPP;
}

static inline int bpf_link_settle(struct bpf_link_primer *primer)
{
        return -EOPNOTSUPP;
}

static inline void bpf_link_cleanup(struct bpf_link_primer *primer)
{
}

static inline void bpf_link_inc(struct bpf_link *link)
{
}

static inline struct bpf_link *bpf_link_inc_not_zero(struct bpf_link *link)
{
        return NULL;
}

static inline void bpf_link_put(struct bpf_link *link)
{
}

static inline int bpf_obj_get_user(const char __user *pathname, int flags)
{
        return -EOPNOTSUPP;
}

static inline bool bpf_token_capable(const struct bpf_token *token, int cap)
{
        return capable(cap) || (cap != CAP_SYS_ADMIN && capable(CAP_SYS_ADMIN));
}

static inline void bpf_token_inc(struct bpf_token *token)
{
}

static inline void bpf_token_put(struct bpf_token *token)
{
}

static inline struct bpf_token *bpf_token_get_from_fd(u32 ufd)
{
        return ERR_PTR(-EOPNOTSUPP);
}

static inline void __dev_flush(struct list_head *flush_list)
{
}

struct xdp_frame;
struct bpf_dtab_netdev;
struct bpf_cpu_map_entry;

static inline
int dev_xdp_enqueue(struct net_device *dev, struct xdp_frame *xdpf,
                    struct net_device *dev_rx)
{
        return 0;
}

static inline
int dev_map_enqueue(struct bpf_dtab_netdev *dst, struct xdp_frame *xdpf,
                    struct net_device *dev_rx)
{
        return 0;
}

static inline
int dev_map_enqueue_multi(struct xdp_frame *xdpf, struct net_device *dev_rx,
                          struct bpf_map *map, bool exclude_ingress)
{
        return 0;
}

struct sk_buff;

static inline int dev_map_generic_redirect(struct bpf_dtab_netdev *dst,
                                           struct sk_buff *skb,
                                           struct bpf_prog *xdp_prog)
{
        return 0;
}

static inline
int dev_map_redirect_multi(struct net_device *dev, struct sk_buff *skb,
                           struct bpf_prog *xdp_prog, struct bpf_map *map,
                           bool exclude_ingress)
{
        return 0;
}

static inline void __cpu_map_flush(struct list_head *flush_list)
{
}

static inline int cpu_map_enqueue(struct bpf_cpu_map_entry *rcpu,
                                  struct xdp_frame *xdpf,
                                  struct net_device *dev_rx)
{
        return 0;
}

static inline int cpu_map_generic_redirect(struct bpf_cpu_map_entry *rcpu,
                                           struct sk_buff *skb)
{
        return -EOPNOTSUPP;
}

static inline struct bpf_prog *bpf_prog_get_type_path(const char *name,
                                enum bpf_prog_type type)
{
        return ERR_PTR(-EOPNOTSUPP);
}

static inline int bpf_prog_test_run_xdp(struct bpf_prog *prog,
                                        const union bpf_attr *kattr,
                                        union bpf_attr __user *uattr)
{
        return -ENOTSUPP;
}

static inline int bpf_prog_test_run_skb(struct bpf_prog *prog,
                                        const union bpf_attr *kattr,
                                        union bpf_attr __user *uattr)
{
        return -ENOTSUPP;
}

static inline int bpf_prog_test_run_tracing(struct bpf_prog *prog,
                                            const union bpf_attr *kattr,
                                            union bpf_attr __user *uattr)
{
        return -ENOTSUPP;
}

static inline int bpf_prog_test_run_flow_dissector(struct bpf_prog *prog,
                                                   const union bpf_attr *kattr,
                                                   union bpf_attr __user *uattr)
{
        return -ENOTSUPP;
}

static inline int bpf_prog_test_run_sk_lookup(struct bpf_prog *prog,
                                              const union bpf_attr *kattr,
                                              union bpf_attr __user *uattr)
{
        return -ENOTSUPP;
}

static inline void bpf_map_put(struct bpf_map *map)
{
}

static inline struct bpf_prog *bpf_prog_by_id(u32 id)
{
        return ERR_PTR(-ENOTSUPP);
}

static inline int btf_struct_access(struct bpf_verifier_log *log,
                                    const struct bpf_reg_state *reg,
                                    int off, int size, enum bpf_access_type atype,
                                    u32 *next_btf_id, enum bpf_type_flag *flag,
                                    const char **field_name)
{
        return -EACCES;
}

static inline const struct bpf_func_proto *
bpf_base_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
{
        return NULL;
}

static inline void bpf_task_storage_free(struct task_struct *task)
{
}

static inline bool bpf_prog_has_kfunc_call(const struct bpf_prog *prog)
{
        return false;
}

static inline const struct btf_func_model *
bpf_jit_find_kfunc_model(const struct bpf_prog *prog,
                         const struct bpf_insn *insn)
{
        return NULL;
}

static inline int
bpf_get_kfunc_addr(const struct bpf_prog *prog, u32 func_id,
                   u16 btf_fd_idx, u8 **func_addr)
{
        return -ENOTSUPP;
}

static inline bool unprivileged_ebpf_enabled(void)
{
        return false;
}

static inline bool has_current_bpf_ctx(void)
{
        return false;
}

static inline void bpf_prog_inc_misses_counter(struct bpf_prog *prog)
{
}

static inline void bpf_cgrp_storage_free(struct cgroup *cgroup)
{
}

static inline void bpf_dynptr_init(struct bpf_dynptr_kern *ptr, void *data,
                                   enum bpf_dynptr_type type, u32 offset, u32 size)
{
}

static inline void bpf_dynptr_set_null(struct bpf_dynptr_kern *ptr)
{
}

static inline void bpf_dynptr_set_rdonly(struct bpf_dynptr_kern *ptr)
{
}
#endif /* CONFIG_BPF_SYSCALL */

static __always_inline int
bpf_probe_read_kernel_common(void *dst, u32 size, const void *unsafe_ptr)
{
        int ret = -EFAULT;

        if (IS_ENABLED(CONFIG_BPF_EVENTS))
                ret = copy_from_kernel_nofault(dst, unsafe_ptr, size);
        if (unlikely(ret < 0))
                memset(dst, 0, size);
        return ret;
}

void __bpf_free_used_btfs(struct btf_mod_pair *used_btfs, u32 len);

static inline struct bpf_prog *bpf_prog_get_type(u32 ufd,
                                                 enum bpf_prog_type type)
{
        return bpf_prog_get_type_dev(ufd, type, false);
}

void __bpf_free_used_maps(struct bpf_prog_aux *aux,
                          struct bpf_map **used_maps, u32 len);

bool bpf_prog_get_ok(struct bpf_prog *, enum bpf_prog_type *, bool);

int bpf_prog_offload_compile(struct bpf_prog *prog);
void bpf_prog_dev_bound_destroy(struct bpf_prog *prog);
int bpf_prog_offload_info_fill(struct bpf_prog_info *info,
                               struct bpf_prog *prog);

int bpf_map_offload_info_fill(struct bpf_map_info *info, struct bpf_map *map);

int bpf_map_offload_lookup_elem(struct bpf_map *map, void *key, void *value);
int bpf_map_offload_update_elem(struct bpf_map *map,
                                void *key, void *value, u64 flags);
int bpf_map_offload_delete_elem(struct bpf_map *map, void *key);
int bpf_map_offload_get_next_key(struct bpf_map *map,
                                 void *key, void *next_key);

bool bpf_offload_prog_map_match(struct bpf_prog *prog, struct bpf_map *map);

struct bpf_offload_dev *
bpf_offload_dev_create(const struct bpf_prog_offload_ops *ops, void *priv);
void bpf_offload_dev_destroy(struct bpf_offload_dev *offdev);
void *bpf_offload_dev_priv(struct bpf_offload_dev *offdev);
int bpf_offload_dev_netdev_register(struct bpf_offload_dev *offdev,
                                    struct net_device *netdev);
void bpf_offload_dev_netdev_unregister(struct bpf_offload_dev *offdev,
                                       struct net_device *netdev);
bool bpf_offload_dev_match(struct bpf_prog *prog, struct net_device *netdev);

void unpriv_ebpf_notify(int new_state);

#if defined(CONFIG_NET) && defined(CONFIG_BPF_SYSCALL)
int bpf_dev_bound_kfunc_check(struct bpf_verifier_log *log,
                              struct bpf_prog_aux *prog_aux);
void *bpf_dev_bound_resolve_kfunc(struct bpf_prog *prog, u32 func_id);
int bpf_prog_dev_bound_init(struct bpf_prog *prog, union bpf_attr *attr);
int bpf_prog_dev_bound_inherit(struct bpf_prog *new_prog, struct bpf_prog *old_prog);
void bpf_dev_bound_netdev_unregister(struct net_device *dev);

static inline bool bpf_prog_is_dev_bound(const struct bpf_prog_aux *aux)
{
        return aux->dev_bound;
}

static inline bool bpf_prog_is_offloaded(const struct bpf_prog_aux *aux)
{
        return aux->offload_requested;
}

bool bpf_prog_dev_bound_match(const struct bpf_prog *lhs, const struct bpf_prog *rhs);

static inline bool bpf_map_is_offloaded(struct bpf_map *map)
{
        return unlikely(map->ops == &bpf_map_offload_ops);
}

struct bpf_map *bpf_map_offload_map_alloc(union bpf_attr *attr);
void bpf_map_offload_map_free(struct bpf_map *map);
u64 bpf_map_offload_map_mem_usage(const struct bpf_map *map);
int bpf_prog_test_run_syscall(struct bpf_prog *prog,
                              const union bpf_attr *kattr,
                              union bpf_attr __user *uattr);

int sock_map_get_from_fd(const union bpf_attr *attr, struct bpf_prog *prog);
int sock_map_prog_detach(const union bpf_attr *attr, enum bpf_prog_type ptype);
int sock_map_update_elem_sys(struct bpf_map *map, void *key, void *value, u64 flags);
int sock_map_bpf_prog_query(const union bpf_attr *attr,
                            union bpf_attr __user *uattr);
int sock_map_link_create(const union bpf_attr *attr, struct bpf_prog *prog);

void sock_map_unhash(struct sock *sk);
void sock_map_destroy(struct sock *sk);
void sock_map_close(struct sock *sk, long timeout);
#else
static inline int bpf_dev_bound_kfunc_check(struct bpf_verifier_log *log,
                                            struct bpf_prog_aux *prog_aux)
{
        return -EOPNOTSUPP;
}

static inline void *bpf_dev_bound_resolve_kfunc(struct bpf_prog *prog,
                                                u32 func_id)
{
        return NULL;
}

static inline int bpf_prog_dev_bound_init(struct bpf_prog *prog,
                                          union bpf_attr *attr)
{
        return -EOPNOTSUPP;
}

static inline int bpf_prog_dev_bound_inherit(struct bpf_prog *new_prog,
                                             struct bpf_prog *old_prog)
{
        return -EOPNOTSUPP;
}

static inline void bpf_dev_bound_netdev_unregister(struct net_device *dev)
{
}

static inline bool bpf_prog_is_dev_bound(const struct bpf_prog_aux *aux)
{
        return false;
}

static inline bool bpf_prog_is_offloaded(struct bpf_prog_aux *aux)
{
        return false;
}

static inline bool bpf_prog_dev_bound_match(const struct bpf_prog *lhs, const struct bpf_prog *rhs)
{
        return false;
}

static inline bool bpf_map_is_offloaded(struct bpf_map *map)
{
        return false;
}

static inline struct bpf_map *bpf_map_offload_map_alloc(union bpf_attr *attr)
{
        return ERR_PTR(-EOPNOTSUPP);
}

static inline void bpf_map_offload_map_free(struct bpf_map *map)
{
}

static inline u64 bpf_map_offload_map_mem_usage(const struct bpf_map *map)
{
        return 0;
}

static inline int bpf_prog_test_run_syscall(struct bpf_prog *prog,
                                            const union bpf_attr *kattr,
                                            union bpf_attr __user *uattr)
{
        return -ENOTSUPP;
}

#ifdef CONFIG_BPF_SYSCALL
static inline int sock_map_get_from_fd(const union bpf_attr *attr,
                                       struct bpf_prog *prog)
{
        return -EINVAL;
}

static inline int sock_map_prog_detach(const union bpf_attr *attr,
                                       enum bpf_prog_type ptype)
{
        return -EOPNOTSUPP;
}

static inline int sock_map_update_elem_sys(struct bpf_map *map, void *key, void *value,
                                           u64 flags)
{
        return -EOPNOTSUPP;
}

static inline int sock_map_bpf_prog_query(const union bpf_attr *attr,
                                          union bpf_attr __user *uattr)
{
        return -EINVAL;
}

static inline int sock_map_link_create(const union bpf_attr *attr, struct bpf_prog *prog)
{
        return -EOPNOTSUPP;
}
#endif /* CONFIG_BPF_SYSCALL */
#endif /* CONFIG_NET && CONFIG_BPF_SYSCALL */

static __always_inline void
bpf_prog_inc_misses_counters(const struct bpf_prog_array *array)
{
        const struct bpf_prog_array_item *item;
        struct bpf_prog *prog;

        if (unlikely(!array))
                return;

        item = &array->items[0];
        while ((prog = READ_ONCE(item->prog))) {
                bpf_prog_inc_misses_counter(prog);
                item++;
        }
}

#if defined(CONFIG_INET) && defined(CONFIG_BPF_SYSCALL)
void bpf_sk_reuseport_detach(struct sock *sk);
int bpf_fd_reuseport_array_lookup_elem(struct bpf_map *map, void *key,
                                       void *value);
int bpf_fd_reuseport_array_update_elem(struct bpf_map *map, void *key,
                                       void *value, u64 map_flags);
#else
static inline void bpf_sk_reuseport_detach(struct sock *sk)
{
}

#ifdef CONFIG_BPF_SYSCALL
static inline int bpf_fd_reuseport_array_lookup_elem(struct bpf_map *map,
                                                     void *key, void *value)
{
        return -EOPNOTSUPP;
}

static inline int bpf_fd_reuseport_array_update_elem(struct bpf_map *map,
                                                     void *key, void *value,
                                                     u64 map_flags)
{
        return -EOPNOTSUPP;
}
#endif /* CONFIG_BPF_SYSCALL */
#endif /* defined(CONFIG_INET) && defined(CONFIG_BPF_SYSCALL) */

/* verifier prototypes for helper functions called from eBPF programs */
extern const struct bpf_func_proto bpf_map_lookup_elem_proto;
extern const struct bpf_func_proto bpf_map_update_elem_proto;
extern const struct bpf_func_proto bpf_map_delete_elem_proto;
extern const struct bpf_func_proto bpf_map_push_elem_proto;
extern const struct bpf_func_proto bpf_map_pop_elem_proto;
extern const struct bpf_func_proto bpf_map_peek_elem_proto;
extern const struct bpf_func_proto bpf_map_lookup_percpu_elem_proto;

extern const struct bpf_func_proto bpf_get_prandom_u32_proto;
extern const struct bpf_func_proto bpf_get_smp_processor_id_proto;
extern const struct bpf_func_proto bpf_get_numa_node_id_proto;
extern const struct bpf_func_proto bpf_tail_call_proto;
extern const struct bpf_func_proto bpf_ktime_get_ns_proto;
extern const struct bpf_func_proto bpf_ktime_get_boot_ns_proto;
extern const struct bpf_func_proto bpf_ktime_get_tai_ns_proto;
extern const struct bpf_func_proto bpf_get_current_pid_tgid_proto;
extern const struct bpf_func_proto bpf_get_current_uid_gid_proto;
extern const struct bpf_func_proto bpf_get_current_comm_proto;
extern const struct bpf_func_proto bpf_get_stackid_proto;
extern const struct bpf_func_proto bpf_get_stack_proto;
extern const struct bpf_func_proto bpf_get_stack_sleepable_proto;
extern const struct bpf_func_proto bpf_get_task_stack_proto;
extern const struct bpf_func_proto bpf_get_task_stack_sleepable_proto;
extern const struct bpf_func_proto bpf_get_stackid_proto_pe;
extern const struct bpf_func_proto bpf_get_stack_proto_pe;
extern const struct bpf_func_proto bpf_sock_map_update_proto;
extern const struct bpf_func_proto bpf_sock_hash_update_proto;
extern const struct bpf_func_proto bpf_get_current_cgroup_id_proto;
extern const struct bpf_func_proto bpf_get_current_ancestor_cgroup_id_proto;
extern const struct bpf_func_proto bpf_get_cgroup_classid_curr_proto;
extern const struct bpf_func_proto bpf_current_task_under_cgroup_proto;
extern const struct bpf_func_proto bpf_msg_redirect_hash_proto;
extern const struct bpf_func_proto bpf_msg_redirect_map_proto;
extern const struct bpf_func_proto bpf_sk_redirect_hash_proto;
extern const struct bpf_func_proto bpf_sk_redirect_map_proto;
extern const struct bpf_func_proto bpf_spin_lock_proto;
extern const struct bpf_func_proto bpf_spin_unlock_proto;
extern const struct bpf_func_proto bpf_get_local_storage_proto;
extern const struct bpf_func_proto bpf_strtol_proto;
extern const struct bpf_func_proto bpf_strtoul_proto;
extern const struct bpf_func_proto bpf_tcp_sock_proto;
extern const struct bpf_func_proto bpf_jiffies64_proto;
extern const struct bpf_func_proto bpf_get_ns_current_pid_tgid_proto;
extern const struct bpf_func_proto bpf_event_output_data_proto;
extern const struct bpf_func_proto bpf_ringbuf_output_proto;
extern const struct bpf_func_proto bpf_ringbuf_reserve_proto;
extern const struct bpf_func_proto bpf_ringbuf_submit_proto;
extern const struct bpf_func_proto bpf_ringbuf_discard_proto;
extern const struct bpf_func_proto bpf_ringbuf_query_proto;
extern const struct bpf_func_proto bpf_ringbuf_reserve_dynptr_proto;
extern const struct bpf_func_proto bpf_ringbuf_submit_dynptr_proto;
extern const struct bpf_func_proto bpf_ringbuf_discard_dynptr_proto;
extern const struct bpf_func_proto bpf_skc_to_tcp6_sock_proto;
extern const struct bpf_func_proto bpf_skc_to_tcp_sock_proto;
extern const struct bpf_func_proto bpf_skc_to_tcp_timewait_sock_proto;
extern const struct bpf_func_proto bpf_skc_to_tcp_request_sock_proto;
extern const struct bpf_func_proto bpf_skc_to_udp6_sock_proto;
extern const struct bpf_func_proto bpf_skc_to_unix_sock_proto;
extern const struct bpf_func_proto bpf_skc_to_mptcp_sock_proto;
extern const struct bpf_func_proto bpf_copy_from_user_proto;
extern const struct bpf_func_proto bpf_snprintf_btf_proto;
extern const struct bpf_func_proto bpf_snprintf_proto;
extern const struct bpf_func_proto bpf_per_cpu_ptr_proto;
extern const struct bpf_func_proto bpf_this_cpu_ptr_proto;
extern const struct bpf_func_proto bpf_ktime_get_coarse_ns_proto;
extern const struct bpf_func_proto bpf_sock_from_file_proto;
extern const struct bpf_func_proto bpf_get_socket_ptr_cookie_proto;
extern const struct bpf_func_proto bpf_task_storage_get_recur_proto;
extern const struct bpf_func_proto bpf_task_storage_get_proto;
extern const struct bpf_func_proto bpf_task_storage_delete_recur_proto;
extern const struct bpf_func_proto bpf_task_storage_delete_proto;
extern const struct bpf_func_proto bpf_for_each_map_elem_proto;
extern const struct bpf_func_proto bpf_btf_find_by_name_kind_proto;
extern const struct bpf_func_proto bpf_sk_setsockopt_proto;
extern const struct bpf_func_proto bpf_sk_getsockopt_proto;
extern const struct bpf_func_proto bpf_unlocked_sk_setsockopt_proto;
extern const struct bpf_func_proto bpf_unlocked_sk_getsockopt_proto;
extern const struct bpf_func_proto bpf_find_vma_proto;
extern const struct bpf_func_proto bpf_loop_proto;
extern const struct bpf_func_proto bpf_copy_from_user_task_proto;
extern const struct bpf_func_proto bpf_set_retval_proto;
extern const struct bpf_func_proto bpf_get_retval_proto;
extern const struct bpf_func_proto bpf_user_ringbuf_drain_proto;
extern const struct bpf_func_proto bpf_cgrp_storage_get_proto;
extern const struct bpf_func_proto bpf_cgrp_storage_delete_proto;

const struct bpf_func_proto *tracing_prog_func_proto(
  enum bpf_func_id func_id, const struct bpf_prog *prog);

/* Shared helpers among cBPF and eBPF. */
void bpf_user_rnd_init_once(void);
u64 bpf_user_rnd_u32(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5);
u64 bpf_get_raw_cpu_id(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5);

#if defined(CONFIG_NET)
bool bpf_sock_common_is_valid_access(int off, int size,
                                     enum bpf_access_type type,
                                     struct bpf_insn_access_aux *info);
bool bpf_sock_is_valid_access(int off, int size, enum bpf_access_type type,
                              struct bpf_insn_access_aux *info);
u32 bpf_sock_convert_ctx_access(enum bpf_access_type type,
                                const struct bpf_insn *si,
                                struct bpf_insn *insn_buf,
                                struct bpf_prog *prog,
                                u32 *target_size);
int bpf_dynptr_from_skb_rdonly(struct __sk_buff *skb, u64 flags,
                               struct bpf_dynptr *ptr);
#else
static inline bool bpf_sock_common_is_valid_access(int off, int size,
                                                   enum bpf_access_type type,
                                                   struct bpf_insn_access_aux *info)
{
        return false;
}
static inline bool bpf_sock_is_valid_access(int off, int size,
                                            enum bpf_access_type type,
                                            struct bpf_insn_access_aux *info)
{
        return false;
}
static inline u32 bpf_sock_convert_ctx_access(enum bpf_access_type type,
                                              const struct bpf_insn *si,
                                              struct bpf_insn *insn_buf,
                                              struct bpf_prog *prog,
                                              u32 *target_size)
{
        return 0;
}
static inline int bpf_dynptr_from_skb_rdonly(struct __sk_buff *skb, u64 flags,
                                             struct bpf_dynptr *ptr)
{
        return -EOPNOTSUPP;
}
#endif

#ifdef CONFIG_INET
struct sk_reuseport_kern {
        struct sk_buff *skb;
        struct sock *sk;
        struct sock *selected_sk;
        struct sock *migrating_sk;
        void *data_end;
        u32 hash;
        u32 reuseport_id;
        bool bind_inany;
};
bool bpf_tcp_sock_is_valid_access(int off, int size, enum bpf_access_type type,
                                  struct bpf_insn_access_aux *info);

u32 bpf_tcp_sock_convert_ctx_access(enum bpf_access_type type,
                                    const struct bpf_insn *si,
                                    struct bpf_insn *insn_buf,
                                    struct bpf_prog *prog,
                                    u32 *target_size);

bool bpf_xdp_sock_is_valid_access(int off, int size, enum bpf_access_type type,
                                  struct bpf_insn_access_aux *info);

u32 bpf_xdp_sock_convert_ctx_access(enum bpf_access_type type,
                                    const struct bpf_insn *si,
                                    struct bpf_insn *insn_buf,
                                    struct bpf_prog *prog,
                                    u32 *target_size);
#else
static inline bool bpf_tcp_sock_is_valid_access(int off, int size,
                                                enum bpf_access_type type,
                                                struct bpf_insn_access_aux *info)
{
        return false;
}

static inline u32 bpf_tcp_sock_convert_ctx_access(enum bpf_access_type type,
                                                  const struct bpf_insn *si,
                                                  struct bpf_insn *insn_buf,
                                                  struct bpf_prog *prog,
                                                  u32 *target_size)
{
        return 0;
}
static inline bool bpf_xdp_sock_is_valid_access(int off, int size,
                                                enum bpf_access_type type,
                                                struct bpf_insn_access_aux *info)
{
        return false;
}

static inline u32 bpf_xdp_sock_convert_ctx_access(enum bpf_access_type type,
                                                  const struct bpf_insn *si,
                                                  struct bpf_insn *insn_buf,
                                                  struct bpf_prog *prog,
                                                  u32 *target_size)
{
        return 0;
}
#endif /* CONFIG_INET */

enum bpf_text_poke_type {
        BPF_MOD_CALL,
        BPF_MOD_JUMP,
};

int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type t,
                       void *addr1, void *addr2);

void bpf_arch_poke_desc_update(struct bpf_jit_poke_descriptor *poke,
                               struct bpf_prog *new, struct bpf_prog *old);

void *bpf_arch_text_copy(void *dst, void *src, size_t len);
int bpf_arch_text_invalidate(void *dst, size_t len);

struct btf_id_set;
bool btf_id_set_contains(const struct btf_id_set *set, u32 id);

#define MAX_BPRINTF_VARARGS                12
#define MAX_BPRINTF_BUF                        1024

struct bpf_bprintf_data {
        u32 *bin_args;
        char *buf;
        bool get_bin_args;
        bool get_buf;
};

int bpf_bprintf_prepare(char *fmt, u32 fmt_size, const u64 *raw_args,
                        u32 num_args, struct bpf_bprintf_data *data);
void bpf_bprintf_cleanup(struct bpf_bprintf_data *data);

#ifdef CONFIG_BPF_LSM
void bpf_cgroup_atype_get(u32 attach_btf_id, int cgroup_atype);
void bpf_cgroup_atype_put(int cgroup_atype);
#else
static inline void bpf_cgroup_atype_get(u32 attach_btf_id, int cgroup_atype) {}
static inline void bpf_cgroup_atype_put(int cgroup_atype) {}
#endif /* CONFIG_BPF_LSM */

struct key;

#ifdef CONFIG_KEYS
struct bpf_key {
        struct key *key;
        bool has_ref;
};
#endif /* CONFIG_KEYS */

static inline bool type_is_alloc(u32 type)
{
        return type & MEM_ALLOC;
}

static inline gfp_t bpf_memcg_flags(gfp_t flags)
{
        if (memcg_bpf_enabled())
                return flags | __GFP_ACCOUNT;
        return flags;
}

static inline bool bpf_is_subprog(const struct bpf_prog *prog)
{
        return prog->aux->func_idx != 0;
}

static inline bool bpf_prog_is_raw_tp(const struct bpf_prog *prog)
{
        return prog->type == BPF_PROG_TYPE_TRACING &&
               prog->expected_attach_type == BPF_TRACE_RAW_TP;
}

#endif /* _LINUX_BPF_H */







































































































































































    6 

    6 









































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
// SPDX-License-Identifier: GPL-2.0-only
/*
 * fs/kernfs/mount.c - kernfs mount implementation
 *
 * Copyright (c) 2001-3 Patrick Mochel
 * Copyright (c) 2007 SUSE Linux Products GmbH
 * Copyright (c) 2007, 2013 Tejun Heo <tj@kernel.org>
 */

#include <linux/fs.h>
#include <linux/mount.h>
#include <linux/init.h>
#include <linux/magic.h>
#include <linux/slab.h>
#include <linux/pagemap.h>
#include <linux/namei.h>
#include <linux/seq_file.h>
#include <linux/exportfs.h>
#include <linux/uuid.h>
#include <linux/statfs.h>

#include "kernfs-internal.h"

struct kmem_cache *kernfs_node_cache __ro_after_init;
struct kmem_cache *kernfs_iattrs_cache __ro_after_init;
struct kernfs_global_locks *kernfs_locks __ro_after_init;

static int kernfs_sop_show_options(struct seq_file *sf, struct dentry *dentry)
{
        struct kernfs_root *root = kernfs_root(kernfs_dentry_node(dentry));
        struct kernfs_syscall_ops *scops = root->syscall_ops;

        if (scops && scops->show_options)
                return scops->show_options(sf, root);
        return 0;
}

static int kernfs_sop_show_path(struct seq_file *sf, struct dentry *dentry)
{
        struct kernfs_node *node = kernfs_dentry_node(dentry);
        struct kernfs_root *root = kernfs_root(node);
        struct kernfs_syscall_ops *scops = root->syscall_ops;

        if (scops && scops->show_path)
                return scops->show_path(sf, node, root);

        seq_dentry(sf, dentry, " \t\n\\");
        return 0;
}

static int kernfs_statfs(struct dentry *dentry, struct kstatfs *buf)
{
        simple_statfs(dentry, buf);
        buf->f_fsid = uuid_to_fsid(dentry->d_sb->s_uuid.b);
        return 0;
}

const struct super_operations kernfs_sops = {
        .statfs                = kernfs_statfs,
        .drop_inode        = generic_delete_inode,
        .evict_inode        = kernfs_evict_inode,

        .show_options        = kernfs_sop_show_options,
        .show_path        = kernfs_sop_show_path,
};

static int kernfs_encode_fh(struct inode *inode, __u32 *fh, int *max_len,
                            struct inode *parent)
{
        struct kernfs_node *kn = inode->i_private;

        if (*max_len < 2) {
                *max_len = 2;
                return FILEID_INVALID;
        }

        *max_len = 2;
        *(u64 *)fh = kn->id;
        return FILEID_KERNFS;
}

static struct dentry *__kernfs_fh_to_dentry(struct super_block *sb,
                                            struct fid *fid, int fh_len,
                                            int fh_type, bool get_parent)
{
        struct kernfs_super_info *info = kernfs_info(sb);
        struct kernfs_node *kn;
        struct inode *inode;
        u64 id;

        if (fh_len < 2)
                return NULL;

        switch (fh_type) {
        case FILEID_KERNFS:
                id = *(u64 *)fid;
                break;
        case FILEID_INO32_GEN:
        case FILEID_INO32_GEN_PARENT:
                /*
                 * blk_log_action() exposes "LOW32,HIGH32" pair without
                 * type and userland can call us with generic fid
                 * constructed from them.  Combine it back to ID.  See
                 * blk_log_action().
                 */
                id = ((u64)fid->i32.gen << 32) | fid->i32.ino;
                break;
        default:
                return NULL;
        }

        kn = kernfs_find_and_get_node_by_id(info->root, id);
        if (!kn)
                return ERR_PTR(-ESTALE);

        if (get_parent) {
                struct kernfs_node *parent;

                parent = kernfs_get_parent(kn);
                kernfs_put(kn);
                kn = parent;
                if (!kn)
                        return ERR_PTR(-ESTALE);
        }

        inode = kernfs_get_inode(sb, kn);
        kernfs_put(kn);
        return d_obtain_alias(inode);
}

static struct dentry *kernfs_fh_to_dentry(struct super_block *sb,
                                          struct fid *fid, int fh_len,
                                          int fh_type)
{
        return __kernfs_fh_to_dentry(sb, fid, fh_len, fh_type, false);
}

static struct dentry *kernfs_fh_to_parent(struct super_block *sb,
                                          struct fid *fid, int fh_len,
                                          int fh_type)
{
        return __kernfs_fh_to_dentry(sb, fid, fh_len, fh_type, true);
}

static struct dentry *kernfs_get_parent_dentry(struct dentry *child)
{
        struct kernfs_node *kn = kernfs_dentry_node(child);

        return d_obtain_alias(kernfs_get_inode(child->d_sb, kn->parent));
}

static const struct export_operations kernfs_export_ops = {
        .encode_fh        = kernfs_encode_fh,
        .fh_to_dentry        = kernfs_fh_to_dentry,
        .fh_to_parent        = kernfs_fh_to_parent,
        .get_parent        = kernfs_get_parent_dentry,
};

/**
 * kernfs_root_from_sb - determine kernfs_root associated with a super_block
 * @sb: the super_block in question
 *
 * Return: the kernfs_root associated with @sb.  If @sb is not a kernfs one,
 * %NULL is returned.
 */
struct kernfs_root *kernfs_root_from_sb(struct super_block *sb)
{
        if (sb->s_op == &kernfs_sops)
                return kernfs_info(sb)->root;
        return NULL;
}

/*
 * find the next ancestor in the path down to @child, where @parent was the
 * ancestor whose descendant we want to find.
 *
 * Say the path is /a/b/c/d.  @child is d, @parent is %NULL.  We return the root
 * node.  If @parent is b, then we return the node for c.
 * Passing in d as @parent is not ok.
 */
static struct kernfs_node *find_next_ancestor(struct kernfs_node *child,
                                              struct kernfs_node *parent)
{
        if (child == parent) {
                pr_crit_once("BUG in find_next_ancestor: called with parent == child");
                return NULL;
        }

        while (child->parent != parent) {
                if (!child->parent)
                        return NULL;
                child = child->parent;
        }

        return child;
}

/**
 * kernfs_node_dentry - get a dentry for the given kernfs_node
 * @kn: kernfs_node for which a dentry is needed
 * @sb: the kernfs super_block
 *
 * Return: the dentry pointer
 */
struct dentry *kernfs_node_dentry(struct kernfs_node *kn,
                                  struct super_block *sb)
{
        struct dentry *dentry;
        struct kernfs_node *knparent;

        BUG_ON(sb->s_op != &kernfs_sops);

        dentry = dget(sb->s_root);

        /* Check if this is the root kernfs_node */
        if (!kn->parent)
                return dentry;

        knparent = find_next_ancestor(kn, NULL);
        if (WARN_ON(!knparent)) {
                dput(dentry);
                return ERR_PTR(-EINVAL);
        }

        do {
                struct dentry *dtmp;
                struct kernfs_node *kntmp;

                if (kn == knparent)
                        return dentry;
                kntmp = find_next_ancestor(kn, knparent);
                if (WARN_ON(!kntmp)) {
                        dput(dentry);
                        return ERR_PTR(-EINVAL);
                }
                dtmp = lookup_positive_unlocked(kntmp->name, dentry,
                                               strlen(kntmp->name));
                dput(dentry);
                if (IS_ERR(dtmp))
                        return dtmp;
                knparent = kntmp;
                dentry = dtmp;
        } while (true);
}

static int kernfs_fill_super(struct super_block *sb, struct kernfs_fs_context *kfc)
{
        struct kernfs_super_info *info = kernfs_info(sb);
        struct kernfs_root *kf_root = kfc->root;
        struct inode *inode;
        struct dentry *root;

        info->sb = sb;
        /* Userspace would break if executables or devices appear on sysfs */
        sb->s_iflags |= SB_I_NOEXEC | SB_I_NODEV;
        sb->s_blocksize = PAGE_SIZE;
        sb->s_blocksize_bits = PAGE_SHIFT;
        sb->s_magic = kfc->magic;
        sb->s_op = &kernfs_sops;
        sb->s_xattr = kernfs_xattr_handlers;
        if (info->root->flags & KERNFS_ROOT_SUPPORT_EXPORTOP)
                sb->s_export_op = &kernfs_export_ops;
        sb->s_time_gran = 1;

        /* sysfs dentries and inodes don't require IO to create */
        sb->s_shrink->seeks = 0;

        /* get root inode, initialize and unlock it */
        down_read(&kf_root->kernfs_rwsem);
        inode = kernfs_get_inode(sb, info->root->kn);
        up_read(&kf_root->kernfs_rwsem);
        if (!inode) {
                pr_debug("kernfs: could not get root inode\n");
                return -ENOMEM;
        }

        /* instantiate and link root dentry */
        root = d_make_root(inode);
        if (!root) {
                pr_debug("%s: could not get root dentry!\n", __func__);
                return -ENOMEM;
        }
        sb->s_root = root;
        sb->s_d_op = &kernfs_dops;
        return 0;
}

static int kernfs_test_super(struct super_block *sb, struct fs_context *fc)
{
        struct kernfs_super_info *sb_info = kernfs_info(sb);
        struct kernfs_super_info *info = fc->s_fs_info;

        return sb_info->root == info->root && sb_info->ns == info->ns;
}

static int kernfs_set_super(struct super_block *sb, struct fs_context *fc)
{
        struct kernfs_fs_context *kfc = fc->fs_private;

        kfc->ns_tag = NULL;
        return set_anon_super_fc(sb, fc);
}

/**
 * kernfs_super_ns - determine the namespace tag of a kernfs super_block
 * @sb: super_block of interest
 *
 * Return: the namespace tag associated with kernfs super_block @sb.
 */
const void *kernfs_super_ns(struct super_block *sb)
{
        struct kernfs_super_info *info = kernfs_info(sb);

        return info->ns;
}

/**
 * kernfs_get_tree - kernfs filesystem access/retrieval helper
 * @fc: The filesystem context.
 *
 * This is to be called from each kernfs user's fs_context->ops->get_tree()
 * implementation, which should set the specified ->@fs_type and ->@flags, and
 * specify the hierarchy and namespace tag to mount via ->@root and ->@ns,
 * respectively.
 *
 * Return: %0 on success, -errno on failure.
 */
int kernfs_get_tree(struct fs_context *fc)
{
        struct kernfs_fs_context *kfc = fc->fs_private;
        struct super_block *sb;
        struct kernfs_super_info *info;
        int error;

        info = kzalloc(sizeof(*info), GFP_KERNEL);
        if (!info)
                return -ENOMEM;

        info->root = kfc->root;
        info->ns = kfc->ns_tag;
        INIT_LIST_HEAD(&info->node);

        fc->s_fs_info = info;
        sb = sget_fc(fc, kernfs_test_super, kernfs_set_super);
        if (IS_ERR(sb))
                return PTR_ERR(sb);

        if (!sb->s_root) {
                struct kernfs_super_info *info = kernfs_info(sb);
                struct kernfs_root *root = kfc->root;

                kfc->new_sb_created = true;

                error = kernfs_fill_super(sb, kfc);
                if (error) {
                        deactivate_locked_super(sb);
                        return error;
                }
                sb->s_flags |= SB_ACTIVE;

                uuid_t uuid;
                uuid_gen(&uuid);
                super_set_uuid(sb, uuid.b, sizeof(uuid));

                down_write(&root->kernfs_supers_rwsem);
                list_add(&info->node, &info->root->supers);
                up_write(&root->kernfs_supers_rwsem);
        }

        fc->root = dget(sb->s_root);
        return 0;
}

void kernfs_free_fs_context(struct fs_context *fc)
{
        /* Note that we don't deal with kfc->ns_tag here. */
        kfree(fc->s_fs_info);
        fc->s_fs_info = NULL;
}

/**
 * kernfs_kill_sb - kill_sb for kernfs
 * @sb: super_block being killed
 *
 * This can be used directly for file_system_type->kill_sb().  If a kernfs
 * user needs extra cleanup, it can implement its own kill_sb() and call
 * this function at the end.
 */
void kernfs_kill_sb(struct super_block *sb)
{
        struct kernfs_super_info *info = kernfs_info(sb);
        struct kernfs_root *root = info->root;

        down_write(&root->kernfs_supers_rwsem);
        list_del(&info->node);
        up_write(&root->kernfs_supers_rwsem);

        /*
         * Remove the superblock from fs_supers/s_instances
         * so we can't find it, before freeing kernfs_super_info.
         */
        kill_anon_super(sb);
        kfree(info);
}

static void __init kernfs_mutex_init(void)
{
        int count;

        for (count = 0; count < NR_KERNFS_LOCKS; count++)
                mutex_init(&kernfs_locks->open_file_mutex[count]);
}

static void __init kernfs_lock_init(void)
{
        kernfs_locks = kmalloc(sizeof(struct kernfs_global_locks), GFP_KERNEL);
        WARN_ON(!kernfs_locks);

        kernfs_mutex_init();
}

void __init kernfs_init(void)
{
        kernfs_node_cache = kmem_cache_create("kernfs_node_cache",
                                              sizeof(struct kernfs_node),
                                              0, SLAB_PANIC, NULL);

        /* Creates slab cache for kernfs inode attributes */
        kernfs_iattrs_cache  = kmem_cache_create("kernfs_iattrs_cache",
                                              sizeof(struct kernfs_iattrs),
                                              0, SLAB_PANIC, NULL);

        kernfs_lock_init();
}




























    3 
    3 








    3 


    3 

    3 









    3 























    3 




    3 

    3 

    3 
    3 














    3 


















    3 

    3 
    3 


    3 











    3 






    3 








    3 





    3 




















    3 
















    3 








    3 





    3 












    3 









    3 
















    3 







    3 










    3 






    3 



    3 


    3 








    3 
    3 












    3 




















    3 


    3 





    3 






















    3 







    3 



    3 
    3 

    3 
    3 

    3 








    3 









    3 
    3 























    3 



    3 
    3 






















    3 




    3 
    3 


    3 
    3 


    3 



    3 


    3 



    3 
    3 






















    3 





















    3 










    3 
    3 





    3 













    3 







    3 




    3 
    3 
















    3 

    3 


    3 




    3 














    3 

































    3 

















    3 
    3 






































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
/*
 * Copyright (c) 2017, Mellanox Technologies inc.  All rights reserved.
 */
#include <rdma/uverbs_ioctl.h>
#include <rdma/rdma_user_ioctl.h>
#include <linux/bitops.h>
#include "rdma_core.h"
#include "uverbs.h"

static int ib_uverbs_notsupp(struct uverbs_attr_bundle *attrs)
{
        return -EOPNOTSUPP;
}

static void *uapi_add_elm(struct uverbs_api *uapi, u32 key, size_t alloc_size)
{
        void *elm;
        int rc;

        if (key == UVERBS_API_KEY_ERR)
                return ERR_PTR(-EOVERFLOW);

        elm = kzalloc(alloc_size, GFP_KERNEL);
        if (!elm)
                return ERR_PTR(-ENOMEM);
        rc = radix_tree_insert(&uapi->radix, key, elm);
        if (rc) {
                kfree(elm);
                return ERR_PTR(rc);
        }

        return elm;
}

static void *uapi_add_get_elm(struct uverbs_api *uapi, u32 key,
                              size_t alloc_size, bool *exists)
{
        void *elm;

        elm = uapi_add_elm(uapi, key, alloc_size);
        if (!IS_ERR(elm)) {
                *exists = false;
                return elm;
        }

        if (elm != ERR_PTR(-EEXIST))
                return elm;

        elm = radix_tree_lookup(&uapi->radix, key);
        if (WARN_ON(!elm))
                return ERR_PTR(-EINVAL);
        *exists = true;
        return elm;
}

static int uapi_create_write(struct uverbs_api *uapi,
                             struct ib_device *ibdev,
                             const struct uapi_definition *def,
                             u32 obj_key,
                             u32 *cur_method_key)
{
        struct uverbs_api_write_method *method_elm;
        u32 method_key = obj_key;
        bool exists;

        if (def->write.is_ex)
                method_key |= uapi_key_write_ex_method(def->write.command_num);
        else
                method_key |= uapi_key_write_method(def->write.command_num);

        method_elm = uapi_add_get_elm(uapi, method_key, sizeof(*method_elm),
                                      &exists);
        if (IS_ERR(method_elm))
                return PTR_ERR(method_elm);

        if (WARN_ON(exists && (def->write.is_ex != method_elm->is_ex)))
                return -EINVAL;

        method_elm->is_ex = def->write.is_ex;
        method_elm->handler = def->func_write;
        if (!def->write.is_ex)
                method_elm->disabled = !(ibdev->uverbs_cmd_mask &
                                         BIT_ULL(def->write.command_num));

        if (!def->write.is_ex && def->func_write) {
                method_elm->has_udata = def->write.has_udata;
                method_elm->has_resp = def->write.has_resp;
                method_elm->req_size = def->write.req_size;
                method_elm->resp_size = def->write.resp_size;
        }

        *cur_method_key = method_key;
        return 0;
}

static int uapi_merge_method(struct uverbs_api *uapi,
                             struct uverbs_api_object *obj_elm, u32 obj_key,
                             const struct uverbs_method_def *method,
                             bool is_driver)
{
        u32 method_key = obj_key | uapi_key_ioctl_method(method->id);
        struct uverbs_api_ioctl_method *method_elm;
        unsigned int i;
        bool exists;

        if (!method->attrs)
                return 0;

        method_elm = uapi_add_get_elm(uapi, method_key, sizeof(*method_elm),
                                      &exists);
        if (IS_ERR(method_elm))
                return PTR_ERR(method_elm);
        if (exists) {
                /*
                 * This occurs when a driver uses ADD_UVERBS_ATTRIBUTES_SIMPLE
                 */
                if (WARN_ON(method->handler))
                        return -EINVAL;
        } else {
                WARN_ON(!method->handler);
                rcu_assign_pointer(method_elm->handler, method->handler);
                if (method->handler != uverbs_destroy_def_handler)
                        method_elm->driver_method = is_driver;
        }

        for (i = 0; i != method->num_attrs; i++) {
                const struct uverbs_attr_def *attr = (*method->attrs)[i];
                struct uverbs_api_attr *attr_slot;

                if (!attr)
                        continue;

                /*
                 * ENUM_IN contains the 'ids' pointer to the driver's .rodata,
                 * so if it is specified by a driver then it always makes this
                 * into a driver method.
                 */
                if (attr->attr.type == UVERBS_ATTR_TYPE_ENUM_IN)
                        method_elm->driver_method |= is_driver;

                /*
                 * Like other uobject based things we only support a single
                 * uobject being NEW'd or DESTROY'd
                 */
                if (attr->attr.type == UVERBS_ATTR_TYPE_IDRS_ARRAY) {
                        u8 access = attr->attr.u2.objs_arr.access;

                        if (WARN_ON(access == UVERBS_ACCESS_NEW ||
                                    access == UVERBS_ACCESS_DESTROY))
                                return -EINVAL;
                }

                attr_slot =
                        uapi_add_elm(uapi, method_key | uapi_key_attr(attr->id),
                                     sizeof(*attr_slot));
                /* Attributes are not allowed to be modified by drivers */
                if (IS_ERR(attr_slot))
                        return PTR_ERR(attr_slot);

                attr_slot->spec = attr->attr;
        }

        return 0;
}

static int uapi_merge_obj_tree(struct uverbs_api *uapi,
                               const struct uverbs_object_def *obj,
                               bool is_driver)
{
        struct uverbs_api_object *obj_elm;
        unsigned int i;
        u32 obj_key;
        bool exists;
        int rc;

        obj_key = uapi_key_obj(obj->id);
        obj_elm = uapi_add_get_elm(uapi, obj_key, sizeof(*obj_elm), &exists);
        if (IS_ERR(obj_elm))
                return PTR_ERR(obj_elm);

        if (obj->type_attrs) {
                if (WARN_ON(obj_elm->type_attrs))
                        return -EINVAL;

                obj_elm->id = obj->id;
                obj_elm->type_attrs = obj->type_attrs;
                obj_elm->type_class = obj->type_attrs->type_class;
                /*
                 * Today drivers are only permitted to use idr_class and
                 * fd_class types. We can revoke the IDR types during
                 * disassociation, and the FD types require the driver to use
                 * struct file_operations.owner to prevent the driver module
                 * code from unloading while the file is open. This provides
                 * enough safety that uverbs_uobject_fd_release() will
                 * continue to work.  Drivers using FD are responsible to
                 * handle disassociation of the device on their own.
                 */
                if (WARN_ON(is_driver &&
                            obj->type_attrs->type_class != &uverbs_idr_class &&
                            obj->type_attrs->type_class != &uverbs_fd_class))
                        return -EINVAL;
        }

        if (!obj->methods)
                return 0;

        for (i = 0; i != obj->num_methods; i++) {
                const struct uverbs_method_def *method = (*obj->methods)[i];

                if (!method)
                        continue;

                rc = uapi_merge_method(uapi, obj_elm, obj_key, method,
                                       is_driver);
                if (rc)
                        return rc;
        }

        return 0;
}

static int uapi_disable_elm(struct uverbs_api *uapi,
                            const struct uapi_definition *def,
                            u32 obj_key,
                            u32 method_key)
{
        bool exists;

        if (def->scope == UAPI_SCOPE_OBJECT) {
                struct uverbs_api_object *obj_elm;

                obj_elm = uapi_add_get_elm(
                        uapi, obj_key, sizeof(*obj_elm), &exists);
                if (IS_ERR(obj_elm))
                        return PTR_ERR(obj_elm);
                obj_elm->disabled = 1;
                return 0;
        }

        if (def->scope == UAPI_SCOPE_METHOD &&
            uapi_key_is_ioctl_method(method_key)) {
                struct uverbs_api_ioctl_method *method_elm;

                method_elm = uapi_add_get_elm(uapi, method_key,
                                              sizeof(*method_elm), &exists);
                if (IS_ERR(method_elm))
                        return PTR_ERR(method_elm);
                method_elm->disabled = 1;
                return 0;
        }

        if (def->scope == UAPI_SCOPE_METHOD &&
            (uapi_key_is_write_method(method_key) ||
             uapi_key_is_write_ex_method(method_key))) {
                struct uverbs_api_write_method *write_elm;

                write_elm = uapi_add_get_elm(uapi, method_key,
                                             sizeof(*write_elm), &exists);
                if (IS_ERR(write_elm))
                        return PTR_ERR(write_elm);
                write_elm->disabled = 1;
                return 0;
        }

        WARN_ON(true);
        return -EINVAL;
}

static int uapi_merge_def(struct uverbs_api *uapi, struct ib_device *ibdev,
                          const struct uapi_definition *def_list,
                          bool is_driver)
{
        const struct uapi_definition *def = def_list;
        u32 cur_obj_key = UVERBS_API_KEY_ERR;
        u32 cur_method_key = UVERBS_API_KEY_ERR;
        bool exists;
        int rc;

        if (!def_list)
                return 0;

        for (;; def++) {
                switch ((enum uapi_definition_kind)def->kind) {
                case UAPI_DEF_CHAIN:
                        rc = uapi_merge_def(uapi, ibdev, def->chain, is_driver);
                        if (rc)
                                return rc;
                        continue;

                case UAPI_DEF_CHAIN_OBJ_TREE:
                        if (WARN_ON(def->object_start.object_id !=
                                    def->chain_obj_tree->id))
                                return -EINVAL;

                        cur_obj_key = uapi_key_obj(def->object_start.object_id);
                        rc = uapi_merge_obj_tree(uapi, def->chain_obj_tree,
                                                 is_driver);
                        if (rc)
                                return rc;
                        continue;

                case UAPI_DEF_END:
                        return 0;

                case UAPI_DEF_IS_SUPPORTED_DEV_FN: {
                        void **ibdev_fn =
                                (void *)(&ibdev->ops) + def->needs_fn_offset;

                        if (*ibdev_fn)
                                continue;
                        rc = uapi_disable_elm(
                                uapi, def, cur_obj_key, cur_method_key);
                        if (rc)
                                return rc;
                        continue;
                }

                case UAPI_DEF_IS_SUPPORTED_FUNC:
                        if (def->func_is_supported(ibdev))
                                continue;
                        rc = uapi_disable_elm(
                                uapi, def, cur_obj_key, cur_method_key);
                        if (rc)
                                return rc;
                        continue;

                case UAPI_DEF_OBJECT_START: {
                        struct uverbs_api_object *obj_elm;

                        cur_obj_key = uapi_key_obj(def->object_start.object_id);
                        obj_elm = uapi_add_get_elm(uapi, cur_obj_key,
                                                   sizeof(*obj_elm), &exists);
                        if (IS_ERR(obj_elm))
                                return PTR_ERR(obj_elm);
                        continue;
                }

                case UAPI_DEF_WRITE:
                        rc = uapi_create_write(
                                uapi, ibdev, def, cur_obj_key, &cur_method_key);
                        if (rc)
                                return rc;
                        continue;
                }
                WARN_ON(true);
                return -EINVAL;
        }
}

static int
uapi_finalize_ioctl_method(struct uverbs_api *uapi,
                           struct uverbs_api_ioctl_method *method_elm,
                           u32 method_key)
{
        struct radix_tree_iter iter;
        unsigned int num_attrs = 0;
        unsigned int max_bkey = 0;
        bool single_uobj = false;
        void __rcu **slot;

        method_elm->destroy_bkey = UVERBS_API_ATTR_BKEY_LEN;
        radix_tree_for_each_slot (slot, &uapi->radix, &iter,
                                  uapi_key_attrs_start(method_key)) {
                struct uverbs_api_attr *elm =
                        rcu_dereference_protected(*slot, true);
                u32 attr_key = iter.index & UVERBS_API_ATTR_KEY_MASK;
                u32 attr_bkey = uapi_bkey_attr(attr_key);
                u8 type = elm->spec.type;

                if (uapi_key_attr_to_ioctl_method(iter.index) !=
                    uapi_key_attr_to_ioctl_method(method_key))
                        break;

                if (elm->spec.mandatory)
                        __set_bit(attr_bkey, method_elm->attr_mandatory);

                if (elm->spec.is_udata)
                        method_elm->has_udata = true;

                if (type == UVERBS_ATTR_TYPE_IDR ||
                    type == UVERBS_ATTR_TYPE_FD) {
                        u8 access = elm->spec.u.obj.access;

                        /*
                         * Verbs specs may only have one NEW/DESTROY, we don't
                         * have the infrastructure to abort multiple NEW's or
                         * cope with multiple DESTROY failure.
                         */
                        if (access == UVERBS_ACCESS_NEW ||
                            access == UVERBS_ACCESS_DESTROY) {
                                if (WARN_ON(single_uobj))
                                        return -EINVAL;

                                single_uobj = true;
                                if (WARN_ON(!elm->spec.mandatory))
                                        return -EINVAL;
                        }

                        if (access == UVERBS_ACCESS_DESTROY)
                                method_elm->destroy_bkey = attr_bkey;
                }

                max_bkey = max(max_bkey, attr_bkey);
                num_attrs++;
        }

        method_elm->key_bitmap_len = max_bkey + 1;
        WARN_ON(method_elm->key_bitmap_len > UVERBS_API_ATTR_BKEY_LEN);

        uapi_compute_bundle_size(method_elm, num_attrs);
        return 0;
}

static int uapi_finalize(struct uverbs_api *uapi)
{
        const struct uverbs_api_write_method **data;
        unsigned long max_write_ex = 0;
        unsigned long max_write = 0;
        struct radix_tree_iter iter;
        void __rcu **slot;
        int rc;
        int i;

        radix_tree_for_each_slot (slot, &uapi->radix, &iter, 0) {
                struct uverbs_api_ioctl_method *method_elm =
                        rcu_dereference_protected(*slot, true);

                if (uapi_key_is_ioctl_method(iter.index)) {
                        rc = uapi_finalize_ioctl_method(uapi, method_elm,
                                                        iter.index);
                        if (rc)
                                return rc;
                }

                if (uapi_key_is_write_method(iter.index))
                        max_write = max(max_write,
                                        iter.index & UVERBS_API_ATTR_KEY_MASK);
                if (uapi_key_is_write_ex_method(iter.index))
                        max_write_ex =
                                max(max_write_ex,
                                    iter.index & UVERBS_API_ATTR_KEY_MASK);
        }

        uapi->notsupp_method.handler = ib_uverbs_notsupp;
        uapi->num_write = max_write + 1;
        uapi->num_write_ex = max_write_ex + 1;
        data = kmalloc_array(uapi->num_write + uapi->num_write_ex,
                             sizeof(*uapi->write_methods), GFP_KERNEL);
        if (!data)
                return -ENOMEM;

        for (i = 0; i != uapi->num_write + uapi->num_write_ex; i++)
                data[i] = &uapi->notsupp_method;
        uapi->write_methods = data;
        uapi->write_ex_methods = data + uapi->num_write;

        radix_tree_for_each_slot (slot, &uapi->radix, &iter, 0) {
                if (uapi_key_is_write_method(iter.index))
                        uapi->write_methods[iter.index &
                                            UVERBS_API_ATTR_KEY_MASK] =
                                rcu_dereference_protected(*slot, true);
                if (uapi_key_is_write_ex_method(iter.index))
                        uapi->write_ex_methods[iter.index &
                                               UVERBS_API_ATTR_KEY_MASK] =
                                rcu_dereference_protected(*slot, true);
        }

        return 0;
}

static void uapi_remove_range(struct uverbs_api *uapi, u32 start, u32 last)
{
        struct radix_tree_iter iter;
        void __rcu **slot;

        radix_tree_for_each_slot (slot, &uapi->radix, &iter, start) {
                if (iter.index > last)
                        return;
                kfree(rcu_dereference_protected(*slot, true));
                radix_tree_iter_delete(&uapi->radix, &iter, slot);
        }
}

static void uapi_remove_object(struct uverbs_api *uapi, u32 obj_key)
{
        uapi_remove_range(uapi, obj_key,
                          obj_key | UVERBS_API_METHOD_KEY_MASK |
                                  UVERBS_API_ATTR_KEY_MASK);
}

static void uapi_remove_method(struct uverbs_api *uapi, u32 method_key)
{
        uapi_remove_range(uapi, method_key,
                          method_key | UVERBS_API_ATTR_KEY_MASK);
}


static u32 uapi_get_obj_id(struct uverbs_attr_spec *spec)
{
        if (spec->type == UVERBS_ATTR_TYPE_IDR ||
            spec->type == UVERBS_ATTR_TYPE_FD)
                return spec->u.obj.obj_type;
        if (spec->type == UVERBS_ATTR_TYPE_IDRS_ARRAY)
                return spec->u2.objs_arr.obj_type;
        return UVERBS_API_KEY_ERR;
}

static void uapi_key_okay(u32 key)
{
        unsigned int count = 0;

        if (uapi_key_is_object(key))
                count++;
        if (uapi_key_is_ioctl_method(key))
                count++;
        if (uapi_key_is_write_method(key))
                count++;
        if (uapi_key_is_write_ex_method(key))
                count++;
        if (uapi_key_is_attr(key))
                count++;
        WARN(count != 1, "Bad count %u key=%x", count, key);
}

static void uapi_finalize_disable(struct uverbs_api *uapi)
{
        struct radix_tree_iter iter;
        u32 starting_key = 0;
        bool scan_again = false;
        void __rcu **slot;

again:
        radix_tree_for_each_slot (slot, &uapi->radix, &iter, starting_key) {
                uapi_key_okay(iter.index);

                if (uapi_key_is_object(iter.index)) {
                        struct uverbs_api_object *obj_elm =
                                rcu_dereference_protected(*slot, true);

                        if (obj_elm->disabled) {
                                /* Have to check all the attrs again */
                                scan_again = true;
                                starting_key = iter.index;
                                uapi_remove_object(uapi, iter.index);
                                goto again;
                        }
                        continue;
                }

                if (uapi_key_is_ioctl_method(iter.index)) {
                        struct uverbs_api_ioctl_method *method_elm =
                                rcu_dereference_protected(*slot, true);

                        if (method_elm->disabled) {
                                starting_key = iter.index;
                                uapi_remove_method(uapi, iter.index);
                                goto again;
                        }
                        continue;
                }

                if (uapi_key_is_write_method(iter.index) ||
                    uapi_key_is_write_ex_method(iter.index)) {
                        struct uverbs_api_write_method *method_elm =
                                rcu_dereference_protected(*slot, true);

                        if (method_elm->disabled) {
                                kfree(method_elm);
                                radix_tree_iter_delete(&uapi->radix, &iter, slot);
                        }
                        continue;
                }

                if (uapi_key_is_attr(iter.index)) {
                        struct uverbs_api_attr *attr_elm =
                                rcu_dereference_protected(*slot, true);
                        const struct uverbs_api_object *tmp_obj;
                        u32 obj_key;

                        /*
                         * If the method has a mandatory object handle
                         * attribute which relies on an object which is not
                         * present then the entire method is uncallable.
                         */
                        if (!attr_elm->spec.mandatory)
                                continue;
                        obj_key = uapi_get_obj_id(&attr_elm->spec);
                        if (obj_key == UVERBS_API_KEY_ERR)
                                continue;
                        tmp_obj = uapi_get_object(uapi, obj_key);
                        if (IS_ERR(tmp_obj)) {
                                if (PTR_ERR(tmp_obj) == -ENOMSG)
                                        continue;
                        } else {
                                if (!tmp_obj->disabled)
                                        continue;
                        }

                        starting_key = iter.index;
                        uapi_remove_method(
                                uapi,
                                iter.index & (UVERBS_API_OBJ_KEY_MASK |
                                              UVERBS_API_METHOD_KEY_MASK));
                        goto again;
                }

                WARN_ON(false);
        }

        if (!scan_again)
                return;
        scan_again = false;
        starting_key = 0;
        goto again;
}

void uverbs_destroy_api(struct uverbs_api *uapi)
{
        if (!uapi)
                return;

        uapi_remove_range(uapi, 0, U32_MAX);
        kfree(uapi->write_methods);
        kfree(uapi);
}

static const struct uapi_definition uverbs_core_api[] = {
        UAPI_DEF_CHAIN(uverbs_def_obj_async_fd),
        UAPI_DEF_CHAIN(uverbs_def_obj_counters),
        UAPI_DEF_CHAIN(uverbs_def_obj_cq),
        UAPI_DEF_CHAIN(uverbs_def_obj_device),
        UAPI_DEF_CHAIN(uverbs_def_obj_dm),
        UAPI_DEF_CHAIN(uverbs_def_obj_flow_action),
        UAPI_DEF_CHAIN(uverbs_def_obj_intf),
        UAPI_DEF_CHAIN(uverbs_def_obj_mr),
        UAPI_DEF_CHAIN(uverbs_def_obj_qp),
        UAPI_DEF_CHAIN(uverbs_def_obj_srq),
        UAPI_DEF_CHAIN(uverbs_def_obj_wq),
        UAPI_DEF_CHAIN(uverbs_def_write_intf),
        {},
};

struct uverbs_api *uverbs_alloc_api(struct ib_device *ibdev)
{
        struct uverbs_api *uapi;
        int rc;

        uapi = kzalloc(sizeof(*uapi), GFP_KERNEL);
        if (!uapi)
                return ERR_PTR(-ENOMEM);

        INIT_RADIX_TREE(&uapi->radix, GFP_KERNEL);
        uapi->driver_id = ibdev->ops.driver_id;

        rc = uapi_merge_def(uapi, ibdev, uverbs_core_api, false);
        if (rc)
                goto err;
        rc = uapi_merge_def(uapi, ibdev, ibdev->driver_def, true);
        if (rc)
                goto err;

        uapi_finalize_disable(uapi);
        rc = uapi_finalize(uapi);
        if (rc)
                goto err;

        return uapi;
err:
        if (rc != -ENOMEM)
                dev_err(&ibdev->dev,
                        "Setup of uverbs_api failed, kernel parsing tree description is not valid (%d)??\n",
                        rc);

        uverbs_destroy_api(uapi);
        return ERR_PTR(rc);
}

/*
 * The pre version is done before destroying the HW objects, it only blocks
 * off method access. All methods that require the ib_dev or the module data
 * must test one of these assignments prior to continuing.
 */
void uverbs_disassociate_api_pre(struct ib_uverbs_device *uverbs_dev)
{
        struct uverbs_api *uapi = uverbs_dev->uapi;
        struct radix_tree_iter iter;
        void __rcu **slot;

        rcu_assign_pointer(uverbs_dev->ib_dev, NULL);

        radix_tree_for_each_slot (slot, &uapi->radix, &iter, 0) {
                if (uapi_key_is_ioctl_method(iter.index)) {
                        struct uverbs_api_ioctl_method *method_elm =
                                rcu_dereference_protected(*slot, true);

                        if (method_elm->driver_method)
                                rcu_assign_pointer(method_elm->handler, NULL);
                }
        }

        synchronize_srcu(&uverbs_dev->disassociate_srcu);
}

/*
 * Called when a driver disassociates from the ib_uverbs_device. The
 * assumption is that the driver module will unload after. Replace everything
 * related to the driver with NULL as a safety measure.
 */
void uverbs_disassociate_api(struct uverbs_api *uapi)
{
        struct radix_tree_iter iter;
        void __rcu **slot;

        radix_tree_for_each_slot (slot, &uapi->radix, &iter, 0) {
                if (uapi_key_is_object(iter.index)) {
                        struct uverbs_api_object *object_elm =
                                rcu_dereference_protected(*slot, true);

                        /*
                         * Some type_attrs are in the driver module. We don't
                         * bother to keep track of which since there should be
                         * no use of this after disassociate.
                         */
                        object_elm->type_attrs = NULL;
                } else if (uapi_key_is_attr(iter.index)) {
                        struct uverbs_api_attr *elm =
                                rcu_dereference_protected(*slot, true);

                        if (elm->spec.type == UVERBS_ATTR_TYPE_ENUM_IN)
                                elm->spec.u2.enum_def.ids = NULL;
                }
        }
}
































































































































































































































































































































































































































































































































































   14 


























   58 







































































































































































































    8 























































































































































































































































































































   93 
  214 























































   47 



























































































































































































































































































































    6 

















    8 

































    6 

















    8 

































































































































































































  339 




  257 




  122 




































































 1702 












  229 
  339 














































  232 
  257 




  122 































































































































































































































































































































































































































































































   80 

   79 







 1817 





















































































































































































    3 
  104 





































































































































































































































































































































































































































   63 
 1601 

  182 
















   63 
 1598 

  179 





















































































  123 




































   68 
    1 























  758 




























































































  781 








































































































































































































































































































   17 






































   16 






    8 

    5 





























  275 



























































































































    1 







    1 
   31 
















































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965
2966
2967
2968
2969
2970
2971
2972
2973
2974
2975
2976
2977
2978
2979
2980
2981
2982
2983
2984
2985
2986
2987
2988
2989
2990
2991
2992
2993
2994
2995
2996
2997
2998
2999
3000
3001
3002
3003
3004
3005
3006
3007
3008
3009
3010
3011
3012
3013
3014
3015
3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029
3030
3031
3032
3033
3034
3035
3036
3037
3038
3039
3040
3041
3042
3043
3044
3045
3046
3047
3048
3049
3050
3051
3052
3053
3054
3055
3056
3057
3058
3059
3060
3061
3062
3063
3064
3065
3066
3067
3068
3069
3070
3071
3072
3073
3074
3075
3076
3077
3078
3079
3080
3081
3082
3083
3084
3085
3086
3087
3088
3089
3090
3091
3092
3093
3094
3095
3096
3097
3098
3099
3100
3101
3102
3103
3104
3105
3106
3107
3108
3109
3110
3111
3112
3113
3114
3115
3116
3117
3118
3119
3120
3121
3122
3123
3124
3125
3126
3127
3128
3129
3130
3131
3132
3133
3134
3135
3136
3137
3138
3139
3140
3141
3142
3143
3144
3145
3146
3147
3148
3149
3150
3151
3152
3153
3154
3155
3156
3157
3158
3159
3160
3161
3162
3163
3164
3165
3166
3167
3168
3169
3170
3171
3172
3173
3174
3175
3176
3177
3178
3179
3180
3181
3182
3183
3184
3185
3186
3187
3188
3189
3190
3191
3192
3193
3194
3195
3196
3197
3198
3199
3200
3201
3202
3203
3204
3205
3206
3207
3208
3209
3210
3211
3212
3213
3214
3215
3216
3217
3218
3219
3220
3221
3222
3223
3224
3225
3226
3227
3228
3229
3230
3231
3232
3233
3234
3235
3236
3237
3238
3239
3240
3241
3242
3243
3244
3245
3246
3247
3248
3249
3250
3251
3252
3253
3254
3255
3256
3257
3258
3259
3260
3261
3262
3263
3264
3265
3266
3267
3268
3269
3270
3271
3272
3273
3274
3275
3276
3277
3278
3279
3280
3281
3282
3283
3284
3285
3286
3287
3288
3289
3290
3291
3292
3293
3294
3295
3296
3297
3298
3299
3300
3301
3302
3303
3304
3305
3306
3307
3308
3309
3310
3311
3312
3313
3314
3315
3316
3317
3318
3319
3320
3321
3322
3323
3324
3325
3326
3327
3328
3329
3330
3331
3332
3333
3334
3335
3336
3337
3338
3339
3340
3341
3342
3343
3344
3345
3346
3347
3348
3349
3350
3351
3352
3353
3354
3355
3356
3357
3358
3359
3360
3361
3362
3363
3364
3365
3366
3367
3368
3369
3370
3371
3372
3373
3374
3375
3376
3377
3378
3379
3380
3381
3382
3383
3384
3385
3386
3387
3388
3389
3390
3391
3392
3393
3394
3395
3396
3397
3398
3399
3400
3401
3402
3403
3404
3405
3406
3407
3408
3409
3410
3411
3412
3413
3414
3415
3416
3417
3418
3419
3420
3421
3422
3423
3424
3425
3426
3427
3428
3429
3430
3431
3432
3433
3434
3435
3436
3437
3438
3439
3440
3441
3442
3443
3444
3445
3446
3447
3448
3449
3450
3451
3452
3453
3454
3455
3456
3457
3458
3459
3460
3461
3462
3463
3464
3465
3466
3467
3468
3469
3470
3471
3472
3473
3474
3475
3476
3477
3478
3479
3480
3481
3482
3483
3484
3485
3486
3487
3488
3489
3490
3491
3492
3493
3494
3495
3496
3497
3498
3499
3500
3501
3502
3503
3504
3505
3506
3507
3508
3509
3510
3511
3512
3513
3514
3515
3516
3517
3518
3519
3520
3521
3522
3523
3524
3525
3526
3527
3528
3529
3530
3531
3532
3533
3534
3535
3536
3537
3538
3539
3540
3541
3542
3543
3544
3545
3546
3547
3548
3549
3550
3551
3552
3553
3554
3555
3556
3557
3558
3559
3560
3561
3562
3563
3564
3565
3566
3567
3568
3569
3570
3571
3572
3573
3574
3575
3576
3577
3578
3579
3580
3581
3582
3583
3584
3585
3586
3587
3588
3589
3590
3591
3592
3593
3594
3595
3596
3597
3598
3599
3600
3601
3602
3603
3604
3605
3606
3607
3608
3609
3610
3611
3612
3613
3614
3615
3616
3617
3618
3619
3620
3621
3622
3623
3624
3625
3626
3627
3628
3629
3630
3631
3632
3633
3634
3635
3636
3637
3638
3639
3640
3641
3642
3643
3644
3645
3646
3647
3648
3649
3650
3651
3652
3653
3654
3655
3656
3657
3658
3659
3660
3661
3662
3663
3664
3665
3666
3667
3668
3669
3670
3671
3672
3673
3674
3675
3676
3677
3678
3679
3680
3681
3682
3683
3684
3685
3686
3687
3688
3689
3690
3691
3692
3693
3694
3695
3696
3697
3698
3699
3700
3701
3702
3703
3704
3705
3706
3707
3708
3709
3710
3711
3712
3713
3714
3715
3716
3717
3718
3719
3720
3721
3722
3723
3724
3725
3726
3727
3728
3729
3730
3731
3732
3733
3734
3735
3736
3737
3738
3739
3740
3741
3742
3743
3744
3745
3746
3747
3748
3749
3750
3751
3752
3753
3754
3755
3756
3757
3758
3759
3760
3761
3762
3763
3764
3765
3766
3767
3768
3769
3770
3771
3772
3773
3774
3775
3776
3777
3778
3779
3780
3781
3782
3783
3784
3785
3786
3787
3788
3789
3790
3791
3792
3793
3794
3795
3796
3797
3798
3799
3800
3801
3802
3803
3804
3805
3806
3807
3808
3809
3810
3811
3812
3813
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_FS_H
#define _LINUX_FS_H

#include <linux/linkage.h>
#include <linux/wait_bit.h>
#include <linux/kdev_t.h>
#include <linux/dcache.h>
#include <linux/path.h>
#include <linux/stat.h>
#include <linux/cache.h>
#include <linux/list.h>
#include <linux/list_lru.h>
#include <linux/llist.h>
#include <linux/radix-tree.h>
#include <linux/xarray.h>
#include <linux/rbtree.h>
#include <linux/init.h>
#include <linux/pid.h>
#include <linux/bug.h>
#include <linux/mutex.h>
#include <linux/rwsem.h>
#include <linux/mm_types.h>
#include <linux/capability.h>
#include <linux/semaphore.h>
#include <linux/fcntl.h>
#include <linux/rculist_bl.h>
#include <linux/atomic.h>
#include <linux/shrinker.h>
#include <linux/migrate_mode.h>
#include <linux/uidgid.h>
#include <linux/lockdep.h>
#include <linux/percpu-rwsem.h>
#include <linux/workqueue.h>
#include <linux/delayed_call.h>
#include <linux/uuid.h>
#include <linux/errseq.h>
#include <linux/ioprio.h>
#include <linux/fs_types.h>
#include <linux/build_bug.h>
#include <linux/stddef.h>
#include <linux/mount.h>
#include <linux/cred.h>
#include <linux/mnt_idmapping.h>
#include <linux/slab.h>
#include <linux/maple_tree.h>
#include <linux/rw_hint.h>
#include <linux/file_ref.h>
#include <linux/unicode.h>

#include <asm/byteorder.h>
#include <uapi/linux/fs.h>

struct backing_dev_info;
struct bdi_writeback;
struct bio;
struct io_comp_batch;
struct export_operations;
struct fiemap_extent_info;
struct hd_geometry;
struct iovec;
struct kiocb;
struct kobject;
struct pipe_inode_info;
struct poll_table_struct;
struct kstatfs;
struct vm_area_struct;
struct vfsmount;
struct cred;
struct swap_info_struct;
struct seq_file;
struct workqueue_struct;
struct iov_iter;
struct fscrypt_inode_info;
struct fscrypt_operations;
struct fsverity_info;
struct fsverity_operations;
struct fsnotify_mark_connector;
struct fsnotify_sb_info;
struct fs_context;
struct fs_parameter_spec;
struct fileattr;
struct iomap_ops;

extern void __init inode_init(void);
extern void __init inode_init_early(void);
extern void __init files_init(void);
extern void __init files_maxfiles_init(void);

extern unsigned long get_max_files(void);
extern unsigned int sysctl_nr_open;

typedef __kernel_rwf_t rwf_t;

struct buffer_head;
typedef int (get_block_t)(struct inode *inode, sector_t iblock,
                        struct buffer_head *bh_result, int create);
typedef int (dio_iodone_t)(struct kiocb *iocb, loff_t offset,
                        ssize_t bytes, void *private);

#define MAY_EXEC                0x00000001
#define MAY_WRITE                0x00000002
#define MAY_READ                0x00000004
#define MAY_APPEND                0x00000008
#define MAY_ACCESS                0x00000010
#define MAY_OPEN                0x00000020
#define MAY_CHDIR                0x00000040
/* called from RCU mode, don't block */
#define MAY_NOT_BLOCK                0x00000080

/*
 * flags in file.f_mode.  Note that FMODE_READ and FMODE_WRITE must correspond
 * to O_WRONLY and O_RDWR via the strange trick in do_dentry_open()
 */

/* file is open for reading */
#define FMODE_READ                ((__force fmode_t)(1 << 0))
/* file is open for writing */
#define FMODE_WRITE                ((__force fmode_t)(1 << 1))
/* file is seekable */
#define FMODE_LSEEK                ((__force fmode_t)(1 << 2))
/* file can be accessed using pread */
#define FMODE_PREAD                ((__force fmode_t)(1 << 3))
/* file can be accessed using pwrite */
#define FMODE_PWRITE                ((__force fmode_t)(1 << 4))
/* File is opened for execution with sys_execve / sys_uselib */
#define FMODE_EXEC                ((__force fmode_t)(1 << 5))
/* File writes are restricted (block device specific) */
#define FMODE_WRITE_RESTRICTED        ((__force fmode_t)(1 << 6))
/* File supports atomic writes */
#define FMODE_CAN_ATOMIC_WRITE        ((__force fmode_t)(1 << 7))

/* FMODE_* bit 8 */

/* 32bit hashes as llseek() offset (for directories) */
#define FMODE_32BITHASH         ((__force fmode_t)(1 << 9))
/* 64bit hashes as llseek() offset (for directories) */
#define FMODE_64BITHASH         ((__force fmode_t)(1 << 10))

/*
 * Don't update ctime and mtime.
 *
 * Currently a special hack for the XFS open_by_handle ioctl, but we'll
 * hopefully graduate it to a proper O_CMTIME flag supported by open(2) soon.
 */
#define FMODE_NOCMTIME                ((__force fmode_t)(1 << 11))

/* Expect random access pattern */
#define FMODE_RANDOM                ((__force fmode_t)(1 << 12))

/* FMODE_* bit 13 */

/* File is opened with O_PATH; almost nothing can be done with it */
#define FMODE_PATH                ((__force fmode_t)(1 << 14))

/* File needs atomic accesses to f_pos */
#define FMODE_ATOMIC_POS        ((__force fmode_t)(1 << 15))
/* Write access to underlying fs */
#define FMODE_WRITER                ((__force fmode_t)(1 << 16))
/* Has read method(s) */
#define FMODE_CAN_READ          ((__force fmode_t)(1 << 17))
/* Has write method(s) */
#define FMODE_CAN_WRITE         ((__force fmode_t)(1 << 18))

#define FMODE_OPENED                ((__force fmode_t)(1 << 19))
#define FMODE_CREATED                ((__force fmode_t)(1 << 20))

/* File is stream-like */
#define FMODE_STREAM                ((__force fmode_t)(1 << 21))

/* File supports DIRECT IO */
#define        FMODE_CAN_ODIRECT        ((__force fmode_t)(1 << 22))

#define        FMODE_NOREUSE                ((__force fmode_t)(1 << 23))

/* FMODE_* bit 24 */

/* File is embedded in backing_file object */
#define FMODE_BACKING                ((__force fmode_t)(1 << 25))

/* File was opened by fanotify and shouldn't generate fanotify events */
#define FMODE_NONOTIFY                ((__force fmode_t)(1 << 26))

/* File is capable of returning -EAGAIN if I/O will block */
#define FMODE_NOWAIT                ((__force fmode_t)(1 << 27))

/* File represents mount that needs unmounting */
#define FMODE_NEED_UNMOUNT        ((__force fmode_t)(1 << 28))

/* File does not contribute to nr_files count */
#define FMODE_NOACCOUNT                ((__force fmode_t)(1 << 29))

/*
 * Attribute flags.  These should be or-ed together to figure out what
 * has been changed!
 */
#define ATTR_MODE        (1 << 0)
#define ATTR_UID        (1 << 1)
#define ATTR_GID        (1 << 2)
#define ATTR_SIZE        (1 << 3)
#define ATTR_ATIME        (1 << 4)
#define ATTR_MTIME        (1 << 5)
#define ATTR_CTIME        (1 << 6)
#define ATTR_ATIME_SET        (1 << 7)
#define ATTR_MTIME_SET        (1 << 8)
#define ATTR_FORCE        (1 << 9) /* Not a change, but a change it */
#define ATTR_KILL_SUID        (1 << 11)
#define ATTR_KILL_SGID        (1 << 12)
#define ATTR_FILE        (1 << 13)
#define ATTR_KILL_PRIV        (1 << 14)
#define ATTR_OPEN        (1 << 15) /* Truncating from open(O_TRUNC) */
#define ATTR_TIMES_SET        (1 << 16)
#define ATTR_TOUCH        (1 << 17)
#define ATTR_DELEG        (1 << 18) /* Delegated attrs. Don't break write delegations */

/*
 * Whiteout is represented by a char device.  The following constants define the
 * mode and device number to use.
 */
#define WHITEOUT_MODE 0
#define WHITEOUT_DEV 0

/*
 * This is the Inode Attributes structure, used for notify_change().  It
 * uses the above definitions as flags, to know which values have changed.
 * Also, in this manner, a Filesystem can look at only the values it cares
 * about.  Basically, these are the attributes that the VFS layer can
 * request to change from the FS layer.
 *
 * Derek Atkins <warlord@MIT.EDU> 94-10-20
 */
struct iattr {
        unsigned int        ia_valid;
        umode_t                ia_mode;
        /*
         * The two anonymous unions wrap structures with the same member.
         *
         * Filesystems raising FS_ALLOW_IDMAP need to use ia_vfs{g,u}id which
         * are a dedicated type requiring the filesystem to use the dedicated
         * helpers. Other filesystem can continue to use ia_{g,u}id until they
         * have been ported.
         *
         * They always contain the same value. In other words FS_ALLOW_IDMAP
         * pass down the same value on idmapped mounts as they would on regular
         * mounts.
         */
        union {
                kuid_t                ia_uid;
                vfsuid_t        ia_vfsuid;
        };
        union {
                kgid_t                ia_gid;
                vfsgid_t        ia_vfsgid;
        };
        loff_t                ia_size;
        struct timespec64 ia_atime;
        struct timespec64 ia_mtime;
        struct timespec64 ia_ctime;

        /*
         * Not an attribute, but an auxiliary info for filesystems wanting to
         * implement an ftruncate() like method.  NOTE: filesystem should
         * check for (ia_valid & ATTR_FILE), and not for (ia_file != NULL).
         */
        struct file        *ia_file;
};

/*
 * Includes for diskquotas.
 */
#include <linux/quota.h>

/*
 * Maximum number of layers of fs stack.  Needs to be limited to
 * prevent kernel stack overflow
 */
#define FILESYSTEM_MAX_STACK_DEPTH 2

/** 
 * enum positive_aop_returns - aop return codes with specific semantics
 *
 * @AOP_WRITEPAGE_ACTIVATE: Informs the caller that page writeback has
 *                             completed, that the page is still locked, and
 *                             should be considered active.  The VM uses this hint
 *                             to return the page to the active list -- it won't
 *                             be a candidate for writeback again in the near
 *                             future.  Other callers must be careful to unlock
 *                             the page if they get this return.  Returned by
 *                             writepage(); 
 *
 * @AOP_TRUNCATED_PAGE: The AOP method that was handed a locked page has
 *                          unlocked it and the page might have been truncated.
 *                          The caller should back up to acquiring a new page and
 *                          trying again.  The aop will be taking reasonable
 *                          precautions not to livelock.  If the caller held a page
 *                          reference, it should drop it before retrying.  Returned
 *                          by read_folio().
 *
 * address_space_operation functions return these large constants to indicate
 * special semantics to the caller.  These are much larger than the bytes in a
 * page to allow for functions that return the number of bytes operated on in a
 * given page.
 */

enum positive_aop_returns {
        AOP_WRITEPAGE_ACTIVATE        = 0x80000,
        AOP_TRUNCATED_PAGE        = 0x80001,
};

/*
 * oh the beauties of C type declarations.
 */
struct page;
struct address_space;
struct writeback_control;
struct readahead_control;

/* Match RWF_* bits to IOCB bits */
#define IOCB_HIPRI                (__force int) RWF_HIPRI
#define IOCB_DSYNC                (__force int) RWF_DSYNC
#define IOCB_SYNC                (__force int) RWF_SYNC
#define IOCB_NOWAIT                (__force int) RWF_NOWAIT
#define IOCB_APPEND                (__force int) RWF_APPEND
#define IOCB_ATOMIC                (__force int) RWF_ATOMIC

/* non-RWF related bits - start at 16 */
#define IOCB_EVENTFD                (1 << 16)
#define IOCB_DIRECT                (1 << 17)
#define IOCB_WRITE                (1 << 18)
/* iocb->ki_waitq is valid */
#define IOCB_WAITQ                (1 << 19)
#define IOCB_NOIO                (1 << 20)
/* can use bio alloc cache */
#define IOCB_ALLOC_CACHE        (1 << 21)
/*
 * IOCB_DIO_CALLER_COMP can be set by the iocb owner, to indicate that the
 * iocb completion can be passed back to the owner for execution from a safe
 * context rather than needing to be punted through a workqueue. If this
 * flag is set, the bio completion handling may set iocb->dio_complete to a
 * handler function and iocb->private to context information for that handler.
 * The issuer should call the handler with that context information from task
 * context to complete the processing of the iocb. Note that while this
 * provides a task context for the dio_complete() callback, it should only be
 * used on the completion side for non-IO generating completions. It's fine to
 * call blocking functions from this callback, but they should not wait for
 * unrelated IO (like cache flushing, new IO generation, etc).
 */
#define IOCB_DIO_CALLER_COMP        (1 << 22)
/* kiocb is a read or write operation submitted by fs/aio.c. */
#define IOCB_AIO_RW                (1 << 23)

/* for use in trace events */
#define TRACE_IOCB_STRINGS \
        { IOCB_HIPRI,                "HIPRI" }, \
        { IOCB_DSYNC,                "DSYNC" }, \
        { IOCB_SYNC,                "SYNC" }, \
        { IOCB_NOWAIT,                "NOWAIT" }, \
        { IOCB_APPEND,                "APPEND" }, \
        { IOCB_ATOMIC,                "ATOMIC"}, \
        { IOCB_EVENTFD,                "EVENTFD"}, \
        { IOCB_DIRECT,                "DIRECT" }, \
        { IOCB_WRITE,                "WRITE" }, \
        { IOCB_WAITQ,                "WAITQ" }, \
        { IOCB_NOIO,                "NOIO" }, \
        { IOCB_ALLOC_CACHE,        "ALLOC_CACHE" }, \
        { IOCB_DIO_CALLER_COMP,        "CALLER_COMP" }

struct kiocb {
        struct file                *ki_filp;
        loff_t                        ki_pos;
        void (*ki_complete)(struct kiocb *iocb, long ret);
        void                        *private;
        int                        ki_flags;
        u16                        ki_ioprio; /* See linux/ioprio.h */
        union {
                /*
                 * Only used for async buffered reads, where it denotes the
                 * page waitqueue associated with completing the read. Valid
                 * IFF IOCB_WAITQ is set.
                 */
                struct wait_page_queue        *ki_waitq;
                /*
                 * Can be used for O_DIRECT IO, where the completion handling
                 * is punted back to the issuer of the IO. May only be set
                 * if IOCB_DIO_CALLER_COMP is set by the issuer, and the issuer
                 * must then check for presence of this handler when ki_complete
                 * is invoked. The data passed in to this handler must be
                 * assigned to ->private when dio_complete is assigned.
                 */
                ssize_t (*dio_complete)(void *data);
        };
};

static inline bool is_sync_kiocb(struct kiocb *kiocb)
{
        return kiocb->ki_complete == NULL;
}

struct address_space_operations {
        int (*writepage)(struct page *page, struct writeback_control *wbc);
        int (*read_folio)(struct file *, struct folio *);

        /* Write back some dirty pages from this mapping. */
        int (*writepages)(struct address_space *, struct writeback_control *);

        /* Mark a folio dirty.  Return true if this dirtied it */
        bool (*dirty_folio)(struct address_space *, struct folio *);

        void (*readahead)(struct readahead_control *);

        int (*write_begin)(struct file *, struct address_space *mapping,
                                loff_t pos, unsigned len,
                                struct folio **foliop, void **fsdata);
        int (*write_end)(struct file *, struct address_space *mapping,
                                loff_t pos, unsigned len, unsigned copied,
                                struct folio *folio, void *fsdata);

        /* Unfortunately this kludge is needed for FIBMAP. Don't use it */
        sector_t (*bmap)(struct address_space *, sector_t);
        void (*invalidate_folio) (struct folio *, size_t offset, size_t len);
        bool (*release_folio)(struct folio *, gfp_t);
        void (*free_folio)(struct folio *folio);
        ssize_t (*direct_IO)(struct kiocb *, struct iov_iter *iter);
        /*
         * migrate the contents of a folio to the specified target. If
         * migrate_mode is MIGRATE_ASYNC, it must not block.
         */
        int (*migrate_folio)(struct address_space *, struct folio *dst,
                        struct folio *src, enum migrate_mode);
        int (*launder_folio)(struct folio *);
        bool (*is_partially_uptodate) (struct folio *, size_t from,
                        size_t count);
        void (*is_dirty_writeback) (struct folio *, bool *dirty, bool *wb);
        int (*error_remove_folio)(struct address_space *, struct folio *);

        /* swapfile support */
        int (*swap_activate)(struct swap_info_struct *sis, struct file *file,
                                sector_t *span);
        void (*swap_deactivate)(struct file *file);
        int (*swap_rw)(struct kiocb *iocb, struct iov_iter *iter);
};

extern const struct address_space_operations empty_aops;

/**
 * struct address_space - Contents of a cacheable, mappable object.
 * @host: Owner, either the inode or the block_device.
 * @i_pages: Cached pages.
 * @invalidate_lock: Guards coherency between page cache contents and
 *   file offset->disk block mappings in the filesystem during invalidates.
 *   It is also used to block modification of page cache contents through
 *   memory mappings.
 * @gfp_mask: Memory allocation flags to use for allocating pages.
 * @i_mmap_writable: Number of VM_SHARED, VM_MAYWRITE mappings.
 * @nr_thps: Number of THPs in the pagecache (non-shmem only).
 * @i_mmap: Tree of private and shared mappings.
 * @i_mmap_rwsem: Protects @i_mmap and @i_mmap_writable.
 * @nrpages: Number of page entries, protected by the i_pages lock.
 * @writeback_index: Writeback starts here.
 * @a_ops: Methods.
 * @flags: Error bits and flags (AS_*).
 * @wb_err: The most recent error which has occurred.
 * @i_private_lock: For use by the owner of the address_space.
 * @i_private_list: For use by the owner of the address_space.
 * @i_private_data: For use by the owner of the address_space.
 */
struct address_space {
        struct inode                *host;
        struct xarray                i_pages;
        struct rw_semaphore        invalidate_lock;
        gfp_t                        gfp_mask;
        atomic_t                i_mmap_writable;
#ifdef CONFIG_READ_ONLY_THP_FOR_FS
        /* number of thp, only for non-shmem files */
        atomic_t                nr_thps;
#endif
        struct rb_root_cached        i_mmap;
        unsigned long                nrpages;
        pgoff_t                        writeback_index;
        const struct address_space_operations *a_ops;
        unsigned long                flags;
        errseq_t                wb_err;
        spinlock_t                i_private_lock;
        struct list_head        i_private_list;
        struct rw_semaphore        i_mmap_rwsem;
        void *                        i_private_data;
} __attribute__((aligned(sizeof(long)))) __randomize_layout;
        /*
         * On most architectures that alignment is already the case; but
         * must be enforced here for CRIS, to let the least significant bit
         * of struct page's "mapping" pointer be used for PAGE_MAPPING_ANON.
         */

/* XArray tags, for tagging dirty and writeback pages in the pagecache. */
#define PAGECACHE_TAG_DIRTY        XA_MARK_0
#define PAGECACHE_TAG_WRITEBACK        XA_MARK_1
#define PAGECACHE_TAG_TOWRITE        XA_MARK_2

/*
 * Returns true if any of the pages in the mapping are marked with the tag.
 */
static inline bool mapping_tagged(struct address_space *mapping, xa_mark_t tag)
{
        return xa_marked(&mapping->i_pages, tag);
}

static inline void i_mmap_lock_write(struct address_space *mapping)
{
        down_write(&mapping->i_mmap_rwsem);
}

static inline int i_mmap_trylock_write(struct address_space *mapping)
{
        return down_write_trylock(&mapping->i_mmap_rwsem);
}

static inline void i_mmap_unlock_write(struct address_space *mapping)
{
        up_write(&mapping->i_mmap_rwsem);
}

static inline int i_mmap_trylock_read(struct address_space *mapping)
{
        return down_read_trylock(&mapping->i_mmap_rwsem);
}

static inline void i_mmap_lock_read(struct address_space *mapping)
{
        down_read(&mapping->i_mmap_rwsem);
}

static inline void i_mmap_unlock_read(struct address_space *mapping)
{
        up_read(&mapping->i_mmap_rwsem);
}

static inline void i_mmap_assert_locked(struct address_space *mapping)
{
        lockdep_assert_held(&mapping->i_mmap_rwsem);
}

static inline void i_mmap_assert_write_locked(struct address_space *mapping)
{
        lockdep_assert_held_write(&mapping->i_mmap_rwsem);
}

/*
 * Might pages of this file be mapped into userspace?
 */
static inline int mapping_mapped(struct address_space *mapping)
{
        return        !RB_EMPTY_ROOT(&mapping->i_mmap.rb_root);
}

/*
 * Might pages of this file have been modified in userspace?
 * Note that i_mmap_writable counts all VM_SHARED, VM_MAYWRITE vmas: do_mmap
 * marks vma as VM_SHARED if it is shared, and the file was opened for
 * writing i.e. vma may be mprotected writable even if now readonly.
 *
 * If i_mmap_writable is negative, no new writable mappings are allowed. You
 * can only deny writable mappings, if none exists right now.
 */
static inline int mapping_writably_mapped(struct address_space *mapping)
{
        return atomic_read(&mapping->i_mmap_writable) > 0;
}

static inline int mapping_map_writable(struct address_space *mapping)
{
        return atomic_inc_unless_negative(&mapping->i_mmap_writable) ?
                0 : -EPERM;
}

static inline void mapping_unmap_writable(struct address_space *mapping)
{
        atomic_dec(&mapping->i_mmap_writable);
}

static inline int mapping_deny_writable(struct address_space *mapping)
{
        return atomic_dec_unless_positive(&mapping->i_mmap_writable) ?
                0 : -EBUSY;
}

static inline void mapping_allow_writable(struct address_space *mapping)
{
        atomic_inc(&mapping->i_mmap_writable);
}

/*
 * Use sequence counter to get consistent i_size on 32-bit processors.
 */
#if BITS_PER_LONG==32 && defined(CONFIG_SMP)
#include <linux/seqlock.h>
#define __NEED_I_SIZE_ORDERED
#define i_size_ordered_init(inode) seqcount_init(&inode->i_size_seqcount)
#else
#define i_size_ordered_init(inode) do { } while (0)
#endif

struct posix_acl;
#define ACL_NOT_CACHED ((void *)(-1))
/*
 * ACL_DONT_CACHE is for stacked filesystems, that rely on underlying fs to
 * cache the ACL.  This also means that ->get_inode_acl() can be called in RCU
 * mode with the LOOKUP_RCU flag.
 */
#define ACL_DONT_CACHE ((void *)(-3))

static inline struct posix_acl *
uncached_acl_sentinel(struct task_struct *task)
{
        return (void *)task + 1;
}

static inline bool
is_uncached_acl(struct posix_acl *acl)
{
        return (long)acl & 1;
}

#define IOP_FASTPERM        0x0001
#define IOP_LOOKUP        0x0002
#define IOP_NOFOLLOW        0x0004
#define IOP_XATTR        0x0008
#define IOP_DEFAULT_READLINK        0x0010
#define IOP_MGTIME        0x0020

/*
 * Keep mostly read-only and often accessed (especially for
 * the RCU path lookup and 'stat' data) fields at the beginning
 * of the 'struct inode'
 */
struct inode {
        umode_t                        i_mode;
        unsigned short                i_opflags;
        kuid_t                        i_uid;
        kgid_t                        i_gid;
        unsigned int                i_flags;

#ifdef CONFIG_FS_POSIX_ACL
        struct posix_acl        *i_acl;
        struct posix_acl        *i_default_acl;
#endif

        const struct inode_operations        *i_op;
        struct super_block        *i_sb;
        struct address_space        *i_mapping;

#ifdef CONFIG_SECURITY
        void                        *i_security;
#endif

        /* Stat data, not accessed from path walking */
        unsigned long                i_ino;
        /*
         * Filesystems may only read i_nlink directly.  They shall use the
         * following functions for modification:
         *
         *    (set|clear|inc|drop)_nlink
         *    inode_(inc|dec)_link_count
         */
        union {
                const unsigned int i_nlink;
                unsigned int __i_nlink;
        };
        dev_t                        i_rdev;
        loff_t                        i_size;
        time64_t                i_atime_sec;
        time64_t                i_mtime_sec;
        time64_t                i_ctime_sec;
        u32                        i_atime_nsec;
        u32                        i_mtime_nsec;
        u32                        i_ctime_nsec;
        u32                        i_generation;
        spinlock_t                i_lock;        /* i_blocks, i_bytes, maybe i_size */
        unsigned short          i_bytes;
        u8                        i_blkbits;
        enum rw_hint                i_write_hint;
        blkcnt_t                i_blocks;

#ifdef __NEED_I_SIZE_ORDERED
        seqcount_t                i_size_seqcount;
#endif

        /* Misc */
        u32                        i_state;
        /* 32-bit hole */
        struct rw_semaphore        i_rwsem;

        unsigned long                dirtied_when;        /* jiffies of first dirtying */
        unsigned long                dirtied_time_when;

        struct hlist_node        i_hash;
        struct list_head        i_io_list;        /* backing dev IO list */
#ifdef CONFIG_CGROUP_WRITEBACK
        struct bdi_writeback        *i_wb;                /* the associated cgroup wb */

        /* foreign inode detection, see wbc_detach_inode() */
        int                        i_wb_frn_winner;
        u16                        i_wb_frn_avg_time;
        u16                        i_wb_frn_history;
#endif
        struct list_head        i_lru;                /* inode LRU list */
        struct list_head        i_sb_list;
        struct list_head        i_wb_list;        /* backing dev writeback list */
        union {
                struct hlist_head        i_dentry;
                struct rcu_head                i_rcu;
        };
        atomic64_t                i_version;
        atomic64_t                i_sequence; /* see futex */
        atomic_t                i_count;
        atomic_t                i_dio_count;
        atomic_t                i_writecount;
#if defined(CONFIG_IMA) || defined(CONFIG_FILE_LOCKING)
        atomic_t                i_readcount; /* struct files open RO */
#endif
        union {
                const struct file_operations        *i_fop;        /* former ->i_op->default_file_ops */
                void (*free_inode)(struct inode *);
        };
        struct file_lock_context        *i_flctx;
        struct address_space        i_data;
        struct list_head        i_devices;
        union {
                struct pipe_inode_info        *i_pipe;
                struct cdev                *i_cdev;
                char                        *i_link;
                unsigned                i_dir_seq;
        };


#ifdef CONFIG_FSNOTIFY
        __u32                        i_fsnotify_mask; /* all events this inode cares about */
        /* 32-bit hole reserved for expanding i_fsnotify_mask */
        struct fsnotify_mark_connector __rcu        *i_fsnotify_marks;
#endif

#ifdef CONFIG_FS_ENCRYPTION
        struct fscrypt_inode_info        *i_crypt_info;
#endif

#ifdef CONFIG_FS_VERITY
        struct fsverity_info        *i_verity_info;
#endif

        void                        *i_private; /* fs or device private pointer */
} __randomize_layout;

/*
 * Get bit address from inode->i_state to use with wait_var_event()
 * infrastructre.
 */
#define inode_state_wait_address(inode, bit) ((char *)&(inode)->i_state + (bit))

struct wait_queue_head *inode_bit_waitqueue(struct wait_bit_queue_entry *wqe,
                                            struct inode *inode, u32 bit);

static inline void inode_wake_up_bit(struct inode *inode, u32 bit)
{
        /* Caller is responsible for correct memory barriers. */
        wake_up_var(inode_state_wait_address(inode, bit));
}

struct timespec64 timestamp_truncate(struct timespec64 t, struct inode *inode);

static inline unsigned int i_blocksize(const struct inode *node)
{
        return (1 << node->i_blkbits);
}

static inline int inode_unhashed(struct inode *inode)
{
        return hlist_unhashed(&inode->i_hash);
}

/*
 * __mark_inode_dirty expects inodes to be hashed.  Since we don't
 * want special inodes in the fileset inode space, we make them
 * appear hashed, but do not put on any lists.  hlist_del()
 * will work fine and require no locking.
 */
static inline void inode_fake_hash(struct inode *inode)
{
        hlist_add_fake(&inode->i_hash);
}

/*
 * inode->i_mutex nesting subclasses for the lock validator:
 *
 * 0: the object of the current VFS operation
 * 1: parent
 * 2: child/target
 * 3: xattr
 * 4: second non-directory
 * 5: second parent (when locking independent directories in rename)
 *
 * I_MUTEX_NONDIR2 is for certain operations (such as rename) which lock two
 * non-directories at once.
 *
 * The locking order between these classes is
 * parent[2] -> child -> grandchild -> normal -> xattr -> second non-directory
 */
enum inode_i_mutex_lock_class
{
        I_MUTEX_NORMAL,
        I_MUTEX_PARENT,
        I_MUTEX_CHILD,
        I_MUTEX_XATTR,
        I_MUTEX_NONDIR2,
        I_MUTEX_PARENT2,
};

static inline void inode_lock(struct inode *inode)
{
        down_write(&inode->i_rwsem);
}

static inline void inode_unlock(struct inode *inode)
{
        up_write(&inode->i_rwsem);
}

static inline void inode_lock_shared(struct inode *inode)
{
        down_read(&inode->i_rwsem);
}

static inline void inode_unlock_shared(struct inode *inode)
{
        up_read(&inode->i_rwsem);
}

static inline int inode_trylock(struct inode *inode)
{
        return down_write_trylock(&inode->i_rwsem);
}

static inline int inode_trylock_shared(struct inode *inode)
{
        return down_read_trylock(&inode->i_rwsem);
}

static inline int inode_is_locked(struct inode *inode)
{
        return rwsem_is_locked(&inode->i_rwsem);
}

static inline void inode_lock_nested(struct inode *inode, unsigned subclass)
{
        down_write_nested(&inode->i_rwsem, subclass);
}

static inline void inode_lock_shared_nested(struct inode *inode, unsigned subclass)
{
        down_read_nested(&inode->i_rwsem, subclass);
}

static inline void filemap_invalidate_lock(struct address_space *mapping)
{
        down_write(&mapping->invalidate_lock);
}

static inline void filemap_invalidate_unlock(struct address_space *mapping)
{
        up_write(&mapping->invalidate_lock);
}

static inline void filemap_invalidate_lock_shared(struct address_space *mapping)
{
        down_read(&mapping->invalidate_lock);
}

static inline int filemap_invalidate_trylock_shared(
                                        struct address_space *mapping)
{
        return down_read_trylock(&mapping->invalidate_lock);
}

static inline void filemap_invalidate_unlock_shared(
                                        struct address_space *mapping)
{
        up_read(&mapping->invalidate_lock);
}

void lock_two_nondirectories(struct inode *, struct inode*);
void unlock_two_nondirectories(struct inode *, struct inode*);

void filemap_invalidate_lock_two(struct address_space *mapping1,
                                 struct address_space *mapping2);
void filemap_invalidate_unlock_two(struct address_space *mapping1,
                                   struct address_space *mapping2);


/*
 * NOTE: in a 32bit arch with a preemptable kernel and
 * an UP compile the i_size_read/write must be atomic
 * with respect to the local cpu (unlike with preempt disabled),
 * but they don't need to be atomic with respect to other cpus like in
 * true SMP (so they need either to either locally disable irq around
 * the read or for example on x86 they can be still implemented as a
 * cmpxchg8b without the need of the lock prefix). For SMP compiles
 * and 64bit archs it makes no difference if preempt is enabled or not.
 */
static inline loff_t i_size_read(const struct inode *inode)
{
#if BITS_PER_LONG==32 && defined(CONFIG_SMP)
        loff_t i_size;
        unsigned int seq;

        do {
                seq = read_seqcount_begin(&inode->i_size_seqcount);
                i_size = inode->i_size;
        } while (read_seqcount_retry(&inode->i_size_seqcount, seq));
        return i_size;
#elif BITS_PER_LONG==32 && defined(CONFIG_PREEMPTION)
        loff_t i_size;

        preempt_disable();
        i_size = inode->i_size;
        preempt_enable();
        return i_size;
#else
        /* Pairs with smp_store_release() in i_size_write() */
        return smp_load_acquire(&inode->i_size);
#endif
}

/*
 * NOTE: unlike i_size_read(), i_size_write() does need locking around it
 * (normally i_mutex), otherwise on 32bit/SMP an update of i_size_seqcount
 * can be lost, resulting in subsequent i_size_read() calls spinning forever.
 */
static inline void i_size_write(struct inode *inode, loff_t i_size)
{
#if BITS_PER_LONG==32 && defined(CONFIG_SMP)
        preempt_disable();
        write_seqcount_begin(&inode->i_size_seqcount);
        inode->i_size = i_size;
        write_seqcount_end(&inode->i_size_seqcount);
        preempt_enable();
#elif BITS_PER_LONG==32 && defined(CONFIG_PREEMPTION)
        preempt_disable();
        inode->i_size = i_size;
        preempt_enable();
#else
        /*
         * Pairs with smp_load_acquire() in i_size_read() to ensure
         * changes related to inode size (such as page contents) are
         * visible before we see the changed inode size.
         */
        smp_store_release(&inode->i_size, i_size);
#endif
}

static inline unsigned iminor(const struct inode *inode)
{
        return MINOR(inode->i_rdev);
}

static inline unsigned imajor(const struct inode *inode)
{
        return MAJOR(inode->i_rdev);
}

struct fown_struct {
        struct file *file;        /* backpointer for security modules */
        rwlock_t lock;          /* protects pid, uid, euid fields */
        struct pid *pid;        /* pid or -pgrp where SIGIO should be sent */
        enum pid_type pid_type;        /* Kind of process group SIGIO should be sent to */
        kuid_t uid, euid;        /* uid/euid of process setting the owner */
        int signum;                /* posix.1b rt signal to be delivered on IO */
};

/**
 * struct file_ra_state - Track a file's readahead state.
 * @start: Where the most recent readahead started.
 * @size: Number of pages read in the most recent readahead.
 * @async_size: Numer of pages that were/are not needed immediately
 *      and so were/are genuinely "ahead".  Start next readahead when
 *      the first of these pages is accessed.
 * @ra_pages: Maximum size of a readahead request, copied from the bdi.
 * @mmap_miss: How many mmap accesses missed in the page cache.
 * @prev_pos: The last byte in the most recent read request.
 *
 * When this structure is passed to ->readahead(), the "most recent"
 * readahead means the current readahead.
 */
struct file_ra_state {
        pgoff_t start;
        unsigned int size;
        unsigned int async_size;
        unsigned int ra_pages;
        unsigned int mmap_miss;
        loff_t prev_pos;
};

/*
 * Check if @index falls in the readahead windows.
 */
static inline int ra_has_index(struct file_ra_state *ra, pgoff_t index)
{
        return (index >= ra->start &&
                index <  ra->start + ra->size);
}

/**
 * struct file - Represents a file
 * @f_ref: reference count
 * @f_lock: Protects f_ep, f_flags. Must not be taken from IRQ context.
 * @f_mode: FMODE_* flags often used in hotpaths
 * @f_op: file operations
 * @f_mapping: Contents of a cacheable, mappable object.
 * @private_data: filesystem or driver specific data
 * @f_inode: cached inode
 * @f_flags: file flags
 * @f_iocb_flags: iocb flags
 * @f_cred: stashed credentials of creator/opener
 * @f_path: path of the file
 * @f_pos_lock: lock protecting file position
 * @f_pipe: specific to pipes
 * @f_pos: file position
 * @f_security: LSM security context of this file
 * @f_owner: file owner
 * @f_wb_err: writeback error
 * @f_sb_err: per sb writeback errors
 * @f_ep: link of all epoll hooks for this file
 * @f_task_work: task work entry point
 * @f_llist: work queue entrypoint
 * @f_ra: file's readahead state
 * @f_freeptr: Pointer used by SLAB_TYPESAFE_BY_RCU file cache (don't touch.)
 */
struct file {
        file_ref_t                        f_ref;
        spinlock_t                        f_lock;
        fmode_t                                f_mode;
        const struct file_operations        *f_op;
        struct address_space                *f_mapping;
        void                                *private_data;
        struct inode                        *f_inode;
        unsigned int                        f_flags;
        unsigned int                        f_iocb_flags;
        const struct cred                *f_cred;
        /* --- cacheline 1 boundary (64 bytes) --- */
        struct path                        f_path;
        union {
                /* regular files (with FMODE_ATOMIC_POS) and directories */
                struct mutex                f_pos_lock;
                /* pipes */
                u64                        f_pipe;
        };
        loff_t                                f_pos;
#ifdef CONFIG_SECURITY
        void                                *f_security;
#endif
        /* --- cacheline 2 boundary (128 bytes) --- */
        struct fown_struct                *f_owner;
        errseq_t                        f_wb_err;
        errseq_t                        f_sb_err;
#ifdef CONFIG_EPOLL
        struct hlist_head                *f_ep;
#endif
        union {
                struct callback_head        f_task_work;
                struct llist_node        f_llist;
                struct file_ra_state        f_ra;
                freeptr_t                f_freeptr;
        };
        /* --- cacheline 3 boundary (192 bytes) --- */
} __randomize_layout
  __attribute__((aligned(4)));        /* lest something weird decides that 2 is OK */

struct file_handle {
        __u32 handle_bytes;
        int handle_type;
        /* file identifier */
        unsigned char f_handle[] __counted_by(handle_bytes);
};

static inline struct file *get_file(struct file *f)
{
        file_ref_inc(&f->f_ref);
        return f;
}

struct file *get_file_rcu(struct file __rcu **f);
struct file *get_file_active(struct file **f);

#define file_count(f)        file_ref_read(&(f)->f_ref)

#define        MAX_NON_LFS        ((1UL<<31) - 1)

/* Page cache limit. The filesystems should put that into their s_maxbytes 
   limits, otherwise bad things can happen in VM. */ 
#if BITS_PER_LONG==32
#define MAX_LFS_FILESIZE        ((loff_t)ULONG_MAX << PAGE_SHIFT)
#elif BITS_PER_LONG==64
#define MAX_LFS_FILESIZE         ((loff_t)LLONG_MAX)
#endif

/* legacy typedef, should eventually be removed */
typedef void *fl_owner_t;

struct file_lock;
struct file_lease;

/* The following constant reflects the upper bound of the file/locking space */
#ifndef OFFSET_MAX
#define OFFSET_MAX        type_max(loff_t)
#define OFFT_OFFSET_MAX        type_max(off_t)
#endif

int file_f_owner_allocate(struct file *file);
static inline struct fown_struct *file_f_owner(const struct file *file)
{
        return READ_ONCE(file->f_owner);
}

extern void send_sigio(struct fown_struct *fown, int fd, int band);

static inline struct inode *file_inode(const struct file *f)
{
        return f->f_inode;
}

/*
 * file_dentry() is a relic from the days that overlayfs was using files with a
 * "fake" path, meaning, f_path on overlayfs and f_inode on underlying fs.
 * In those days, file_dentry() was needed to get the underlying fs dentry that
 * matches f_inode.
 * Files with "fake" path should not exist nowadays, so use an assertion to make
 * sure that file_dentry() was not papering over filesystem bugs.
 */
static inline struct dentry *file_dentry(const struct file *file)
{
        struct dentry *dentry = file->f_path.dentry;

        WARN_ON_ONCE(d_inode(dentry) != file_inode(file));
        return dentry;
}

struct fasync_struct {
        rwlock_t                fa_lock;
        int                        magic;
        int                        fa_fd;
        struct fasync_struct        *fa_next; /* singly linked list */
        struct file                *fa_file;
        struct rcu_head                fa_rcu;
};

#define FASYNC_MAGIC 0x4601

/* SMP safe fasync helpers: */
extern int fasync_helper(int, struct file *, int, struct fasync_struct **);
extern struct fasync_struct *fasync_insert_entry(int, struct file *, struct fasync_struct **, struct fasync_struct *);
extern int fasync_remove_entry(struct file *, struct fasync_struct **);
extern struct fasync_struct *fasync_alloc(void);
extern void fasync_free(struct fasync_struct *);

/* can be called from interrupts */
extern void kill_fasync(struct fasync_struct **, int, int);

extern void __f_setown(struct file *filp, struct pid *, enum pid_type, int force);
extern int f_setown(struct file *filp, int who, int force);
extern void f_delown(struct file *filp);
extern pid_t f_getown(struct file *filp);
extern int send_sigurg(struct file *file);

/*
 * sb->s_flags.  Note that these mirror the equivalent MS_* flags where
 * represented in both.
 */
#define SB_RDONLY       BIT(0)        /* Mount read-only */
#define SB_NOSUID       BIT(1)        /* Ignore suid and sgid bits */
#define SB_NODEV        BIT(2)        /* Disallow access to device special files */
#define SB_NOEXEC       BIT(3)        /* Disallow program execution */
#define SB_SYNCHRONOUS  BIT(4)        /* Writes are synced at once */
#define SB_MANDLOCK     BIT(6)        /* Allow mandatory locks on an FS */
#define SB_DIRSYNC      BIT(7)        /* Directory modifications are synchronous */
#define SB_NOATIME      BIT(10)        /* Do not update access times. */
#define SB_NODIRATIME   BIT(11)        /* Do not update directory access times */
#define SB_SILENT       BIT(15)
#define SB_POSIXACL     BIT(16)        /* Supports POSIX ACLs */
#define SB_INLINECRYPT  BIT(17)        /* Use blk-crypto for encrypted files */
#define SB_KERNMOUNT    BIT(22)        /* this is a kern_mount call */
#define SB_I_VERSION    BIT(23)        /* Update inode I_version field */
#define SB_LAZYTIME     BIT(25)        /* Update the on-disk [acm]times lazily */

/* These sb flags are internal to the kernel */
#define SB_DEAD         BIT(21)
#define SB_DYING        BIT(24)
#define SB_SUBMOUNT     BIT(26)
#define SB_FORCE        BIT(27)
#define SB_NOSEC        BIT(28)
#define SB_BORN         BIT(29)
#define SB_ACTIVE       BIT(30)
#define SB_NOUSER       BIT(31)

/* These flags relate to encoding and casefolding */
#define SB_ENC_STRICT_MODE_FL        (1 << 0)

#define sb_has_strict_encoding(sb) \
        (sb->s_encoding_flags & SB_ENC_STRICT_MODE_FL)

/*
 *        Umount options
 */

#define MNT_FORCE        0x00000001        /* Attempt to forcibily umount */
#define MNT_DETACH        0x00000002        /* Just detach from the tree */
#define MNT_EXPIRE        0x00000004        /* Mark for expiry */
#define UMOUNT_NOFOLLOW        0x00000008        /* Don't follow symlink on umount */
#define UMOUNT_UNUSED        0x80000000        /* Flag guaranteed to be unused */

/* sb->s_iflags */
#define SB_I_CGROUPWB        0x00000001        /* cgroup-aware writeback enabled */
#define SB_I_NOEXEC        0x00000002        /* Ignore executables on this fs */
#define SB_I_NODEV        0x00000004        /* Ignore devices on this fs */
#define SB_I_STABLE_WRITES 0x00000008        /* don't modify blks until WB is done */

/* sb->s_iflags to limit user namespace mounts */
#define SB_I_USERNS_VISIBLE                0x00000010 /* fstype already mounted */
#define SB_I_IMA_UNVERIFIABLE_SIGNATURE        0x00000020
#define SB_I_UNTRUSTED_MOUNTER                0x00000040
#define SB_I_EVM_HMAC_UNSUPPORTED        0x00000080

#define SB_I_SKIP_SYNC        0x00000100        /* Skip superblock at global sync */
#define SB_I_PERSB_BDI        0x00000200        /* has a per-sb bdi */
#define SB_I_TS_EXPIRY_WARNED 0x00000400 /* warned about timestamp range expiry */
#define SB_I_RETIRED        0x00000800        /* superblock shouldn't be reused */
#define SB_I_NOUMASK        0x00001000        /* VFS does not apply umask */
#define SB_I_NOIDMAP        0x00002000        /* No idmapped mounts on this superblock */

/* Possible states of 'frozen' field */
enum {
        SB_UNFROZEN = 0,                /* FS is unfrozen */
        SB_FREEZE_WRITE        = 1,                /* Writes, dir ops, ioctls frozen */
        SB_FREEZE_PAGEFAULT = 2,        /* Page faults stopped as well */
        SB_FREEZE_FS = 3,                /* For internal FS use (e.g. to stop
                                         * internal threads if needed) */
        SB_FREEZE_COMPLETE = 4,                /* ->freeze_fs finished successfully */
};

#define SB_FREEZE_LEVELS (SB_FREEZE_COMPLETE - 1)

struct sb_writers {
        unsigned short                        frozen;                /* Is sb frozen? */
        int                                freeze_kcount;        /* How many kernel freeze requests? */
        int                                freeze_ucount;        /* How many userspace freeze requests? */
        struct percpu_rw_semaphore        rw_sem[SB_FREEZE_LEVELS];
};

struct super_block {
        struct list_head        s_list;                /* Keep this first */
        dev_t                        s_dev;                /* search index; _not_ kdev_t */
        unsigned char                s_blocksize_bits;
        unsigned long                s_blocksize;
        loff_t                        s_maxbytes;        /* Max file size */
        struct file_system_type        *s_type;
        const struct super_operations        *s_op;
        const struct dquot_operations        *dq_op;
        const struct quotactl_ops        *s_qcop;
        const struct export_operations *s_export_op;
        unsigned long                s_flags;
        unsigned long                s_iflags;        /* internal SB_I_* flags */
        unsigned long                s_magic;
        struct dentry                *s_root;
        struct rw_semaphore        s_umount;
        int                        s_count;
        atomic_t                s_active;
#ifdef CONFIG_SECURITY
        void                    *s_security;
#endif
        const struct xattr_handler * const *s_xattr;
#ifdef CONFIG_FS_ENCRYPTION
        const struct fscrypt_operations        *s_cop;
        struct fscrypt_keyring        *s_master_keys; /* master crypto keys in use */
#endif
#ifdef CONFIG_FS_VERITY
        const struct fsverity_operations *s_vop;
#endif
#if IS_ENABLED(CONFIG_UNICODE)
        struct unicode_map *s_encoding;
        __u16 s_encoding_flags;
#endif
        struct hlist_bl_head        s_roots;        /* alternate root dentries for NFS */
        struct list_head        s_mounts;        /* list of mounts; _not_ for fs use */
        struct block_device        *s_bdev;        /* can go away once we use an accessor for @s_bdev_file */
        struct file                *s_bdev_file;
        struct backing_dev_info *s_bdi;
        struct mtd_info                *s_mtd;
        struct hlist_node        s_instances;
        unsigned int                s_quota_types;        /* Bitmask of supported quota types */
        struct quota_info        s_dquot;        /* Diskquota specific options */

        struct sb_writers        s_writers;

        /*
         * Keep s_fs_info, s_time_gran, s_fsnotify_mask, and
         * s_fsnotify_info together for cache efficiency. They are frequently
         * accessed and rarely modified.
         */
        void                        *s_fs_info;        /* Filesystem private info */

        /* Granularity of c/m/atime in ns (cannot be worse than a second) */
        u32                        s_time_gran;
        /* Time limits for c/m/atime in seconds */
        time64_t                   s_time_min;
        time64_t                   s_time_max;
#ifdef CONFIG_FSNOTIFY
        u32                        s_fsnotify_mask;
        struct fsnotify_sb_info        *s_fsnotify_info;
#endif

        /*
         * q: why are s_id and s_sysfs_name not the same? both are human
         * readable strings that identify the filesystem
         * a: s_id is allowed to change at runtime; it's used in log messages,
         * and we want to when a device starts out as single device (s_id is dev
         * name) but then a device is hot added and we have to switch to
         * identifying it by UUID
         * but s_sysfs_name is a handle for programmatic access, and can't
         * change at runtime
         */
        char                        s_id[32];        /* Informational name */
        uuid_t                        s_uuid;                /* UUID */
        u8                        s_uuid_len;        /* Default 16, possibly smaller for weird filesystems */

        /* if set, fs shows up under sysfs at /sys/fs/$FSTYP/s_sysfs_name */
        char                        s_sysfs_name[UUID_STRING_LEN + 1];

        unsigned int                s_max_links;

        /*
         * The next field is for VFS *only*. No filesystems have any business
         * even looking at it. You had been warned.
         */
        struct mutex s_vfs_rename_mutex;        /* Kludge */

        /*
         * Filesystem subtype.  If non-empty the filesystem type field
         * in /proc/mounts will be "type.subtype"
         */
        const char *s_subtype;

        const struct dentry_operations *s_d_op; /* default d_op for dentries */

        struct shrinker *s_shrink;        /* per-sb shrinker handle */

        /* Number of inodes with nlink == 0 but still referenced */
        atomic_long_t s_remove_count;

        /* Read-only state of the superblock is being changed */
        int s_readonly_remount;

        /* per-sb errseq_t for reporting writeback errors via syncfs */
        errseq_t s_wb_err;

        /* AIO completions deferred from interrupt context */
        struct workqueue_struct *s_dio_done_wq;
        struct hlist_head s_pins;

        /*
         * Owning user namespace and default context in which to
         * interpret filesystem uids, gids, quotas, device nodes,
         * xattrs and security labels.
         */
        struct user_namespace *s_user_ns;

        /*
         * The list_lru structure is essentially just a pointer to a table
         * of per-node lru lists, each of which has its own spinlock.
         * There is no need to put them into separate cachelines.
         */
        struct list_lru                s_dentry_lru;
        struct list_lru                s_inode_lru;
        struct rcu_head                rcu;
        struct work_struct        destroy_work;

        struct mutex                s_sync_lock;        /* sync serialisation lock */

        /*
         * Indicates how deep in a filesystem stack this SB is
         */
        int s_stack_depth;

        /* s_inode_list_lock protects s_inodes */
        spinlock_t                s_inode_list_lock ____cacheline_aligned_in_smp;
        struct list_head        s_inodes;        /* all inodes */

        spinlock_t                s_inode_wblist_lock;
        struct list_head        s_inodes_wb;        /* writeback inodes */
} __randomize_layout;

static inline struct user_namespace *i_user_ns(const struct inode *inode)
{
        return inode->i_sb->s_user_ns;
}

/* Helper functions so that in most cases filesystems will
 * not need to deal directly with kuid_t and kgid_t and can
 * instead deal with the raw numeric values that are stored
 * in the filesystem.
 */
static inline uid_t i_uid_read(const struct inode *inode)
{
        return from_kuid(i_user_ns(inode), inode->i_uid);
}

static inline gid_t i_gid_read(const struct inode *inode)
{
        return from_kgid(i_user_ns(inode), inode->i_gid);
}

static inline void i_uid_write(struct inode *inode, uid_t uid)
{
        inode->i_uid = make_kuid(i_user_ns(inode), uid);
}

static inline void i_gid_write(struct inode *inode, gid_t gid)
{
        inode->i_gid = make_kgid(i_user_ns(inode), gid);
}

/**
 * i_uid_into_vfsuid - map an inode's i_uid down according to an idmapping
 * @idmap: idmap of the mount the inode was found from
 * @inode: inode to map
 *
 * Return: whe inode's i_uid mapped down according to @idmap.
 * If the inode's i_uid has no mapping INVALID_VFSUID is returned.
 */
static inline vfsuid_t i_uid_into_vfsuid(struct mnt_idmap *idmap,
                                         const struct inode *inode)
{
        return make_vfsuid(idmap, i_user_ns(inode), inode->i_uid);
}

/**
 * i_uid_needs_update - check whether inode's i_uid needs to be updated
 * @idmap: idmap of the mount the inode was found from
 * @attr: the new attributes of @inode
 * @inode: the inode to update
 *
 * Check whether the $inode's i_uid field needs to be updated taking idmapped
 * mounts into account if the filesystem supports it.
 *
 * Return: true if @inode's i_uid field needs to be updated, false if not.
 */
static inline bool i_uid_needs_update(struct mnt_idmap *idmap,
                                      const struct iattr *attr,
                                      const struct inode *inode)
{
        return ((attr->ia_valid & ATTR_UID) &&
                !vfsuid_eq(attr->ia_vfsuid,
                           i_uid_into_vfsuid(idmap, inode)));
}

/**
 * i_uid_update - update @inode's i_uid field
 * @idmap: idmap of the mount the inode was found from
 * @attr: the new attributes of @inode
 * @inode: the inode to update
 *
 * Safely update @inode's i_uid field translating the vfsuid of any idmapped
 * mount into the filesystem kuid.
 */
static inline void i_uid_update(struct mnt_idmap *idmap,
                                const struct iattr *attr,
                                struct inode *inode)
{
        if (attr->ia_valid & ATTR_UID)
                inode->i_uid = from_vfsuid(idmap, i_user_ns(inode),
                                           attr->ia_vfsuid);
}

/**
 * i_gid_into_vfsgid - map an inode's i_gid down according to an idmapping
 * @idmap: idmap of the mount the inode was found from
 * @inode: inode to map
 *
 * Return: the inode's i_gid mapped down according to @idmap.
 * If the inode's i_gid has no mapping INVALID_VFSGID is returned.
 */
static inline vfsgid_t i_gid_into_vfsgid(struct mnt_idmap *idmap,
                                         const struct inode *inode)
{
        return make_vfsgid(idmap, i_user_ns(inode), inode->i_gid);
}

/**
 * i_gid_needs_update - check whether inode's i_gid needs to be updated
 * @idmap: idmap of the mount the inode was found from
 * @attr: the new attributes of @inode
 * @inode: the inode to update
 *
 * Check whether the $inode's i_gid field needs to be updated taking idmapped
 * mounts into account if the filesystem supports it.
 *
 * Return: true if @inode's i_gid field needs to be updated, false if not.
 */
static inline bool i_gid_needs_update(struct mnt_idmap *idmap,
                                      const struct iattr *attr,
                                      const struct inode *inode)
{
        return ((attr->ia_valid & ATTR_GID) &&
                !vfsgid_eq(attr->ia_vfsgid,
                           i_gid_into_vfsgid(idmap, inode)));
}

/**
 * i_gid_update - update @inode's i_gid field
 * @idmap: idmap of the mount the inode was found from
 * @attr: the new attributes of @inode
 * @inode: the inode to update
 *
 * Safely update @inode's i_gid field translating the vfsgid of any idmapped
 * mount into the filesystem kgid.
 */
static inline void i_gid_update(struct mnt_idmap *idmap,
                                const struct iattr *attr,
                                struct inode *inode)
{
        if (attr->ia_valid & ATTR_GID)
                inode->i_gid = from_vfsgid(idmap, i_user_ns(inode),
                                           attr->ia_vfsgid);
}

/**
 * inode_fsuid_set - initialize inode's i_uid field with callers fsuid
 * @inode: inode to initialize
 * @idmap: idmap of the mount the inode was found from
 *
 * Initialize the i_uid field of @inode. If the inode was found/created via
 * an idmapped mount map the caller's fsuid according to @idmap.
 */
static inline void inode_fsuid_set(struct inode *inode,
                                   struct mnt_idmap *idmap)
{
        inode->i_uid = mapped_fsuid(idmap, i_user_ns(inode));
}

/**
 * inode_fsgid_set - initialize inode's i_gid field with callers fsgid
 * @inode: inode to initialize
 * @idmap: idmap of the mount the inode was found from
 *
 * Initialize the i_gid field of @inode. If the inode was found/created via
 * an idmapped mount map the caller's fsgid according to @idmap.
 */
static inline void inode_fsgid_set(struct inode *inode,
                                   struct mnt_idmap *idmap)
{
        inode->i_gid = mapped_fsgid(idmap, i_user_ns(inode));
}

/**
 * fsuidgid_has_mapping() - check whether caller's fsuid/fsgid is mapped
 * @sb: the superblock we want a mapping in
 * @idmap: idmap of the relevant mount
 *
 * Check whether the caller's fsuid and fsgid have a valid mapping in the
 * s_user_ns of the superblock @sb. If the caller is on an idmapped mount map
 * the caller's fsuid and fsgid according to the @idmap first.
 *
 * Return: true if fsuid and fsgid is mapped, false if not.
 */
static inline bool fsuidgid_has_mapping(struct super_block *sb,
                                        struct mnt_idmap *idmap)
{
        struct user_namespace *fs_userns = sb->s_user_ns;
        kuid_t kuid;
        kgid_t kgid;

        kuid = mapped_fsuid(idmap, fs_userns);
        if (!uid_valid(kuid))
                return false;
        kgid = mapped_fsgid(idmap, fs_userns);
        if (!gid_valid(kgid))
                return false;
        return kuid_has_mapping(fs_userns, kuid) &&
               kgid_has_mapping(fs_userns, kgid);
}

struct timespec64 current_time(struct inode *inode);
struct timespec64 inode_set_ctime_current(struct inode *inode);
struct timespec64 inode_set_ctime_deleg(struct inode *inode,
                                        struct timespec64 update);

static inline time64_t inode_get_atime_sec(const struct inode *inode)
{
        return inode->i_atime_sec;
}

static inline long inode_get_atime_nsec(const struct inode *inode)
{
        return inode->i_atime_nsec;
}

static inline struct timespec64 inode_get_atime(const struct inode *inode)
{
        struct timespec64 ts = { .tv_sec  = inode_get_atime_sec(inode),
                                 .tv_nsec = inode_get_atime_nsec(inode) };

        return ts;
}

static inline struct timespec64 inode_set_atime_to_ts(struct inode *inode,
                                                      struct timespec64 ts)
{
        inode->i_atime_sec = ts.tv_sec;
        inode->i_atime_nsec = ts.tv_nsec;
        return ts;
}

static inline struct timespec64 inode_set_atime(struct inode *inode,
                                                time64_t sec, long nsec)
{
        struct timespec64 ts = { .tv_sec  = sec,
                                 .tv_nsec = nsec };

        return inode_set_atime_to_ts(inode, ts);
}

static inline time64_t inode_get_mtime_sec(const struct inode *inode)
{
        return inode->i_mtime_sec;
}

static inline long inode_get_mtime_nsec(const struct inode *inode)
{
        return inode->i_mtime_nsec;
}

static inline struct timespec64 inode_get_mtime(const struct inode *inode)
{
        struct timespec64 ts = { .tv_sec  = inode_get_mtime_sec(inode),
                                 .tv_nsec = inode_get_mtime_nsec(inode) };
        return ts;
}

static inline struct timespec64 inode_set_mtime_to_ts(struct inode *inode,
                                                      struct timespec64 ts)
{
        inode->i_mtime_sec = ts.tv_sec;
        inode->i_mtime_nsec = ts.tv_nsec;
        return ts;
}

static inline struct timespec64 inode_set_mtime(struct inode *inode,
                                                time64_t sec, long nsec)
{
        struct timespec64 ts = { .tv_sec  = sec,
                                 .tv_nsec = nsec };
        return inode_set_mtime_to_ts(inode, ts);
}

/*
 * Multigrain timestamps
 *
 * Conditionally use fine-grained ctime and mtime timestamps when there
 * are users actively observing them via getattr. The primary use-case
 * for this is NFS clients that use the ctime to distinguish between
 * different states of the file, and that are often fooled by multiple
 * operations that occur in the same coarse-grained timer tick.
 */
#define I_CTIME_QUERIED                ((u32)BIT(31))

static inline time64_t inode_get_ctime_sec(const struct inode *inode)
{
        return inode->i_ctime_sec;
}

static inline long inode_get_ctime_nsec(const struct inode *inode)
{
        return inode->i_ctime_nsec & ~I_CTIME_QUERIED;
}

static inline struct timespec64 inode_get_ctime(const struct inode *inode)
{
        struct timespec64 ts = { .tv_sec  = inode_get_ctime_sec(inode),
                                 .tv_nsec = inode_get_ctime_nsec(inode) };

        return ts;
}

struct timespec64 inode_set_ctime_to_ts(struct inode *inode, struct timespec64 ts);

/**
 * inode_set_ctime - set the ctime in the inode
 * @inode: inode in which to set the ctime
 * @sec: tv_sec value to set
 * @nsec: tv_nsec value to set
 *
 * Set the ctime in @inode to { @sec, @nsec }
 */
static inline struct timespec64 inode_set_ctime(struct inode *inode,
                                                time64_t sec, long nsec)
{
        struct timespec64 ts = { .tv_sec  = sec,
                                 .tv_nsec = nsec };

        return inode_set_ctime_to_ts(inode, ts);
}

struct timespec64 simple_inode_init_ts(struct inode *inode);

/*
 * Snapshotting support.
 */

/*
 * These are internal functions, please use sb_start_{write,pagefault,intwrite}
 * instead.
 */
static inline void __sb_end_write(struct super_block *sb, int level)
{
        percpu_up_read(sb->s_writers.rw_sem + level-1);
}

static inline void __sb_start_write(struct super_block *sb, int level)
{
        percpu_down_read(sb->s_writers.rw_sem + level - 1);
}

static inline bool __sb_start_write_trylock(struct super_block *sb, int level)
{
        return percpu_down_read_trylock(sb->s_writers.rw_sem + level - 1);
}

#define __sb_writers_acquired(sb, lev)        \
        percpu_rwsem_acquire(&(sb)->s_writers.rw_sem[(lev)-1], 1, _THIS_IP_)
#define __sb_writers_release(sb, lev)        \
        percpu_rwsem_release(&(sb)->s_writers.rw_sem[(lev)-1], _THIS_IP_)

/**
 * __sb_write_started - check if sb freeze level is held
 * @sb: the super we write to
 * @level: the freeze level
 *
 * * > 0 - sb freeze level is held
 * *   0 - sb freeze level is not held
 * * < 0 - !CONFIG_LOCKDEP/LOCK_STATE_UNKNOWN
 */
static inline int __sb_write_started(const struct super_block *sb, int level)
{
        return lockdep_is_held_type(sb->s_writers.rw_sem + level - 1, 1);
}

/**
 * sb_write_started - check if SB_FREEZE_WRITE is held
 * @sb: the super we write to
 *
 * May be false positive with !CONFIG_LOCKDEP/LOCK_STATE_UNKNOWN.
 */
static inline bool sb_write_started(const struct super_block *sb)
{
        return __sb_write_started(sb, SB_FREEZE_WRITE);
}

/**
 * sb_write_not_started - check if SB_FREEZE_WRITE is not held
 * @sb: the super we write to
 *
 * May be false positive with !CONFIG_LOCKDEP/LOCK_STATE_UNKNOWN.
 */
static inline bool sb_write_not_started(const struct super_block *sb)
{
        return __sb_write_started(sb, SB_FREEZE_WRITE) <= 0;
}

/**
 * file_write_started - check if SB_FREEZE_WRITE is held
 * @file: the file we write to
 *
 * May be false positive with !CONFIG_LOCKDEP/LOCK_STATE_UNKNOWN.
 * May be false positive with !S_ISREG, because file_start_write() has
 * no effect on !S_ISREG.
 */
static inline bool file_write_started(const struct file *file)
{
        if (!S_ISREG(file_inode(file)->i_mode))
                return true;
        return sb_write_started(file_inode(file)->i_sb);
}

/**
 * file_write_not_started - check if SB_FREEZE_WRITE is not held
 * @file: the file we write to
 *
 * May be false positive with !CONFIG_LOCKDEP/LOCK_STATE_UNKNOWN.
 * May be false positive with !S_ISREG, because file_start_write() has
 * no effect on !S_ISREG.
 */
static inline bool file_write_not_started(const struct file *file)
{
        if (!S_ISREG(file_inode(file)->i_mode))
                return true;
        return sb_write_not_started(file_inode(file)->i_sb);
}

/**
 * sb_end_write - drop write access to a superblock
 * @sb: the super we wrote to
 *
 * Decrement number of writers to the filesystem. Wake up possible waiters
 * wanting to freeze the filesystem.
 */
static inline void sb_end_write(struct super_block *sb)
{
        __sb_end_write(sb, SB_FREEZE_WRITE);
}

/**
 * sb_end_pagefault - drop write access to a superblock from a page fault
 * @sb: the super we wrote to
 *
 * Decrement number of processes handling write page fault to the filesystem.
 * Wake up possible waiters wanting to freeze the filesystem.
 */
static inline void sb_end_pagefault(struct super_block *sb)
{
        __sb_end_write(sb, SB_FREEZE_PAGEFAULT);
}

/**
 * sb_end_intwrite - drop write access to a superblock for internal fs purposes
 * @sb: the super we wrote to
 *
 * Decrement fs-internal number of writers to the filesystem.  Wake up possible
 * waiters wanting to freeze the filesystem.
 */
static inline void sb_end_intwrite(struct super_block *sb)
{
        __sb_end_write(sb, SB_FREEZE_FS);
}

/**
 * sb_start_write - get write access to a superblock
 * @sb: the super we write to
 *
 * When a process wants to write data or metadata to a file system (i.e. dirty
 * a page or an inode), it should embed the operation in a sb_start_write() -
 * sb_end_write() pair to get exclusion against file system freezing. This
 * function increments number of writers preventing freezing. If the file
 * system is already frozen, the function waits until the file system is
 * thawed.
 *
 * Since freeze protection behaves as a lock, users have to preserve
 * ordering of freeze protection and other filesystem locks. Generally,
 * freeze protection should be the outermost lock. In particular, we have:
 *
 * sb_start_write
 *   -> i_mutex                        (write path, truncate, directory ops, ...)
 *   -> s_umount                (freeze_super, thaw_super)
 */
static inline void sb_start_write(struct super_block *sb)
{
        __sb_start_write(sb, SB_FREEZE_WRITE);
}

static inline bool sb_start_write_trylock(struct super_block *sb)
{
        return __sb_start_write_trylock(sb, SB_FREEZE_WRITE);
}

/**
 * sb_start_pagefault - get write access to a superblock from a page fault
 * @sb: the super we write to
 *
 * When a process starts handling write page fault, it should embed the
 * operation into sb_start_pagefault() - sb_end_pagefault() pair to get
 * exclusion against file system freezing. This is needed since the page fault
 * is going to dirty a page. This function increments number of running page
 * faults preventing freezing. If the file system is already frozen, the
 * function waits until the file system is thawed.
 *
 * Since page fault freeze protection behaves as a lock, users have to preserve
 * ordering of freeze protection and other filesystem locks. It is advised to
 * put sb_start_pagefault() close to mmap_lock in lock ordering. Page fault
 * handling code implies lock dependency:
 *
 * mmap_lock
 *   -> sb_start_pagefault
 */
static inline void sb_start_pagefault(struct super_block *sb)
{
        __sb_start_write(sb, SB_FREEZE_PAGEFAULT);
}

/**
 * sb_start_intwrite - get write access to a superblock for internal fs purposes
 * @sb: the super we write to
 *
 * This is the third level of protection against filesystem freezing. It is
 * free for use by a filesystem. The only requirement is that it must rank
 * below sb_start_pagefault.
 *
 * For example filesystem can call sb_start_intwrite() when starting a
 * transaction which somewhat eases handling of freezing for internal sources
 * of filesystem changes (internal fs threads, discarding preallocation on file
 * close, etc.).
 */
static inline void sb_start_intwrite(struct super_block *sb)
{
        __sb_start_write(sb, SB_FREEZE_FS);
}

static inline bool sb_start_intwrite_trylock(struct super_block *sb)
{
        return __sb_start_write_trylock(sb, SB_FREEZE_FS);
}

bool inode_owner_or_capable(struct mnt_idmap *idmap,
                            const struct inode *inode);

/*
 * VFS helper functions..
 */
int vfs_create(struct mnt_idmap *, struct inode *,
               struct dentry *, umode_t, bool);
int vfs_mkdir(struct mnt_idmap *, struct inode *,
              struct dentry *, umode_t);
int vfs_mknod(struct mnt_idmap *, struct inode *, struct dentry *,
              umode_t, dev_t);
int vfs_symlink(struct mnt_idmap *, struct inode *,
                struct dentry *, const char *);
int vfs_link(struct dentry *, struct mnt_idmap *, struct inode *,
             struct dentry *, struct inode **);
int vfs_rmdir(struct mnt_idmap *, struct inode *, struct dentry *);
int vfs_unlink(struct mnt_idmap *, struct inode *, struct dentry *,
               struct inode **);

/**
 * struct renamedata - contains all information required for renaming
 * @old_mnt_idmap:     idmap of the old mount the inode was found from
 * @old_dir:           parent of source
 * @old_dentry:                source
 * @new_mnt_idmap:     idmap of the new mount the inode was found from
 * @new_dir:           parent of destination
 * @new_dentry:                destination
 * @delegated_inode:   returns an inode needing a delegation break
 * @flags:             rename flags
 */
struct renamedata {
        struct mnt_idmap *old_mnt_idmap;
        struct inode *old_dir;
        struct dentry *old_dentry;
        struct mnt_idmap *new_mnt_idmap;
        struct inode *new_dir;
        struct dentry *new_dentry;
        struct inode **delegated_inode;
        unsigned int flags;
} __randomize_layout;

int vfs_rename(struct renamedata *);

static inline int vfs_whiteout(struct mnt_idmap *idmap,
                               struct inode *dir, struct dentry *dentry)
{
        return vfs_mknod(idmap, dir, dentry, S_IFCHR | WHITEOUT_MODE,
                         WHITEOUT_DEV);
}

struct file *kernel_tmpfile_open(struct mnt_idmap *idmap,
                                 const struct path *parentpath,
                                 umode_t mode, int open_flag,
                                 const struct cred *cred);
struct file *kernel_file_open(const struct path *path, int flags,
                              const struct cred *cred);

int vfs_mkobj(struct dentry *, umode_t,
                int (*f)(struct dentry *, umode_t, void *),
                void *);

int vfs_fchown(struct file *file, uid_t user, gid_t group);
int vfs_fchmod(struct file *file, umode_t mode);
int vfs_utimes(const struct path *path, struct timespec64 *times);

extern long vfs_ioctl(struct file *file, unsigned int cmd, unsigned long arg);

#ifdef CONFIG_COMPAT
extern long compat_ptr_ioctl(struct file *file, unsigned int cmd,
                                        unsigned long arg);
#else
#define compat_ptr_ioctl NULL
#endif

/*
 * VFS file helper functions.
 */
void inode_init_owner(struct mnt_idmap *idmap, struct inode *inode,
                      const struct inode *dir, umode_t mode);
extern bool may_open_dev(const struct path *path);
umode_t mode_strip_sgid(struct mnt_idmap *idmap,
                        const struct inode *dir, umode_t mode);
bool in_group_or_capable(struct mnt_idmap *idmap,
                         const struct inode *inode, vfsgid_t vfsgid);

/*
 * This is the "filldir" function type, used by readdir() to let
 * the kernel specify what kind of dirent layout it wants to have.
 * This allows the kernel to read directories into kernel space or
 * to have different dirent layouts depending on the binary type.
 * Return 'true' to keep going and 'false' if there are no more entries.
 */
struct dir_context;
typedef bool (*filldir_t)(struct dir_context *, const char *, int, loff_t, u64,
                         unsigned);

struct dir_context {
        filldir_t actor;
        loff_t pos;
};

/*
 * These flags let !MMU mmap() govern direct device mapping vs immediate
 * copying more easily for MAP_PRIVATE, especially for ROM filesystems.
 *
 * NOMMU_MAP_COPY:        Copy can be mapped (MAP_PRIVATE)
 * NOMMU_MAP_DIRECT:        Can be mapped directly (MAP_SHARED)
 * NOMMU_MAP_READ:        Can be mapped for reading
 * NOMMU_MAP_WRITE:        Can be mapped for writing
 * NOMMU_MAP_EXEC:        Can be mapped for execution
 */
#define NOMMU_MAP_COPY                0x00000001
#define NOMMU_MAP_DIRECT        0x00000008
#define NOMMU_MAP_READ                VM_MAYREAD
#define NOMMU_MAP_WRITE                VM_MAYWRITE
#define NOMMU_MAP_EXEC                VM_MAYEXEC

#define NOMMU_VMFLAGS \
        (NOMMU_MAP_READ | NOMMU_MAP_WRITE | NOMMU_MAP_EXEC)

/*
 * These flags control the behavior of the remap_file_range function pointer.
 * If it is called with len == 0 that means "remap to end of source file".
 * See Documentation/filesystems/vfs.rst for more details about this call.
 *
 * REMAP_FILE_DEDUP: only remap if contents identical (i.e. deduplicate)
 * REMAP_FILE_CAN_SHORTEN: caller can handle a shortened request
 */
#define REMAP_FILE_DEDUP                (1 << 0)
#define REMAP_FILE_CAN_SHORTEN                (1 << 1)

/*
 * These flags signal that the caller is ok with altering various aspects of
 * the behavior of the remap operation.  The changes must be made by the
 * implementation; the vfs remap helper functions can take advantage of them.
 * Flags in this category exist to preserve the quirky behavior of the hoisted
 * btrfs clone/dedupe ioctls.
 */
#define REMAP_FILE_ADVISORY                (REMAP_FILE_CAN_SHORTEN)

/*
 * These flags control the behavior of vfs_copy_file_range().
 * They are not available to the user via syscall.
 *
 * COPY_FILE_SPLICE: call splice direct instead of fs clone/copy ops
 */
#define COPY_FILE_SPLICE                (1 << 0)

struct iov_iter;
struct io_uring_cmd;
struct offset_ctx;

typedef unsigned int __bitwise fop_flags_t;

struct file_operations {
        struct module *owner;
        fop_flags_t fop_flags;
        loff_t (*llseek) (struct file *, loff_t, int);
        ssize_t (*read) (struct file *, char __user *, size_t, loff_t *);
        ssize_t (*write) (struct file *, const char __user *, size_t, loff_t *);
        ssize_t (*read_iter) (struct kiocb *, struct iov_iter *);
        ssize_t (*write_iter) (struct kiocb *, struct iov_iter *);
        int (*iopoll)(struct kiocb *kiocb, struct io_comp_batch *,
                        unsigned int flags);
        int (*iterate_shared) (struct file *, struct dir_context *);
        __poll_t (*poll) (struct file *, struct poll_table_struct *);
        long (*unlocked_ioctl) (struct file *, unsigned int, unsigned long);
        long (*compat_ioctl) (struct file *, unsigned int, unsigned long);
        int (*mmap) (struct file *, struct vm_area_struct *);
        int (*open) (struct inode *, struct file *);
        int (*flush) (struct file *, fl_owner_t id);
        int (*release) (struct inode *, struct file *);
        int (*fsync) (struct file *, loff_t, loff_t, int datasync);
        int (*fasync) (int, struct file *, int);
        int (*lock) (struct file *, int, struct file_lock *);
        unsigned long (*get_unmapped_area)(struct file *, unsigned long, unsigned long, unsigned long, unsigned long);
        int (*check_flags)(int);
        int (*flock) (struct file *, int, struct file_lock *);
        ssize_t (*splice_write)(struct pipe_inode_info *, struct file *, loff_t *, size_t, unsigned int);
        ssize_t (*splice_read)(struct file *, loff_t *, struct pipe_inode_info *, size_t, unsigned int);
        void (*splice_eof)(struct file *file);
        int (*setlease)(struct file *, int, struct file_lease **, void **);
        long (*fallocate)(struct file *file, int mode, loff_t offset,
                          loff_t len);
        void (*show_fdinfo)(struct seq_file *m, struct file *f);
#ifndef CONFIG_MMU
        unsigned (*mmap_capabilities)(struct file *);
#endif
        ssize_t (*copy_file_range)(struct file *, loff_t, struct file *,
                        loff_t, size_t, unsigned int);
        loff_t (*remap_file_range)(struct file *file_in, loff_t pos_in,
                                   struct file *file_out, loff_t pos_out,
                                   loff_t len, unsigned int remap_flags);
        int (*fadvise)(struct file *, loff_t, loff_t, int);
        int (*uring_cmd)(struct io_uring_cmd *ioucmd, unsigned int issue_flags);
        int (*uring_cmd_iopoll)(struct io_uring_cmd *, struct io_comp_batch *,
                                unsigned int poll_flags);
} __randomize_layout;

/* Supports async buffered reads */
#define FOP_BUFFER_RASYNC        ((__force fop_flags_t)(1 << 0))
/* Supports async buffered writes */
#define FOP_BUFFER_WASYNC        ((__force fop_flags_t)(1 << 1))
/* Supports synchronous page faults for mappings */
#define FOP_MMAP_SYNC                ((__force fop_flags_t)(1 << 2))
/* Supports non-exclusive O_DIRECT writes from multiple threads */
#define FOP_DIO_PARALLEL_WRITE        ((__force fop_flags_t)(1 << 3))
/* Contains huge pages */
#define FOP_HUGE_PAGES                ((__force fop_flags_t)(1 << 4))
/* Treat loff_t as unsigned (e.g., /dev/mem) */
#define FOP_UNSIGNED_OFFSET        ((__force fop_flags_t)(1 << 5))
/* Supports asynchronous lock callbacks */
#define FOP_ASYNC_LOCK                ((__force fop_flags_t)(1 << 6))

/* Wrap a directory iterator that needs exclusive inode access */
int wrap_directory_iterator(struct file *, struct dir_context *,
                            int (*) (struct file *, struct dir_context *));
#define WRAP_DIR_ITER(x) \
        static int shared_##x(struct file *file , struct dir_context *ctx) \
        { return wrap_directory_iterator(file, ctx, x); }

struct inode_operations {
        struct dentry * (*lookup) (struct inode *,struct dentry *, unsigned int);
        const char * (*get_link) (struct dentry *, struct inode *, struct delayed_call *);
        int (*permission) (struct mnt_idmap *, struct inode *, int);
        struct posix_acl * (*get_inode_acl)(struct inode *, int, bool);

        int (*readlink) (struct dentry *, char __user *,int);

        int (*create) (struct mnt_idmap *, struct inode *,struct dentry *,
                       umode_t, bool);
        int (*link) (struct dentry *,struct inode *,struct dentry *);
        int (*unlink) (struct inode *,struct dentry *);
        int (*symlink) (struct mnt_idmap *, struct inode *,struct dentry *,
                        const char *);
        int (*mkdir) (struct mnt_idmap *, struct inode *,struct dentry *,
                      umode_t);
        int (*rmdir) (struct inode *,struct dentry *);
        int (*mknod) (struct mnt_idmap *, struct inode *,struct dentry *,
                      umode_t,dev_t);
        int (*rename) (struct mnt_idmap *, struct inode *, struct dentry *,
                        struct inode *, struct dentry *, unsigned int);
        int (*setattr) (struct mnt_idmap *, struct dentry *, struct iattr *);
        int (*getattr) (struct mnt_idmap *, const struct path *,
                        struct kstat *, u32, unsigned int);
        ssize_t (*listxattr) (struct dentry *, char *, size_t);
        int (*fiemap)(struct inode *, struct fiemap_extent_info *, u64 start,
                      u64 len);
        int (*update_time)(struct inode *, int);
        int (*atomic_open)(struct inode *, struct dentry *,
                           struct file *, unsigned open_flag,
                           umode_t create_mode);
        int (*tmpfile) (struct mnt_idmap *, struct inode *,
                        struct file *, umode_t);
        struct posix_acl *(*get_acl)(struct mnt_idmap *, struct dentry *,
                                     int);
        int (*set_acl)(struct mnt_idmap *, struct dentry *,
                       struct posix_acl *, int);
        int (*fileattr_set)(struct mnt_idmap *idmap,
                            struct dentry *dentry, struct fileattr *fa);
        int (*fileattr_get)(struct dentry *dentry, struct fileattr *fa);
        struct offset_ctx *(*get_offset_ctx)(struct inode *inode);
} ____cacheline_aligned;

static inline int call_mmap(struct file *file, struct vm_area_struct *vma)
{
        return file->f_op->mmap(file, vma);
}

extern ssize_t vfs_read(struct file *, char __user *, size_t, loff_t *);
extern ssize_t vfs_write(struct file *, const char __user *, size_t, loff_t *);
extern ssize_t vfs_copy_file_range(struct file *, loff_t , struct file *,
                                   loff_t, size_t, unsigned int);
int remap_verify_area(struct file *file, loff_t pos, loff_t len, bool write);
int __generic_remap_file_range_prep(struct file *file_in, loff_t pos_in,
                                    struct file *file_out, loff_t pos_out,
                                    loff_t *len, unsigned int remap_flags,
                                    const struct iomap_ops *dax_read_ops);
int generic_remap_file_range_prep(struct file *file_in, loff_t pos_in,
                                  struct file *file_out, loff_t pos_out,
                                  loff_t *count, unsigned int remap_flags);
extern loff_t vfs_clone_file_range(struct file *file_in, loff_t pos_in,
                                   struct file *file_out, loff_t pos_out,
                                   loff_t len, unsigned int remap_flags);
extern int vfs_dedupe_file_range(struct file *file,
                                 struct file_dedupe_range *same);
extern loff_t vfs_dedupe_file_range_one(struct file *src_file, loff_t src_pos,
                                        struct file *dst_file, loff_t dst_pos,
                                        loff_t len, unsigned int remap_flags);

/**
 * enum freeze_holder - holder of the freeze
 * @FREEZE_HOLDER_KERNEL: kernel wants to freeze or thaw filesystem
 * @FREEZE_HOLDER_USERSPACE: userspace wants to freeze or thaw filesystem
 * @FREEZE_MAY_NEST: whether nesting freeze and thaw requests is allowed
 *
 * Indicate who the owner of the freeze or thaw request is and whether
 * the freeze needs to be exclusive or can nest.
 * Without @FREEZE_MAY_NEST, multiple freeze and thaw requests from the
 * same holder aren't allowed. It is however allowed to hold a single
 * @FREEZE_HOLDER_USERSPACE and a single @FREEZE_HOLDER_KERNEL freeze at
 * the same time. This is relied upon by some filesystems during online
 * repair or similar.
 */
enum freeze_holder {
        FREEZE_HOLDER_KERNEL        = (1U << 0),
        FREEZE_HOLDER_USERSPACE        = (1U << 1),
        FREEZE_MAY_NEST                = (1U << 2),
};

struct super_operations {
           struct inode *(*alloc_inode)(struct super_block *sb);
        void (*destroy_inode)(struct inode *);
        void (*free_inode)(struct inode *);

           void (*dirty_inode) (struct inode *, int flags);
        int (*write_inode) (struct inode *, struct writeback_control *wbc);
        int (*drop_inode) (struct inode *);
        void (*evict_inode) (struct inode *);
        void (*put_super) (struct super_block *);
        int (*sync_fs)(struct super_block *sb, int wait);
        int (*freeze_super) (struct super_block *, enum freeze_holder who);
        int (*freeze_fs) (struct super_block *);
        int (*thaw_super) (struct super_block *, enum freeze_holder who);
        int (*unfreeze_fs) (struct super_block *);
        int (*statfs) (struct dentry *, struct kstatfs *);
        int (*remount_fs) (struct super_block *, int *, char *);
        void (*umount_begin) (struct super_block *);

        int (*show_options)(struct seq_file *, struct dentry *);
        int (*show_devname)(struct seq_file *, struct dentry *);
        int (*show_path)(struct seq_file *, struct dentry *);
        int (*show_stats)(struct seq_file *, struct dentry *);
#ifdef CONFIG_QUOTA
        ssize_t (*quota_read)(struct super_block *, int, char *, size_t, loff_t);
        ssize_t (*quota_write)(struct super_block *, int, const char *, size_t, loff_t);
        struct dquot __rcu **(*get_dquots)(struct inode *);
#endif
        long (*nr_cached_objects)(struct super_block *,
                                  struct shrink_control *);
        long (*free_cached_objects)(struct super_block *,
                                    struct shrink_control *);
        void (*shutdown)(struct super_block *sb);
};

/*
 * Inode flags - they have no relation to superblock flags now
 */
#define S_SYNC                (1 << 0)  /* Writes are synced at once */
#define S_NOATIME        (1 << 1)  /* Do not update access times */
#define S_APPEND        (1 << 2)  /* Append-only file */
#define S_IMMUTABLE        (1 << 3)  /* Immutable file */
#define S_DEAD                (1 << 4)  /* removed, but still open directory */
#define S_NOQUOTA        (1 << 5)  /* Inode is not counted to quota */
#define S_DIRSYNC        (1 << 6)  /* Directory modifications are synchronous */
#define S_NOCMTIME        (1 << 7)  /* Do not update file c/mtime */
#define S_SWAPFILE        (1 << 8)  /* Do not truncate: swapon got its bmaps */
#define S_PRIVATE        (1 << 9)  /* Inode is fs-internal */
#define S_IMA                (1 << 10) /* Inode has an associated IMA struct */
#define S_AUTOMOUNT        (1 << 11) /* Automount/referral quasi-directory */
#define S_NOSEC                (1 << 12) /* no suid or xattr security attributes */
#ifdef CONFIG_FS_DAX
#define S_DAX                (1 << 13) /* Direct Access, avoiding the page cache */
#else
#define S_DAX                0          /* Make all the DAX code disappear */
#endif
#define S_ENCRYPTED        (1 << 14) /* Encrypted file (using fs/crypto/) */
#define S_CASEFOLD        (1 << 15) /* Casefolded file */
#define S_VERITY        (1 << 16) /* Verity file (using fs/verity/) */
#define S_KERNEL_FILE        (1 << 17) /* File is in use by the kernel (eg. fs/cachefiles) */

/*
 * Note that nosuid etc flags are inode-specific: setting some file-system
 * flags just means all the inodes inherit those flags by default. It might be
 * possible to override it selectively if you really wanted to with some
 * ioctl() that is not currently implemented.
 *
 * Exception: SB_RDONLY is always applied to the entire file system.
 *
 * Unfortunately, it is possible to change a filesystems flags with it mounted
 * with files in use.  This means that all of the inodes will not have their
 * i_flags updated.  Hence, i_flags no longer inherit the superblock mount
 * flags, so these have to be checked separately. -- rmk@arm.uk.linux.org
 */
#define __IS_FLG(inode, flg)        ((inode)->i_sb->s_flags & (flg))

static inline bool sb_rdonly(const struct super_block *sb) { return sb->s_flags & SB_RDONLY; }
#define IS_RDONLY(inode)        sb_rdonly((inode)->i_sb)
#define IS_SYNC(inode)                (__IS_FLG(inode, SB_SYNCHRONOUS) || \
                                        ((inode)->i_flags & S_SYNC))
#define IS_DIRSYNC(inode)        (__IS_FLG(inode, SB_SYNCHRONOUS|SB_DIRSYNC) || \
                                        ((inode)->i_flags & (S_SYNC|S_DIRSYNC)))
#define IS_MANDLOCK(inode)        __IS_FLG(inode, SB_MANDLOCK)
#define IS_NOATIME(inode)        __IS_FLG(inode, SB_RDONLY|SB_NOATIME)
#define IS_I_VERSION(inode)        __IS_FLG(inode, SB_I_VERSION)

#define IS_NOQUOTA(inode)        ((inode)->i_flags & S_NOQUOTA)
#define IS_APPEND(inode)        ((inode)->i_flags & S_APPEND)
#define IS_IMMUTABLE(inode)        ((inode)->i_flags & S_IMMUTABLE)

#ifdef CONFIG_FS_POSIX_ACL
#define IS_POSIXACL(inode)        __IS_FLG(inode, SB_POSIXACL)
#else
#define IS_POSIXACL(inode)        0
#endif

#define IS_DEADDIR(inode)        ((inode)->i_flags & S_DEAD)
#define IS_NOCMTIME(inode)        ((inode)->i_flags & S_NOCMTIME)

#ifdef CONFIG_SWAP
#define IS_SWAPFILE(inode)        ((inode)->i_flags & S_SWAPFILE)
#else
#define IS_SWAPFILE(inode)        ((void)(inode), 0U)
#endif

#define IS_PRIVATE(inode)        ((inode)->i_flags & S_PRIVATE)
#define IS_IMA(inode)                ((inode)->i_flags & S_IMA)
#define IS_AUTOMOUNT(inode)        ((inode)->i_flags & S_AUTOMOUNT)
#define IS_NOSEC(inode)                ((inode)->i_flags & S_NOSEC)
#define IS_DAX(inode)                ((inode)->i_flags & S_DAX)
#define IS_ENCRYPTED(inode)        ((inode)->i_flags & S_ENCRYPTED)
#define IS_CASEFOLDED(inode)        ((inode)->i_flags & S_CASEFOLD)
#define IS_VERITY(inode)        ((inode)->i_flags & S_VERITY)

#define IS_WHITEOUT(inode)        (S_ISCHR(inode->i_mode) && \
                                 (inode)->i_rdev == WHITEOUT_DEV)

static inline bool HAS_UNMAPPED_ID(struct mnt_idmap *idmap,
                                   struct inode *inode)
{
        return !vfsuid_valid(i_uid_into_vfsuid(idmap, inode)) ||
               !vfsgid_valid(i_gid_into_vfsgid(idmap, inode));
}

static inline void init_sync_kiocb(struct kiocb *kiocb, struct file *filp)
{
        *kiocb = (struct kiocb) {
                .ki_filp = filp,
                .ki_flags = filp->f_iocb_flags,
                .ki_ioprio = get_current_ioprio(),
        };
}

static inline void kiocb_clone(struct kiocb *kiocb, struct kiocb *kiocb_src,
                               struct file *filp)
{
        *kiocb = (struct kiocb) {
                .ki_filp = filp,
                .ki_flags = kiocb_src->ki_flags,
                .ki_ioprio = kiocb_src->ki_ioprio,
                .ki_pos = kiocb_src->ki_pos,
        };
}

/*
 * Inode state bits.  Protected by inode->i_lock
 *
 * Four bits determine the dirty state of the inode: I_DIRTY_SYNC,
 * I_DIRTY_DATASYNC, I_DIRTY_PAGES, and I_DIRTY_TIME.
 *
 * Four bits define the lifetime of an inode.  Initially, inodes are I_NEW,
 * until that flag is cleared.  I_WILL_FREE, I_FREEING and I_CLEAR are set at
 * various stages of removing an inode.
 *
 * Two bits are used for locking and completion notification, I_NEW and I_SYNC.
 *
 * I_DIRTY_SYNC                Inode is dirty, but doesn't have to be written on
 *                        fdatasync() (unless I_DIRTY_DATASYNC is also set).
 *                        Timestamp updates are the usual cause.
 * I_DIRTY_DATASYNC        Data-related inode changes pending.  We keep track of
 *                        these changes separately from I_DIRTY_SYNC so that we
 *                        don't have to write inode on fdatasync() when only
 *                        e.g. the timestamps have changed.
 * I_DIRTY_PAGES        Inode has dirty pages.  Inode itself may be clean.
 * I_DIRTY_TIME                The inode itself has dirty timestamps, and the
 *                        lazytime mount option is enabled.  We keep track of this
 *                        separately from I_DIRTY_SYNC in order to implement
 *                        lazytime.  This gets cleared if I_DIRTY_INODE
 *                        (I_DIRTY_SYNC and/or I_DIRTY_DATASYNC) gets set. But
 *                        I_DIRTY_TIME can still be set if I_DIRTY_SYNC is already
 *                        in place because writeback might already be in progress
 *                        and we don't want to lose the time update
 * I_NEW                Serves as both a mutex and completion notification.
 *                        New inodes set I_NEW.  If two processes both create
 *                        the same inode, one of them will release its inode and
 *                        wait for I_NEW to be released before returning.
 *                        Inodes in I_WILL_FREE, I_FREEING or I_CLEAR state can
 *                        also cause waiting on I_NEW, without I_NEW actually
 *                        being set.  find_inode() uses this to prevent returning
 *                        nearly-dead inodes.
 * I_WILL_FREE                Must be set when calling write_inode_now() if i_count
 *                        is zero.  I_FREEING must be set when I_WILL_FREE is
 *                        cleared.
 * I_FREEING                Set when inode is about to be freed but still has dirty
 *                        pages or buffers attached or the inode itself is still
 *                        dirty.
 * I_CLEAR                Added by clear_inode().  In this state the inode is
 *                        clean and can be destroyed.  Inode keeps I_FREEING.
 *
 *                        Inodes that are I_WILL_FREE, I_FREEING or I_CLEAR are
 *                        prohibited for many purposes.  iget() must wait for
 *                        the inode to be completely released, then create it
 *                        anew.  Other functions will just ignore such inodes,
 *                        if appropriate.  I_NEW is used for waiting.
 *
 * I_SYNC                Writeback of inode is running. The bit is set during
 *                        data writeback, and cleared with a wakeup on the bit
 *                        address once it is done. The bit is also used to pin
 *                        the inode in memory for flusher thread.
 *
 * I_REFERENCED                Marks the inode as recently references on the LRU list.
 *
 * I_WB_SWITCH                Cgroup bdi_writeback switching in progress.  Used to
 *                        synchronize competing switching instances and to tell
 *                        wb stat updates to grab the i_pages lock.  See
 *                        inode_switch_wbs_work_fn() for details.
 *
 * I_OVL_INUSE                Used by overlayfs to get exclusive ownership on upper
 *                        and work dirs among overlayfs mounts.
 *
 * I_CREATING                New object's inode in the middle of setting up.
 *
 * I_DONTCACHE                Evict inode as soon as it is not used anymore.
 *
 * I_SYNC_QUEUED        Inode is queued in b_io or b_more_io writeback lists.
 *                        Used to detect that mark_inode_dirty() should not move
 *                         inode between dirty lists.
 *
 * I_PINNING_FSCACHE_WB        Inode is pinning an fscache object for writeback.
 *
 * I_LRU_ISOLATING        Inode is pinned being isolated from LRU without holding
 *                        i_count.
 *
 * Q: What is the difference between I_WILL_FREE and I_FREEING?
 *
 * __I_{SYNC,NEW,LRU_ISOLATING} are used to derive unique addresses to wait
 * upon. There's one free address left.
 */
#define __I_NEW                        0
#define I_NEW                        (1 << __I_NEW)
#define __I_SYNC                1
#define I_SYNC                        (1 << __I_SYNC)
#define __I_LRU_ISOLATING        2
#define I_LRU_ISOLATING                (1 << __I_LRU_ISOLATING)

#define I_DIRTY_SYNC                (1 << 3)
#define I_DIRTY_DATASYNC        (1 << 4)
#define I_DIRTY_PAGES                (1 << 5)
#define I_WILL_FREE                (1 << 6)
#define I_FREEING                (1 << 7)
#define I_CLEAR                        (1 << 8)
#define I_REFERENCED                (1 << 9)
#define I_LINKABLE                (1 << 10)
#define I_DIRTY_TIME                (1 << 11)
#define I_WB_SWITCH                (1 << 12)
#define I_OVL_INUSE                (1 << 13)
#define I_CREATING                (1 << 14)
#define I_DONTCACHE                (1 << 15)
#define I_SYNC_QUEUED                (1 << 16)
#define I_PINNING_NETFS_WB        (1 << 17)

#define I_DIRTY_INODE (I_DIRTY_SYNC | I_DIRTY_DATASYNC)
#define I_DIRTY (I_DIRTY_INODE | I_DIRTY_PAGES)
#define I_DIRTY_ALL (I_DIRTY | I_DIRTY_TIME)

extern void __mark_inode_dirty(struct inode *, int);
static inline void mark_inode_dirty(struct inode *inode)
{
        __mark_inode_dirty(inode, I_DIRTY);
}

static inline void mark_inode_dirty_sync(struct inode *inode)
{
        __mark_inode_dirty(inode, I_DIRTY_SYNC);
}

/*
 * Returns true if the given inode itself only has dirty timestamps (its pages
 * may still be dirty) and isn't currently being allocated or freed.
 * Filesystems should call this if when writing an inode when lazytime is
 * enabled, they want to opportunistically write the timestamps of other inodes
 * located very nearby on-disk, e.g. in the same inode block.  This returns true
 * if the given inode is in need of such an opportunistic update.  Requires
 * i_lock, or at least later re-checking under i_lock.
 */
static inline bool inode_is_dirtytime_only(struct inode *inode)
{
        return (inode->i_state & (I_DIRTY_TIME | I_NEW |
                                  I_FREEING | I_WILL_FREE)) == I_DIRTY_TIME;
}

extern void inc_nlink(struct inode *inode);
extern void drop_nlink(struct inode *inode);
extern void clear_nlink(struct inode *inode);
extern void set_nlink(struct inode *inode, unsigned int nlink);

static inline void inode_inc_link_count(struct inode *inode)
{
        inc_nlink(inode);
        mark_inode_dirty(inode);
}

static inline void inode_dec_link_count(struct inode *inode)
{
        drop_nlink(inode);
        mark_inode_dirty(inode);
}

enum file_time_flags {
        S_ATIME = 1,
        S_MTIME = 2,
        S_CTIME = 4,
        S_VERSION = 8,
};

extern bool atime_needs_update(const struct path *, struct inode *);
extern void touch_atime(const struct path *);
int inode_update_time(struct inode *inode, int flags);

static inline void file_accessed(struct file *file)
{
        if (!(file->f_flags & O_NOATIME))
                touch_atime(&file->f_path);
}

extern int file_modified(struct file *file);
int kiocb_modified(struct kiocb *iocb);

int sync_inode_metadata(struct inode *inode, int wait);

struct file_system_type {
        const char *name;
        int fs_flags;
#define FS_REQUIRES_DEV                1 
#define FS_BINARY_MOUNTDATA        2
#define FS_HAS_SUBTYPE                4
#define FS_USERNS_MOUNT                8        /* Can be mounted by userns root */
#define FS_DISALLOW_NOTIFY_PERM        16        /* Disable fanotify permission events */
#define FS_ALLOW_IDMAP         32      /* FS has been updated to handle vfs idmappings. */
#define FS_MGTIME                64        /* FS uses multigrain timestamps */
#define FS_RENAME_DOES_D_MOVE        32768        /* FS will handle d_move() during rename() internally. */
        int (*init_fs_context)(struct fs_context *);
        const struct fs_parameter_spec *parameters;
        struct dentry *(*mount) (struct file_system_type *, int,
                       const char *, void *);
        void (*kill_sb) (struct super_block *);
        struct module *owner;
        struct file_system_type * next;
        struct hlist_head fs_supers;

        struct lock_class_key s_lock_key;
        struct lock_class_key s_umount_key;
        struct lock_class_key s_vfs_rename_key;
        struct lock_class_key s_writers_key[SB_FREEZE_LEVELS];

        struct lock_class_key i_lock_key;
        struct lock_class_key i_mutex_key;
        struct lock_class_key invalidate_lock_key;
        struct lock_class_key i_mutex_dir_key;
};

#define MODULE_ALIAS_FS(NAME) MODULE_ALIAS("fs-" NAME)

/**
 * is_mgtime: is this inode using multigrain timestamps
 * @inode: inode to test for multigrain timestamps
 *
 * Return true if the inode uses multigrain timestamps, false otherwise.
 */
static inline bool is_mgtime(const struct inode *inode)
{
        return inode->i_opflags & IOP_MGTIME;
}

extern struct dentry *mount_bdev(struct file_system_type *fs_type,
        int flags, const char *dev_name, void *data,
        int (*fill_super)(struct super_block *, void *, int));
extern struct dentry *mount_single(struct file_system_type *fs_type,
        int flags, void *data,
        int (*fill_super)(struct super_block *, void *, int));
extern struct dentry *mount_nodev(struct file_system_type *fs_type,
        int flags, void *data,
        int (*fill_super)(struct super_block *, void *, int));
extern struct dentry *mount_subtree(struct vfsmount *mnt, const char *path);
void retire_super(struct super_block *sb);
void generic_shutdown_super(struct super_block *sb);
void kill_block_super(struct super_block *sb);
void kill_anon_super(struct super_block *sb);
void kill_litter_super(struct super_block *sb);
void deactivate_super(struct super_block *sb);
void deactivate_locked_super(struct super_block *sb);
int set_anon_super(struct super_block *s, void *data);
int set_anon_super_fc(struct super_block *s, struct fs_context *fc);
int get_anon_bdev(dev_t *);
void free_anon_bdev(dev_t);
struct super_block *sget_fc(struct fs_context *fc,
                            int (*test)(struct super_block *, struct fs_context *),
                            int (*set)(struct super_block *, struct fs_context *));
struct super_block *sget(struct file_system_type *type,
                        int (*test)(struct super_block *,void *),
                        int (*set)(struct super_block *,void *),
                        int flags, void *data);
struct super_block *sget_dev(struct fs_context *fc, dev_t dev);

/* Alas, no aliases. Too much hassle with bringing module.h everywhere */
#define fops_get(fops) ({                                                \
        const struct file_operations *_fops = (fops);                        \
        (((_fops) && try_module_get((_fops)->owner) ? (_fops) : NULL));        \
})

#define fops_put(fops) ({                                                \
        const struct file_operations *_fops = (fops);                        \
        if (_fops)                                                        \
                module_put((_fops)->owner);                                \
})

/*
 * This one is to be used *ONLY* from ->open() instances.
 * fops must be non-NULL, pinned down *and* module dependencies
 * should be sufficient to pin the caller down as well.
 */
#define replace_fops(f, fops) \
        do {        \
                struct file *__file = (f); \
                fops_put(__file->f_op); \
                BUG_ON(!(__file->f_op = (fops))); \
        } while(0)

extern int register_filesystem(struct file_system_type *);
extern int unregister_filesystem(struct file_system_type *);
extern int vfs_statfs(const struct path *, struct kstatfs *);
extern int user_statfs(const char __user *, struct kstatfs *);
extern int fd_statfs(int, struct kstatfs *);
int freeze_super(struct super_block *super, enum freeze_holder who);
int thaw_super(struct super_block *super, enum freeze_holder who);
extern __printf(2, 3)
int super_setup_bdi_name(struct super_block *sb, char *fmt, ...);
extern int super_setup_bdi(struct super_block *sb);

static inline void super_set_uuid(struct super_block *sb, const u8 *uuid, unsigned len)
{
        if (WARN_ON(len > sizeof(sb->s_uuid)))
                len = sizeof(sb->s_uuid);
        sb->s_uuid_len = len;
        memcpy(&sb->s_uuid, uuid, len);
}

/* set sb sysfs name based on sb->s_bdev */
static inline void super_set_sysfs_name_bdev(struct super_block *sb)
{
        snprintf(sb->s_sysfs_name, sizeof(sb->s_sysfs_name), "%pg", sb->s_bdev);
}

/* set sb sysfs name based on sb->s_uuid */
static inline void super_set_sysfs_name_uuid(struct super_block *sb)
{
        WARN_ON(sb->s_uuid_len != sizeof(sb->s_uuid));
        snprintf(sb->s_sysfs_name, sizeof(sb->s_sysfs_name), "%pU", sb->s_uuid.b);
}

/* set sb sysfs name based on sb->s_id */
static inline void super_set_sysfs_name_id(struct super_block *sb)
{
        strscpy(sb->s_sysfs_name, sb->s_id, sizeof(sb->s_sysfs_name));
}

/* try to use something standard before you use this */
__printf(2, 3)
static inline void super_set_sysfs_name_generic(struct super_block *sb, const char *fmt, ...)
{
        va_list args;

        va_start(args, fmt);
        vsnprintf(sb->s_sysfs_name, sizeof(sb->s_sysfs_name), fmt, args);
        va_end(args);
}

extern int current_umask(void);

extern void ihold(struct inode * inode);
extern void iput(struct inode *);
int inode_update_timestamps(struct inode *inode, int flags);
int generic_update_time(struct inode *, int);

/* /sys/fs */
extern struct kobject *fs_kobj;

#define MAX_RW_COUNT (INT_MAX & PAGE_MASK)

/* fs/open.c */
struct audit_names;
struct filename {
        const char                *name;        /* pointer to actual string */
        const __user char        *uptr;        /* original userland pointer */
        atomic_t                refcnt;
        struct audit_names        *aname;
        const char                iname[];
};
static_assert(offsetof(struct filename, iname) % sizeof(long) == 0);

static inline struct mnt_idmap *file_mnt_idmap(const struct file *file)
{
        return mnt_idmap(file->f_path.mnt);
}

/**
 * is_idmapped_mnt - check whether a mount is mapped
 * @mnt: the mount to check
 *
 * If @mnt has an non @nop_mnt_idmap attached to it then @mnt is mapped.
 *
 * Return: true if mount is mapped, false if not.
 */
static inline bool is_idmapped_mnt(const struct vfsmount *mnt)
{
        return mnt_idmap(mnt) != &nop_mnt_idmap;
}

extern long vfs_truncate(const struct path *, loff_t);
int do_truncate(struct mnt_idmap *, struct dentry *, loff_t start,
                unsigned int time_attrs, struct file *filp);
extern int vfs_fallocate(struct file *file, int mode, loff_t offset,
                        loff_t len);
extern long do_sys_open(int dfd, const char __user *filename, int flags,
                        umode_t mode);
extern struct file *file_open_name(struct filename *, int, umode_t);
extern struct file *filp_open(const char *, int, umode_t);
extern struct file *file_open_root(const struct path *,
                                   const char *, int, umode_t);
static inline struct file *file_open_root_mnt(struct vfsmount *mnt,
                                   const char *name, int flags, umode_t mode)
{
        return file_open_root(&(struct path){.mnt = mnt, .dentry = mnt->mnt_root},
                              name, flags, mode);
}
struct file *dentry_open(const struct path *path, int flags,
                         const struct cred *creds);
struct file *dentry_create(const struct path *path, int flags, umode_t mode,
                           const struct cred *cred);
struct path *backing_file_user_path(struct file *f);

/*
 * When mmapping a file on a stackable filesystem (e.g., overlayfs), the file
 * stored in ->vm_file is a backing file whose f_inode is on the underlying
 * filesystem.  When the mapped file path and inode number are displayed to
 * user (e.g. via /proc/<pid>/maps), these helpers should be used to get the
 * path and inode number to display to the user, which is the path of the fd
 * that user has requested to map and the inode number that would be returned
 * by fstat() on that same fd.
 */
/* Get the path to display in /proc/<pid>/maps */
static inline const struct path *file_user_path(struct file *f)
{
        if (unlikely(f->f_mode & FMODE_BACKING))
                return backing_file_user_path(f);
        return &f->f_path;
}
/* Get the inode whose inode number to display in /proc/<pid>/maps */
static inline const struct inode *file_user_inode(struct file *f)
{
        if (unlikely(f->f_mode & FMODE_BACKING))
                return d_inode(backing_file_user_path(f)->dentry);
        return file_inode(f);
}

static inline struct file *file_clone_open(struct file *file)
{
        return dentry_open(&file->f_path, file->f_flags, file->f_cred);
}
extern int filp_close(struct file *, fl_owner_t id);

extern struct filename *getname_flags(const char __user *, int);
extern struct filename *getname_uflags(const char __user *, int);
extern struct filename *getname(const char __user *);
extern struct filename *getname_kernel(const char *);
extern struct filename *__getname_maybe_null(const char __user *);
static inline struct filename *getname_maybe_null(const char __user *name, int flags)
{
        if (!(flags & AT_EMPTY_PATH))
                return getname(name);

        if (!name)
                return NULL;
        return __getname_maybe_null(name);
}
extern void putname(struct filename *name);

extern int finish_open(struct file *file, struct dentry *dentry,
                        int (*open)(struct inode *, struct file *));
extern int finish_no_open(struct file *file, struct dentry *dentry);

/* Helper for the simple case when original dentry is used */
static inline int finish_open_simple(struct file *file, int error)
{
        if (error)
                return error;

        return finish_open(file, file->f_path.dentry, NULL);
}

/* fs/dcache.c */
extern void __init vfs_caches_init_early(void);
extern void __init vfs_caches_init(void);

extern struct kmem_cache *names_cachep;

#define __getname()                kmem_cache_alloc(names_cachep, GFP_KERNEL)
#define __putname(name)                kmem_cache_free(names_cachep, (void *)(name))

extern struct super_block *blockdev_superblock;
static inline bool sb_is_blkdev_sb(struct super_block *sb)
{
        return IS_ENABLED(CONFIG_BLOCK) && sb == blockdev_superblock;
}

void emergency_thaw_all(void);
extern int sync_filesystem(struct super_block *);
extern const struct file_operations def_blk_fops;
extern const struct file_operations def_chr_fops;

/* fs/char_dev.c */
#define CHRDEV_MAJOR_MAX 512
/* Marks the bottom of the first segment of free char majors */
#define CHRDEV_MAJOR_DYN_END 234
/* Marks the top and bottom of the second segment of free char majors */
#define CHRDEV_MAJOR_DYN_EXT_START 511
#define CHRDEV_MAJOR_DYN_EXT_END 384

extern int alloc_chrdev_region(dev_t *, unsigned, unsigned, const char *);
extern int register_chrdev_region(dev_t, unsigned, const char *);
extern int __register_chrdev(unsigned int major, unsigned int baseminor,
                             unsigned int count, const char *name,
                             const struct file_operations *fops);
extern void __unregister_chrdev(unsigned int major, unsigned int baseminor,
                                unsigned int count, const char *name);
extern void unregister_chrdev_region(dev_t, unsigned);
extern void chrdev_show(struct seq_file *,off_t);

static inline int register_chrdev(unsigned int major, const char *name,
                                  const struct file_operations *fops)
{
        return __register_chrdev(major, 0, 256, name, fops);
}

static inline void unregister_chrdev(unsigned int major, const char *name)
{
        __unregister_chrdev(major, 0, 256, name);
}

extern void init_special_inode(struct inode *, umode_t, dev_t);

/* Invalid inode operations -- fs/bad_inode.c */
extern void make_bad_inode(struct inode *);
extern bool is_bad_inode(struct inode *);

extern int __must_check file_fdatawait_range(struct file *file, loff_t lstart,
                                                loff_t lend);
extern int __must_check file_check_and_advance_wb_err(struct file *file);
extern int __must_check file_write_and_wait_range(struct file *file,
                                                loff_t start, loff_t end);

static inline int file_write_and_wait(struct file *file)
{
        return file_write_and_wait_range(file, 0, LLONG_MAX);
}

extern int vfs_fsync_range(struct file *file, loff_t start, loff_t end,
                           int datasync);
extern int vfs_fsync(struct file *file, int datasync);

extern int sync_file_range(struct file *file, loff_t offset, loff_t nbytes,
                                unsigned int flags);

static inline bool iocb_is_dsync(const struct kiocb *iocb)
{
        return (iocb->ki_flags & IOCB_DSYNC) ||
                IS_SYNC(iocb->ki_filp->f_mapping->host);
}

/*
 * Sync the bytes written if this was a synchronous write.  Expect ki_pos
 * to already be updated for the write, and will return either the amount
 * of bytes passed in, or an error if syncing the file failed.
 */
static inline ssize_t generic_write_sync(struct kiocb *iocb, ssize_t count)
{
        if (iocb_is_dsync(iocb)) {
                int ret = vfs_fsync_range(iocb->ki_filp,
                                iocb->ki_pos - count, iocb->ki_pos - 1,
                                (iocb->ki_flags & IOCB_SYNC) ? 0 : 1);
                if (ret)
                        return ret;
        }

        return count;
}

extern void emergency_sync(void);
extern void emergency_remount(void);

#ifdef CONFIG_BLOCK
extern int bmap(struct inode *inode, sector_t *block);
#else
static inline int bmap(struct inode *inode,  sector_t *block)
{
        return -EINVAL;
}
#endif

int notify_change(struct mnt_idmap *, struct dentry *,
                  struct iattr *, struct inode **);
int inode_permission(struct mnt_idmap *, struct inode *, int);
int generic_permission(struct mnt_idmap *, struct inode *, int);
static inline int file_permission(struct file *file, int mask)
{
        return inode_permission(file_mnt_idmap(file),
                                file_inode(file), mask);
}
static inline int path_permission(const struct path *path, int mask)
{
        return inode_permission(mnt_idmap(path->mnt),
                                d_inode(path->dentry), mask);
}
int __check_sticky(struct mnt_idmap *idmap, struct inode *dir,
                   struct inode *inode);

static inline bool execute_ok(struct inode *inode)
{
        return (inode->i_mode & S_IXUGO) || S_ISDIR(inode->i_mode);
}

static inline bool inode_wrong_type(const struct inode *inode, umode_t mode)
{
        return (inode->i_mode ^ mode) & S_IFMT;
}

/**
 * file_start_write - get write access to a superblock for regular file io
 * @file: the file we want to write to
 *
 * This is a variant of sb_start_write() which is a noop on non-regualr file.
 * Should be matched with a call to file_end_write().
 */
static inline void file_start_write(struct file *file)
{
        if (!S_ISREG(file_inode(file)->i_mode))
                return;
        sb_start_write(file_inode(file)->i_sb);
}

static inline bool file_start_write_trylock(struct file *file)
{
        if (!S_ISREG(file_inode(file)->i_mode))
                return true;
        return sb_start_write_trylock(file_inode(file)->i_sb);
}

/**
 * file_end_write - drop write access to a superblock of a regular file
 * @file: the file we wrote to
 *
 * Should be matched with a call to file_start_write().
 */
static inline void file_end_write(struct file *file)
{
        if (!S_ISREG(file_inode(file)->i_mode))
                return;
        sb_end_write(file_inode(file)->i_sb);
}

/**
 * kiocb_start_write - get write access to a superblock for async file io
 * @iocb: the io context we want to submit the write with
 *
 * This is a variant of sb_start_write() for async io submission.
 * Should be matched with a call to kiocb_end_write().
 */
static inline void kiocb_start_write(struct kiocb *iocb)
{
        struct inode *inode = file_inode(iocb->ki_filp);

        sb_start_write(inode->i_sb);
        /*
         * Fool lockdep by telling it the lock got released so that it
         * doesn't complain about the held lock when we return to userspace.
         */
        __sb_writers_release(inode->i_sb, SB_FREEZE_WRITE);
}

/**
 * kiocb_end_write - drop write access to a superblock after async file io
 * @iocb: the io context we sumbitted the write with
 *
 * Should be matched with a call to kiocb_start_write().
 */
static inline void kiocb_end_write(struct kiocb *iocb)
{
        struct inode *inode = file_inode(iocb->ki_filp);

        /*
         * Tell lockdep we inherited freeze protection from submission thread.
         */
        __sb_writers_acquired(inode->i_sb, SB_FREEZE_WRITE);
        sb_end_write(inode->i_sb);
}

/*
 * This is used for regular files where some users -- especially the
 * currently executed binary in a process, previously handled via
 * VM_DENYWRITE -- cannot handle concurrent write (and maybe mmap
 * read-write shared) accesses.
 *
 * get_write_access() gets write permission for a file.
 * put_write_access() releases this write permission.
 * deny_write_access() denies write access to a file.
 * allow_write_access() re-enables write access to a file.
 *
 * The i_writecount field of an inode can have the following values:
 * 0: no write access, no denied write access
 * < 0: (-i_writecount) users that denied write access to the file.
 * > 0: (i_writecount) users that have write access to the file.
 *
 * Normally we operate on that counter with atomic_{inc,dec} and it's safe
 * except for the cases where we don't hold i_writecount yet. Then we need to
 * use {get,deny}_write_access() - these functions check the sign and refuse
 * to do the change if sign is wrong.
 */
static inline int get_write_access(struct inode *inode)
{
        return atomic_inc_unless_negative(&inode->i_writecount) ? 0 : -ETXTBSY;
}
static inline int deny_write_access(struct file *file)
{
        struct inode *inode = file_inode(file);
        return atomic_dec_unless_positive(&inode->i_writecount) ? 0 : -ETXTBSY;
}
static inline void put_write_access(struct inode * inode)
{
        atomic_dec(&inode->i_writecount);
}
static inline void allow_write_access(struct file *file)
{
        if (file)
                atomic_inc(&file_inode(file)->i_writecount);
}
static inline bool inode_is_open_for_write(const struct inode *inode)
{
        return atomic_read(&inode->i_writecount) > 0;
}

#if defined(CONFIG_IMA) || defined(CONFIG_FILE_LOCKING)
static inline void i_readcount_dec(struct inode *inode)
{
        BUG_ON(atomic_dec_return(&inode->i_readcount) < 0);
}
static inline void i_readcount_inc(struct inode *inode)
{
        atomic_inc(&inode->i_readcount);
}
#else
static inline void i_readcount_dec(struct inode *inode)
{
        return;
}
static inline void i_readcount_inc(struct inode *inode)
{
        return;
}
#endif
extern int do_pipe_flags(int *, int);

extern ssize_t kernel_read(struct file *, void *, size_t, loff_t *);
ssize_t __kernel_read(struct file *file, void *buf, size_t count, loff_t *pos);
extern ssize_t kernel_write(struct file *, const void *, size_t, loff_t *);
extern ssize_t __kernel_write(struct file *, const void *, size_t, loff_t *);
extern struct file * open_exec(const char *);
 
/* fs/dcache.c -- generic fs support functions */
extern bool is_subdir(struct dentry *, struct dentry *);
extern bool path_is_under(const struct path *, const struct path *);

extern char *file_path(struct file *, char *, int);

/**
 * is_dot_dotdot - returns true only if @name is "." or ".."
 * @name: file name to check
 * @len: length of file name, in bytes
 */
static inline bool is_dot_dotdot(const char *name, size_t len)
{
        return len && unlikely(name[0] == '.') &&
                (len == 1 || (len == 2 && name[1] == '.'));
}

#include <linux/err.h>

/* needed for stackable file system support */
extern loff_t default_llseek(struct file *file, loff_t offset, int whence);

extern loff_t vfs_llseek(struct file *file, loff_t offset, int whence);

extern int inode_init_always_gfp(struct super_block *, struct inode *, gfp_t);
static inline int inode_init_always(struct super_block *sb, struct inode *inode)
{
        return inode_init_always_gfp(sb, inode, GFP_NOFS);
}

extern void inode_init_once(struct inode *);
extern void address_space_init_once(struct address_space *mapping);
extern struct inode * igrab(struct inode *);
extern ino_t iunique(struct super_block *, ino_t);
extern int inode_needs_sync(struct inode *inode);
extern int generic_delete_inode(struct inode *inode);
static inline int generic_drop_inode(struct inode *inode)
{
        return !inode->i_nlink || inode_unhashed(inode);
}
extern void d_mark_dontcache(struct inode *inode);

extern struct inode *ilookup5_nowait(struct super_block *sb,
                unsigned long hashval, int (*test)(struct inode *, void *),
                void *data);
extern struct inode *ilookup5(struct super_block *sb, unsigned long hashval,
                int (*test)(struct inode *, void *), void *data);
extern struct inode *ilookup(struct super_block *sb, unsigned long ino);

extern struct inode *inode_insert5(struct inode *inode, unsigned long hashval,
                int (*test)(struct inode *, void *),
                int (*set)(struct inode *, void *),
                void *data);
struct inode *iget5_locked(struct super_block *, unsigned long,
                           int (*test)(struct inode *, void *),
                           int (*set)(struct inode *, void *), void *);
struct inode *iget5_locked_rcu(struct super_block *, unsigned long,
                               int (*test)(struct inode *, void *),
                               int (*set)(struct inode *, void *), void *);
extern struct inode * iget_locked(struct super_block *, unsigned long);
extern struct inode *find_inode_nowait(struct super_block *,
                                       unsigned long,
                                       int (*match)(struct inode *,
                                                    unsigned long, void *),
                                       void *data);
extern struct inode *find_inode_rcu(struct super_block *, unsigned long,
                                    int (*)(struct inode *, void *), void *);
extern struct inode *find_inode_by_ino_rcu(struct super_block *, unsigned long);
extern int insert_inode_locked4(struct inode *, unsigned long, int (*test)(struct inode *, void *), void *);
extern int insert_inode_locked(struct inode *);
#ifdef CONFIG_DEBUG_LOCK_ALLOC
extern void lockdep_annotate_inode_mutex_key(struct inode *inode);
#else
static inline void lockdep_annotate_inode_mutex_key(struct inode *inode) { };
#endif
extern void unlock_new_inode(struct inode *);
extern void discard_new_inode(struct inode *);
extern unsigned int get_next_ino(void);
extern void evict_inodes(struct super_block *sb);
void dump_mapping(const struct address_space *);

/*
 * Userspace may rely on the inode number being non-zero. For example, glibc
 * simply ignores files with zero i_ino in unlink() and other places.
 *
 * As an additional complication, if userspace was compiled with
 * _FILE_OFFSET_BITS=32 on a 64-bit kernel we'll only end up reading out the
 * lower 32 bits, so we need to check that those aren't zero explicitly. With
 * _FILE_OFFSET_BITS=64, this may cause some harmless false-negatives, but
 * better safe than sorry.
 */
static inline bool is_zero_ino(ino_t ino)
{
        return (u32)ino == 0;
}

/*
 * inode->i_lock must be held
 */
static inline void __iget(struct inode *inode)
{
        atomic_inc(&inode->i_count);
}

extern void iget_failed(struct inode *);
extern void clear_inode(struct inode *);
extern void __destroy_inode(struct inode *);
extern struct inode *new_inode_pseudo(struct super_block *sb);
extern struct inode *new_inode(struct super_block *sb);
extern void free_inode_nonrcu(struct inode *inode);
extern int setattr_should_drop_suidgid(struct mnt_idmap *, struct inode *);
extern int file_remove_privs_flags(struct file *file, unsigned int flags);
extern int file_remove_privs(struct file *);
int setattr_should_drop_sgid(struct mnt_idmap *idmap,
                             const struct inode *inode);

/*
 * This must be used for allocating filesystems specific inodes to set
 * up the inode reclaim context correctly.
 */
#define alloc_inode_sb(_sb, _cache, _gfp) kmem_cache_alloc_lru(_cache, &_sb->s_inode_lru, _gfp)

extern void __insert_inode_hash(struct inode *, unsigned long hashval);
static inline void insert_inode_hash(struct inode *inode)
{
        __insert_inode_hash(inode, inode->i_ino);
}

extern void __remove_inode_hash(struct inode *);
static inline void remove_inode_hash(struct inode *inode)
{
        if (!inode_unhashed(inode) && !hlist_fake(&inode->i_hash))
                __remove_inode_hash(inode);
}

extern void inode_sb_list_add(struct inode *inode);
extern void inode_add_lru(struct inode *inode);

extern int sb_set_blocksize(struct super_block *, int);
extern int sb_min_blocksize(struct super_block *, int);

extern int generic_file_mmap(struct file *, struct vm_area_struct *);
extern int generic_file_readonly_mmap(struct file *, struct vm_area_struct *);
extern ssize_t generic_write_checks(struct kiocb *, struct iov_iter *);
int generic_write_checks_count(struct kiocb *iocb, loff_t *count);
extern int generic_write_check_limits(struct file *file, loff_t pos,
                loff_t *count);
extern int generic_file_rw_checks(struct file *file_in, struct file *file_out);
ssize_t filemap_read(struct kiocb *iocb, struct iov_iter *to,
                ssize_t already_read);
extern ssize_t generic_file_read_iter(struct kiocb *, struct iov_iter *);
extern ssize_t __generic_file_write_iter(struct kiocb *, struct iov_iter *);
extern ssize_t generic_file_write_iter(struct kiocb *, struct iov_iter *);
extern ssize_t generic_file_direct_write(struct kiocb *, struct iov_iter *);
ssize_t generic_perform_write(struct kiocb *, struct iov_iter *);
ssize_t direct_write_fallback(struct kiocb *iocb, struct iov_iter *iter,
                ssize_t direct_written, ssize_t buffered_written);

ssize_t vfs_iter_read(struct file *file, struct iov_iter *iter, loff_t *ppos,
                rwf_t flags);
ssize_t vfs_iter_write(struct file *file, struct iov_iter *iter, loff_t *ppos,
                rwf_t flags);
ssize_t vfs_iocb_iter_read(struct file *file, struct kiocb *iocb,
                           struct iov_iter *iter);
ssize_t vfs_iocb_iter_write(struct file *file, struct kiocb *iocb,
                            struct iov_iter *iter);

/* fs/splice.c */
ssize_t filemap_splice_read(struct file *in, loff_t *ppos,
                            struct pipe_inode_info *pipe,
                            size_t len, unsigned int flags);
ssize_t copy_splice_read(struct file *in, loff_t *ppos,
                         struct pipe_inode_info *pipe,
                         size_t len, unsigned int flags);
extern ssize_t iter_file_splice_write(struct pipe_inode_info *,
                struct file *, loff_t *, size_t, unsigned int);


extern void
file_ra_state_init(struct file_ra_state *ra, struct address_space *mapping);
extern loff_t noop_llseek(struct file *file, loff_t offset, int whence);
extern loff_t vfs_setpos(struct file *file, loff_t offset, loff_t maxsize);
extern loff_t generic_file_llseek(struct file *file, loff_t offset, int whence);
extern loff_t generic_file_llseek_size(struct file *file, loff_t offset,
                int whence, loff_t maxsize, loff_t eof);
loff_t generic_llseek_cookie(struct file *file, loff_t offset, int whence,
                             u64 *cookie);
extern loff_t fixed_size_llseek(struct file *file, loff_t offset,
                int whence, loff_t size);
extern loff_t no_seek_end_llseek_size(struct file *, loff_t, int, loff_t);
extern loff_t no_seek_end_llseek(struct file *, loff_t, int);
int rw_verify_area(int, struct file *, const loff_t *, size_t);
extern int generic_file_open(struct inode * inode, struct file * filp);
extern int nonseekable_open(struct inode * inode, struct file * filp);
extern int stream_open(struct inode * inode, struct file * filp);

#ifdef CONFIG_BLOCK
typedef void (dio_submit_t)(struct bio *bio, struct inode *inode,
                            loff_t file_offset);

enum {
        /* need locking between buffered and direct access */
        DIO_LOCKING        = 0x01,

        /* filesystem does not support filling holes */
        DIO_SKIP_HOLES        = 0x02,
};

ssize_t __blockdev_direct_IO(struct kiocb *iocb, struct inode *inode,
                             struct block_device *bdev, struct iov_iter *iter,
                             get_block_t get_block,
                             dio_iodone_t end_io,
                             int flags);

static inline ssize_t blockdev_direct_IO(struct kiocb *iocb,
                                         struct inode *inode,
                                         struct iov_iter *iter,
                                         get_block_t get_block)
{
        return __blockdev_direct_IO(iocb, inode, inode->i_sb->s_bdev, iter,
                        get_block, NULL, DIO_LOCKING | DIO_SKIP_HOLES);
}
#endif

bool inode_dio_finished(const struct inode *inode);
void inode_dio_wait(struct inode *inode);
void inode_dio_wait_interruptible(struct inode *inode);

/**
 * inode_dio_begin - signal start of a direct I/O requests
 * @inode: inode the direct I/O happens on
 *
 * This is called once we've finished processing a direct I/O request,
 * and is used to wake up callers waiting for direct I/O to be quiesced.
 */
static inline void inode_dio_begin(struct inode *inode)
{
        atomic_inc(&inode->i_dio_count);
}

/**
 * inode_dio_end - signal finish of a direct I/O requests
 * @inode: inode the direct I/O happens on
 *
 * This is called once we've finished processing a direct I/O request,
 * and is used to wake up callers waiting for direct I/O to be quiesced.
 */
static inline void inode_dio_end(struct inode *inode)
{
        if (atomic_dec_and_test(&inode->i_dio_count))
                wake_up_var(&inode->i_dio_count);
}

extern void inode_set_flags(struct inode *inode, unsigned int flags,
                            unsigned int mask);

extern const struct file_operations generic_ro_fops;

#define special_file(m) (S_ISCHR(m)||S_ISBLK(m)||S_ISFIFO(m)||S_ISSOCK(m))

extern int readlink_copy(char __user *, int, const char *);
extern int page_readlink(struct dentry *, char __user *, int);
extern const char *page_get_link(struct dentry *, struct inode *,
                                 struct delayed_call *);
extern void page_put_link(void *);
extern int page_symlink(struct inode *inode, const char *symname, int len);
extern const struct inode_operations page_symlink_inode_operations;
extern void kfree_link(void *);
void fill_mg_cmtime(struct kstat *stat, u32 request_mask, struct inode *inode);
void generic_fillattr(struct mnt_idmap *, u32, struct inode *, struct kstat *);
void generic_fill_statx_attr(struct inode *inode, struct kstat *stat);
void generic_fill_statx_atomic_writes(struct kstat *stat,
                                      unsigned int unit_min,
                                      unsigned int unit_max);
extern int vfs_getattr_nosec(const struct path *, struct kstat *, u32, unsigned int);
extern int vfs_getattr(const struct path *, struct kstat *, u32, unsigned int);
void __inode_add_bytes(struct inode *inode, loff_t bytes);
void inode_add_bytes(struct inode *inode, loff_t bytes);
void __inode_sub_bytes(struct inode *inode, loff_t bytes);
void inode_sub_bytes(struct inode *inode, loff_t bytes);
static inline loff_t __inode_get_bytes(struct inode *inode)
{
        return (((loff_t)inode->i_blocks) << 9) + inode->i_bytes;
}
loff_t inode_get_bytes(struct inode *inode);
void inode_set_bytes(struct inode *inode, loff_t bytes);
const char *simple_get_link(struct dentry *, struct inode *,
                            struct delayed_call *);
extern const struct inode_operations simple_symlink_inode_operations;

extern int iterate_dir(struct file *, struct dir_context *);

int vfs_fstatat(int dfd, const char __user *filename, struct kstat *stat,
                int flags);
int vfs_fstat(int fd, struct kstat *stat);

static inline int vfs_stat(const char __user *filename, struct kstat *stat)
{
        return vfs_fstatat(AT_FDCWD, filename, stat, 0);
}
static inline int vfs_lstat(const char __user *name, struct kstat *stat)
{
        return vfs_fstatat(AT_FDCWD, name, stat, AT_SYMLINK_NOFOLLOW);
}

extern const char *vfs_get_link(struct dentry *, struct delayed_call *);
extern int vfs_readlink(struct dentry *, char __user *, int);

extern struct file_system_type *get_filesystem(struct file_system_type *fs);
extern void put_filesystem(struct file_system_type *fs);
extern struct file_system_type *get_fs_type(const char *name);
extern void drop_super(struct super_block *sb);
extern void drop_super_exclusive(struct super_block *sb);
extern void iterate_supers(void (*)(struct super_block *, void *), void *);
extern void iterate_supers_type(struct file_system_type *,
                                void (*)(struct super_block *, void *), void *);

extern int dcache_dir_open(struct inode *, struct file *);
extern int dcache_dir_close(struct inode *, struct file *);
extern loff_t dcache_dir_lseek(struct file *, loff_t, int);
extern int dcache_readdir(struct file *, struct dir_context *);
extern int simple_setattr(struct mnt_idmap *, struct dentry *,
                          struct iattr *);
extern int simple_getattr(struct mnt_idmap *, const struct path *,
                          struct kstat *, u32, unsigned int);
extern int simple_statfs(struct dentry *, struct kstatfs *);
extern int simple_open(struct inode *inode, struct file *file);
extern int simple_link(struct dentry *, struct inode *, struct dentry *);
extern int simple_unlink(struct inode *, struct dentry *);
extern int simple_rmdir(struct inode *, struct dentry *);
void simple_rename_timestamp(struct inode *old_dir, struct dentry *old_dentry,
                             struct inode *new_dir, struct dentry *new_dentry);
extern int simple_rename_exchange(struct inode *old_dir, struct dentry *old_dentry,
                                  struct inode *new_dir, struct dentry *new_dentry);
extern int simple_rename(struct mnt_idmap *, struct inode *,
                         struct dentry *, struct inode *, struct dentry *,
                         unsigned int);
extern void simple_recursive_removal(struct dentry *,
                              void (*callback)(struct dentry *));
extern int noop_fsync(struct file *, loff_t, loff_t, int);
extern ssize_t noop_direct_IO(struct kiocb *iocb, struct iov_iter *iter);
extern int simple_empty(struct dentry *);
extern int simple_write_begin(struct file *file, struct address_space *mapping,
                        loff_t pos, unsigned len,
                        struct folio **foliop, void **fsdata);
extern const struct address_space_operations ram_aops;
extern int always_delete_dentry(const struct dentry *);
extern struct inode *alloc_anon_inode(struct super_block *);
extern int simple_nosetlease(struct file *, int, struct file_lease **, void **);
extern const struct dentry_operations simple_dentry_operations;

extern struct dentry *simple_lookup(struct inode *, struct dentry *, unsigned int flags);
extern ssize_t generic_read_dir(struct file *, char __user *, size_t, loff_t *);
extern const struct file_operations simple_dir_operations;
extern const struct inode_operations simple_dir_inode_operations;
extern void make_empty_dir_inode(struct inode *inode);
extern bool is_empty_dir_inode(struct inode *inode);
struct tree_descr { const char *name; const struct file_operations *ops; int mode; };
struct dentry *d_alloc_name(struct dentry *, const char *);
extern int simple_fill_super(struct super_block *, unsigned long,
                             const struct tree_descr *);
extern int simple_pin_fs(struct file_system_type *, struct vfsmount **mount, int *count);
extern void simple_release_fs(struct vfsmount **mount, int *count);

extern ssize_t simple_read_from_buffer(void __user *to, size_t count,
                        loff_t *ppos, const void *from, size_t available);
extern ssize_t simple_write_to_buffer(void *to, size_t available, loff_t *ppos,
                const void __user *from, size_t count);

struct offset_ctx {
        struct maple_tree        mt;
        unsigned long                next_offset;
};

void simple_offset_init(struct offset_ctx *octx);
int simple_offset_add(struct offset_ctx *octx, struct dentry *dentry);
void simple_offset_remove(struct offset_ctx *octx, struct dentry *dentry);
int simple_offset_empty(struct dentry *dentry);
int simple_offset_rename(struct inode *old_dir, struct dentry *old_dentry,
                         struct inode *new_dir, struct dentry *new_dentry);
int simple_offset_rename_exchange(struct inode *old_dir,
                                  struct dentry *old_dentry,
                                  struct inode *new_dir,
                                  struct dentry *new_dentry);
void simple_offset_destroy(struct offset_ctx *octx);

extern const struct file_operations simple_offset_dir_operations;

extern int __generic_file_fsync(struct file *, loff_t, loff_t, int);
extern int generic_file_fsync(struct file *, loff_t, loff_t, int);

extern int generic_check_addressable(unsigned, u64);

extern void generic_set_sb_d_ops(struct super_block *sb);
extern int generic_ci_match(const struct inode *parent,
                            const struct qstr *name,
                            const struct qstr *folded_name,
                            const u8 *de_name, u32 de_name_len);

#if IS_ENABLED(CONFIG_UNICODE)
int generic_ci_d_hash(const struct dentry *dentry, struct qstr *str);
int generic_ci_d_compare(const struct dentry *dentry, unsigned int len,
                         const char *str, const struct qstr *name);

/**
 * generic_ci_validate_strict_name - Check if a given name is suitable
 * for a directory
 *
 * This functions checks if the proposed filename is valid for the
 * parent directory. That means that only valid UTF-8 filenames will be
 * accepted for casefold directories from filesystems created with the
 * strict encoding flag.  That also means that any name will be
 * accepted for directories that doesn't have casefold enabled, or
 * aren't being strict with the encoding.
 *
 * @dir: inode of the directory where the new file will be created
 * @name: name of the new file
 *
 * Return:
 * * True: if the filename is suitable for this directory. It can be
 *   true if a given name is not suitable for a strict encoding
 *   directory, but the directory being used isn't strict
 * * False if the filename isn't suitable for this directory. This only
 *   happens when a directory is casefolded and the filesystem is strict
 *   about its encoding.
 */
static inline bool generic_ci_validate_strict_name(struct inode *dir, struct qstr *name)
{
        if (!IS_CASEFOLDED(dir) || !sb_has_strict_encoding(dir->i_sb))
                return true;

        /*
         * A casefold dir must have a encoding set, unless the filesystem
         * is corrupted
         */
        if (WARN_ON_ONCE(!dir->i_sb->s_encoding))
                return true;

        return !utf8_validate(dir->i_sb->s_encoding, name);
}
#else
static inline bool generic_ci_validate_strict_name(struct inode *dir, struct qstr *name)
{
        return true;
}
#endif

static inline bool sb_has_encoding(const struct super_block *sb)
{
#if IS_ENABLED(CONFIG_UNICODE)
        return !!sb->s_encoding;
#else
        return false;
#endif
}

int may_setattr(struct mnt_idmap *idmap, struct inode *inode,
                unsigned int ia_valid);
int setattr_prepare(struct mnt_idmap *, struct dentry *, struct iattr *);
extern int inode_newsize_ok(const struct inode *, loff_t offset);
void setattr_copy(struct mnt_idmap *, struct inode *inode,
                  const struct iattr *attr);

extern int file_update_time(struct file *file);

static inline bool vma_is_dax(const struct vm_area_struct *vma)
{
        return vma->vm_file && IS_DAX(vma->vm_file->f_mapping->host);
}

static inline bool vma_is_fsdax(struct vm_area_struct *vma)
{
        struct inode *inode;

        if (!IS_ENABLED(CONFIG_FS_DAX) || !vma->vm_file)
                return false;
        if (!vma_is_dax(vma))
                return false;
        inode = file_inode(vma->vm_file);
        if (S_ISCHR(inode->i_mode))
                return false; /* device-dax */
        return true;
}

static inline int iocb_flags(struct file *file)
{
        int res = 0;
        if (file->f_flags & O_APPEND)
                res |= IOCB_APPEND;
        if (file->f_flags & O_DIRECT)
                res |= IOCB_DIRECT;
        if (file->f_flags & O_DSYNC)
                res |= IOCB_DSYNC;
        if (file->f_flags & __O_SYNC)
                res |= IOCB_SYNC;
        return res;
}

static inline int kiocb_set_rw_flags(struct kiocb *ki, rwf_t flags,
                                     int rw_type)
{
        int kiocb_flags = 0;

        /* make sure there's no overlap between RWF and private IOCB flags */
        BUILD_BUG_ON((__force int) RWF_SUPPORTED & IOCB_EVENTFD);

        if (!flags)
                return 0;
        if (unlikely(flags & ~RWF_SUPPORTED))
                return -EOPNOTSUPP;
        if (unlikely((flags & RWF_APPEND) && (flags & RWF_NOAPPEND)))
                return -EINVAL;

        if (flags & RWF_NOWAIT) {
                if (!(ki->ki_filp->f_mode & FMODE_NOWAIT))
                        return -EOPNOTSUPP;
        }
        if (flags & RWF_ATOMIC) {
                if (rw_type != WRITE)
                        return -EOPNOTSUPP;
                if (!(ki->ki_filp->f_mode & FMODE_CAN_ATOMIC_WRITE))
                        return -EOPNOTSUPP;
        }
        kiocb_flags |= (__force int) (flags & RWF_SUPPORTED);
        if (flags & RWF_SYNC)
                kiocb_flags |= IOCB_DSYNC;

        if ((flags & RWF_NOAPPEND) && (ki->ki_flags & IOCB_APPEND)) {
                if (IS_APPEND(file_inode(ki->ki_filp)))
                        return -EPERM;
                ki->ki_flags &= ~IOCB_APPEND;
        }

        ki->ki_flags |= kiocb_flags;
        return 0;
}

/* Transaction based IO helpers */

/*
 * An argresp is stored in an allocated page and holds the
 * size of the argument or response, along with its content
 */
struct simple_transaction_argresp {
        ssize_t size;
        char data[];
};

#define SIMPLE_TRANSACTION_LIMIT (PAGE_SIZE - sizeof(struct simple_transaction_argresp))

char *simple_transaction_get(struct file *file, const char __user *buf,
                                size_t size);
ssize_t simple_transaction_read(struct file *file, char __user *buf,
                                size_t size, loff_t *pos);
int simple_transaction_release(struct inode *inode, struct file *file);

void simple_transaction_set(struct file *file, size_t n);

/*
 * simple attribute files
 *
 * These attributes behave similar to those in sysfs:
 *
 * Writing to an attribute immediately sets a value, an open file can be
 * written to multiple times.
 *
 * Reading from an attribute creates a buffer from the value that might get
 * read with multiple read calls. When the attribute has been read
 * completely, no further read calls are possible until the file is opened
 * again.
 *
 * All attributes contain a text representation of a numeric value
 * that are accessed with the get() and set() functions.
 */
#define DEFINE_SIMPLE_ATTRIBUTE_XSIGNED(__fops, __get, __set, __fmt, __is_signed)        \
static int __fops ## _open(struct inode *inode, struct file *file)        \
{                                                                        \
        __simple_attr_check_format(__fmt, 0ull);                        \
        return simple_attr_open(inode, file, __get, __set, __fmt);        \
}                                                                        \
static const struct file_operations __fops = {                                \
        .owner         = THIS_MODULE,                                                \
        .open         = __fops ## _open,                                        \
        .release = simple_attr_release,                                        \
        .read         = simple_attr_read,                                        \
        .write         = (__is_signed) ? simple_attr_write_signed : simple_attr_write,        \
        .llseek         = generic_file_llseek,                                        \
}

#define DEFINE_SIMPLE_ATTRIBUTE(__fops, __get, __set, __fmt)                \
        DEFINE_SIMPLE_ATTRIBUTE_XSIGNED(__fops, __get, __set, __fmt, false)

#define DEFINE_SIMPLE_ATTRIBUTE_SIGNED(__fops, __get, __set, __fmt)        \
        DEFINE_SIMPLE_ATTRIBUTE_XSIGNED(__fops, __get, __set, __fmt, true)

static inline __printf(1, 2)
void __simple_attr_check_format(const char *fmt, ...)
{
        /* don't do anything, just let the compiler check the arguments; */
}

int simple_attr_open(struct inode *inode, struct file *file,
                     int (*get)(void *, u64 *), int (*set)(void *, u64),
                     const char *fmt);
int simple_attr_release(struct inode *inode, struct file *file);
ssize_t simple_attr_read(struct file *file, char __user *buf,
                         size_t len, loff_t *ppos);
ssize_t simple_attr_write(struct file *file, const char __user *buf,
                          size_t len, loff_t *ppos);
ssize_t simple_attr_write_signed(struct file *file, const char __user *buf,
                                 size_t len, loff_t *ppos);

struct ctl_table;
int __init list_bdev_fs_names(char *buf, size_t size);

#define __FMODE_EXEC                ((__force int) FMODE_EXEC)
#define __FMODE_NONOTIFY        ((__force int) FMODE_NONOTIFY)

#define ACC_MODE(x) ("\004\002\006\006"[(x)&O_ACCMODE])
#define OPEN_FMODE(flag) ((__force fmode_t)(((flag + 1) & O_ACCMODE) | \
                                            (flag & __FMODE_NONOTIFY)))

static inline bool is_sxid(umode_t mode)
{
        return mode & (S_ISUID | S_ISGID);
}

static inline int check_sticky(struct mnt_idmap *idmap,
                               struct inode *dir, struct inode *inode)
{
        if (!(dir->i_mode & S_ISVTX))
                return 0;

        return __check_sticky(idmap, dir, inode);
}

static inline void inode_has_no_xattr(struct inode *inode)
{
        if (!is_sxid(inode->i_mode) && (inode->i_sb->s_flags & SB_NOSEC))
                inode->i_flags |= S_NOSEC;
}

static inline bool is_root_inode(struct inode *inode)
{
        return inode == inode->i_sb->s_root->d_inode;
}

static inline bool dir_emit(struct dir_context *ctx,
                            const char *name, int namelen,
                            u64 ino, unsigned type)
{
        return ctx->actor(ctx, name, namelen, ctx->pos, ino, type);
}
static inline bool dir_emit_dot(struct file *file, struct dir_context *ctx)
{
        return ctx->actor(ctx, ".", 1, ctx->pos,
                          file->f_path.dentry->d_inode->i_ino, DT_DIR);
}
static inline bool dir_emit_dotdot(struct file *file, struct dir_context *ctx)
{
        return ctx->actor(ctx, "..", 2, ctx->pos,
                          d_parent_ino(file->f_path.dentry), DT_DIR);
}
static inline bool dir_emit_dots(struct file *file, struct dir_context *ctx)
{
        if (ctx->pos == 0) {
                if (!dir_emit_dot(file, ctx))
                        return false;
                ctx->pos = 1;
        }
        if (ctx->pos == 1) {
                if (!dir_emit_dotdot(file, ctx))
                        return false;
                ctx->pos = 2;
        }
        return true;
}
static inline bool dir_relax(struct inode *inode)
{
        inode_unlock(inode);
        inode_lock(inode);
        return !IS_DEADDIR(inode);
}

static inline bool dir_relax_shared(struct inode *inode)
{
        inode_unlock_shared(inode);
        inode_lock_shared(inode);
        return !IS_DEADDIR(inode);
}

extern bool path_noexec(const struct path *path);
extern void inode_nohighmem(struct inode *inode);

/* mm/fadvise.c */
extern int vfs_fadvise(struct file *file, loff_t offset, loff_t len,
                       int advice);
extern int generic_fadvise(struct file *file, loff_t offset, loff_t len,
                           int advice);

static inline bool vfs_empty_path(int dfd, const char __user *path)
{
        char c;

        if (dfd < 0)
                return false;

        /* We now allow NULL to be used for empty path. */
        if (!path)
                return true;

        if (unlikely(get_user(c, path)))
                return false;

        return !c;
}

int generic_atomic_write_valid(struct kiocb *iocb, struct iov_iter *iter);

#endif /* _LINUX_FS_H */



























































































































































































































































































































































































































































































































































































    7 














    1 






    6 
    6 
    6 
    6 
    5 
    5 
    6 
    6 
    6 
    6 
    6 
    6 
    6 
    6 
    6 
    6 

    6 

    4 
    4 
    4 
    4 
    4 
    4 
    4 
    4 




    3 
    3 
    3 
    3 
    3 
    3 
    3 
    3 


    2 













    3 


    3 




    1 




    1 


    1 




    2 




    2 


    2 









    6 






1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * Common Twofish algorithm parts shared between the c and assembler
 * implementations
 *
 * Originally Twofish for GPG
 * By Matthew Skala <mskala@ansuz.sooke.bc.ca>, July 26, 1998
 * 256-bit key length added March 20, 1999
 * Some modifications to reduce the text size by Werner Koch, April, 1998
 * Ported to the kerneli patch by Marc Mutz <Marc@Mutz.com>
 * Ported to CryptoAPI by Colin Slater <hoho@tacomeat.net>
 *
 * The original author has disclaimed all copyright interest in this
 * code and thus put it in the public domain. The subsequent authors
 * have put this under the GNU General Public License.
 *
 * This code is a "clean room" implementation, written from the paper
 * _Twofish: A 128-Bit Block Cipher_ by Bruce Schneier, John Kelsey,
 * Doug Whiting, David Wagner, Chris Hall, and Niels Ferguson, available
 * through http://www.counterpane.com/twofish.html
 *
 * For background information on multiplication in finite fields, used for
 * the matrix operations in the key schedule, see the book _Contemporary
 * Abstract Algebra_ by Joseph A. Gallian, especially chapter 22 in the
 * Third Edition.
 */

#include <crypto/algapi.h>
#include <crypto/twofish.h>
#include <linux/bitops.h>
#include <linux/errno.h>
#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/types.h>


/* The large precomputed tables for the Twofish cipher (twofish.c)
 * Taken from the same source as twofish.c
 * Marc Mutz <Marc@Mutz.com>
 */

/* These two tables are the q0 and q1 permutations, exactly as described in
 * the Twofish paper. */

static const u8 q0[256] = {
        0xA9, 0x67, 0xB3, 0xE8, 0x04, 0xFD, 0xA3, 0x76, 0x9A, 0x92, 0x80, 0x78,
        0xE4, 0xDD, 0xD1, 0x38, 0x0D, 0xC6, 0x35, 0x98, 0x18, 0xF7, 0xEC, 0x6C,
        0x43, 0x75, 0x37, 0x26, 0xFA, 0x13, 0x94, 0x48, 0xF2, 0xD0, 0x8B, 0x30,
        0x84, 0x54, 0xDF, 0x23, 0x19, 0x5B, 0x3D, 0x59, 0xF3, 0xAE, 0xA2, 0x82,
        0x63, 0x01, 0x83, 0x2E, 0xD9, 0x51, 0x9B, 0x7C, 0xA6, 0xEB, 0xA5, 0xBE,
        0x16, 0x0C, 0xE3, 0x61, 0xC0, 0x8C, 0x3A, 0xF5, 0x73, 0x2C, 0x25, 0x0B,
        0xBB, 0x4E, 0x89, 0x6B, 0x53, 0x6A, 0xB4, 0xF1, 0xE1, 0xE6, 0xBD, 0x45,
        0xE2, 0xF4, 0xB6, 0x66, 0xCC, 0x95, 0x03, 0x56, 0xD4, 0x1C, 0x1E, 0xD7,
        0xFB, 0xC3, 0x8E, 0xB5, 0xE9, 0xCF, 0xBF, 0xBA, 0xEA, 0x77, 0x39, 0xAF,
        0x33, 0xC9, 0x62, 0x71, 0x81, 0x79, 0x09, 0xAD, 0x24, 0xCD, 0xF9, 0xD8,
        0xE5, 0xC5, 0xB9, 0x4D, 0x44, 0x08, 0x86, 0xE7, 0xA1, 0x1D, 0xAA, 0xED,
        0x06, 0x70, 0xB2, 0xD2, 0x41, 0x7B, 0xA0, 0x11, 0x31, 0xC2, 0x27, 0x90,
        0x20, 0xF6, 0x60, 0xFF, 0x96, 0x5C, 0xB1, 0xAB, 0x9E, 0x9C, 0x52, 0x1B,
        0x5F, 0x93, 0x0A, 0xEF, 0x91, 0x85, 0x49, 0xEE, 0x2D, 0x4F, 0x8F, 0x3B,
        0x47, 0x87, 0x6D, 0x46, 0xD6, 0x3E, 0x69, 0x64, 0x2A, 0xCE, 0xCB, 0x2F,
        0xFC, 0x97, 0x05, 0x7A, 0xAC, 0x7F, 0xD5, 0x1A, 0x4B, 0x0E, 0xA7, 0x5A,
        0x28, 0x14, 0x3F, 0x29, 0x88, 0x3C, 0x4C, 0x02, 0xB8, 0xDA, 0xB0, 0x17,
        0x55, 0x1F, 0x8A, 0x7D, 0x57, 0xC7, 0x8D, 0x74, 0xB7, 0xC4, 0x9F, 0x72,
        0x7E, 0x15, 0x22, 0x12, 0x58, 0x07, 0x99, 0x34, 0x6E, 0x50, 0xDE, 0x68,
        0x65, 0xBC, 0xDB, 0xF8, 0xC8, 0xA8, 0x2B, 0x40, 0xDC, 0xFE, 0x32, 0xA4,
        0xCA, 0x10, 0x21, 0xF0, 0xD3, 0x5D, 0x0F, 0x00, 0x6F, 0x9D, 0x36, 0x42,
        0x4A, 0x5E, 0xC1, 0xE0
};

static const u8 q1[256] = {
        0x75, 0xF3, 0xC6, 0xF4, 0xDB, 0x7B, 0xFB, 0xC8, 0x4A, 0xD3, 0xE6, 0x6B,
        0x45, 0x7D, 0xE8, 0x4B, 0xD6, 0x32, 0xD8, 0xFD, 0x37, 0x71, 0xF1, 0xE1,
        0x30, 0x0F, 0xF8, 0x1B, 0x87, 0xFA, 0x06, 0x3F, 0x5E, 0xBA, 0xAE, 0x5B,
        0x8A, 0x00, 0xBC, 0x9D, 0x6D, 0xC1, 0xB1, 0x0E, 0x80, 0x5D, 0xD2, 0xD5,
        0xA0, 0x84, 0x07, 0x14, 0xB5, 0x90, 0x2C, 0xA3, 0xB2, 0x73, 0x4C, 0x54,
        0x92, 0x74, 0x36, 0x51, 0x38, 0xB0, 0xBD, 0x5A, 0xFC, 0x60, 0x62, 0x96,
        0x6C, 0x42, 0xF7, 0x10, 0x7C, 0x28, 0x27, 0x8C, 0x13, 0x95, 0x9C, 0xC7,
        0x24, 0x46, 0x3B, 0x70, 0xCA, 0xE3, 0x85, 0xCB, 0x11, 0xD0, 0x93, 0xB8,
        0xA6, 0x83, 0x20, 0xFF, 0x9F, 0x77, 0xC3, 0xCC, 0x03, 0x6F, 0x08, 0xBF,
        0x40, 0xE7, 0x2B, 0xE2, 0x79, 0x0C, 0xAA, 0x82, 0x41, 0x3A, 0xEA, 0xB9,
        0xE4, 0x9A, 0xA4, 0x97, 0x7E, 0xDA, 0x7A, 0x17, 0x66, 0x94, 0xA1, 0x1D,
        0x3D, 0xF0, 0xDE, 0xB3, 0x0B, 0x72, 0xA7, 0x1C, 0xEF, 0xD1, 0x53, 0x3E,
        0x8F, 0x33, 0x26, 0x5F, 0xEC, 0x76, 0x2A, 0x49, 0x81, 0x88, 0xEE, 0x21,
        0xC4, 0x1A, 0xEB, 0xD9, 0xC5, 0x39, 0x99, 0xCD, 0xAD, 0x31, 0x8B, 0x01,
        0x18, 0x23, 0xDD, 0x1F, 0x4E, 0x2D, 0xF9, 0x48, 0x4F, 0xF2, 0x65, 0x8E,
        0x78, 0x5C, 0x58, 0x19, 0x8D, 0xE5, 0x98, 0x57, 0x67, 0x7F, 0x05, 0x64,
        0xAF, 0x63, 0xB6, 0xFE, 0xF5, 0xB7, 0x3C, 0xA5, 0xCE, 0xE9, 0x68, 0x44,
        0xE0, 0x4D, 0x43, 0x69, 0x29, 0x2E, 0xAC, 0x15, 0x59, 0xA8, 0x0A, 0x9E,
        0x6E, 0x47, 0xDF, 0x34, 0x35, 0x6A, 0xCF, 0xDC, 0x22, 0xC9, 0xC0, 0x9B,
        0x89, 0xD4, 0xED, 0xAB, 0x12, 0xA2, 0x0D, 0x52, 0xBB, 0x02, 0x2F, 0xA9,
        0xD7, 0x61, 0x1E, 0xB4, 0x50, 0x04, 0xF6, 0xC2, 0x16, 0x25, 0x86, 0x56,
        0x55, 0x09, 0xBE, 0x91
};

/* These MDS tables are actually tables of MDS composed with q0 and q1,
 * because it is only ever used that way and we can save some time by
 * precomputing.  Of course the main saving comes from precomputing the
 * GF(2^8) multiplication involved in the MDS matrix multiply; by looking
 * things up in these tables we reduce the matrix multiply to four lookups
 * and three XORs.  Semi-formally, the definition of these tables is:
 * mds[0][i] = MDS (q1[i] 0 0 0)^T  mds[1][i] = MDS (0 q0[i] 0 0)^T
 * mds[2][i] = MDS (0 0 q1[i] 0)^T  mds[3][i] = MDS (0 0 0 q0[i])^T
 * where ^T means "transpose", the matrix multiply is performed in GF(2^8)
 * represented as GF(2)[x]/v(x) where v(x)=x^8+x^6+x^5+x^3+1 as described
 * by Schneier et al, and I'm casually glossing over the byte/word
 * conversion issues. */

static const u32 mds[4][256] = {
        {
        0xBCBC3275, 0xECEC21F3, 0x202043C6, 0xB3B3C9F4, 0xDADA03DB, 0x02028B7B,
        0xE2E22BFB, 0x9E9EFAC8, 0xC9C9EC4A, 0xD4D409D3, 0x18186BE6, 0x1E1E9F6B,
        0x98980E45, 0xB2B2387D, 0xA6A6D2E8, 0x2626B74B, 0x3C3C57D6, 0x93938A32,
        0x8282EED8, 0x525298FD, 0x7B7BD437, 0xBBBB3771, 0x5B5B97F1, 0x474783E1,
        0x24243C30, 0x5151E20F, 0xBABAC6F8, 0x4A4AF31B, 0xBFBF4887, 0x0D0D70FA,
        0xB0B0B306, 0x7575DE3F, 0xD2D2FD5E, 0x7D7D20BA, 0x666631AE, 0x3A3AA35B,
        0x59591C8A, 0x00000000, 0xCDCD93BC, 0x1A1AE09D, 0xAEAE2C6D, 0x7F7FABC1,
        0x2B2BC7B1, 0xBEBEB90E, 0xE0E0A080, 0x8A8A105D, 0x3B3B52D2, 0x6464BAD5,
        0xD8D888A0, 0xE7E7A584, 0x5F5FE807, 0x1B1B1114, 0x2C2CC2B5, 0xFCFCB490,
        0x3131272C, 0x808065A3, 0x73732AB2, 0x0C0C8173, 0x79795F4C, 0x6B6B4154,
        0x4B4B0292, 0x53536974, 0x94948F36, 0x83831F51, 0x2A2A3638, 0xC4C49CB0,
        0x2222C8BD, 0xD5D5F85A, 0xBDBDC3FC, 0x48487860, 0xFFFFCE62, 0x4C4C0796,
        0x4141776C, 0xC7C7E642, 0xEBEB24F7, 0x1C1C1410, 0x5D5D637C, 0x36362228,
        0x6767C027, 0xE9E9AF8C, 0x4444F913, 0x1414EA95, 0xF5F5BB9C, 0xCFCF18C7,
        0x3F3F2D24, 0xC0C0E346, 0x7272DB3B, 0x54546C70, 0x29294CCA, 0xF0F035E3,
        0x0808FE85, 0xC6C617CB, 0xF3F34F11, 0x8C8CE4D0, 0xA4A45993, 0xCACA96B8,
        0x68683BA6, 0xB8B84D83, 0x38382820, 0xE5E52EFF, 0xADAD569F, 0x0B0B8477,
        0xC8C81DC3, 0x9999FFCC, 0x5858ED03, 0x19199A6F, 0x0E0E0A08, 0x95957EBF,
        0x70705040, 0xF7F730E7, 0x6E6ECF2B, 0x1F1F6EE2, 0xB5B53D79, 0x09090F0C,
        0x616134AA, 0x57571682, 0x9F9F0B41, 0x9D9D803A, 0x111164EA, 0x2525CDB9,
        0xAFAFDDE4, 0x4545089A, 0xDFDF8DA4, 0xA3A35C97, 0xEAEAD57E, 0x353558DA,
        0xEDEDD07A, 0x4343FC17, 0xF8F8CB66, 0xFBFBB194, 0x3737D3A1, 0xFAFA401D,
        0xC2C2683D, 0xB4B4CCF0, 0x32325DDE, 0x9C9C71B3, 0x5656E70B, 0xE3E3DA72,
        0x878760A7, 0x15151B1C, 0xF9F93AEF, 0x6363BFD1, 0x3434A953, 0x9A9A853E,
        0xB1B1428F, 0x7C7CD133, 0x88889B26, 0x3D3DA65F, 0xA1A1D7EC, 0xE4E4DF76,
        0x8181942A, 0x91910149, 0x0F0FFB81, 0xEEEEAA88, 0x161661EE, 0xD7D77321,
        0x9797F5C4, 0xA5A5A81A, 0xFEFE3FEB, 0x6D6DB5D9, 0x7878AEC5, 0xC5C56D39,
        0x1D1DE599, 0x7676A4CD, 0x3E3EDCAD, 0xCBCB6731, 0xB6B6478B, 0xEFEF5B01,
        0x12121E18, 0x6060C523, 0x6A6AB0DD, 0x4D4DF61F, 0xCECEE94E, 0xDEDE7C2D,
        0x55559DF9, 0x7E7E5A48, 0x2121B24F, 0x03037AF2, 0xA0A02665, 0x5E5E198E,
        0x5A5A6678, 0x65654B5C, 0x62624E58, 0xFDFD4519, 0x0606F48D, 0x404086E5,
        0xF2F2BE98, 0x3333AC57, 0x17179067, 0x05058E7F, 0xE8E85E05, 0x4F4F7D64,
        0x89896AAF, 0x10109563, 0x74742FB6, 0x0A0A75FE, 0x5C5C92F5, 0x9B9B74B7,
        0x2D2D333C, 0x3030D6A5, 0x2E2E49CE, 0x494989E9, 0x46467268, 0x77775544,
        0xA8A8D8E0, 0x9696044D, 0x2828BD43, 0xA9A92969, 0xD9D97929, 0x8686912E,
        0xD1D187AC, 0xF4F44A15, 0x8D8D1559, 0xD6D682A8, 0xB9B9BC0A, 0x42420D9E,
        0xF6F6C16E, 0x2F2FB847, 0xDDDD06DF, 0x23233934, 0xCCCC6235, 0xF1F1C46A,
        0xC1C112CF, 0x8585EBDC, 0x8F8F9E22, 0x7171A1C9, 0x9090F0C0, 0xAAAA539B,
        0x0101F189, 0x8B8BE1D4, 0x4E4E8CED, 0x8E8E6FAB, 0xABABA212, 0x6F6F3EA2,
        0xE6E6540D, 0xDBDBF252, 0x92927BBB, 0xB7B7B602, 0x6969CA2F, 0x3939D9A9,
        0xD3D30CD7, 0xA7A72361, 0xA2A2AD1E, 0xC3C399B4, 0x6C6C4450, 0x07070504,
        0x04047FF6, 0x272746C2, 0xACACA716, 0xD0D07625, 0x50501386, 0xDCDCF756,
        0x84841A55, 0xE1E15109, 0x7A7A25BE, 0x1313EF91},

        {
        0xA9D93939, 0x67901717, 0xB3719C9C, 0xE8D2A6A6, 0x04050707, 0xFD985252,
        0xA3658080, 0x76DFE4E4, 0x9A084545, 0x92024B4B, 0x80A0E0E0, 0x78665A5A,
        0xE4DDAFAF, 0xDDB06A6A, 0xD1BF6363, 0x38362A2A, 0x0D54E6E6, 0xC6432020,
        0x3562CCCC, 0x98BEF2F2, 0x181E1212, 0xF724EBEB, 0xECD7A1A1, 0x6C774141,
        0x43BD2828, 0x7532BCBC, 0x37D47B7B, 0x269B8888, 0xFA700D0D, 0x13F94444,
        0x94B1FBFB, 0x485A7E7E, 0xF27A0303, 0xD0E48C8C, 0x8B47B6B6, 0x303C2424,
        0x84A5E7E7, 0x54416B6B, 0xDF06DDDD, 0x23C56060, 0x1945FDFD, 0x5BA33A3A,
        0x3D68C2C2, 0x59158D8D, 0xF321ECEC, 0xAE316666, 0xA23E6F6F, 0x82165757,
        0x63951010, 0x015BEFEF, 0x834DB8B8, 0x2E918686, 0xD9B56D6D, 0x511F8383,
        0x9B53AAAA, 0x7C635D5D, 0xA63B6868, 0xEB3FFEFE, 0xA5D63030, 0xBE257A7A,
        0x16A7ACAC, 0x0C0F0909, 0xE335F0F0, 0x6123A7A7, 0xC0F09090, 0x8CAFE9E9,
        0x3A809D9D, 0xF5925C5C, 0x73810C0C, 0x2C273131, 0x2576D0D0, 0x0BE75656,
        0xBB7B9292, 0x4EE9CECE, 0x89F10101, 0x6B9F1E1E, 0x53A93434, 0x6AC4F1F1,
        0xB499C3C3, 0xF1975B5B, 0xE1834747, 0xE66B1818, 0xBDC82222, 0x450E9898,
        0xE26E1F1F, 0xF4C9B3B3, 0xB62F7474, 0x66CBF8F8, 0xCCFF9999, 0x95EA1414,
        0x03ED5858, 0x56F7DCDC, 0xD4E18B8B, 0x1C1B1515, 0x1EADA2A2, 0xD70CD3D3,
        0xFB2BE2E2, 0xC31DC8C8, 0x8E195E5E, 0xB5C22C2C, 0xE9894949, 0xCF12C1C1,
        0xBF7E9595, 0xBA207D7D, 0xEA641111, 0x77840B0B, 0x396DC5C5, 0xAF6A8989,
        0x33D17C7C, 0xC9A17171, 0x62CEFFFF, 0x7137BBBB, 0x81FB0F0F, 0x793DB5B5,
        0x0951E1E1, 0xADDC3E3E, 0x242D3F3F, 0xCDA47676, 0xF99D5555, 0xD8EE8282,
        0xE5864040, 0xC5AE7878, 0xB9CD2525, 0x4D049696, 0x44557777, 0x080A0E0E,
        0x86135050, 0xE730F7F7, 0xA1D33737, 0x1D40FAFA, 0xAA346161, 0xED8C4E4E,
        0x06B3B0B0, 0x706C5454, 0xB22A7373, 0xD2523B3B, 0x410B9F9F, 0x7B8B0202,
        0xA088D8D8, 0x114FF3F3, 0x3167CBCB, 0xC2462727, 0x27C06767, 0x90B4FCFC,
        0x20283838, 0xF67F0404, 0x60784848, 0xFF2EE5E5, 0x96074C4C, 0x5C4B6565,
        0xB1C72B2B, 0xAB6F8E8E, 0x9E0D4242, 0x9CBBF5F5, 0x52F2DBDB, 0x1BF34A4A,
        0x5FA63D3D, 0x9359A4A4, 0x0ABCB9B9, 0xEF3AF9F9, 0x91EF1313, 0x85FE0808,
        0x49019191, 0xEE611616, 0x2D7CDEDE, 0x4FB22121, 0x8F42B1B1, 0x3BDB7272,
        0x47B82F2F, 0x8748BFBF, 0x6D2CAEAE, 0x46E3C0C0, 0xD6573C3C, 0x3E859A9A,
        0x6929A9A9, 0x647D4F4F, 0x2A948181, 0xCE492E2E, 0xCB17C6C6, 0x2FCA6969,
        0xFCC3BDBD, 0x975CA3A3, 0x055EE8E8, 0x7AD0EDED, 0xAC87D1D1, 0x7F8E0505,
        0xD5BA6464, 0x1AA8A5A5, 0x4BB72626, 0x0EB9BEBE, 0xA7608787, 0x5AF8D5D5,
        0x28223636, 0x14111B1B, 0x3FDE7575, 0x2979D9D9, 0x88AAEEEE, 0x3C332D2D,
        0x4C5F7979, 0x02B6B7B7, 0xB896CACA, 0xDA583535, 0xB09CC4C4, 0x17FC4343,
        0x551A8484, 0x1FF64D4D, 0x8A1C5959, 0x7D38B2B2, 0x57AC3333, 0xC718CFCF,
        0x8DF40606, 0x74695353, 0xB7749B9B, 0xC4F59797, 0x9F56ADAD, 0x72DAE3E3,
        0x7ED5EAEA, 0x154AF4F4, 0x229E8F8F, 0x12A2ABAB, 0x584E6262, 0x07E85F5F,
        0x99E51D1D, 0x34392323, 0x6EC1F6F6, 0x50446C6C, 0xDE5D3232, 0x68724646,
        0x6526A0A0, 0xBC93CDCD, 0xDB03DADA, 0xF8C6BABA, 0xC8FA9E9E, 0xA882D6D6,
        0x2BCF6E6E, 0x40507070, 0xDCEB8585, 0xFE750A0A, 0x328A9393, 0xA48DDFDF,
        0xCA4C2929, 0x10141C1C, 0x2173D7D7, 0xF0CCB4B4, 0xD309D4D4, 0x5D108A8A,
        0x0FE25151, 0x00000000, 0x6F9A1919, 0x9DE01A1A, 0x368F9494, 0x42E6C7C7,
        0x4AECC9C9, 0x5EFDD2D2, 0xC1AB7F7F, 0xE0D8A8A8},

        {
        0xBC75BC32, 0xECF3EC21, 0x20C62043, 0xB3F4B3C9, 0xDADBDA03, 0x027B028B,
        0xE2FBE22B, 0x9EC89EFA, 0xC94AC9EC, 0xD4D3D409, 0x18E6186B, 0x1E6B1E9F,
        0x9845980E, 0xB27DB238, 0xA6E8A6D2, 0x264B26B7, 0x3CD63C57, 0x9332938A,
        0x82D882EE, 0x52FD5298, 0x7B377BD4, 0xBB71BB37, 0x5BF15B97, 0x47E14783,
        0x2430243C, 0x510F51E2, 0xBAF8BAC6, 0x4A1B4AF3, 0xBF87BF48, 0x0DFA0D70,
        0xB006B0B3, 0x753F75DE, 0xD25ED2FD, 0x7DBA7D20, 0x66AE6631, 0x3A5B3AA3,
        0x598A591C, 0x00000000, 0xCDBCCD93, 0x1A9D1AE0, 0xAE6DAE2C, 0x7FC17FAB,
        0x2BB12BC7, 0xBE0EBEB9, 0xE080E0A0, 0x8A5D8A10, 0x3BD23B52, 0x64D564BA,
        0xD8A0D888, 0xE784E7A5, 0x5F075FE8, 0x1B141B11, 0x2CB52CC2, 0xFC90FCB4,
        0x312C3127, 0x80A38065, 0x73B2732A, 0x0C730C81, 0x794C795F, 0x6B546B41,
        0x4B924B02, 0x53745369, 0x9436948F, 0x8351831F, 0x2A382A36, 0xC4B0C49C,
        0x22BD22C8, 0xD55AD5F8, 0xBDFCBDC3, 0x48604878, 0xFF62FFCE, 0x4C964C07,
        0x416C4177, 0xC742C7E6, 0xEBF7EB24, 0x1C101C14, 0x5D7C5D63, 0x36283622,
        0x672767C0, 0xE98CE9AF, 0x441344F9, 0x149514EA, 0xF59CF5BB, 0xCFC7CF18,
        0x3F243F2D, 0xC046C0E3, 0x723B72DB, 0x5470546C, 0x29CA294C, 0xF0E3F035,
        0x088508FE, 0xC6CBC617, 0xF311F34F, 0x8CD08CE4, 0xA493A459, 0xCAB8CA96,
        0x68A6683B, 0xB883B84D, 0x38203828, 0xE5FFE52E, 0xAD9FAD56, 0x0B770B84,
        0xC8C3C81D, 0x99CC99FF, 0x580358ED, 0x196F199A, 0x0E080E0A, 0x95BF957E,
        0x70407050, 0xF7E7F730, 0x6E2B6ECF, 0x1FE21F6E, 0xB579B53D, 0x090C090F,
        0x61AA6134, 0x57825716, 0x9F419F0B, 0x9D3A9D80, 0x11EA1164, 0x25B925CD,
        0xAFE4AFDD, 0x459A4508, 0xDFA4DF8D, 0xA397A35C, 0xEA7EEAD5, 0x35DA3558,
        0xED7AEDD0, 0x431743FC, 0xF866F8CB, 0xFB94FBB1, 0x37A137D3, 0xFA1DFA40,
        0xC23DC268, 0xB4F0B4CC, 0x32DE325D, 0x9CB39C71, 0x560B56E7, 0xE372E3DA,
        0x87A78760, 0x151C151B, 0xF9EFF93A, 0x63D163BF, 0x345334A9, 0x9A3E9A85,
        0xB18FB142, 0x7C337CD1, 0x8826889B, 0x3D5F3DA6, 0xA1ECA1D7, 0xE476E4DF,
        0x812A8194, 0x91499101, 0x0F810FFB, 0xEE88EEAA, 0x16EE1661, 0xD721D773,
        0x97C497F5, 0xA51AA5A8, 0xFEEBFE3F, 0x6DD96DB5, 0x78C578AE, 0xC539C56D,
        0x1D991DE5, 0x76CD76A4, 0x3EAD3EDC, 0xCB31CB67, 0xB68BB647, 0xEF01EF5B,
        0x1218121E, 0x602360C5, 0x6ADD6AB0, 0x4D1F4DF6, 0xCE4ECEE9, 0xDE2DDE7C,
        0x55F9559D, 0x7E487E5A, 0x214F21B2, 0x03F2037A, 0xA065A026, 0x5E8E5E19,
        0x5A785A66, 0x655C654B, 0x6258624E, 0xFD19FD45, 0x068D06F4, 0x40E54086,
        0xF298F2BE, 0x335733AC, 0x17671790, 0x057F058E, 0xE805E85E, 0x4F644F7D,
        0x89AF896A, 0x10631095, 0x74B6742F, 0x0AFE0A75, 0x5CF55C92, 0x9BB79B74,
        0x2D3C2D33, 0x30A530D6, 0x2ECE2E49, 0x49E94989, 0x46684672, 0x77447755,
        0xA8E0A8D8, 0x964D9604, 0x284328BD, 0xA969A929, 0xD929D979, 0x862E8691,
        0xD1ACD187, 0xF415F44A, 0x8D598D15, 0xD6A8D682, 0xB90AB9BC, 0x429E420D,
        0xF66EF6C1, 0x2F472FB8, 0xDDDFDD06, 0x23342339, 0xCC35CC62, 0xF16AF1C4,
        0xC1CFC112, 0x85DC85EB, 0x8F228F9E, 0x71C971A1, 0x90C090F0, 0xAA9BAA53,
        0x018901F1, 0x8BD48BE1, 0x4EED4E8C, 0x8EAB8E6F, 0xAB12ABA2, 0x6FA26F3E,
        0xE60DE654, 0xDB52DBF2, 0x92BB927B, 0xB702B7B6, 0x692F69CA, 0x39A939D9,
        0xD3D7D30C, 0xA761A723, 0xA21EA2AD, 0xC3B4C399, 0x6C506C44, 0x07040705,
        0x04F6047F, 0x27C22746, 0xAC16ACA7, 0xD025D076, 0x50865013, 0xDC56DCF7,
        0x8455841A, 0xE109E151, 0x7ABE7A25, 0x139113EF},

        {
        0xD939A9D9, 0x90176790, 0x719CB371, 0xD2A6E8D2, 0x05070405, 0x9852FD98,
        0x6580A365, 0xDFE476DF, 0x08459A08, 0x024B9202, 0xA0E080A0, 0x665A7866,
        0xDDAFE4DD, 0xB06ADDB0, 0xBF63D1BF, 0x362A3836, 0x54E60D54, 0x4320C643,
        0x62CC3562, 0xBEF298BE, 0x1E12181E, 0x24EBF724, 0xD7A1ECD7, 0x77416C77,
        0xBD2843BD, 0x32BC7532, 0xD47B37D4, 0x9B88269B, 0x700DFA70, 0xF94413F9,
        0xB1FB94B1, 0x5A7E485A, 0x7A03F27A, 0xE48CD0E4, 0x47B68B47, 0x3C24303C,
        0xA5E784A5, 0x416B5441, 0x06DDDF06, 0xC56023C5, 0x45FD1945, 0xA33A5BA3,
        0x68C23D68, 0x158D5915, 0x21ECF321, 0x3166AE31, 0x3E6FA23E, 0x16578216,
        0x95106395, 0x5BEF015B, 0x4DB8834D, 0x91862E91, 0xB56DD9B5, 0x1F83511F,
        0x53AA9B53, 0x635D7C63, 0x3B68A63B, 0x3FFEEB3F, 0xD630A5D6, 0x257ABE25,
        0xA7AC16A7, 0x0F090C0F, 0x35F0E335, 0x23A76123, 0xF090C0F0, 0xAFE98CAF,
        0x809D3A80, 0x925CF592, 0x810C7381, 0x27312C27, 0x76D02576, 0xE7560BE7,
        0x7B92BB7B, 0xE9CE4EE9, 0xF10189F1, 0x9F1E6B9F, 0xA93453A9, 0xC4F16AC4,
        0x99C3B499, 0x975BF197, 0x8347E183, 0x6B18E66B, 0xC822BDC8, 0x0E98450E,
        0x6E1FE26E, 0xC9B3F4C9, 0x2F74B62F, 0xCBF866CB, 0xFF99CCFF, 0xEA1495EA,
        0xED5803ED, 0xF7DC56F7, 0xE18BD4E1, 0x1B151C1B, 0xADA21EAD, 0x0CD3D70C,
        0x2BE2FB2B, 0x1DC8C31D, 0x195E8E19, 0xC22CB5C2, 0x8949E989, 0x12C1CF12,
        0x7E95BF7E, 0x207DBA20, 0x6411EA64, 0x840B7784, 0x6DC5396D, 0x6A89AF6A,
        0xD17C33D1, 0xA171C9A1, 0xCEFF62CE, 0x37BB7137, 0xFB0F81FB, 0x3DB5793D,
        0x51E10951, 0xDC3EADDC, 0x2D3F242D, 0xA476CDA4, 0x9D55F99D, 0xEE82D8EE,
        0x8640E586, 0xAE78C5AE, 0xCD25B9CD, 0x04964D04, 0x55774455, 0x0A0E080A,
        0x13508613, 0x30F7E730, 0xD337A1D3, 0x40FA1D40, 0x3461AA34, 0x8C4EED8C,
        0xB3B006B3, 0x6C54706C, 0x2A73B22A, 0x523BD252, 0x0B9F410B, 0x8B027B8B,
        0x88D8A088, 0x4FF3114F, 0x67CB3167, 0x4627C246, 0xC06727C0, 0xB4FC90B4,
        0x28382028, 0x7F04F67F, 0x78486078, 0x2EE5FF2E, 0x074C9607, 0x4B655C4B,
        0xC72BB1C7, 0x6F8EAB6F, 0x0D429E0D, 0xBBF59CBB, 0xF2DB52F2, 0xF34A1BF3,
        0xA63D5FA6, 0x59A49359, 0xBCB90ABC, 0x3AF9EF3A, 0xEF1391EF, 0xFE0885FE,
        0x01914901, 0x6116EE61, 0x7CDE2D7C, 0xB2214FB2, 0x42B18F42, 0xDB723BDB,
        0xB82F47B8, 0x48BF8748, 0x2CAE6D2C, 0xE3C046E3, 0x573CD657, 0x859A3E85,
        0x29A96929, 0x7D4F647D, 0x94812A94, 0x492ECE49, 0x17C6CB17, 0xCA692FCA,
        0xC3BDFCC3, 0x5CA3975C, 0x5EE8055E, 0xD0ED7AD0, 0x87D1AC87, 0x8E057F8E,
        0xBA64D5BA, 0xA8A51AA8, 0xB7264BB7, 0xB9BE0EB9, 0x6087A760, 0xF8D55AF8,
        0x22362822, 0x111B1411, 0xDE753FDE, 0x79D92979, 0xAAEE88AA, 0x332D3C33,
        0x5F794C5F, 0xB6B702B6, 0x96CAB896, 0x5835DA58, 0x9CC4B09C, 0xFC4317FC,
        0x1A84551A, 0xF64D1FF6, 0x1C598A1C, 0x38B27D38, 0xAC3357AC, 0x18CFC718,
        0xF4068DF4, 0x69537469, 0x749BB774, 0xF597C4F5, 0x56AD9F56, 0xDAE372DA,
        0xD5EA7ED5, 0x4AF4154A, 0x9E8F229E, 0xA2AB12A2, 0x4E62584E, 0xE85F07E8,
        0xE51D99E5, 0x39233439, 0xC1F66EC1, 0x446C5044, 0x5D32DE5D, 0x72466872,
        0x26A06526, 0x93CDBC93, 0x03DADB03, 0xC6BAF8C6, 0xFA9EC8FA, 0x82D6A882,
        0xCF6E2BCF, 0x50704050, 0xEB85DCEB, 0x750AFE75, 0x8A93328A, 0x8DDFA48D,
        0x4C29CA4C, 0x141C1014, 0x73D72173, 0xCCB4F0CC, 0x09D4D309, 0x108A5D10,
        0xE2510FE2, 0x00000000, 0x9A196F9A, 0xE01A9DE0, 0x8F94368F, 0xE6C742E6,
        0xECC94AEC, 0xFDD25EFD, 0xAB7FC1AB, 0xD8A8E0D8}
};

/* The exp_to_poly and poly_to_exp tables are used to perform efficient
 * operations in GF(2^8) represented as GF(2)[x]/w(x) where
 * w(x)=x^8+x^6+x^3+x^2+1.  We care about doing that because it's part of the
 * definition of the RS matrix in the key schedule.  Elements of that field
 * are polynomials of degree not greater than 7 and all coefficients 0 or 1,
 * which can be represented naturally by bytes (just substitute x=2).  In that
 * form, GF(2^8) addition is the same as bitwise XOR, but GF(2^8)
 * multiplication is inefficient without hardware support.  To multiply
 * faster, I make use of the fact x is a generator for the nonzero elements,
 * so that every element p of GF(2)[x]/w(x) is either 0 or equal to (x)^n for
 * some n in 0..254.  Note that caret is exponentiation in GF(2^8),
 * *not* polynomial notation.  So if I want to compute pq where p and q are
 * in GF(2^8), I can just say:
 *    1. if p=0 or q=0 then pq=0
 *    2. otherwise, find m and n such that p=x^m and q=x^n
 *    3. pq=(x^m)(x^n)=x^(m+n), so add m and n and find pq
 * The translations in steps 2 and 3 are looked up in the tables
 * poly_to_exp (for step 2) and exp_to_poly (for step 3).  To see this
 * in action, look at the CALC_S macro.  As additional wrinkles, note that
 * one of my operands is always a constant, so the poly_to_exp lookup on it
 * is done in advance; I included the original values in the comments so
 * readers can have some chance of recognizing that this *is* the RS matrix
 * from the Twofish paper.  I've only included the table entries I actually
 * need; I never do a lookup on a variable input of zero and the biggest
 * exponents I'll ever see are 254 (variable) and 237 (constant), so they'll
 * never sum to more than 491.        I'm repeating part of the exp_to_poly table
 * so that I don't have to do mod-255 reduction in the exponent arithmetic.
 * Since I know my constant operands are never zero, I only have to worry
 * about zero values in the variable operand, and I do it with a simple
 * conditional branch.        I know conditionals are expensive, but I couldn't
 * see a non-horrible way of avoiding them, and I did manage to group the
 * statements so that each if covers four group multiplications. */

static const u8 poly_to_exp[255] = {
        0x00, 0x01, 0x17, 0x02, 0x2E, 0x18, 0x53, 0x03, 0x6A, 0x2F, 0x93, 0x19,
        0x34, 0x54, 0x45, 0x04, 0x5C, 0x6B, 0xB6, 0x30, 0xA6, 0x94, 0x4B, 0x1A,
        0x8C, 0x35, 0x81, 0x55, 0xAA, 0x46, 0x0D, 0x05, 0x24, 0x5D, 0x87, 0x6C,
        0x9B, 0xB7, 0xC1, 0x31, 0x2B, 0xA7, 0xA3, 0x95, 0x98, 0x4C, 0xCA, 0x1B,
        0xE6, 0x8D, 0x73, 0x36, 0xCD, 0x82, 0x12, 0x56, 0x62, 0xAB, 0xF0, 0x47,
        0x4F, 0x0E, 0xBD, 0x06, 0xD4, 0x25, 0xD2, 0x5E, 0x27, 0x88, 0x66, 0x6D,
        0xD6, 0x9C, 0x79, 0xB8, 0x08, 0xC2, 0xDF, 0x32, 0x68, 0x2C, 0xFD, 0xA8,
        0x8A, 0xA4, 0x5A, 0x96, 0x29, 0x99, 0x22, 0x4D, 0x60, 0xCB, 0xE4, 0x1C,
        0x7B, 0xE7, 0x3B, 0x8E, 0x9E, 0x74, 0xF4, 0x37, 0xD8, 0xCE, 0xF9, 0x83,
        0x6F, 0x13, 0xB2, 0x57, 0xE1, 0x63, 0xDC, 0xAC, 0xC4, 0xF1, 0xAF, 0x48,
        0x0A, 0x50, 0x42, 0x0F, 0xBA, 0xBE, 0xC7, 0x07, 0xDE, 0xD5, 0x78, 0x26,
        0x65, 0xD3, 0xD1, 0x5F, 0xE3, 0x28, 0x21, 0x89, 0x59, 0x67, 0xFC, 0x6E,
        0xB1, 0xD7, 0xF8, 0x9D, 0xF3, 0x7A, 0x3A, 0xB9, 0xC6, 0x09, 0x41, 0xC3,
        0xAE, 0xE0, 0xDB, 0x33, 0x44, 0x69, 0x92, 0x2D, 0x52, 0xFE, 0x16, 0xA9,
        0x0C, 0x8B, 0x80, 0xA5, 0x4A, 0x5B, 0xB5, 0x97, 0xC9, 0x2A, 0xA2, 0x9A,
        0xC0, 0x23, 0x86, 0x4E, 0xBC, 0x61, 0xEF, 0xCC, 0x11, 0xE5, 0x72, 0x1D,
        0x3D, 0x7C, 0xEB, 0xE8, 0xE9, 0x3C, 0xEA, 0x8F, 0x7D, 0x9F, 0xEC, 0x75,
        0x1E, 0xF5, 0x3E, 0x38, 0xF6, 0xD9, 0x3F, 0xCF, 0x76, 0xFA, 0x1F, 0x84,
        0xA0, 0x70, 0xED, 0x14, 0x90, 0xB3, 0x7E, 0x58, 0xFB, 0xE2, 0x20, 0x64,
        0xD0, 0xDD, 0x77, 0xAD, 0xDA, 0xC5, 0x40, 0xF2, 0x39, 0xB0, 0xF7, 0x49,
        0xB4, 0x0B, 0x7F, 0x51, 0x15, 0x43, 0x91, 0x10, 0x71, 0xBB, 0xEE, 0xBF,
        0x85, 0xC8, 0xA1
};

static const u8 exp_to_poly[492] = {
        0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x4D, 0x9A, 0x79, 0xF2,
        0xA9, 0x1F, 0x3E, 0x7C, 0xF8, 0xBD, 0x37, 0x6E, 0xDC, 0xF5, 0xA7, 0x03,
        0x06, 0x0C, 0x18, 0x30, 0x60, 0xC0, 0xCD, 0xD7, 0xE3, 0x8B, 0x5B, 0xB6,
        0x21, 0x42, 0x84, 0x45, 0x8A, 0x59, 0xB2, 0x29, 0x52, 0xA4, 0x05, 0x0A,
        0x14, 0x28, 0x50, 0xA0, 0x0D, 0x1A, 0x34, 0x68, 0xD0, 0xED, 0x97, 0x63,
        0xC6, 0xC1, 0xCF, 0xD3, 0xEB, 0x9B, 0x7B, 0xF6, 0xA1, 0x0F, 0x1E, 0x3C,
        0x78, 0xF0, 0xAD, 0x17, 0x2E, 0x5C, 0xB8, 0x3D, 0x7A, 0xF4, 0xA5, 0x07,
        0x0E, 0x1C, 0x38, 0x70, 0xE0, 0x8D, 0x57, 0xAE, 0x11, 0x22, 0x44, 0x88,
        0x5D, 0xBA, 0x39, 0x72, 0xE4, 0x85, 0x47, 0x8E, 0x51, 0xA2, 0x09, 0x12,
        0x24, 0x48, 0x90, 0x6D, 0xDA, 0xF9, 0xBF, 0x33, 0x66, 0xCC, 0xD5, 0xE7,
        0x83, 0x4B, 0x96, 0x61, 0xC2, 0xC9, 0xDF, 0xF3, 0xAB, 0x1B, 0x36, 0x6C,
        0xD8, 0xFD, 0xB7, 0x23, 0x46, 0x8C, 0x55, 0xAA, 0x19, 0x32, 0x64, 0xC8,
        0xDD, 0xF7, 0xA3, 0x0B, 0x16, 0x2C, 0x58, 0xB0, 0x2D, 0x5A, 0xB4, 0x25,
        0x4A, 0x94, 0x65, 0xCA, 0xD9, 0xFF, 0xB3, 0x2B, 0x56, 0xAC, 0x15, 0x2A,
        0x54, 0xA8, 0x1D, 0x3A, 0x74, 0xE8, 0x9D, 0x77, 0xEE, 0x91, 0x6F, 0xDE,
        0xF1, 0xAF, 0x13, 0x26, 0x4C, 0x98, 0x7D, 0xFA, 0xB9, 0x3F, 0x7E, 0xFC,
        0xB5, 0x27, 0x4E, 0x9C, 0x75, 0xEA, 0x99, 0x7F, 0xFE, 0xB1, 0x2F, 0x5E,
        0xBC, 0x35, 0x6A, 0xD4, 0xE5, 0x87, 0x43, 0x86, 0x41, 0x82, 0x49, 0x92,
        0x69, 0xD2, 0xE9, 0x9F, 0x73, 0xE6, 0x81, 0x4F, 0x9E, 0x71, 0xE2, 0x89,
        0x5F, 0xBE, 0x31, 0x62, 0xC4, 0xC5, 0xC7, 0xC3, 0xCB, 0xDB, 0xFB, 0xBB,
        0x3B, 0x76, 0xEC, 0x95, 0x67, 0xCE, 0xD1, 0xEF, 0x93, 0x6B, 0xD6, 0xE1,
        0x8F, 0x53, 0xA6, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x4D,
        0x9A, 0x79, 0xF2, 0xA9, 0x1F, 0x3E, 0x7C, 0xF8, 0xBD, 0x37, 0x6E, 0xDC,
        0xF5, 0xA7, 0x03, 0x06, 0x0C, 0x18, 0x30, 0x60, 0xC0, 0xCD, 0xD7, 0xE3,
        0x8B, 0x5B, 0xB6, 0x21, 0x42, 0x84, 0x45, 0x8A, 0x59, 0xB2, 0x29, 0x52,
        0xA4, 0x05, 0x0A, 0x14, 0x28, 0x50, 0xA0, 0x0D, 0x1A, 0x34, 0x68, 0xD0,
        0xED, 0x97, 0x63, 0xC6, 0xC1, 0xCF, 0xD3, 0xEB, 0x9B, 0x7B, 0xF6, 0xA1,
        0x0F, 0x1E, 0x3C, 0x78, 0xF0, 0xAD, 0x17, 0x2E, 0x5C, 0xB8, 0x3D, 0x7A,
        0xF4, 0xA5, 0x07, 0x0E, 0x1C, 0x38, 0x70, 0xE0, 0x8D, 0x57, 0xAE, 0x11,
        0x22, 0x44, 0x88, 0x5D, 0xBA, 0x39, 0x72, 0xE4, 0x85, 0x47, 0x8E, 0x51,
        0xA2, 0x09, 0x12, 0x24, 0x48, 0x90, 0x6D, 0xDA, 0xF9, 0xBF, 0x33, 0x66,
        0xCC, 0xD5, 0xE7, 0x83, 0x4B, 0x96, 0x61, 0xC2, 0xC9, 0xDF, 0xF3, 0xAB,
        0x1B, 0x36, 0x6C, 0xD8, 0xFD, 0xB7, 0x23, 0x46, 0x8C, 0x55, 0xAA, 0x19,
        0x32, 0x64, 0xC8, 0xDD, 0xF7, 0xA3, 0x0B, 0x16, 0x2C, 0x58, 0xB0, 0x2D,
        0x5A, 0xB4, 0x25, 0x4A, 0x94, 0x65, 0xCA, 0xD9, 0xFF, 0xB3, 0x2B, 0x56,
        0xAC, 0x15, 0x2A, 0x54, 0xA8, 0x1D, 0x3A, 0x74, 0xE8, 0x9D, 0x77, 0xEE,
        0x91, 0x6F, 0xDE, 0xF1, 0xAF, 0x13, 0x26, 0x4C, 0x98, 0x7D, 0xFA, 0xB9,
        0x3F, 0x7E, 0xFC, 0xB5, 0x27, 0x4E, 0x9C, 0x75, 0xEA, 0x99, 0x7F, 0xFE,
        0xB1, 0x2F, 0x5E, 0xBC, 0x35, 0x6A, 0xD4, 0xE5, 0x87, 0x43, 0x86, 0x41,
        0x82, 0x49, 0x92, 0x69, 0xD2, 0xE9, 0x9F, 0x73, 0xE6, 0x81, 0x4F, 0x9E,
        0x71, 0xE2, 0x89, 0x5F, 0xBE, 0x31, 0x62, 0xC4, 0xC5, 0xC7, 0xC3, 0xCB
};


/* The table constants are indices of
 * S-box entries, preprocessed through q0 and q1. */
static const u8 calc_sb_tbl[512] = {
        0xA9, 0x75, 0x67, 0xF3, 0xB3, 0xC6, 0xE8, 0xF4,
        0x04, 0xDB, 0xFD, 0x7B, 0xA3, 0xFB, 0x76, 0xC8,
        0x9A, 0x4A, 0x92, 0xD3, 0x80, 0xE6, 0x78, 0x6B,
        0xE4, 0x45, 0xDD, 0x7D, 0xD1, 0xE8, 0x38, 0x4B,
        0x0D, 0xD6, 0xC6, 0x32, 0x35, 0xD8, 0x98, 0xFD,
        0x18, 0x37, 0xF7, 0x71, 0xEC, 0xF1, 0x6C, 0xE1,
        0x43, 0x30, 0x75, 0x0F, 0x37, 0xF8, 0x26, 0x1B,
        0xFA, 0x87, 0x13, 0xFA, 0x94, 0x06, 0x48, 0x3F,
        0xF2, 0x5E, 0xD0, 0xBA, 0x8B, 0xAE, 0x30, 0x5B,
        0x84, 0x8A, 0x54, 0x00, 0xDF, 0xBC, 0x23, 0x9D,
        0x19, 0x6D, 0x5B, 0xC1, 0x3D, 0xB1, 0x59, 0x0E,
        0xF3, 0x80, 0xAE, 0x5D, 0xA2, 0xD2, 0x82, 0xD5,
        0x63, 0xA0, 0x01, 0x84, 0x83, 0x07, 0x2E, 0x14,
        0xD9, 0xB5, 0x51, 0x90, 0x9B, 0x2C, 0x7C, 0xA3,
        0xA6, 0xB2, 0xEB, 0x73, 0xA5, 0x4C, 0xBE, 0x54,
        0x16, 0x92, 0x0C, 0x74, 0xE3, 0x36, 0x61, 0x51,
        0xC0, 0x38, 0x8C, 0xB0, 0x3A, 0xBD, 0xF5, 0x5A,
        0x73, 0xFC, 0x2C, 0x60, 0x25, 0x62, 0x0B, 0x96,
        0xBB, 0x6C, 0x4E, 0x42, 0x89, 0xF7, 0x6B, 0x10,
        0x53, 0x7C, 0x6A, 0x28, 0xB4, 0x27, 0xF1, 0x8C,
        0xE1, 0x13, 0xE6, 0x95, 0xBD, 0x9C, 0x45, 0xC7,
        0xE2, 0x24, 0xF4, 0x46, 0xB6, 0x3B, 0x66, 0x70,
        0xCC, 0xCA, 0x95, 0xE3, 0x03, 0x85, 0x56, 0xCB,
        0xD4, 0x11, 0x1C, 0xD0, 0x1E, 0x93, 0xD7, 0xB8,
        0xFB, 0xA6, 0xC3, 0x83, 0x8E, 0x20, 0xB5, 0xFF,
        0xE9, 0x9F, 0xCF, 0x77, 0xBF, 0xC3, 0xBA, 0xCC,
        0xEA, 0x03, 0x77, 0x6F, 0x39, 0x08, 0xAF, 0xBF,
        0x33, 0x40, 0xC9, 0xE7, 0x62, 0x2B, 0x71, 0xE2,
        0x81, 0x79, 0x79, 0x0C, 0x09, 0xAA, 0xAD, 0x82,
        0x24, 0x41, 0xCD, 0x3A, 0xF9, 0xEA, 0xD8, 0xB9,
        0xE5, 0xE4, 0xC5, 0x9A, 0xB9, 0xA4, 0x4D, 0x97,
        0x44, 0x7E, 0x08, 0xDA, 0x86, 0x7A, 0xE7, 0x17,
        0xA1, 0x66, 0x1D, 0x94, 0xAA, 0xA1, 0xED, 0x1D,
        0x06, 0x3D, 0x70, 0xF0, 0xB2, 0xDE, 0xD2, 0xB3,
        0x41, 0x0B, 0x7B, 0x72, 0xA0, 0xA7, 0x11, 0x1C,
        0x31, 0xEF, 0xC2, 0xD1, 0x27, 0x53, 0x90, 0x3E,
        0x20, 0x8F, 0xF6, 0x33, 0x60, 0x26, 0xFF, 0x5F,
        0x96, 0xEC, 0x5C, 0x76, 0xB1, 0x2A, 0xAB, 0x49,
        0x9E, 0x81, 0x9C, 0x88, 0x52, 0xEE, 0x1B, 0x21,
        0x5F, 0xC4, 0x93, 0x1A, 0x0A, 0xEB, 0xEF, 0xD9,
        0x91, 0xC5, 0x85, 0x39, 0x49, 0x99, 0xEE, 0xCD,
        0x2D, 0xAD, 0x4F, 0x31, 0x8F, 0x8B, 0x3B, 0x01,
        0x47, 0x18, 0x87, 0x23, 0x6D, 0xDD, 0x46, 0x1F,
        0xD6, 0x4E, 0x3E, 0x2D, 0x69, 0xF9, 0x64, 0x48,
        0x2A, 0x4F, 0xCE, 0xF2, 0xCB, 0x65, 0x2F, 0x8E,
        0xFC, 0x78, 0x97, 0x5C, 0x05, 0x58, 0x7A, 0x19,
        0xAC, 0x8D, 0x7F, 0xE5, 0xD5, 0x98, 0x1A, 0x57,
        0x4B, 0x67, 0x0E, 0x7F, 0xA7, 0x05, 0x5A, 0x64,
        0x28, 0xAF, 0x14, 0x63, 0x3F, 0xB6, 0x29, 0xFE,
        0x88, 0xF5, 0x3C, 0xB7, 0x4C, 0x3C, 0x02, 0xA5,
        0xB8, 0xCE, 0xDA, 0xE9, 0xB0, 0x68, 0x17, 0x44,
        0x55, 0xE0, 0x1F, 0x4D, 0x8A, 0x43, 0x7D, 0x69,
        0x57, 0x29, 0xC7, 0x2E, 0x8D, 0xAC, 0x74, 0x15,
        0xB7, 0x59, 0xC4, 0xA8, 0x9F, 0x0A, 0x72, 0x9E,
        0x7E, 0x6E, 0x15, 0x47, 0x22, 0xDF, 0x12, 0x34,
        0x58, 0x35, 0x07, 0x6A, 0x99, 0xCF, 0x34, 0xDC,
        0x6E, 0x22, 0x50, 0xC9, 0xDE, 0xC0, 0x68, 0x9B,
        0x65, 0x89, 0xBC, 0xD4, 0xDB, 0xED, 0xF8, 0xAB,
        0xC8, 0x12, 0xA8, 0xA2, 0x2B, 0x0D, 0x40, 0x52,
        0xDC, 0xBB, 0xFE, 0x02, 0x32, 0x2F, 0xA4, 0xA9,
        0xCA, 0xD7, 0x10, 0x61, 0x21, 0x1E, 0xF0, 0xB4,
        0xD3, 0x50, 0x5D, 0x04, 0x0F, 0xF6, 0x00, 0xC2,
        0x6F, 0x16, 0x9D, 0x25, 0x36, 0x86, 0x42, 0x56,
        0x4A, 0x55, 0x5E, 0x09, 0xC1, 0xBE, 0xE0, 0x91
};

/* Macro to perform one column of the RS matrix multiplication.  The
 * parameters a, b, c, and d are the four bytes of output; i is the index
 * of the key bytes, and w, x, y, and z, are the column of constants from
 * the RS matrix, preprocessed through the poly_to_exp table. */

#define CALC_S(a, b, c, d, i, w, x, y, z) \
   if (key[i]) { \
      tmp = poly_to_exp[key[i] - 1]; \
      (a) ^= exp_to_poly[tmp + (w)]; \
      (b) ^= exp_to_poly[tmp + (x)]; \
      (c) ^= exp_to_poly[tmp + (y)]; \
      (d) ^= exp_to_poly[tmp + (z)]; \
   }

/* Macros to calculate the key-dependent S-boxes for a 128-bit key using
 * the S vector from CALC_S.  CALC_SB_2 computes a single entry in all
 * four S-boxes, where i is the index of the entry to compute, and a and b
 * are the index numbers preprocessed through the q0 and q1 tables
 * respectively. */

#define CALC_SB_2(i, a, b) \
   ctx->s[0][i] = mds[0][q0[(a) ^ sa] ^ se]; \
   ctx->s[1][i] = mds[1][q0[(b) ^ sb] ^ sf]; \
   ctx->s[2][i] = mds[2][q1[(a) ^ sc] ^ sg]; \
   ctx->s[3][i] = mds[3][q1[(b) ^ sd] ^ sh]

/* Macro exactly like CALC_SB_2, but for 192-bit keys. */

#define CALC_SB192_2(i, a, b) \
   ctx->s[0][i] = mds[0][q0[q0[(b) ^ sa] ^ se] ^ si]; \
   ctx->s[1][i] = mds[1][q0[q1[(b) ^ sb] ^ sf] ^ sj]; \
   ctx->s[2][i] = mds[2][q1[q0[(a) ^ sc] ^ sg] ^ sk]; \
   ctx->s[3][i] = mds[3][q1[q1[(a) ^ sd] ^ sh] ^ sl];

/* Macro exactly like CALC_SB_2, but for 256-bit keys. */

#define CALC_SB256_2(i, a, b) \
   ctx->s[0][i] = mds[0][q0[q0[q1[(b) ^ sa] ^ se] ^ si] ^ sm]; \
   ctx->s[1][i] = mds[1][q0[q1[q1[(a) ^ sb] ^ sf] ^ sj] ^ sn]; \
   ctx->s[2][i] = mds[2][q1[q0[q0[(a) ^ sc] ^ sg] ^ sk] ^ so]; \
   ctx->s[3][i] = mds[3][q1[q1[q0[(b) ^ sd] ^ sh] ^ sl] ^ sp];

/* Macros to calculate the whitening and round subkeys.  CALC_K_2 computes the
 * last two stages of the h() function for a given index (either 2i or 2i+1).
 * a, b, c, and d are the four bytes going into the last two stages.  For
 * 128-bit keys, this is the entire h() function and a and c are the index
 * preprocessed through q0 and q1 respectively; for longer keys they are the
 * output of previous stages.  j is the index of the first key byte to use.
 * CALC_K computes a pair of subkeys for 128-bit Twofish, by calling CALC_K_2
 * twice, doing the Pseudo-Hadamard Transform, and doing the necessary
 * rotations.  Its parameters are: a, the array to write the results into,
 * j, the index of the first output entry, k and l, the preprocessed indices
 * for index 2i, and m and n, the preprocessed indices for index 2i+1.
 * CALC_K192_2 expands CALC_K_2 to handle 192-bit keys, by doing an
 * additional lookup-and-XOR stage.  The parameters a, b, c and d are the
 * four bytes going into the last three stages.  For 192-bit keys, c = d
 * are the index preprocessed through q0, and a = b are the index
 * preprocessed through q1; j is the index of the first key byte to use.
 * CALC_K192 is identical to CALC_K but for using the CALC_K192_2 macro
 * instead of CALC_K_2.
 * CALC_K256_2 expands CALC_K192_2 to handle 256-bit keys, by doing an
 * additional lookup-and-XOR stage.  The parameters a and b are the index
 * preprocessed through q0 and q1 respectively; j is the index of the first
 * key byte to use.  CALC_K256 is identical to CALC_K but for using the
 * CALC_K256_2 macro instead of CALC_K_2. */

#define CALC_K_2(a, b, c, d, j) \
     mds[0][q0[a ^ key[(j) + 8]] ^ key[j]] \
   ^ mds[1][q0[b ^ key[(j) + 9]] ^ key[(j) + 1]] \
   ^ mds[2][q1[c ^ key[(j) + 10]] ^ key[(j) + 2]] \
   ^ mds[3][q1[d ^ key[(j) + 11]] ^ key[(j) + 3]]

#define CALC_K(a, j, k, l, m, n) \
   x = CALC_K_2 (k, l, k, l, 0); \
   y = CALC_K_2 (m, n, m, n, 4); \
   y = rol32(y, 8); \
   x += y; y += x; ctx->a[j] = x; \
   ctx->a[(j) + 1] = rol32(y, 9)

#define CALC_K192_2(a, b, c, d, j) \
   CALC_K_2 (q0[a ^ key[(j) + 16]], \
             q1[b ^ key[(j) + 17]], \
             q0[c ^ key[(j) + 18]], \
             q1[d ^ key[(j) + 19]], j)

#define CALC_K192(a, j, k, l, m, n) \
   x = CALC_K192_2 (l, l, k, k, 0); \
   y = CALC_K192_2 (n, n, m, m, 4); \
   y = rol32(y, 8); \
   x += y; y += x; ctx->a[j] = x; \
   ctx->a[(j) + 1] = rol32(y, 9)

#define CALC_K256_2(a, b, j) \
   CALC_K192_2 (q1[b ^ key[(j) + 24]], \
                q1[a ^ key[(j) + 25]], \
                q0[a ^ key[(j) + 26]], \
                q0[b ^ key[(j) + 27]], j)

#define CALC_K256(a, j, k, l, m, n) \
   x = CALC_K256_2 (k, l, 0); \
   y = CALC_K256_2 (m, n, 4); \
   y = rol32(y, 8); \
   x += y; y += x; ctx->a[j] = x; \
   ctx->a[(j) + 1] = rol32(y, 9)

/* Perform the key setup. */
int __twofish_setkey(struct twofish_ctx *ctx, const u8 *key,
                     unsigned int key_len)
{
        int i, j, k;

        /* Temporaries for CALC_K. */
        u32 x, y;

        /* The S vector used to key the S-boxes, split up into individual bytes.
         * 128-bit keys use only sa through sh; 256-bit use all of them. */
        u8 sa = 0, sb = 0, sc = 0, sd = 0, se = 0, sf = 0, sg = 0, sh = 0;
        u8 si = 0, sj = 0, sk = 0, sl = 0, sm = 0, sn = 0, so = 0, sp = 0;

        /* Temporary for CALC_S. */
        u8 tmp;

        /* Check key length. */
        if (key_len % 8)
                return -EINVAL; /* unsupported key length */

        /* Compute the first two words of the S vector.  The magic numbers are
         * the entries of the RS matrix, preprocessed through poly_to_exp. The
         * numbers in the comments are the original (polynomial form) matrix
         * entries. */
        CALC_S (sa, sb, sc, sd, 0, 0x00, 0x2D, 0x01, 0x2D); /* 01 A4 02 A4 */
        CALC_S (sa, sb, sc, sd, 1, 0x2D, 0xA4, 0x44, 0x8A); /* A4 56 A1 55 */
        CALC_S (sa, sb, sc, sd, 2, 0x8A, 0xD5, 0xBF, 0xD1); /* 55 82 FC 87 */
        CALC_S (sa, sb, sc, sd, 3, 0xD1, 0x7F, 0x3D, 0x99); /* 87 F3 C1 5A */
        CALC_S (sa, sb, sc, sd, 4, 0x99, 0x46, 0x66, 0x96); /* 5A 1E 47 58 */
        CALC_S (sa, sb, sc, sd, 5, 0x96, 0x3C, 0x5B, 0xED); /* 58 C6 AE DB */
        CALC_S (sa, sb, sc, sd, 6, 0xED, 0x37, 0x4F, 0xE0); /* DB 68 3D 9E */
        CALC_S (sa, sb, sc, sd, 7, 0xE0, 0xD0, 0x8C, 0x17); /* 9E E5 19 03 */
        CALC_S (se, sf, sg, sh, 8, 0x00, 0x2D, 0x01, 0x2D); /* 01 A4 02 A4 */
        CALC_S (se, sf, sg, sh, 9, 0x2D, 0xA4, 0x44, 0x8A); /* A4 56 A1 55 */
        CALC_S (se, sf, sg, sh, 10, 0x8A, 0xD5, 0xBF, 0xD1); /* 55 82 FC 87 */
        CALC_S (se, sf, sg, sh, 11, 0xD1, 0x7F, 0x3D, 0x99); /* 87 F3 C1 5A */
        CALC_S (se, sf, sg, sh, 12, 0x99, 0x46, 0x66, 0x96); /* 5A 1E 47 58 */
        CALC_S (se, sf, sg, sh, 13, 0x96, 0x3C, 0x5B, 0xED); /* 58 C6 AE DB */
        CALC_S (se, sf, sg, sh, 14, 0xED, 0x37, 0x4F, 0xE0); /* DB 68 3D 9E */
        CALC_S (se, sf, sg, sh, 15, 0xE0, 0xD0, 0x8C, 0x17); /* 9E E5 19 03 */

        if (key_len == 24 || key_len == 32) { /* 192- or 256-bit key */
                /* Calculate the third word of the S vector */
                CALC_S (si, sj, sk, sl, 16, 0x00, 0x2D, 0x01, 0x2D); /* 01 A4 02 A4 */
                CALC_S (si, sj, sk, sl, 17, 0x2D, 0xA4, 0x44, 0x8A); /* A4 56 A1 55 */
                CALC_S (si, sj, sk, sl, 18, 0x8A, 0xD5, 0xBF, 0xD1); /* 55 82 FC 87 */
                CALC_S (si, sj, sk, sl, 19, 0xD1, 0x7F, 0x3D, 0x99); /* 87 F3 C1 5A */
                CALC_S (si, sj, sk, sl, 20, 0x99, 0x46, 0x66, 0x96); /* 5A 1E 47 58 */
                CALC_S (si, sj, sk, sl, 21, 0x96, 0x3C, 0x5B, 0xED); /* 58 C6 AE DB */
                CALC_S (si, sj, sk, sl, 22, 0xED, 0x37, 0x4F, 0xE0); /* DB 68 3D 9E */
                CALC_S (si, sj, sk, sl, 23, 0xE0, 0xD0, 0x8C, 0x17); /* 9E E5 19 03 */
        }

        if (key_len == 32) { /* 256-bit key */
                /* Calculate the fourth word of the S vector */
                CALC_S (sm, sn, so, sp, 24, 0x00, 0x2D, 0x01, 0x2D); /* 01 A4 02 A4 */
                CALC_S (sm, sn, so, sp, 25, 0x2D, 0xA4, 0x44, 0x8A); /* A4 56 A1 55 */
                CALC_S (sm, sn, so, sp, 26, 0x8A, 0xD5, 0xBF, 0xD1); /* 55 82 FC 87 */
                CALC_S (sm, sn, so, sp, 27, 0xD1, 0x7F, 0x3D, 0x99); /* 87 F3 C1 5A */
                CALC_S (sm, sn, so, sp, 28, 0x99, 0x46, 0x66, 0x96); /* 5A 1E 47 58 */
                CALC_S (sm, sn, so, sp, 29, 0x96, 0x3C, 0x5B, 0xED); /* 58 C6 AE DB */
                CALC_S (sm, sn, so, sp, 30, 0xED, 0x37, 0x4F, 0xE0); /* DB 68 3D 9E */
                CALC_S (sm, sn, so, sp, 31, 0xE0, 0xD0, 0x8C, 0x17); /* 9E E5 19 03 */

                /* Compute the S-boxes. */
                for ( i = j = 0, k = 1; i < 256; i++, j += 2, k += 2 ) {
                        CALC_SB256_2( i, calc_sb_tbl[j], calc_sb_tbl[k] );
                }

                /* CALC_K256/CALC_K192/CALC_K loops were unrolled.
                 * Unrolling produced x2.5 more code (+18k on i386),
                 * and speeded up key setup by 7%:
                 * unrolled: twofish_setkey/sec: 41128
                 *     loop: twofish_setkey/sec: 38148
                 * CALC_K256: ~100 insns each
                 * CALC_K192: ~90 insns
                 *    CALC_K: ~70 insns
                 */
                /* Calculate whitening and round subkeys */
                for ( i = 0; i < 8; i += 2 ) {
                        CALC_K256 (w, i, q0[i], q1[i], q0[i+1], q1[i+1]);
                }
                for ( i = 0; i < 32; i += 2 ) {
                        CALC_K256 (k, i, q0[i+8], q1[i+8], q0[i+9], q1[i+9]);
                }
        } else if (key_len == 24) { /* 192-bit key */
                /* Compute the S-boxes. */
                for ( i = j = 0, k = 1; i < 256; i++, j += 2, k += 2 ) {
                        CALC_SB192_2( i, calc_sb_tbl[j], calc_sb_tbl[k] );
                }

                /* Calculate whitening and round subkeys */
                for ( i = 0; i < 8; i += 2 ) {
                        CALC_K192 (w, i, q0[i], q1[i], q0[i+1], q1[i+1]);
                }
                for ( i = 0; i < 32; i += 2 ) {
                        CALC_K192 (k, i, q0[i+8], q1[i+8], q0[i+9], q1[i+9]);
                }
        } else { /* 128-bit key */
                /* Compute the S-boxes. */
                for ( i = j = 0, k = 1; i < 256; i++, j += 2, k += 2 ) {
                        CALC_SB_2( i, calc_sb_tbl[j], calc_sb_tbl[k] );
                }

                /* Calculate whitening and round subkeys */
                for ( i = 0; i < 8; i += 2 ) {
                        CALC_K (w, i, q0[i], q1[i], q0[i+1], q1[i+1]);
                }
                for ( i = 0; i < 32; i += 2 ) {
                        CALC_K (k, i, q0[i+8], q1[i+8], q0[i+9], q1[i+9]);
                }
        }

        return 0;
}
EXPORT_SYMBOL_GPL(__twofish_setkey);

int twofish_setkey(struct crypto_tfm *tfm, const u8 *key, unsigned int key_len)
{
        return __twofish_setkey(crypto_tfm_ctx(tfm), key, key_len);
}
EXPORT_SYMBOL_GPL(twofish_setkey);

MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("Twofish cipher common functions");

















































    9 














    8 








    2 






    2 












    4 




    8 














    7 










































    8 

















   11 





    7 







    1 
    1 











    1 
    1 













    9 


    1 






    8 












































   11 










   13 



   13 




    4 






    8 








   11 








    7 






















    2 

    2 



    1 



    2 






    5 

    2 










    2 













    5 



    5 
    4 








    1 
    2 
    1 
    2 





    1 



    2 


















   10 







    9 


















































    3 
























    4 








    4 



















   10 











    1 




    6 
    3 

    7 
    2 

    7 
    2 

    7 
    2 

    4 
    5 

    7 
    2 





    5 



    5 






    5 




    5 




   34 

















   76 



    1 
    1 
    1 
    1 

   71 

















































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 *        Linux NET3: IP/IP protocol decoder modified to support
 *                    virtual tunnel interface
 *
 *        Authors:
 *                Saurabh Mohan (saurabh.mohan@vyatta.com) 05/07/2012
 */

/*
   This version of net/ipv4/ip_vti.c is cloned of net/ipv4/ipip.c

   For comments look at net/ipv4/ip_gre.c --ANK
 */


#include <linux/capability.h>
#include <linux/module.h>
#include <linux/types.h>
#include <linux/kernel.h>
#include <linux/uaccess.h>
#include <linux/skbuff.h>
#include <linux/netdevice.h>
#include <linux/in.h>
#include <linux/tcp.h>
#include <linux/udp.h>
#include <linux/if_arp.h>
#include <linux/init.h>
#include <linux/netfilter_ipv4.h>
#include <linux/if_ether.h>
#include <linux/icmpv6.h>

#include <net/sock.h>
#include <net/ip.h>
#include <net/icmp.h>
#include <net/ip_tunnels.h>
#include <net/inet_ecn.h>
#include <net/xfrm.h>
#include <net/net_namespace.h>
#include <net/netns/generic.h>

static struct rtnl_link_ops vti_link_ops __read_mostly;

static unsigned int vti_net_id __read_mostly;
static int vti_tunnel_init(struct net_device *dev);

static int vti_input(struct sk_buff *skb, int nexthdr, __be32 spi,
                     int encap_type, bool update_skb_dev)
{
        struct ip_tunnel *tunnel;
        const struct iphdr *iph = ip_hdr(skb);
        struct net *net = dev_net(skb->dev);
        struct ip_tunnel_net *itn = net_generic(net, vti_net_id);
        IP_TUNNEL_DECLARE_FLAGS(flags) = { };

        __set_bit(IP_TUNNEL_NO_KEY_BIT, flags);

        tunnel = ip_tunnel_lookup(itn, skb->dev->ifindex, flags,
                                  iph->saddr, iph->daddr, 0);
        if (tunnel) {
                if (!xfrm4_policy_check(NULL, XFRM_POLICY_IN, skb))
                        goto drop;

                XFRM_TUNNEL_SKB_CB(skb)->tunnel.ip4 = tunnel;

                if (update_skb_dev)
                        skb->dev = tunnel->dev;

                return xfrm_input(skb, nexthdr, spi, encap_type);
        }

        return -EINVAL;
drop:
        kfree_skb(skb);
        return 0;
}

static int vti_input_proto(struct sk_buff *skb, int nexthdr, __be32 spi,
                           int encap_type)
{
        return vti_input(skb, nexthdr, spi, encap_type, false);
}

static int vti_rcv(struct sk_buff *skb, __be32 spi, bool update_skb_dev)
{
        XFRM_SPI_SKB_CB(skb)->family = AF_INET;
        XFRM_SPI_SKB_CB(skb)->daddroff = offsetof(struct iphdr, daddr);

        return vti_input(skb, ip_hdr(skb)->protocol, spi, 0, update_skb_dev);
}

static int vti_rcv_proto(struct sk_buff *skb)
{
        return vti_rcv(skb, 0, false);
}

static int vti_rcv_cb(struct sk_buff *skb, int err)
{
        unsigned short family;
        struct net_device *dev;
        struct xfrm_state *x;
        const struct xfrm_mode *inner_mode;
        struct ip_tunnel *tunnel = XFRM_TUNNEL_SKB_CB(skb)->tunnel.ip4;
        u32 orig_mark = skb->mark;
        int ret;

        if (!tunnel)
                return 1;

        dev = tunnel->dev;

        if (err) {
                DEV_STATS_INC(dev, rx_errors);
                DEV_STATS_INC(dev, rx_dropped);

                return 0;
        }

        x = xfrm_input_state(skb);

        inner_mode = &x->inner_mode;

        if (x->sel.family == AF_UNSPEC) {
                inner_mode = xfrm_ip2inner_mode(x, XFRM_MODE_SKB_CB(skb)->protocol);
                if (inner_mode == NULL) {
                        XFRM_INC_STATS(dev_net(skb->dev),
                                       LINUX_MIB_XFRMINSTATEMODEERROR);
                        return -EINVAL;
                }
        }

        family = inner_mode->family;

        skb->mark = be32_to_cpu(tunnel->parms.i_key);
        ret = xfrm_policy_check(NULL, XFRM_POLICY_IN, skb, family);
        skb->mark = orig_mark;

        if (!ret)
                return -EPERM;

        skb_scrub_packet(skb, !net_eq(tunnel->net, dev_net(skb->dev)));
        skb->dev = dev;
        dev_sw_netstats_rx_add(dev, skb->len);

        return 0;
}

static bool vti_state_check(const struct xfrm_state *x, __be32 dst, __be32 src)
{
        xfrm_address_t *daddr = (xfrm_address_t *)&dst;
        xfrm_address_t *saddr = (xfrm_address_t *)&src;

        /* if there is no transform then this tunnel is not functional.
         * Or if the xfrm is not mode tunnel.
         */
        if (!x || x->props.mode != XFRM_MODE_TUNNEL ||
            x->props.family != AF_INET)
                return false;

        if (!dst)
                return xfrm_addr_equal(saddr, &x->props.saddr, AF_INET);

        if (!xfrm_state_addr_check(x, daddr, saddr, AF_INET))
                return false;

        return true;
}

static netdev_tx_t vti_xmit(struct sk_buff *skb, struct net_device *dev,
                            struct flowi *fl)
{
        struct ip_tunnel *tunnel = netdev_priv(dev);
        struct ip_tunnel_parm_kern *parms = &tunnel->parms;
        struct dst_entry *dst = skb_dst(skb);
        struct net_device *tdev;        /* Device to other host */
        int pkt_len = skb->len;
        int err;
        int mtu;

        if (!dst) {
                switch (skb->protocol) {
                case htons(ETH_P_IP): {
                        struct rtable *rt;

                        fl->u.ip4.flowi4_oif = dev->ifindex;
                        fl->u.ip4.flowi4_flags |= FLOWI_FLAG_ANYSRC;
                        rt = __ip_route_output_key(dev_net(dev), &fl->u.ip4);
                        if (IS_ERR(rt)) {
                                DEV_STATS_INC(dev, tx_carrier_errors);
                                goto tx_error_icmp;
                        }
                        dst = &rt->dst;
                        skb_dst_set(skb, dst);
                        break;
                }
#if IS_ENABLED(CONFIG_IPV6)
                case htons(ETH_P_IPV6):
                        fl->u.ip6.flowi6_oif = dev->ifindex;
                        fl->u.ip6.flowi6_flags |= FLOWI_FLAG_ANYSRC;
                        dst = ip6_route_output(dev_net(dev), NULL, &fl->u.ip6);
                        if (dst->error) {
                                dst_release(dst);
                                dst = NULL;
                                DEV_STATS_INC(dev, tx_carrier_errors);
                                goto tx_error_icmp;
                        }
                        skb_dst_set(skb, dst);
                        break;
#endif
                default:
                        DEV_STATS_INC(dev, tx_carrier_errors);
                        goto tx_error_icmp;
                }
        }

        dst_hold(dst);
        dst = xfrm_lookup_route(tunnel->net, dst, fl, NULL, 0);
        if (IS_ERR(dst)) {
                DEV_STATS_INC(dev, tx_carrier_errors);
                goto tx_error_icmp;
        }

        if (dst->flags & DST_XFRM_QUEUE)
                goto xmit;

        if (!vti_state_check(dst->xfrm, parms->iph.daddr, parms->iph.saddr)) {
                DEV_STATS_INC(dev, tx_carrier_errors);
                dst_release(dst);
                goto tx_error_icmp;
        }

        tdev = dst->dev;

        if (tdev == dev) {
                dst_release(dst);
                DEV_STATS_INC(dev, collisions);
                goto tx_error;
        }

        mtu = dst_mtu(dst);
        if (skb->len > mtu) {
                skb_dst_update_pmtu_no_confirm(skb, mtu);
                if (skb->protocol == htons(ETH_P_IP)) {
                        if (!(ip_hdr(skb)->frag_off & htons(IP_DF)))
                                goto xmit;
                        icmp_ndo_send(skb, ICMP_DEST_UNREACH, ICMP_FRAG_NEEDED,
                                      htonl(mtu));
                } else {
                        if (mtu < IPV6_MIN_MTU)
                                mtu = IPV6_MIN_MTU;

                        icmpv6_ndo_send(skb, ICMPV6_PKT_TOOBIG, 0, mtu);
                }

                dst_release(dst);
                goto tx_error;
        }

xmit:
        skb_scrub_packet(skb, !net_eq(tunnel->net, dev_net(dev)));
        skb_dst_set(skb, dst);
        skb->dev = skb_dst(skb)->dev;

        err = dst_output(tunnel->net, skb->sk, skb);
        if (net_xmit_eval(err) == 0)
                err = pkt_len;
        iptunnel_xmit_stats(dev, err);
        return NETDEV_TX_OK;

tx_error_icmp:
        dst_link_failure(skb);
tx_error:
        DEV_STATS_INC(dev, tx_errors);
        kfree_skb(skb);
        return NETDEV_TX_OK;
}

/* This function assumes it is being called from dev_queue_xmit()
 * and that skb is filled properly by that function.
 */
static netdev_tx_t vti_tunnel_xmit(struct sk_buff *skb, struct net_device *dev)
{
        struct ip_tunnel *tunnel = netdev_priv(dev);
        struct flowi fl;

        if (!pskb_inet_may_pull(skb))
                goto tx_err;

        memset(&fl, 0, sizeof(fl));

        switch (skb->protocol) {
        case htons(ETH_P_IP):
                memset(IPCB(skb), 0, sizeof(*IPCB(skb)));
                xfrm_decode_session(dev_net(dev), skb, &fl, AF_INET);
                break;
        case htons(ETH_P_IPV6):
                memset(IP6CB(skb), 0, sizeof(*IP6CB(skb)));
                xfrm_decode_session(dev_net(dev), skb, &fl, AF_INET6);
                break;
        default:
                goto tx_err;
        }

        /* override mark with tunnel output key */
        fl.flowi_mark = be32_to_cpu(tunnel->parms.o_key);

        return vti_xmit(skb, dev, &fl);

tx_err:
        DEV_STATS_INC(dev, tx_errors);
        kfree_skb(skb);
        return NETDEV_TX_OK;
}

static int vti4_err(struct sk_buff *skb, u32 info)
{
        __be32 spi;
        __u32 mark;
        struct xfrm_state *x;
        struct ip_tunnel *tunnel;
        struct ip_esp_hdr *esph;
        struct ip_auth_hdr *ah ;
        struct ip_comp_hdr *ipch;
        struct net *net = dev_net(skb->dev);
        const struct iphdr *iph = (const struct iphdr *)skb->data;
        int protocol = iph->protocol;
        struct ip_tunnel_net *itn = net_generic(net, vti_net_id);
        IP_TUNNEL_DECLARE_FLAGS(flags) = { };

        __set_bit(IP_TUNNEL_NO_KEY_BIT, flags);

        tunnel = ip_tunnel_lookup(itn, skb->dev->ifindex, flags,
                                  iph->daddr, iph->saddr, 0);
        if (!tunnel)
                return -1;

        mark = be32_to_cpu(tunnel->parms.o_key);

        switch (protocol) {
        case IPPROTO_ESP:
                esph = (struct ip_esp_hdr *)(skb->data+(iph->ihl<<2));
                spi = esph->spi;
                break;
        case IPPROTO_AH:
                ah = (struct ip_auth_hdr *)(skb->data+(iph->ihl<<2));
                spi = ah->spi;
                break;
        case IPPROTO_COMP:
                ipch = (struct ip_comp_hdr *)(skb->data+(iph->ihl<<2));
                spi = htonl(ntohs(ipch->cpi));
                break;
        default:
                return 0;
        }

        switch (icmp_hdr(skb)->type) {
        case ICMP_DEST_UNREACH:
                if (icmp_hdr(skb)->code != ICMP_FRAG_NEEDED)
                        return 0;
                break;
        case ICMP_REDIRECT:
                break;
        default:
                return 0;
        }

        x = xfrm_state_lookup(net, mark, (const xfrm_address_t *)&iph->daddr,
                              spi, protocol, AF_INET);
        if (!x)
                return 0;

        if (icmp_hdr(skb)->type == ICMP_DEST_UNREACH)
                ipv4_update_pmtu(skb, net, info, 0, protocol);
        else
                ipv4_redirect(skb, net, 0, protocol);
        xfrm_state_put(x);

        return 0;
}

static int
vti_tunnel_ctl(struct net_device *dev, struct ip_tunnel_parm_kern *p, int cmd)
{
        IP_TUNNEL_DECLARE_FLAGS(flags) = { };
        int err = 0;

        if (cmd == SIOCADDTUNNEL || cmd == SIOCCHGTUNNEL) {
                if (p->iph.version != 4 || p->iph.protocol != IPPROTO_IPIP ||
                    p->iph.ihl != 5)
                        return -EINVAL;
        }

        if (!ip_tunnel_flags_is_be16_compat(p->i_flags) ||
            !ip_tunnel_flags_is_be16_compat(p->o_flags))
                return -EOVERFLOW;

        if (!(ip_tunnel_flags_to_be16(p->i_flags) & GRE_KEY))
                p->i_key = 0;
        if (!(ip_tunnel_flags_to_be16(p->o_flags) & GRE_KEY))
                p->o_key = 0;

        __set_bit(IP_TUNNEL_VTI_BIT, flags);
        ip_tunnel_flags_copy(p->i_flags, flags);

        err = ip_tunnel_ctl(dev, p, cmd);
        if (err)
                return err;

        if (cmd != SIOCDELTUNNEL) {
                ip_tunnel_flags_from_be16(flags, GRE_KEY);
                ip_tunnel_flags_or(p->i_flags, p->i_flags, flags);
                ip_tunnel_flags_or(p->o_flags, p->o_flags, flags);
        }
        return 0;
}

static const struct net_device_ops vti_netdev_ops = {
        .ndo_init        = vti_tunnel_init,
        .ndo_uninit        = ip_tunnel_uninit,
        .ndo_start_xmit        = vti_tunnel_xmit,
        .ndo_siocdevprivate = ip_tunnel_siocdevprivate,
        .ndo_change_mtu        = ip_tunnel_change_mtu,
        .ndo_get_stats64 = dev_get_tstats64,
        .ndo_get_iflink = ip_tunnel_get_iflink,
        .ndo_tunnel_ctl        = vti_tunnel_ctl,
};

static void vti_tunnel_setup(struct net_device *dev)
{
        dev->netdev_ops                = &vti_netdev_ops;
        dev->header_ops                = &ip_tunnel_header_ops;
        dev->type                = ARPHRD_TUNNEL;
        ip_tunnel_setup(dev, vti_net_id);
}

static int vti_tunnel_init(struct net_device *dev)
{
        struct ip_tunnel *tunnel = netdev_priv(dev);
        struct iphdr *iph = &tunnel->parms.iph;

        __dev_addr_set(dev, &iph->saddr, 4);
        memcpy(dev->broadcast, &iph->daddr, 4);

        dev->flags                = IFF_NOARP;
        dev->addr_len                = 4;
        dev->lltx                = true;
        netif_keep_dst(dev);

        return ip_tunnel_init(dev);
}

static void __net_init vti_fb_tunnel_init(struct net_device *dev)
{
        struct ip_tunnel *tunnel = netdev_priv(dev);
        struct iphdr *iph = &tunnel->parms.iph;

        iph->version                = 4;
        iph->protocol                = IPPROTO_IPIP;
        iph->ihl                = 5;
}

static struct xfrm4_protocol vti_esp4_protocol __read_mostly = {
        .handler        =        vti_rcv_proto,
        .input_handler        =        vti_input_proto,
        .cb_handler        =        vti_rcv_cb,
        .err_handler        =        vti4_err,
        .priority        =        100,
};

static struct xfrm4_protocol vti_ah4_protocol __read_mostly = {
        .handler        =        vti_rcv_proto,
        .input_handler        =        vti_input_proto,
        .cb_handler        =        vti_rcv_cb,
        .err_handler        =        vti4_err,
        .priority        =        100,
};

static struct xfrm4_protocol vti_ipcomp4_protocol __read_mostly = {
        .handler        =        vti_rcv_proto,
        .input_handler        =        vti_input_proto,
        .cb_handler        =        vti_rcv_cb,
        .err_handler        =        vti4_err,
        .priority        =        100,
};

#if IS_ENABLED(CONFIG_INET_XFRM_TUNNEL)
static int vti_rcv_tunnel(struct sk_buff *skb)
{
        XFRM_SPI_SKB_CB(skb)->family = AF_INET;
        XFRM_SPI_SKB_CB(skb)->daddroff = offsetof(struct iphdr, daddr);

        return vti_input(skb, IPPROTO_IPIP, ip_hdr(skb)->saddr, 0, false);
}

static struct xfrm_tunnel vti_ipip_handler __read_mostly = {
        .handler        =        vti_rcv_tunnel,
        .cb_handler        =        vti_rcv_cb,
        .err_handler        =        vti4_err,
        .priority        =        0,
};

#if IS_ENABLED(CONFIG_IPV6)
static struct xfrm_tunnel vti_ipip6_handler __read_mostly = {
        .handler        =        vti_rcv_tunnel,
        .cb_handler        =        vti_rcv_cb,
        .err_handler        =        vti4_err,
        .priority        =        0,
};
#endif
#endif

static int __net_init vti_init_net(struct net *net)
{
        int err;
        struct ip_tunnel_net *itn;

        err = ip_tunnel_init_net(net, vti_net_id, &vti_link_ops, "ip_vti0");
        if (err)
                return err;
        itn = net_generic(net, vti_net_id);
        if (itn->fb_tunnel_dev)
                vti_fb_tunnel_init(itn->fb_tunnel_dev);
        return 0;
}

static void __net_exit vti_exit_batch_rtnl(struct list_head *list_net,
                                           struct list_head *dev_to_kill)
{
        ip_tunnel_delete_nets(list_net, vti_net_id, &vti_link_ops,
                              dev_to_kill);
}

static struct pernet_operations vti_net_ops = {
        .init = vti_init_net,
        .exit_batch_rtnl = vti_exit_batch_rtnl,
        .id   = &vti_net_id,
        .size = sizeof(struct ip_tunnel_net),
};

static int vti_tunnel_validate(struct nlattr *tb[], struct nlattr *data[],
                               struct netlink_ext_ack *extack)
{
        return 0;
}

static void vti_netlink_parms(struct nlattr *data[],
                              struct ip_tunnel_parm_kern *parms,
                              __u32 *fwmark)
{
        memset(parms, 0, sizeof(*parms));

        parms->iph.protocol = IPPROTO_IPIP;

        if (!data)
                return;

        __set_bit(IP_TUNNEL_VTI_BIT, parms->i_flags);

        if (data[IFLA_VTI_LINK])
                parms->link = nla_get_u32(data[IFLA_VTI_LINK]);

        if (data[IFLA_VTI_IKEY])
                parms->i_key = nla_get_be32(data[IFLA_VTI_IKEY]);

        if (data[IFLA_VTI_OKEY])
                parms->o_key = nla_get_be32(data[IFLA_VTI_OKEY]);

        if (data[IFLA_VTI_LOCAL])
                parms->iph.saddr = nla_get_in_addr(data[IFLA_VTI_LOCAL]);

        if (data[IFLA_VTI_REMOTE])
                parms->iph.daddr = nla_get_in_addr(data[IFLA_VTI_REMOTE]);

        if (data[IFLA_VTI_FWMARK])
                *fwmark = nla_get_u32(data[IFLA_VTI_FWMARK]);
}

static int vti_newlink(struct net *src_net, struct net_device *dev,
                       struct nlattr *tb[], struct nlattr *data[],
                       struct netlink_ext_ack *extack)
{
        struct ip_tunnel_parm_kern parms;
        __u32 fwmark = 0;

        vti_netlink_parms(data, &parms, &fwmark);
        return ip_tunnel_newlink(dev, tb, &parms, fwmark);
}

static int vti_changelink(struct net_device *dev, struct nlattr *tb[],
                          struct nlattr *data[],
                          struct netlink_ext_ack *extack)
{
        struct ip_tunnel *t = netdev_priv(dev);
        struct ip_tunnel_parm_kern p;
        __u32 fwmark = t->fwmark;

        vti_netlink_parms(data, &p, &fwmark);
        return ip_tunnel_changelink(dev, tb, &p, fwmark);
}

static size_t vti_get_size(const struct net_device *dev)
{
        return
                /* IFLA_VTI_LINK */
                nla_total_size(4) +
                /* IFLA_VTI_IKEY */
                nla_total_size(4) +
                /* IFLA_VTI_OKEY */
                nla_total_size(4) +
                /* IFLA_VTI_LOCAL */
                nla_total_size(4) +
                /* IFLA_VTI_REMOTE */
                nla_total_size(4) +
                /* IFLA_VTI_FWMARK */
                nla_total_size(4) +
                0;
}

static int vti_fill_info(struct sk_buff *skb, const struct net_device *dev)
{
        struct ip_tunnel *t = netdev_priv(dev);
        struct ip_tunnel_parm_kern *p = &t->parms;

        if (nla_put_u32(skb, IFLA_VTI_LINK, p->link) ||
            nla_put_be32(skb, IFLA_VTI_IKEY, p->i_key) ||
            nla_put_be32(skb, IFLA_VTI_OKEY, p->o_key) ||
            nla_put_in_addr(skb, IFLA_VTI_LOCAL, p->iph.saddr) ||
            nla_put_in_addr(skb, IFLA_VTI_REMOTE, p->iph.daddr) ||
            nla_put_u32(skb, IFLA_VTI_FWMARK, t->fwmark))
                return -EMSGSIZE;

        return 0;
}

static const struct nla_policy vti_policy[IFLA_VTI_MAX + 1] = {
        [IFLA_VTI_LINK]                = { .type = NLA_U32 },
        [IFLA_VTI_IKEY]                = { .type = NLA_U32 },
        [IFLA_VTI_OKEY]                = { .type = NLA_U32 },
        [IFLA_VTI_LOCAL]        = { .len = sizeof_field(struct iphdr, saddr) },
        [IFLA_VTI_REMOTE]        = { .len = sizeof_field(struct iphdr, daddr) },
        [IFLA_VTI_FWMARK]        = { .type = NLA_U32 },
};

static struct rtnl_link_ops vti_link_ops __read_mostly = {
        .kind                = "vti",
        .maxtype        = IFLA_VTI_MAX,
        .policy                = vti_policy,
        .priv_size        = sizeof(struct ip_tunnel),
        .setup                = vti_tunnel_setup,
        .validate        = vti_tunnel_validate,
        .newlink        = vti_newlink,
        .changelink        = vti_changelink,
        .dellink        = ip_tunnel_dellink,
        .get_size        = vti_get_size,
        .fill_info        = vti_fill_info,
        .get_link_net        = ip_tunnel_get_link_net,
};

static int __init vti_init(void)
{
        const char *msg;
        int err;

        pr_info("IPv4 over IPsec tunneling driver\n");

        msg = "tunnel device";
        err = register_pernet_device(&vti_net_ops);
        if (err < 0)
                goto pernet_dev_failed;

        msg = "tunnel protocols";
        err = xfrm4_protocol_register(&vti_esp4_protocol, IPPROTO_ESP);
        if (err < 0)
                goto xfrm_proto_esp_failed;
        err = xfrm4_protocol_register(&vti_ah4_protocol, IPPROTO_AH);
        if (err < 0)
                goto xfrm_proto_ah_failed;
        err = xfrm4_protocol_register(&vti_ipcomp4_protocol, IPPROTO_COMP);
        if (err < 0)
                goto xfrm_proto_comp_failed;

#if IS_ENABLED(CONFIG_INET_XFRM_TUNNEL)
        msg = "ipip tunnel";
        err = xfrm4_tunnel_register(&vti_ipip_handler, AF_INET);
        if (err < 0)
                goto xfrm_tunnel_ipip_failed;
#if IS_ENABLED(CONFIG_IPV6)
        err = xfrm4_tunnel_register(&vti_ipip6_handler, AF_INET6);
        if (err < 0)
                goto xfrm_tunnel_ipip6_failed;
#endif
#endif

        msg = "netlink interface";
        err = rtnl_link_register(&vti_link_ops);
        if (err < 0)
                goto rtnl_link_failed;

        return err;

rtnl_link_failed:
#if IS_ENABLED(CONFIG_INET_XFRM_TUNNEL)
#if IS_ENABLED(CONFIG_IPV6)
        xfrm4_tunnel_deregister(&vti_ipip6_handler, AF_INET6);
xfrm_tunnel_ipip6_failed:
#endif
        xfrm4_tunnel_deregister(&vti_ipip_handler, AF_INET);
xfrm_tunnel_ipip_failed:
#endif
        xfrm4_protocol_deregister(&vti_ipcomp4_protocol, IPPROTO_COMP);
xfrm_proto_comp_failed:
        xfrm4_protocol_deregister(&vti_ah4_protocol, IPPROTO_AH);
xfrm_proto_ah_failed:
        xfrm4_protocol_deregister(&vti_esp4_protocol, IPPROTO_ESP);
xfrm_proto_esp_failed:
        unregister_pernet_device(&vti_net_ops);
pernet_dev_failed:
        pr_err("vti init: failed to register %s\n", msg);
        return err;
}

static void __exit vti_fini(void)
{
        rtnl_link_unregister(&vti_link_ops);
#if IS_ENABLED(CONFIG_INET_XFRM_TUNNEL)
#if IS_ENABLED(CONFIG_IPV6)
        xfrm4_tunnel_deregister(&vti_ipip6_handler, AF_INET6);
#endif
        xfrm4_tunnel_deregister(&vti_ipip_handler, AF_INET);
#endif
        xfrm4_protocol_deregister(&vti_ipcomp4_protocol, IPPROTO_COMP);
        xfrm4_protocol_deregister(&vti_ah4_protocol, IPPROTO_AH);
        xfrm4_protocol_deregister(&vti_esp4_protocol, IPPROTO_ESP);
        unregister_pernet_device(&vti_net_ops);
}

module_init(vti_init);
module_exit(vti_fini);
MODULE_DESCRIPTION("Virtual (secure) IP tunneling library");
MODULE_LICENSE("GPL");
MODULE_ALIAS_RTNL_LINK("vti");
MODULE_ALIAS_NETDEV("ip_vti0");













































































    5 









1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
// SPDX-License-Identifier: GPL-2.0
/*
 * procfs-based user access to knfsd statistics
 *
 * /proc/net/rpc/nfsd
 *
 * Format:
 *        rc <hits> <misses> <nocache>
 *                        Statistsics for the reply cache
 *        fh <stale> <deprecated filehandle cache stats>
 *                        statistics for filehandle lookup
 *        io <bytes-read> <bytes-written>
 *                        statistics for IO throughput
 *        th <threads> <deprecated thread usage histogram stats>
 *                        number of threads
 *        ra <deprecated ra-cache stats>
 *
 *        plus generic RPC stats (see net/sunrpc/stats.c)
 *
 * Copyright (C) 1995, 1996, 1997 Olaf Kirch <okir@monad.swb.de>
 */

#include <linux/seq_file.h>
#include <linux/module.h>
#include <linux/sunrpc/stats.h>
#include <net/net_namespace.h>

#include "nfsd.h"

static int nfsd_show(struct seq_file *seq, void *v)
{
        struct net *net = pde_data(file_inode(seq->file));
        struct nfsd_net *nn = net_generic(net, nfsd_net_id);
        int i;

        seq_printf(seq, "rc %lld %lld %lld\nfh %lld 0 0 0 0\nio %lld %lld\n",
                   percpu_counter_sum_positive(&nn->counter[NFSD_STATS_RC_HITS]),
                   percpu_counter_sum_positive(&nn->counter[NFSD_STATS_RC_MISSES]),
                   percpu_counter_sum_positive(&nn->counter[NFSD_STATS_RC_NOCACHE]),
                   percpu_counter_sum_positive(&nn->counter[NFSD_STATS_FH_STALE]),
                   percpu_counter_sum_positive(&nn->counter[NFSD_STATS_IO_READ]),
                   percpu_counter_sum_positive(&nn->counter[NFSD_STATS_IO_WRITE]));

        /* thread usage: */
        seq_printf(seq, "th %u 0", atomic_read(&nfsd_th_cnt));

        /* deprecated thread usage histogram stats */
        for (i = 0; i < 10; i++)
                seq_puts(seq, " 0.000");

        /* deprecated ra-cache stats */
        seq_puts(seq, "\nra 0 0 0 0 0 0 0 0 0 0 0 0\n");

        /* show my rpc info */
        svc_seq_show(seq, &nn->nfsd_svcstats);

#ifdef CONFIG_NFSD_V4
        /* Show count for individual nfsv4 operations */
        /* Writing operation numbers 0 1 2 also for maintaining uniformity */
        seq_printf(seq, "proc4ops %u", LAST_NFS4_OP + 1);
        for (i = 0; i <= LAST_NFS4_OP; i++) {
                seq_printf(seq, " %lld",
                           percpu_counter_sum_positive(&nn->counter[NFSD_STATS_NFS4_OP(i)]));
        }
        seq_printf(seq, "\nwdeleg_getattr %lld",
                percpu_counter_sum_positive(&nn->counter[NFSD_STATS_WDELEG_GETATTR]));

        seq_putc(seq, '\n');
#endif

        return 0;
}

DEFINE_PROC_SHOW_ATTRIBUTE(nfsd);

void nfsd_proc_stat_init(struct net *net)
{
        struct nfsd_net *nn = net_generic(net, nfsd_net_id);

        svc_proc_register(net, &nn->nfsd_svcstats, &nfsd_proc_ops);
}

void nfsd_proc_stat_shutdown(struct net *net)
{
        svc_proc_unregister(net, "nfsd");
}














































    1 









































    4 


































































































































































































    5 










    1 



    1 







    2 





    1 


    1 




    1 








    4 




    4 














































































    5 










































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 *  SR-IPv6 implementation -- HMAC functions
 *
 *  Author:
 *  David Lebrun <david.lebrun@uclouvain.be>
 */

#include <linux/errno.h>
#include <linux/kernel.h>
#include <linux/types.h>
#include <linux/socket.h>
#include <linux/sockios.h>
#include <linux/net.h>
#include <linux/netdevice.h>
#include <linux/in6.h>
#include <linux/icmpv6.h>
#include <linux/mroute6.h>
#include <linux/slab.h>
#include <linux/rhashtable.h>

#include <linux/netfilter.h>
#include <linux/netfilter_ipv6.h>

#include <net/sock.h>
#include <net/snmp.h>

#include <net/ipv6.h>
#include <net/protocol.h>
#include <net/transp_v6.h>
#include <net/rawv6.h>
#include <net/ndisc.h>
#include <net/ip6_route.h>
#include <net/addrconf.h>
#include <net/xfrm.h>

#include <crypto/hash.h>
#include <net/seg6.h>
#include <net/genetlink.h>
#include <net/seg6_hmac.h>
#include <linux/random.h>

static DEFINE_PER_CPU(char [SEG6_HMAC_RING_SIZE], hmac_ring);

static int seg6_hmac_cmpfn(struct rhashtable_compare_arg *arg, const void *obj)
{
        const struct seg6_hmac_info *hinfo = obj;

        return (hinfo->hmackeyid != *(__u32 *)arg->key);
}

static inline void seg6_hinfo_release(struct seg6_hmac_info *hinfo)
{
        kfree_rcu(hinfo, rcu);
}

static void seg6_free_hi(void *ptr, void *arg)
{
        struct seg6_hmac_info *hinfo = (struct seg6_hmac_info *)ptr;

        if (hinfo)
                seg6_hinfo_release(hinfo);
}

static const struct rhashtable_params rht_params = {
        .head_offset                = offsetof(struct seg6_hmac_info, node),
        .key_offset                = offsetof(struct seg6_hmac_info, hmackeyid),
        .key_len                = sizeof(u32),
        .automatic_shrinking        = true,
        .obj_cmpfn                = seg6_hmac_cmpfn,
};

static struct seg6_hmac_algo hmac_algos[] = {
        {
                .alg_id = SEG6_HMAC_ALGO_SHA1,
                .name = "hmac(sha1)",
        },
        {
                .alg_id = SEG6_HMAC_ALGO_SHA256,
                .name = "hmac(sha256)",
        },
};

static struct sr6_tlv_hmac *seg6_get_tlv_hmac(struct ipv6_sr_hdr *srh)
{
        struct sr6_tlv_hmac *tlv;

        if (srh->hdrlen < (srh->first_segment + 1) * 2 + 5)
                return NULL;

        if (!sr_has_hmac(srh))
                return NULL;

        tlv = (struct sr6_tlv_hmac *)
              ((char *)srh + ((srh->hdrlen + 1) << 3) - 40);

        if (tlv->tlvhdr.type != SR6_TLV_HMAC || tlv->tlvhdr.len != 38)
                return NULL;

        return tlv;
}

static struct seg6_hmac_algo *__hmac_get_algo(u8 alg_id)
{
        struct seg6_hmac_algo *algo;
        int i, alg_count;

        alg_count = ARRAY_SIZE(hmac_algos);
        for (i = 0; i < alg_count; i++) {
                algo = &hmac_algos[i];
                if (algo->alg_id == alg_id)
                        return algo;
        }

        return NULL;
}

static int __do_hmac(struct seg6_hmac_info *hinfo, const char *text, u8 psize,
                     u8 *output, int outlen)
{
        struct seg6_hmac_algo *algo;
        struct crypto_shash *tfm;
        struct shash_desc *shash;
        int ret, dgsize;

        algo = __hmac_get_algo(hinfo->alg_id);
        if (!algo)
                return -ENOENT;

        tfm = *this_cpu_ptr(algo->tfms);

        dgsize = crypto_shash_digestsize(tfm);
        if (dgsize > outlen) {
                pr_debug("sr-ipv6: __do_hmac: digest size too big (%d / %d)\n",
                         dgsize, outlen);
                return -ENOMEM;
        }

        ret = crypto_shash_setkey(tfm, hinfo->secret, hinfo->slen);
        if (ret < 0) {
                pr_debug("sr-ipv6: crypto_shash_setkey failed: err %d\n", ret);
                goto failed;
        }

        shash = *this_cpu_ptr(algo->shashs);
        shash->tfm = tfm;

        ret = crypto_shash_digest(shash, text, psize, output);
        if (ret < 0) {
                pr_debug("sr-ipv6: crypto_shash_digest failed: err %d\n", ret);
                goto failed;
        }

        return dgsize;

failed:
        return ret;
}

int seg6_hmac_compute(struct seg6_hmac_info *hinfo, struct ipv6_sr_hdr *hdr,
                      struct in6_addr *saddr, u8 *output)
{
        __be32 hmackeyid = cpu_to_be32(hinfo->hmackeyid);
        u8 tmp_out[SEG6_HMAC_MAX_DIGESTSIZE];
        int plen, i, dgsize, wrsize;
        char *ring, *off;

        /* a 160-byte buffer for digest output allows to store highest known
         * hash function (RadioGatun) with up to 1216 bits
         */

        /* saddr(16) + first_seg(1) + flags(1) + keyid(4) + seglist(16n) */
        plen = 16 + 1 + 1 + 4 + (hdr->first_segment + 1) * 16;

        /* this limit allows for 14 segments */
        if (plen >= SEG6_HMAC_RING_SIZE)
                return -EMSGSIZE;

        /* Let's build the HMAC text on the ring buffer. The text is composed
         * as follows, in order:
         *
         * 1. Source IPv6 address (128 bits)
         * 2. first_segment value (8 bits)
         * 3. Flags (8 bits)
         * 4. HMAC Key ID (32 bits)
         * 5. All segments in the segments list (n * 128 bits)
         */

        local_bh_disable();
        ring = this_cpu_ptr(hmac_ring);
        off = ring;

        /* source address */
        memcpy(off, saddr, 16);
        off += 16;

        /* first_segment value */
        *off++ = hdr->first_segment;

        /* flags */
        *off++ = hdr->flags;

        /* HMAC Key ID */
        memcpy(off, &hmackeyid, 4);
        off += 4;

        /* all segments in the list */
        for (i = 0; i < hdr->first_segment + 1; i++) {
                memcpy(off, hdr->segments + i, 16);
                off += 16;
        }

        dgsize = __do_hmac(hinfo, ring, plen, tmp_out,
                           SEG6_HMAC_MAX_DIGESTSIZE);
        local_bh_enable();

        if (dgsize < 0)
                return dgsize;

        wrsize = SEG6_HMAC_FIELD_LEN;
        if (wrsize > dgsize)
                wrsize = dgsize;

        memset(output, 0, SEG6_HMAC_FIELD_LEN);
        memcpy(output, tmp_out, wrsize);

        return 0;
}
EXPORT_SYMBOL(seg6_hmac_compute);

/* checks if an incoming SR-enabled packet's HMAC status matches
 * the incoming policy.
 *
 * called with rcu_read_lock()
 */
bool seg6_hmac_validate_skb(struct sk_buff *skb)
{
        u8 hmac_output[SEG6_HMAC_FIELD_LEN];
        struct net *net = dev_net(skb->dev);
        struct seg6_hmac_info *hinfo;
        struct sr6_tlv_hmac *tlv;
        struct ipv6_sr_hdr *srh;
        struct inet6_dev *idev;
        int require_hmac;

        idev = __in6_dev_get(skb->dev);

        srh = (struct ipv6_sr_hdr *)skb_transport_header(skb);

        tlv = seg6_get_tlv_hmac(srh);

        require_hmac = READ_ONCE(idev->cnf.seg6_require_hmac);
        /* mandatory check but no tlv */
        if (require_hmac > 0 && !tlv)
                return false;

        /* no check */
        if (require_hmac < 0)
                return true;

        /* check only if present */
        if (require_hmac == 0 && !tlv)
                return true;

        /* now, seg6_require_hmac >= 0 && tlv */

        hinfo = seg6_hmac_info_lookup(net, be32_to_cpu(tlv->hmackeyid));
        if (!hinfo)
                return false;

        if (seg6_hmac_compute(hinfo, srh, &ipv6_hdr(skb)->saddr, hmac_output))
                return false;

        if (memcmp(hmac_output, tlv->hmac, SEG6_HMAC_FIELD_LEN) != 0)
                return false;

        return true;
}
EXPORT_SYMBOL(seg6_hmac_validate_skb);

/* called with rcu_read_lock() */
struct seg6_hmac_info *seg6_hmac_info_lookup(struct net *net, u32 key)
{
        struct seg6_pernet_data *sdata = seg6_pernet(net);
        struct seg6_hmac_info *hinfo;

        hinfo = rhashtable_lookup_fast(&sdata->hmac_infos, &key, rht_params);

        return hinfo;
}
EXPORT_SYMBOL(seg6_hmac_info_lookup);

int seg6_hmac_info_add(struct net *net, u32 key, struct seg6_hmac_info *hinfo)
{
        struct seg6_pernet_data *sdata = seg6_pernet(net);
        int err;

        err = rhashtable_lookup_insert_fast(&sdata->hmac_infos, &hinfo->node,
                                            rht_params);

        return err;
}
EXPORT_SYMBOL(seg6_hmac_info_add);

int seg6_hmac_info_del(struct net *net, u32 key)
{
        struct seg6_pernet_data *sdata = seg6_pernet(net);
        struct seg6_hmac_info *hinfo;
        int err = -ENOENT;

        hinfo = rhashtable_lookup_fast(&sdata->hmac_infos, &key, rht_params);
        if (!hinfo)
                goto out;

        err = rhashtable_remove_fast(&sdata->hmac_infos, &hinfo->node,
                                     rht_params);
        if (err)
                goto out;

        seg6_hinfo_release(hinfo);

out:
        return err;
}
EXPORT_SYMBOL(seg6_hmac_info_del);

int seg6_push_hmac(struct net *net, struct in6_addr *saddr,
                   struct ipv6_sr_hdr *srh)
{
        struct seg6_hmac_info *hinfo;
        struct sr6_tlv_hmac *tlv;
        int err = -ENOENT;

        tlv = seg6_get_tlv_hmac(srh);
        if (!tlv)
                return -EINVAL;

        rcu_read_lock();

        hinfo = seg6_hmac_info_lookup(net, be32_to_cpu(tlv->hmackeyid));
        if (!hinfo)
                goto out;

        memset(tlv->hmac, 0, SEG6_HMAC_FIELD_LEN);
        err = seg6_hmac_compute(hinfo, srh, saddr, tlv->hmac);

out:
        rcu_read_unlock();
        return err;
}
EXPORT_SYMBOL(seg6_push_hmac);

static int seg6_hmac_init_algo(void)
{
        struct seg6_hmac_algo *algo;
        struct crypto_shash *tfm;
        struct shash_desc *shash;
        int i, alg_count, cpu;
        int ret = -ENOMEM;

        alg_count = ARRAY_SIZE(hmac_algos);

        for (i = 0; i < alg_count; i++) {
                struct crypto_shash **p_tfm;
                int shsize;

                algo = &hmac_algos[i];
                algo->tfms = alloc_percpu(struct crypto_shash *);
                if (!algo->tfms)
                        goto error_out;

                for_each_possible_cpu(cpu) {
                        tfm = crypto_alloc_shash(algo->name, 0, 0);
                        if (IS_ERR(tfm)) {
                                ret = PTR_ERR(tfm);
                                goto error_out;
                        }
                        p_tfm = per_cpu_ptr(algo->tfms, cpu);
                        *p_tfm = tfm;
                }

                p_tfm = raw_cpu_ptr(algo->tfms);
                tfm = *p_tfm;

                shsize = sizeof(*shash) + crypto_shash_descsize(tfm);

                algo->shashs = alloc_percpu(struct shash_desc *);
                if (!algo->shashs)
                        goto error_out;

                for_each_possible_cpu(cpu) {
                        shash = kzalloc_node(shsize, GFP_KERNEL,
                                             cpu_to_node(cpu));
                        if (!shash)
                                goto error_out;
                        *per_cpu_ptr(algo->shashs, cpu) = shash;
                }
        }

        return 0;

error_out:
        seg6_hmac_exit();
        return ret;
}

int __init seg6_hmac_init(void)
{
        return seg6_hmac_init_algo();
}

int __net_init seg6_hmac_net_init(struct net *net)
{
        struct seg6_pernet_data *sdata = seg6_pernet(net);

        return rhashtable_init(&sdata->hmac_infos, &rht_params);
}

void seg6_hmac_exit(void)
{
        struct seg6_hmac_algo *algo = NULL;
        struct crypto_shash *tfm;
        struct shash_desc *shash;
        int i, alg_count, cpu;

        alg_count = ARRAY_SIZE(hmac_algos);
        for (i = 0; i < alg_count; i++) {
                algo = &hmac_algos[i];

                if (algo->shashs) {
                        for_each_possible_cpu(cpu) {
                                shash = *per_cpu_ptr(algo->shashs, cpu);
                                kfree(shash);
                        }
                        free_percpu(algo->shashs);
                }

                if (algo->tfms) {
                        for_each_possible_cpu(cpu) {
                                tfm = *per_cpu_ptr(algo->tfms, cpu);
                                crypto_free_shash(tfm);
                        }
                        free_percpu(algo->tfms);
                }
        }
}
EXPORT_SYMBOL(seg6_hmac_exit);

void __net_exit seg6_hmac_net_exit(struct net *net)
{
        struct seg6_pernet_data *sdata = seg6_pernet(net);

        rhashtable_free_and_destroy(&sdata->hmac_infos, seg6_free_hi, NULL);
}
EXPORT_SYMBOL(seg6_hmac_net_exit);












































































































































    5 

    5 







    6 




















    5 
    5 



    7 
    8 







    5 
































    1 






   42 





   43 


   43 

   42 
   42 
    2 
















    6 






    1 





    5 



















    5 

    5 

















    2 
    2 




















































  858 

  836 






   37 


    1 

    1 

















































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
// SPDX-License-Identifier: GPL-2.0
// Copyright (c) 2010-2011 EIA Electronics,
//                         Pieter Beyens <pieter.beyens@eia.be>
// Copyright (c) 2010-2011 EIA Electronics,
//                         Kurt Van Dijck <kurt.van.dijck@eia.be>
// Copyright (c) 2018 Protonic,
//                         Robin van der Gracht <robin@protonic.nl>
// Copyright (c) 2017-2019 Pengutronix,
//                         Marc Kleine-Budde <kernel@pengutronix.de>
// Copyright (c) 2017-2019 Pengutronix,
//                         Oleksij Rempel <kernel@pengutronix.de>

/* Core of can-j1939 that links j1939 to CAN. */

#include <linux/can/can-ml.h>
#include <linux/can/core.h>
#include <linux/can/skb.h>
#include <linux/if_arp.h>
#include <linux/module.h>

#include "j1939-priv.h"

MODULE_DESCRIPTION("PF_CAN SAE J1939");
MODULE_LICENSE("GPL v2");
MODULE_AUTHOR("EIA Electronics (Kurt Van Dijck & Pieter Beyens)");
MODULE_ALIAS("can-proto-" __stringify(CAN_J1939));

/* LOWLEVEL CAN interface */

/* CAN_HDR: #bytes before can_frame data part */
#define J1939_CAN_HDR (offsetof(struct can_frame, data))

/* lowest layer */
static void j1939_can_recv(struct sk_buff *iskb, void *data)
{
        struct j1939_priv *priv = data;
        struct sk_buff *skb;
        struct j1939_sk_buff_cb *skcb, *iskcb;
        struct can_frame *cf;

        /* make sure we only get Classical CAN frames */
        if (!can_is_can_skb(iskb))
                return;

        /* create a copy of the skb
         * j1939 only delivers the real data bytes,
         * the header goes into sockaddr.
         * j1939 may not touch the incoming skb in such way
         */
        skb = skb_clone(iskb, GFP_ATOMIC);
        if (!skb)
                return;

        j1939_priv_get(priv);
        can_skb_set_owner(skb, iskb->sk);

        /* get a pointer to the header of the skb
         * the skb payload (pointer) is moved, so that the next skb_data
         * returns the actual payload
         */
        cf = (void *)skb->data;
        skb_pull(skb, J1939_CAN_HDR);

        /* fix length, set to dlc, with 8 maximum */
        skb_trim(skb, min_t(uint8_t, cf->len, 8));

        /* set addr */
        skcb = j1939_skb_to_cb(skb);
        memset(skcb, 0, sizeof(*skcb));

        iskcb = j1939_skb_to_cb(iskb);
        skcb->tskey = iskcb->tskey;
        skcb->priority = (cf->can_id >> 26) & 0x7;
        skcb->addr.sa = cf->can_id;
        skcb->addr.pgn = (cf->can_id >> 8) & J1939_PGN_MAX;
        /* set default message type */
        skcb->addr.type = J1939_TP;

        if (!j1939_address_is_valid(skcb->addr.sa)) {
                netdev_err_once(priv->ndev, "%s: sa is broadcast address, ignoring!\n",
                                __func__);
                goto done;
        }

        if (j1939_pgn_is_pdu1(skcb->addr.pgn)) {
                /* Type 1: with destination address */
                skcb->addr.da = skcb->addr.pgn;
                /* normalize pgn: strip dst address */
                skcb->addr.pgn &= 0x3ff00;
        } else {
                /* set broadcast address */
                skcb->addr.da = J1939_NO_ADDR;
        }

        /* update localflags */
        read_lock_bh(&priv->lock);
        if (j1939_address_is_unicast(skcb->addr.sa) &&
            priv->ents[skcb->addr.sa].nusers)
                skcb->flags |= J1939_ECU_LOCAL_SRC;
        if (j1939_address_is_unicast(skcb->addr.da) &&
            priv->ents[skcb->addr.da].nusers)
                skcb->flags |= J1939_ECU_LOCAL_DST;
        read_unlock_bh(&priv->lock);

        /* deliver into the j1939 stack ... */
        j1939_ac_recv(priv, skb);

        if (j1939_tp_recv(priv, skb))
                /* this means the transport layer processed the message */
                goto done;

        j1939_simple_recv(priv, skb);
        j1939_sk_recv(priv, skb);
 done:
        j1939_priv_put(priv);
        kfree_skb(skb);
}

/* NETDEV MANAGEMENT */

/* values for can_rx_(un)register */
#define J1939_CAN_ID CAN_EFF_FLAG
#define J1939_CAN_MASK (CAN_EFF_FLAG | CAN_RTR_FLAG)

static DEFINE_MUTEX(j1939_netdev_lock);

static struct j1939_priv *j1939_priv_create(struct net_device *ndev)
{
        struct j1939_priv *priv;

        priv = kzalloc(sizeof(*priv), GFP_KERNEL);
        if (!priv)
                return NULL;

        rwlock_init(&priv->lock);
        INIT_LIST_HEAD(&priv->ecus);
        priv->ndev = ndev;
        kref_init(&priv->kref);
        kref_init(&priv->rx_kref);
        dev_hold(ndev);

        netdev_dbg(priv->ndev, "%s : 0x%p\n", __func__, priv);

        return priv;
}

static inline void j1939_priv_set(struct net_device *ndev,
                                  struct j1939_priv *priv)
{
        struct can_ml_priv *can_ml = can_get_ml_priv(ndev);

        can_ml->j1939_priv = priv;
}

static void __j1939_priv_release(struct kref *kref)
{
        struct j1939_priv *priv = container_of(kref, struct j1939_priv, kref);
        struct net_device *ndev = priv->ndev;

        netdev_dbg(priv->ndev, "%s: 0x%p\n", __func__, priv);

        WARN_ON_ONCE(!list_empty(&priv->active_session_list));
        WARN_ON_ONCE(!list_empty(&priv->ecus));
        WARN_ON_ONCE(!list_empty(&priv->j1939_socks));

        dev_put(ndev);
        kfree(priv);
}

void j1939_priv_put(struct j1939_priv *priv)
{
        kref_put(&priv->kref, __j1939_priv_release);
}

void j1939_priv_get(struct j1939_priv *priv)
{
        kref_get(&priv->kref);
}

static int j1939_can_rx_register(struct j1939_priv *priv)
{
        struct net_device *ndev = priv->ndev;
        int ret;

        j1939_priv_get(priv);
        ret = can_rx_register(dev_net(ndev), ndev, J1939_CAN_ID, J1939_CAN_MASK,
                              j1939_can_recv, priv, "j1939", NULL);
        if (ret < 0) {
                j1939_priv_put(priv);
                return ret;
        }

        return 0;
}

static void j1939_can_rx_unregister(struct j1939_priv *priv)
{
        struct net_device *ndev = priv->ndev;

        can_rx_unregister(dev_net(ndev), ndev, J1939_CAN_ID, J1939_CAN_MASK,
                          j1939_can_recv, priv);

        /* The last reference of priv is dropped by the RCU deferred
         * j1939_sk_sock_destruct() of the last socket, so we can
         * safely drop this reference here.
         */
        j1939_priv_put(priv);
}

static void __j1939_rx_release(struct kref *kref)
        __releases(&j1939_netdev_lock)
{
        struct j1939_priv *priv = container_of(kref, struct j1939_priv,
                                               rx_kref);

        j1939_can_rx_unregister(priv);
        j1939_ecu_unmap_all(priv);
        j1939_priv_set(priv->ndev, NULL);
        mutex_unlock(&j1939_netdev_lock);
}

/* get pointer to priv without increasing ref counter */
static inline struct j1939_priv *j1939_ndev_to_priv(struct net_device *ndev)
{
        struct can_ml_priv *can_ml = can_get_ml_priv(ndev);

        return can_ml->j1939_priv;
}

static struct j1939_priv *j1939_priv_get_by_ndev_locked(struct net_device *ndev)
{
        struct j1939_priv *priv;

        lockdep_assert_held(&j1939_netdev_lock);

        priv = j1939_ndev_to_priv(ndev);
        if (priv)
                j1939_priv_get(priv);

        return priv;
}

static struct j1939_priv *j1939_priv_get_by_ndev(struct net_device *ndev)
{
        struct j1939_priv *priv;

        mutex_lock(&j1939_netdev_lock);
        priv = j1939_priv_get_by_ndev_locked(ndev);
        mutex_unlock(&j1939_netdev_lock);

        return priv;
}

struct j1939_priv *j1939_netdev_start(struct net_device *ndev)
{
        struct j1939_priv *priv, *priv_new;
        int ret;

        mutex_lock(&j1939_netdev_lock);
        priv = j1939_priv_get_by_ndev_locked(ndev);
        if (priv) {
                kref_get(&priv->rx_kref);
                mutex_unlock(&j1939_netdev_lock);
                return priv;
        }
        mutex_unlock(&j1939_netdev_lock);

        priv = j1939_priv_create(ndev);
        if (!priv)
                return ERR_PTR(-ENOMEM);

        j1939_tp_init(priv);
        rwlock_init(&priv->j1939_socks_lock);
        INIT_LIST_HEAD(&priv->j1939_socks);

        mutex_lock(&j1939_netdev_lock);
        priv_new = j1939_priv_get_by_ndev_locked(ndev);
        if (priv_new) {
                /* Someone was faster than us, use their priv and roll
                 * back our's.
                 */
                kref_get(&priv_new->rx_kref);
                mutex_unlock(&j1939_netdev_lock);
                dev_put(ndev);
                kfree(priv);
                return priv_new;
        }
        j1939_priv_set(ndev, priv);

        ret = j1939_can_rx_register(priv);
        if (ret < 0)
                goto out_priv_put;

        mutex_unlock(&j1939_netdev_lock);
        return priv;

 out_priv_put:
        j1939_priv_set(ndev, NULL);
        mutex_unlock(&j1939_netdev_lock);

        dev_put(ndev);
        kfree(priv);

        return ERR_PTR(ret);
}

void j1939_netdev_stop(struct j1939_priv *priv)
{
        kref_put_mutex(&priv->rx_kref, __j1939_rx_release, &j1939_netdev_lock);
        j1939_priv_put(priv);
}

int j1939_send_one(struct j1939_priv *priv, struct sk_buff *skb)
{
        int ret, dlc;
        canid_t canid;
        struct j1939_sk_buff_cb *skcb = j1939_skb_to_cb(skb);
        struct can_frame *cf;

        /* apply sanity checks */
        if (j1939_pgn_is_pdu1(skcb->addr.pgn))
                skcb->addr.pgn &= J1939_PGN_PDU1_MAX;
        else
                skcb->addr.pgn &= J1939_PGN_MAX;

        if (skcb->priority > 7)
                skcb->priority = 6;

        ret = j1939_ac_fixup(priv, skb);
        if (unlikely(ret))
                goto failed;
        dlc = skb->len;

        /* re-claim the CAN_HDR from the SKB */
        cf = skb_push(skb, J1939_CAN_HDR);

        /* initialize header structure */
        memset(cf, 0, J1939_CAN_HDR);

        /* make it a full can frame again */
        skb_put_zero(skb, 8 - dlc);

        canid = CAN_EFF_FLAG |
                (skcb->priority << 26) |
                (skcb->addr.pgn << 8) |
                skcb->addr.sa;
        if (j1939_pgn_is_pdu1(skcb->addr.pgn))
                canid |= skcb->addr.da << 8;

        cf->can_id = canid;
        cf->len = dlc;

        return can_send(skb, 1);

 failed:
        kfree_skb(skb);
        return ret;
}

static int j1939_netdev_notify(struct notifier_block *nb,
                               unsigned long msg, void *data)
{
        struct net_device *ndev = netdev_notifier_info_to_dev(data);
        struct can_ml_priv *can_ml = can_get_ml_priv(ndev);
        struct j1939_priv *priv;

        if (!can_ml)
                goto notify_done;

        priv = j1939_priv_get_by_ndev(ndev);
        if (!priv)
                goto notify_done;

        switch (msg) {
        case NETDEV_DOWN:
                j1939_cancel_active_session(priv, NULL);
                j1939_sk_netdev_event_netdown(priv);
                j1939_ecu_unmap_all(priv);
                break;
        }

        j1939_priv_put(priv);

notify_done:
        return NOTIFY_DONE;
}

static struct notifier_block j1939_netdev_notifier = {
        .notifier_call = j1939_netdev_notify,
};

/* MODULE interface */
static __init int j1939_module_init(void)
{
        int ret;

        pr_info("can: SAE J1939\n");

        ret = register_netdevice_notifier(&j1939_netdev_notifier);
        if (ret)
                goto fail_notifier;

        ret = can_proto_register(&j1939_can_proto);
        if (ret < 0) {
                pr_err("can: registration of j1939 protocol failed\n");
                goto fail_sk;
        }

        return 0;

 fail_sk:
        unregister_netdevice_notifier(&j1939_netdev_notifier);
 fail_notifier:
        return ret;
}

static __exit void j1939_module_exit(void)
{
        can_proto_unregister(&j1939_can_proto);

        unregister_netdevice_notifier(&j1939_netdev_notifier);
}

module_init(j1939_module_init);
module_exit(j1939_module_exit);







































    6 





    6 

























































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
// SPDX-License-Identifier: GPL-2.0
/*
 *  linux/fs/filesystems.c
 *
 *  Copyright (C) 1991, 1992  Linus Torvalds
 *
 *  table of configured filesystems
 */

#include <linux/syscalls.h>
#include <linux/fs.h>
#include <linux/proc_fs.h>
#include <linux/seq_file.h>
#include <linux/kmod.h>
#include <linux/init.h>
#include <linux/module.h>
#include <linux/slab.h>
#include <linux/uaccess.h>
#include <linux/fs_parser.h>

/*
 * Handling of filesystem drivers list.
 * Rules:
 *        Inclusion to/removals from/scanning of list are protected by spinlock.
 *        During the unload module must call unregister_filesystem().
 *        We can access the fields of list element if:
 *                1) spinlock is held or
 *                2) we hold the reference to the module.
 *        The latter can be guaranteed by call of try_module_get(); if it
 *        returned 0 we must skip the element, otherwise we got the reference.
 *        Once the reference is obtained we can drop the spinlock.
 */

static struct file_system_type *file_systems;
static DEFINE_RWLOCK(file_systems_lock);

/* WARNING: This can be used only if we _already_ own a reference */
struct file_system_type *get_filesystem(struct file_system_type *fs)
{
        __module_get(fs->owner);
        return fs;
}

void put_filesystem(struct file_system_type *fs)
{
        module_put(fs->owner);
}

static struct file_system_type **find_filesystem(const char *name, unsigned len)
{
        struct file_system_type **p;
        for (p = &file_systems; *p; p = &(*p)->next)
                if (strncmp((*p)->name, name, len) == 0 &&
                    !(*p)->name[len])
                        break;
        return p;
}

/**
 *        register_filesystem - register a new filesystem
 *        @fs: the file system structure
 *
 *        Adds the file system passed to the list of file systems the kernel
 *        is aware of for mount and other syscalls. Returns 0 on success,
 *        or a negative errno code on an error.
 *
 *        The &struct file_system_type that is passed is linked into the kernel 
 *        structures and must not be freed until the file system has been
 *        unregistered.
 */
 
int register_filesystem(struct file_system_type * fs)
{
        int res = 0;
        struct file_system_type ** p;

        if (fs->parameters &&
            !fs_validate_description(fs->name, fs->parameters))
                return -EINVAL;

        BUG_ON(strchr(fs->name, '.'));
        if (fs->next)
                return -EBUSY;
        write_lock(&file_systems_lock);
        p = find_filesystem(fs->name, strlen(fs->name));
        if (*p)
                res = -EBUSY;
        else
                *p = fs;
        write_unlock(&file_systems_lock);
        return res;
}

EXPORT_SYMBOL(register_filesystem);

/**
 *        unregister_filesystem - unregister a file system
 *        @fs: filesystem to unregister
 *
 *        Remove a file system that was previously successfully registered
 *        with the kernel. An error is returned if the file system is not found.
 *        Zero is returned on a success.
 *        
 *        Once this function has returned the &struct file_system_type structure
 *        may be freed or reused.
 */
 
int unregister_filesystem(struct file_system_type * fs)
{
        struct file_system_type ** tmp;

        write_lock(&file_systems_lock);
        tmp = &file_systems;
        while (*tmp) {
                if (fs == *tmp) {
                        *tmp = fs->next;
                        fs->next = NULL;
                        write_unlock(&file_systems_lock);
                        synchronize_rcu();
                        return 0;
                }
                tmp = &(*tmp)->next;
        }
        write_unlock(&file_systems_lock);

        return -EINVAL;
}

EXPORT_SYMBOL(unregister_filesystem);

#ifdef CONFIG_SYSFS_SYSCALL
static int fs_index(const char __user * __name)
{
        struct file_system_type * tmp;
        struct filename *name;
        int err, index;

        name = getname(__name);
        err = PTR_ERR(name);
        if (IS_ERR(name))
                return err;

        err = -EINVAL;
        read_lock(&file_systems_lock);
        for (tmp=file_systems, index=0 ; tmp ; tmp=tmp->next, index++) {
                if (strcmp(tmp->name, name->name) == 0) {
                        err = index;
                        break;
                }
        }
        read_unlock(&file_systems_lock);
        putname(name);
        return err;
}

static int fs_name(unsigned int index, char __user * buf)
{
        struct file_system_type * tmp;
        int len, res;

        read_lock(&file_systems_lock);
        for (tmp = file_systems; tmp; tmp = tmp->next, index--)
                if (index <= 0 && try_module_get(tmp->owner))
                        break;
        read_unlock(&file_systems_lock);
        if (!tmp)
                return -EINVAL;

        /* OK, we got the reference, so we can safely block */
        len = strlen(tmp->name) + 1;
        res = copy_to_user(buf, tmp->name, len) ? -EFAULT : 0;
        put_filesystem(tmp);
        return res;
}

static int fs_maxindex(void)
{
        struct file_system_type * tmp;
        int index;

        read_lock(&file_systems_lock);
        for (tmp = file_systems, index = 0 ; tmp ; tmp = tmp->next, index++)
                ;
        read_unlock(&file_systems_lock);
        return index;
}

/*
 * Whee.. Weird sysv syscall. 
 */
SYSCALL_DEFINE3(sysfs, int, option, unsigned long, arg1, unsigned long, arg2)
{
        int retval = -EINVAL;

        switch (option) {
                case 1:
                        retval = fs_index((const char __user *) arg1);
                        break;

                case 2:
                        retval = fs_name(arg1, (char __user *) arg2);
                        break;

                case 3:
                        retval = fs_maxindex();
                        break;
        }
        return retval;
}
#endif

int __init list_bdev_fs_names(char *buf, size_t size)
{
        struct file_system_type *p;
        size_t len;
        int count = 0;

        read_lock(&file_systems_lock);
        for (p = file_systems; p; p = p->next) {
                if (!(p->fs_flags & FS_REQUIRES_DEV))
                        continue;
                len = strlen(p->name) + 1;
                if (len > size) {
                        pr_warn("%s: truncating file system list\n", __func__);
                        break;
                }
                memcpy(buf, p->name, len);
                buf += len;
                size -= len;
                count++;
        }
        read_unlock(&file_systems_lock);
        return count;
}

#ifdef CONFIG_PROC_FS
static int filesystems_proc_show(struct seq_file *m, void *v)
{
        struct file_system_type * tmp;

        read_lock(&file_systems_lock);
        tmp = file_systems;
        while (tmp) {
                seq_printf(m, "%s\t%s\n",
                        (tmp->fs_flags & FS_REQUIRES_DEV) ? "" : "nodev",
                        tmp->name);
                tmp = tmp->next;
        }
        read_unlock(&file_systems_lock);
        return 0;
}

static int __init proc_filesystems_init(void)
{
        proc_create_single("filesystems", 0, NULL, filesystems_proc_show);
        return 0;
}
module_init(proc_filesystems_init);
#endif

static struct file_system_type *__get_fs_type(const char *name, int len)
{
        struct file_system_type *fs;

        read_lock(&file_systems_lock);
        fs = *(find_filesystem(name, len));
        if (fs && !try_module_get(fs->owner))
                fs = NULL;
        read_unlock(&file_systems_lock);
        return fs;
}

struct file_system_type *get_fs_type(const char *name)
{
        struct file_system_type *fs;
        const char *dot = strchr(name, '.');
        int len = dot ? dot - name : strlen(name);

        fs = __get_fs_type(name, len);
        if (!fs && (request_module("fs-%.*s", len, name) == 0)) {
                fs = __get_fs_type(name, len);
                if (!fs)
                        pr_warn_once("request_module fs-%.*s succeeded, but still no fs?\n",
                                     len, name);
        }

        if (dot && fs && !(fs->fs_flags & FS_HAS_SUBTYPE)) {
                put_filesystem(fs);
                fs = NULL;
        }
        return fs;
}

EXPORT_SYMBOL(get_fs_type);






















































































    9 
  681 














   19 
    1 













































































   21 



































    7 


    7 

    3 
    4 
    4 


    7 






































































































   15 

   29 

   98 










   75 
   24 
   55 












  356 

  356 








  313 








  372 


  375 
    1 













  148 




  272 





1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
 * INET                An implementation of the TCP/IP protocol suite for the LINUX
 *                operating system.  INET  is implemented using the  BSD Socket
 *                interface as the means of communication with the user level.
 *
 *                Definitions for the IP router.
 *
 * Version:        @(#)route.h        1.0.4        05/27/93
 *
 * Authors:        Ross Biro
 *                Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
 * Fixes:
 *                Alan Cox        :        Reformatted. Added ip_rt_local()
 *                Alan Cox        :        Support for TCP parameters.
 *                Alexey Kuznetsov:        Major changes for new routing code.
 *                Mike McLagan    :        Routing by source
 *                Robert Olsson   :        Added rt_cache statistics
 */
#ifndef _ROUTE_H
#define _ROUTE_H

#include <net/dst.h>
#include <net/inetpeer.h>
#include <net/flow.h>
#include <net/inet_sock.h>
#include <net/ip_fib.h>
#include <net/arp.h>
#include <net/ndisc.h>
#include <net/inet_dscp.h>
#include <linux/in_route.h>
#include <linux/rtnetlink.h>
#include <linux/rcupdate.h>
#include <linux/route.h>
#include <linux/ip.h>
#include <linux/cache.h>
#include <linux/security.h>

static inline __u8 ip_sock_rt_scope(const struct sock *sk)
{
        if (sock_flag(sk, SOCK_LOCALROUTE))
                return RT_SCOPE_LINK;

        return RT_SCOPE_UNIVERSE;
}

static inline __u8 ip_sock_rt_tos(const struct sock *sk)
{
        return READ_ONCE(inet_sk(sk)->tos) & INET_DSCP_MASK;
}

struct ip_tunnel_info;
struct fib_nh;
struct fib_info;
struct uncached_list;
struct rtable {
        struct dst_entry        dst;

        int                        rt_genid;
        unsigned int                rt_flags;
        __u16                        rt_type;
        __u8                        rt_is_input;
        __u8                        rt_uses_gateway;

        int                        rt_iif;

        u8                        rt_gw_family;
        /* Info on neighbour */
        union {
                __be32                rt_gw4;
                struct in6_addr        rt_gw6;
        };

        /* Miscellaneous cached information */
        u32                        rt_mtu_locked:1,
                                rt_pmtu:31;
};

#define dst_rtable(_ptr) container_of_const(_ptr, struct rtable, dst)

/**
 * skb_rtable - Returns the skb &rtable
 * @skb: buffer
 */
static inline struct rtable *skb_rtable(const struct sk_buff *skb)
{
        return dst_rtable(skb_dst(skb));
}

static inline bool rt_is_input_route(const struct rtable *rt)
{
        return rt->rt_is_input != 0;
}

static inline bool rt_is_output_route(const struct rtable *rt)
{
        return rt->rt_is_input == 0;
}

static inline __be32 rt_nexthop(const struct rtable *rt, __be32 daddr)
{
        if (rt->rt_gw_family == AF_INET)
                return rt->rt_gw4;
        return daddr;
}

struct ip_rt_acct {
        __u32         o_bytes;
        __u32         o_packets;
        __u32         i_bytes;
        __u32         i_packets;
};

struct rt_cache_stat {
        unsigned int in_slow_tot;
        unsigned int in_slow_mc;
        unsigned int in_no_route;
        unsigned int in_brd;
        unsigned int in_martian_dst;
        unsigned int in_martian_src;
        unsigned int out_slow_tot;
        unsigned int out_slow_mc;
};

extern struct ip_rt_acct __percpu *ip_rt_acct;

struct in_device;

int ip_rt_init(void);
void rt_cache_flush(struct net *net);
void rt_flush_dev(struct net_device *dev);
struct rtable *ip_route_output_key_hash(struct net *net, struct flowi4 *flp,
                                        const struct sk_buff *skb);
struct rtable *ip_route_output_key_hash_rcu(struct net *net, struct flowi4 *flp,
                                            struct fib_result *res,
                                            const struct sk_buff *skb);

static inline struct rtable *__ip_route_output_key(struct net *net,
                                                   struct flowi4 *flp)
{
        return ip_route_output_key_hash(net, flp, NULL);
}

struct rtable *ip_route_output_flow(struct net *, struct flowi4 *flp,
                                    const struct sock *sk);
struct dst_entry *ipv4_blackhole_route(struct net *net,
                                       struct dst_entry *dst_orig);

static inline struct rtable *ip_route_output_key(struct net *net, struct flowi4 *flp)
{
        return ip_route_output_flow(net, flp, NULL);
}

/* Simplistic IPv4 route lookup function.
 * This is only suitable for some particular use cases: since the flowi4
 * structure is only partially set, it may bypass some fib-rules.
 */
static inline struct rtable *ip_route_output(struct net *net, __be32 daddr,
                                             __be32 saddr, dscp_t dscp,
                                             int oif, __u8 scope)
{
        struct flowi4 fl4 = {
                .flowi4_oif = oif,
                .flowi4_tos = inet_dscp_to_dsfield(dscp),
                .flowi4_scope = scope,
                .daddr = daddr,
                .saddr = saddr,
        };

        return ip_route_output_key(net, &fl4);
}

static inline struct rtable *ip_route_output_ports(struct net *net, struct flowi4 *fl4,
                                                   const struct sock *sk,
                                                   __be32 daddr, __be32 saddr,
                                                   __be16 dport, __be16 sport,
                                                   __u8 proto, __u8 tos, int oif)
{
        flowi4_init_output(fl4, oif, sk ? READ_ONCE(sk->sk_mark) : 0, tos,
                           sk ? ip_sock_rt_scope(sk) : RT_SCOPE_UNIVERSE,
                           proto, sk ? inet_sk_flowi_flags(sk) : 0,
                           daddr, saddr, dport, sport, sock_net_uid(net, sk));
        if (sk)
                security_sk_classify_flow(sk, flowi4_to_flowi_common(fl4));
        return ip_route_output_flow(net, fl4, sk);
}

static inline struct rtable *ip_route_output_gre(struct net *net, struct flowi4 *fl4,
                                                 __be32 daddr, __be32 saddr,
                                                 __be32 gre_key, __u8 tos, int oif)
{
        memset(fl4, 0, sizeof(*fl4));
        fl4->flowi4_oif = oif;
        fl4->daddr = daddr;
        fl4->saddr = saddr;
        fl4->flowi4_tos = tos;
        fl4->flowi4_proto = IPPROTO_GRE;
        fl4->fl4_gre_key = gre_key;
        return ip_route_output_key(net, fl4);
}

enum skb_drop_reason
ip_mc_validate_source(struct sk_buff *skb, __be32 daddr, __be32 saddr,
                      dscp_t dscp, struct net_device *dev,
                      struct in_device *in_dev, u32 *itag);
enum skb_drop_reason
ip_route_input_noref(struct sk_buff *skb, __be32 daddr, __be32 saddr,
                     dscp_t dscp, struct net_device *dev);
enum skb_drop_reason
ip_route_use_hint(struct sk_buff *skb, __be32 daddr, __be32 saddr,
                  dscp_t dscp, struct net_device *dev,
                  const struct sk_buff *hint);

static inline enum skb_drop_reason
ip_route_input(struct sk_buff *skb, __be32 dst, __be32 src, dscp_t dscp,
               struct net_device *devin)
{
        enum skb_drop_reason reason;

        rcu_read_lock();
        reason = ip_route_input_noref(skb, dst, src, dscp, devin);
        if (!reason) {
                skb_dst_force(skb);
                if (!skb_dst(skb))
                        reason = SKB_DROP_REASON_NOT_SPECIFIED;
        }
        rcu_read_unlock();

        return reason;
}

void ipv4_update_pmtu(struct sk_buff *skb, struct net *net, u32 mtu, int oif,
                      u8 protocol);
void ipv4_sk_update_pmtu(struct sk_buff *skb, struct sock *sk, u32 mtu);
void ipv4_redirect(struct sk_buff *skb, struct net *net, int oif, u8 protocol);
void ipv4_sk_redirect(struct sk_buff *skb, struct sock *sk);
void ip_rt_send_redirect(struct sk_buff *skb);

unsigned int inet_addr_type(struct net *net, __be32 addr);
unsigned int inet_addr_type_table(struct net *net, __be32 addr, u32 tb_id);
unsigned int inet_dev_addr_type(struct net *net, const struct net_device *dev,
                                __be32 addr);
unsigned int inet_addr_type_dev_table(struct net *net,
                                      const struct net_device *dev,
                                      __be32 addr);
void ip_rt_multicast_event(struct in_device *);
int ip_rt_ioctl(struct net *, unsigned int cmd, struct rtentry *rt);
void ip_rt_get_source(u8 *src, struct sk_buff *skb, struct rtable *rt);
struct rtable *rt_dst_alloc(struct net_device *dev,
                            unsigned int flags, u16 type, bool noxfrm);
struct rtable *rt_dst_clone(struct net_device *dev, struct rtable *rt);

struct in_ifaddr;
void fib_add_ifaddr(struct in_ifaddr *);
void fib_del_ifaddr(struct in_ifaddr *, struct in_ifaddr *);
void fib_modify_prefix_metric(struct in_ifaddr *ifa, u32 new_metric);

void rt_add_uncached_list(struct rtable *rt);
void rt_del_uncached_list(struct rtable *rt);

int fib_dump_info_fnhe(struct sk_buff *skb, struct netlink_callback *cb,
                       u32 table_id, struct fib_info *fi,
                       int *fa_index, int fa_start, unsigned int flags);

static inline void ip_rt_put(struct rtable *rt)
{
        /* dst_release() accepts a NULL parameter.
         * We rely on dst being first structure in struct rtable
         */
        BUILD_BUG_ON(offsetof(struct rtable, dst) != 0);
        dst_release(&rt->dst);
}

extern const __u8 ip_tos2prio[16];

static inline char rt_tos2priority(u8 tos)
{
        return ip_tos2prio[IPTOS_TOS(tos)>>1];
}

/* ip_route_connect() and ip_route_newports() work in tandem whilst
 * binding a socket for a new outgoing connection.
 *
 * In order to use IPSEC properly, we must, in the end, have a
 * route that was looked up using all available keys including source
 * and destination ports.
 *
 * However, if a source port needs to be allocated (the user specified
 * a wildcard source port) we need to obtain addressing information
 * in order to perform that allocation.
 *
 * So ip_route_connect() looks up a route using wildcarded source and
 * destination ports in the key, simply so that we can get a pair of
 * addresses to use for port allocation.
 *
 * Later, once the ports are allocated, ip_route_newports() will make
 * another route lookup if needed to make sure we catch any IPSEC
 * rules keyed on the port information.
 *
 * The callers allocate the flow key on their stack, and must pass in
 * the same flowi4 object to both the ip_route_connect() and the
 * ip_route_newports() calls.
 */

static inline void ip_route_connect_init(struct flowi4 *fl4, __be32 dst,
                                         __be32 src, int oif, u8 protocol,
                                         __be16 sport, __be16 dport,
                                         const struct sock *sk)
{
        __u8 flow_flags = 0;

        if (inet_test_bit(TRANSPARENT, sk))
                flow_flags |= FLOWI_FLAG_ANYSRC;

        flowi4_init_output(fl4, oif, READ_ONCE(sk->sk_mark), ip_sock_rt_tos(sk),
                           ip_sock_rt_scope(sk), protocol, flow_flags, dst,
                           src, dport, sport, sk->sk_uid);
}

static inline struct rtable *ip_route_connect(struct flowi4 *fl4, __be32 dst,
                                              __be32 src, int oif, u8 protocol,
                                              __be16 sport, __be16 dport,
                                              const struct sock *sk)
{
        struct net *net = sock_net(sk);
        struct rtable *rt;

        ip_route_connect_init(fl4, dst, src, oif, protocol, sport, dport, sk);

        if (!dst || !src) {
                rt = __ip_route_output_key(net, fl4);
                if (IS_ERR(rt))
                        return rt;
                ip_rt_put(rt);
                flowi4_update_output(fl4, oif, fl4->daddr, fl4->saddr);
        }
        security_sk_classify_flow(sk, flowi4_to_flowi_common(fl4));
        return ip_route_output_flow(net, fl4, sk);
}

static inline struct rtable *ip_route_newports(struct flowi4 *fl4, struct rtable *rt,
                                               __be16 orig_sport, __be16 orig_dport,
                                               __be16 sport, __be16 dport,
                                               const struct sock *sk)
{
        if (sport != orig_sport || dport != orig_dport) {
                fl4->fl4_dport = dport;
                fl4->fl4_sport = sport;
                ip_rt_put(rt);
                flowi4_update_output(fl4, sk->sk_bound_dev_if, fl4->daddr,
                                     fl4->saddr);
                security_sk_classify_flow(sk, flowi4_to_flowi_common(fl4));
                return ip_route_output_flow(sock_net(sk), fl4, sk);
        }
        return rt;
}

static inline int inet_iif(const struct sk_buff *skb)
{
        struct rtable *rt = skb_rtable(skb);

        if (rt && rt->rt_iif)
                return rt->rt_iif;

        return skb->skb_iif;
}

static inline int ip4_dst_hoplimit(const struct dst_entry *dst)
{
        int hoplimit = dst_metric_raw(dst, RTAX_HOPLIMIT);
        struct net *net = dev_net(dst->dev);

        if (hoplimit == 0)
                hoplimit = READ_ONCE(net->ipv4.sysctl_ip_default_ttl);
        return hoplimit;
}

static inline struct neighbour *ip_neigh_gw4(struct net_device *dev,
                                             __be32 daddr)
{
        struct neighbour *neigh;

        neigh = __ipv4_neigh_lookup_noref(dev, (__force u32)daddr);
        if (unlikely(!neigh))
                neigh = __neigh_create(&arp_tbl, &daddr, dev, false);

        return neigh;
}

static inline struct neighbour *ip_neigh_for_gw(struct rtable *rt,
                                                struct sk_buff *skb,
                                                bool *is_v6gw)
{
        struct net_device *dev = rt->dst.dev;
        struct neighbour *neigh;

        if (likely(rt->rt_gw_family == AF_INET)) {
                neigh = ip_neigh_gw4(dev, rt->rt_gw4);
        } else if (rt->rt_gw_family == AF_INET6) {
                neigh = ip_neigh_gw6(dev, &rt->rt_gw6);
                *is_v6gw = true;
        } else {
                neigh = ip_neigh_gw4(dev, ip_hdr(skb)->daddr);
        }
        return neigh;
}

#endif        /* _ROUTE_H */























































































    8 



    8 


    8 






















































































































    8 
































































































































































































































































    1 
























    7 















    8 

    8 


















    8 
















    8 












    8 





    8 

    8 










    8 











    8 










    8 



















    7 









    8 
    8 


    8 





    8 







    8 

































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright (C) 2005,2006,2007,2008 IBM Corporation
 *
 * Authors:
 * Mimi Zohar <zohar@us.ibm.com>
 * Kylene Hall <kjhall@us.ibm.com>
 *
 * File: ima_crypto.c
 *        Calculates md5/sha1 file hash, template hash, boot-aggreate hash
 */

#include <linux/kernel.h>
#include <linux/moduleparam.h>
#include <linux/ratelimit.h>
#include <linux/file.h>
#include <linux/crypto.h>
#include <linux/scatterlist.h>
#include <linux/err.h>
#include <linux/slab.h>
#include <crypto/hash.h>

#include "ima.h"

/* minimum file size for ahash use */
static unsigned long ima_ahash_minsize;
module_param_named(ahash_minsize, ima_ahash_minsize, ulong, 0644);
MODULE_PARM_DESC(ahash_minsize, "Minimum file size for ahash use");

/* default is 0 - 1 page. */
static int ima_maxorder;
static unsigned int ima_bufsize = PAGE_SIZE;

static int param_set_bufsize(const char *val, const struct kernel_param *kp)
{
        unsigned long long size;
        int order;

        size = memparse(val, NULL);
        order = get_order(size);
        if (order > MAX_PAGE_ORDER)
                return -EINVAL;
        ima_maxorder = order;
        ima_bufsize = PAGE_SIZE << order;
        return 0;
}

static const struct kernel_param_ops param_ops_bufsize = {
        .set = param_set_bufsize,
        .get = param_get_uint,
};
#define param_check_bufsize(name, p) __param_check(name, p, unsigned int)

module_param_named(ahash_bufsize, ima_bufsize, bufsize, 0644);
MODULE_PARM_DESC(ahash_bufsize, "Maximum ahash buffer size");

static struct crypto_shash *ima_shash_tfm;
static struct crypto_ahash *ima_ahash_tfm;

int ima_sha1_idx __ro_after_init;
int ima_hash_algo_idx __ro_after_init;
/*
 * Additional number of slots reserved, as needed, for SHA1
 * and IMA default algo.
 */
int ima_extra_slots __ro_after_init;

struct ima_algo_desc *ima_algo_array __ro_after_init;

static int __init ima_init_ima_crypto(void)
{
        long rc;

        ima_shash_tfm = crypto_alloc_shash(hash_algo_name[ima_hash_algo], 0, 0);
        if (IS_ERR(ima_shash_tfm)) {
                rc = PTR_ERR(ima_shash_tfm);
                pr_err("Can not allocate %s (reason: %ld)\n",
                       hash_algo_name[ima_hash_algo], rc);
                return rc;
        }
        pr_info("Allocated hash algorithm: %s\n",
                hash_algo_name[ima_hash_algo]);
        return 0;
}

static struct crypto_shash *ima_alloc_tfm(enum hash_algo algo)
{
        struct crypto_shash *tfm = ima_shash_tfm;
        int rc, i;

        if (algo < 0 || algo >= HASH_ALGO__LAST)
                algo = ima_hash_algo;

        if (algo == ima_hash_algo)
                return tfm;

        for (i = 0; i < NR_BANKS(ima_tpm_chip) + ima_extra_slots; i++)
                if (ima_algo_array[i].tfm && ima_algo_array[i].algo == algo)
                        return ima_algo_array[i].tfm;

        tfm = crypto_alloc_shash(hash_algo_name[algo], 0, 0);
        if (IS_ERR(tfm)) {
                rc = PTR_ERR(tfm);
                pr_err("Can not allocate %s (reason: %d)\n",
                       hash_algo_name[algo], rc);
        }
        return tfm;
}

int __init ima_init_crypto(void)
{
        enum hash_algo algo;
        long rc;
        int i;

        rc = ima_init_ima_crypto();
        if (rc)
                return rc;

        ima_sha1_idx = -1;
        ima_hash_algo_idx = -1;

        for (i = 0; i < NR_BANKS(ima_tpm_chip); i++) {
                algo = ima_tpm_chip->allocated_banks[i].crypto_id;
                if (algo == HASH_ALGO_SHA1)
                        ima_sha1_idx = i;

                if (algo == ima_hash_algo)
                        ima_hash_algo_idx = i;
        }

        if (ima_sha1_idx < 0) {
                ima_sha1_idx = NR_BANKS(ima_tpm_chip) + ima_extra_slots++;
                if (ima_hash_algo == HASH_ALGO_SHA1)
                        ima_hash_algo_idx = ima_sha1_idx;
        }

        if (ima_hash_algo_idx < 0)
                ima_hash_algo_idx = NR_BANKS(ima_tpm_chip) + ima_extra_slots++;

        ima_algo_array = kcalloc(NR_BANKS(ima_tpm_chip) + ima_extra_slots,
                                 sizeof(*ima_algo_array), GFP_KERNEL);
        if (!ima_algo_array) {
                rc = -ENOMEM;
                goto out;
        }

        for (i = 0; i < NR_BANKS(ima_tpm_chip); i++) {
                algo = ima_tpm_chip->allocated_banks[i].crypto_id;
                ima_algo_array[i].algo = algo;

                /* unknown TPM algorithm */
                if (algo == HASH_ALGO__LAST)
                        continue;

                if (algo == ima_hash_algo) {
                        ima_algo_array[i].tfm = ima_shash_tfm;
                        continue;
                }

                ima_algo_array[i].tfm = ima_alloc_tfm(algo);
                if (IS_ERR(ima_algo_array[i].tfm)) {
                        if (algo == HASH_ALGO_SHA1) {
                                rc = PTR_ERR(ima_algo_array[i].tfm);
                                ima_algo_array[i].tfm = NULL;
                                goto out_array;
                        }

                        ima_algo_array[i].tfm = NULL;
                }
        }

        if (ima_sha1_idx >= NR_BANKS(ima_tpm_chip)) {
                if (ima_hash_algo == HASH_ALGO_SHA1) {
                        ima_algo_array[ima_sha1_idx].tfm = ima_shash_tfm;
                } else {
                        ima_algo_array[ima_sha1_idx].tfm =
                                                ima_alloc_tfm(HASH_ALGO_SHA1);
                        if (IS_ERR(ima_algo_array[ima_sha1_idx].tfm)) {
                                rc = PTR_ERR(ima_algo_array[ima_sha1_idx].tfm);
                                goto out_array;
                        }
                }

                ima_algo_array[ima_sha1_idx].algo = HASH_ALGO_SHA1;
        }

        if (ima_hash_algo_idx >= NR_BANKS(ima_tpm_chip) &&
            ima_hash_algo_idx != ima_sha1_idx) {
                ima_algo_array[ima_hash_algo_idx].tfm = ima_shash_tfm;
                ima_algo_array[ima_hash_algo_idx].algo = ima_hash_algo;
        }

        return 0;
out_array:
        for (i = 0; i < NR_BANKS(ima_tpm_chip) + ima_extra_slots; i++) {
                if (!ima_algo_array[i].tfm ||
                    ima_algo_array[i].tfm == ima_shash_tfm)
                        continue;

                crypto_free_shash(ima_algo_array[i].tfm);
        }
        kfree(ima_algo_array);
out:
        crypto_free_shash(ima_shash_tfm);
        return rc;
}

static void ima_free_tfm(struct crypto_shash *tfm)
{
        int i;

        if (tfm == ima_shash_tfm)
                return;

        for (i = 0; i < NR_BANKS(ima_tpm_chip) + ima_extra_slots; i++)
                if (ima_algo_array[i].tfm == tfm)
                        return;

        crypto_free_shash(tfm);
}

/**
 * ima_alloc_pages() - Allocate contiguous pages.
 * @max_size:       Maximum amount of memory to allocate.
 * @allocated_size: Returned size of actual allocation.
 * @last_warn:      Should the min_size allocation warn or not.
 *
 * Tries to do opportunistic allocation for memory first trying to allocate
 * max_size amount of memory and then splitting that until zero order is
 * reached. Allocation is tried without generating allocation warnings unless
 * last_warn is set. Last_warn set affects only last allocation of zero order.
 *
 * By default, ima_maxorder is 0 and it is equivalent to kmalloc(GFP_KERNEL)
 *
 * Return pointer to allocated memory, or NULL on failure.
 */
static void *ima_alloc_pages(loff_t max_size, size_t *allocated_size,
                             int last_warn)
{
        void *ptr;
        int order = ima_maxorder;
        gfp_t gfp_mask = __GFP_RECLAIM | __GFP_NOWARN | __GFP_NORETRY;

        if (order)
                order = min(get_order(max_size), order);

        for (; order; order--) {
                ptr = (void *)__get_free_pages(gfp_mask, order);
                if (ptr) {
                        *allocated_size = PAGE_SIZE << order;
                        return ptr;
                }
        }

        /* order is zero - one page */

        gfp_mask = GFP_KERNEL;

        if (!last_warn)
                gfp_mask |= __GFP_NOWARN;

        ptr = (void *)__get_free_pages(gfp_mask, 0);
        if (ptr) {
                *allocated_size = PAGE_SIZE;
                return ptr;
        }

        *allocated_size = 0;
        return NULL;
}

/**
 * ima_free_pages() - Free pages allocated by ima_alloc_pages().
 * @ptr:  Pointer to allocated pages.
 * @size: Size of allocated buffer.
 */
static void ima_free_pages(void *ptr, size_t size)
{
        if (!ptr)
                return;
        free_pages((unsigned long)ptr, get_order(size));
}

static struct crypto_ahash *ima_alloc_atfm(enum hash_algo algo)
{
        struct crypto_ahash *tfm = ima_ahash_tfm;
        int rc;

        if (algo < 0 || algo >= HASH_ALGO__LAST)
                algo = ima_hash_algo;

        if (algo != ima_hash_algo || !tfm) {
                tfm = crypto_alloc_ahash(hash_algo_name[algo], 0, 0);
                if (!IS_ERR(tfm)) {
                        if (algo == ima_hash_algo)
                                ima_ahash_tfm = tfm;
                } else {
                        rc = PTR_ERR(tfm);
                        pr_err("Can not allocate %s (reason: %d)\n",
                               hash_algo_name[algo], rc);
                }
        }
        return tfm;
}

static void ima_free_atfm(struct crypto_ahash *tfm)
{
        if (tfm != ima_ahash_tfm)
                crypto_free_ahash(tfm);
}

static inline int ahash_wait(int err, struct crypto_wait *wait)
{

        err = crypto_wait_req(err, wait);

        if (err)
                pr_crit_ratelimited("ahash calculation failed: err: %d\n", err);

        return err;
}

static int ima_calc_file_hash_atfm(struct file *file,
                                   struct ima_digest_data *hash,
                                   struct crypto_ahash *tfm)
{
        loff_t i_size, offset;
        char *rbuf[2] = { NULL, };
        int rc, rbuf_len, active = 0, ahash_rc = 0;
        struct ahash_request *req;
        struct scatterlist sg[1];
        struct crypto_wait wait;
        size_t rbuf_size[2];

        hash->length = crypto_ahash_digestsize(tfm);

        req = ahash_request_alloc(tfm, GFP_KERNEL);
        if (!req)
                return -ENOMEM;

        crypto_init_wait(&wait);
        ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG |
                                   CRYPTO_TFM_REQ_MAY_SLEEP,
                                   crypto_req_done, &wait);

        rc = ahash_wait(crypto_ahash_init(req), &wait);
        if (rc)
                goto out1;

        i_size = i_size_read(file_inode(file));

        if (i_size == 0)
                goto out2;

        /*
         * Try to allocate maximum size of memory.
         * Fail if even a single page cannot be allocated.
         */
        rbuf[0] = ima_alloc_pages(i_size, &rbuf_size[0], 1);
        if (!rbuf[0]) {
                rc = -ENOMEM;
                goto out1;
        }

        /* Only allocate one buffer if that is enough. */
        if (i_size > rbuf_size[0]) {
                /*
                 * Try to allocate secondary buffer. If that fails fallback to
                 * using single buffering. Use previous memory allocation size
                 * as baseline for possible allocation size.
                 */
                rbuf[1] = ima_alloc_pages(i_size - rbuf_size[0],
                                          &rbuf_size[1], 0);
        }

        for (offset = 0; offset < i_size; offset += rbuf_len) {
                if (!rbuf[1] && offset) {
                        /* Not using two buffers, and it is not the first
                         * read/request, wait for the completion of the
                         * previous ahash_update() request.
                         */
                        rc = ahash_wait(ahash_rc, &wait);
                        if (rc)
                                goto out3;
                }
                /* read buffer */
                rbuf_len = min_t(loff_t, i_size - offset, rbuf_size[active]);
                rc = integrity_kernel_read(file, offset, rbuf[active],
                                           rbuf_len);
                if (rc != rbuf_len) {
                        if (rc >= 0)
                                rc = -EINVAL;
                        /*
                         * Forward current rc, do not overwrite with return value
                         * from ahash_wait()
                         */
                        ahash_wait(ahash_rc, &wait);
                        goto out3;
                }

                if (rbuf[1] && offset) {
                        /* Using two buffers, and it is not the first
                         * read/request, wait for the completion of the
                         * previous ahash_update() request.
                         */
                        rc = ahash_wait(ahash_rc, &wait);
                        if (rc)
                                goto out3;
                }

                sg_init_one(&sg[0], rbuf[active], rbuf_len);
                ahash_request_set_crypt(req, sg, NULL, rbuf_len);

                ahash_rc = crypto_ahash_update(req);

                if (rbuf[1])
                        active = !active; /* swap buffers, if we use two */
        }
        /* wait for the last update request to complete */
        rc = ahash_wait(ahash_rc, &wait);
out3:
        ima_free_pages(rbuf[0], rbuf_size[0]);
        ima_free_pages(rbuf[1], rbuf_size[1]);
out2:
        if (!rc) {
                ahash_request_set_crypt(req, NULL, hash->digest, 0);
                rc = ahash_wait(crypto_ahash_final(req), &wait);
        }
out1:
        ahash_request_free(req);
        return rc;
}

static int ima_calc_file_ahash(struct file *file, struct ima_digest_data *hash)
{
        struct crypto_ahash *tfm;
        int rc;

        tfm = ima_alloc_atfm(hash->algo);
        if (IS_ERR(tfm))
                return PTR_ERR(tfm);

        rc = ima_calc_file_hash_atfm(file, hash, tfm);

        ima_free_atfm(tfm);

        return rc;
}

static int ima_calc_file_hash_tfm(struct file *file,
                                  struct ima_digest_data *hash,
                                  struct crypto_shash *tfm)
{
        loff_t i_size, offset = 0;
        char *rbuf;
        int rc;
        SHASH_DESC_ON_STACK(shash, tfm);

        shash->tfm = tfm;

        hash->length = crypto_shash_digestsize(tfm);

        rc = crypto_shash_init(shash);
        if (rc != 0)
                return rc;

        i_size = i_size_read(file_inode(file));

        if (i_size == 0)
                goto out;

        rbuf = kzalloc(PAGE_SIZE, GFP_KERNEL);
        if (!rbuf)
                return -ENOMEM;

        while (offset < i_size) {
                int rbuf_len;

                rbuf_len = integrity_kernel_read(file, offset, rbuf, PAGE_SIZE);
                if (rbuf_len < 0) {
                        rc = rbuf_len;
                        break;
                }
                if (rbuf_len == 0) {        /* unexpected EOF */
                        rc = -EINVAL;
                        break;
                }
                offset += rbuf_len;

                rc = crypto_shash_update(shash, rbuf, rbuf_len);
                if (rc)
                        break;
        }
        kfree(rbuf);
out:
        if (!rc)
                rc = crypto_shash_final(shash, hash->digest);
        return rc;
}

static int ima_calc_file_shash(struct file *file, struct ima_digest_data *hash)
{
        struct crypto_shash *tfm;
        int rc;

        tfm = ima_alloc_tfm(hash->algo);
        if (IS_ERR(tfm))
                return PTR_ERR(tfm);

        rc = ima_calc_file_hash_tfm(file, hash, tfm);

        ima_free_tfm(tfm);

        return rc;
}

/*
 * ima_calc_file_hash - calculate file hash
 *
 * Asynchronous hash (ahash) allows using HW acceleration for calculating
 * a hash. ahash performance varies for different data sizes on different
 * crypto accelerators. shash performance might be better for smaller files.
 * The 'ima.ahash_minsize' module parameter allows specifying the best
 * minimum file size for using ahash on the system.
 *
 * If the ima.ahash_minsize parameter is not specified, this function uses
 * shash for the hash calculation.  If ahash fails, it falls back to using
 * shash.
 */
int ima_calc_file_hash(struct file *file, struct ima_digest_data *hash)
{
        loff_t i_size;
        int rc;
        struct file *f = file;
        bool new_file_instance = false;

        /*
         * For consistency, fail file's opened with the O_DIRECT flag on
         * filesystems mounted with/without DAX option.
         */
        if (file->f_flags & O_DIRECT) {
                hash->length = hash_digest_size[ima_hash_algo];
                hash->algo = ima_hash_algo;
                return -EINVAL;
        }

        /* Open a new file instance in O_RDONLY if we cannot read */
        if (!(file->f_mode & FMODE_READ)) {
                int flags = file->f_flags & ~(O_WRONLY | O_APPEND |
                                O_TRUNC | O_CREAT | O_NOCTTY | O_EXCL);
                flags |= O_RDONLY;
                f = dentry_open(&file->f_path, flags, file->f_cred);
                if (IS_ERR(f))
                        return PTR_ERR(f);

                new_file_instance = true;
        }

        i_size = i_size_read(file_inode(f));

        if (ima_ahash_minsize && i_size >= ima_ahash_minsize) {
                rc = ima_calc_file_ahash(f, hash);
                if (!rc)
                        goto out;
        }

        rc = ima_calc_file_shash(f, hash);
out:
        if (new_file_instance)
                fput(f);
        return rc;
}

/*
 * Calculate the hash of template data
 */
static int ima_calc_field_array_hash_tfm(struct ima_field_data *field_data,
                                         struct ima_template_entry *entry,
                                         int tfm_idx)
{
        SHASH_DESC_ON_STACK(shash, ima_algo_array[tfm_idx].tfm);
        struct ima_template_desc *td = entry->template_desc;
        int num_fields = entry->template_desc->num_fields;
        int rc, i;

        shash->tfm = ima_algo_array[tfm_idx].tfm;

        rc = crypto_shash_init(shash);
        if (rc != 0)
                return rc;

        for (i = 0; i < num_fields; i++) {
                u8 buffer[IMA_EVENT_NAME_LEN_MAX + 1] = { 0 };
                u8 *data_to_hash = field_data[i].data;
                u32 datalen = field_data[i].len;
                u32 datalen_to_hash = !ima_canonical_fmt ?
                                datalen : (__force u32)cpu_to_le32(datalen);

                if (strcmp(td->name, IMA_TEMPLATE_IMA_NAME) != 0) {
                        rc = crypto_shash_update(shash,
                                                (const u8 *) &datalen_to_hash,
                                                sizeof(datalen_to_hash));
                        if (rc)
                                break;
                } else if (strcmp(td->fields[i]->field_id, "n") == 0) {
                        memcpy(buffer, data_to_hash, datalen);
                        data_to_hash = buffer;
                        datalen = IMA_EVENT_NAME_LEN_MAX + 1;
                }
                rc = crypto_shash_update(shash, data_to_hash, datalen);
                if (rc)
                        break;
        }

        if (!rc)
                rc = crypto_shash_final(shash, entry->digests[tfm_idx].digest);

        return rc;
}

int ima_calc_field_array_hash(struct ima_field_data *field_data,
                              struct ima_template_entry *entry)
{
        u16 alg_id;
        int rc, i;

        rc = ima_calc_field_array_hash_tfm(field_data, entry, ima_sha1_idx);
        if (rc)
                return rc;

        entry->digests[ima_sha1_idx].alg_id = TPM_ALG_SHA1;

        for (i = 0; i < NR_BANKS(ima_tpm_chip) + ima_extra_slots; i++) {
                if (i == ima_sha1_idx)
                        continue;

                if (i < NR_BANKS(ima_tpm_chip)) {
                        alg_id = ima_tpm_chip->allocated_banks[i].alg_id;
                        entry->digests[i].alg_id = alg_id;
                }

                /* for unmapped TPM algorithms digest is still a padded SHA1 */
                if (!ima_algo_array[i].tfm) {
                        memcpy(entry->digests[i].digest,
                               entry->digests[ima_sha1_idx].digest,
                               TPM_DIGEST_SIZE);
                        continue;
                }

                rc = ima_calc_field_array_hash_tfm(field_data, entry, i);
                if (rc)
                        return rc;
        }
        return rc;
}

static int calc_buffer_ahash_atfm(const void *buf, loff_t len,
                                  struct ima_digest_data *hash,
                                  struct crypto_ahash *tfm)
{
        struct ahash_request *req;
        struct scatterlist sg;
        struct crypto_wait wait;
        int rc, ahash_rc = 0;

        hash->length = crypto_ahash_digestsize(tfm);

        req = ahash_request_alloc(tfm, GFP_KERNEL);
        if (!req)
                return -ENOMEM;

        crypto_init_wait(&wait);
        ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG |
                                   CRYPTO_TFM_REQ_MAY_SLEEP,
                                   crypto_req_done, &wait);

        rc = ahash_wait(crypto_ahash_init(req), &wait);
        if (rc)
                goto out;

        sg_init_one(&sg, buf, len);
        ahash_request_set_crypt(req, &sg, NULL, len);

        ahash_rc = crypto_ahash_update(req);

        /* wait for the update request to complete */
        rc = ahash_wait(ahash_rc, &wait);
        if (!rc) {
                ahash_request_set_crypt(req, NULL, hash->digest, 0);
                rc = ahash_wait(crypto_ahash_final(req), &wait);
        }
out:
        ahash_request_free(req);
        return rc;
}

static int calc_buffer_ahash(const void *buf, loff_t len,
                             struct ima_digest_data *hash)
{
        struct crypto_ahash *tfm;
        int rc;

        tfm = ima_alloc_atfm(hash->algo);
        if (IS_ERR(tfm))
                return PTR_ERR(tfm);

        rc = calc_buffer_ahash_atfm(buf, len, hash, tfm);

        ima_free_atfm(tfm);

        return rc;
}

static int calc_buffer_shash_tfm(const void *buf, loff_t size,
                                struct ima_digest_data *hash,
                                struct crypto_shash *tfm)
{
        SHASH_DESC_ON_STACK(shash, tfm);
        unsigned int len;
        int rc;

        shash->tfm = tfm;

        hash->length = crypto_shash_digestsize(tfm);

        rc = crypto_shash_init(shash);
        if (rc != 0)
                return rc;

        while (size) {
                len = size < PAGE_SIZE ? size : PAGE_SIZE;
                rc = crypto_shash_update(shash, buf, len);
                if (rc)
                        break;
                buf += len;
                size -= len;
        }

        if (!rc)
                rc = crypto_shash_final(shash, hash->digest);
        return rc;
}

static int calc_buffer_shash(const void *buf, loff_t len,
                             struct ima_digest_data *hash)
{
        struct crypto_shash *tfm;
        int rc;

        tfm = ima_alloc_tfm(hash->algo);
        if (IS_ERR(tfm))
                return PTR_ERR(tfm);

        rc = calc_buffer_shash_tfm(buf, len, hash, tfm);

        ima_free_tfm(tfm);
        return rc;
}

int ima_calc_buffer_hash(const void *buf, loff_t len,
                         struct ima_digest_data *hash)
{
        int rc;

        if (ima_ahash_minsize && len >= ima_ahash_minsize) {
                rc = calc_buffer_ahash(buf, len, hash);
                if (!rc)
                        return 0;
        }

        return calc_buffer_shash(buf, len, hash);
}

static void ima_pcrread(u32 idx, struct tpm_digest *d)
{
        if (!ima_tpm_chip)
                return;

        if (tpm_pcr_read(ima_tpm_chip, idx, d) != 0)
                pr_err("Error Communicating to TPM chip\n");
}

/*
 * The boot_aggregate is a cumulative hash over TPM registers 0 - 7.  With
 * TPM 1.2 the boot_aggregate was based on reading the SHA1 PCRs, but with
 * TPM 2.0 hash agility, TPM chips could support multiple TPM PCR banks,
 * allowing firmware to configure and enable different banks.
 *
 * Knowing which TPM bank is read to calculate the boot_aggregate digest
 * needs to be conveyed to a verifier.  For this reason, use the same
 * hash algorithm for reading the TPM PCRs as for calculating the boot
 * aggregate digest as stored in the measurement list.
 */
static int ima_calc_boot_aggregate_tfm(char *digest, u16 alg_id,
                                       struct crypto_shash *tfm)
{
        struct tpm_digest d = { .alg_id = alg_id, .digest = {0} };
        int rc;
        u32 i;
        SHASH_DESC_ON_STACK(shash, tfm);

        shash->tfm = tfm;

        pr_devel("calculating the boot-aggregate based on TPM bank: %04x\n",
                 d.alg_id);

        rc = crypto_shash_init(shash);
        if (rc != 0)
                return rc;

        /* cumulative digest over TPM registers 0-7 */
        for (i = TPM_PCR0; i < TPM_PCR8; i++) {
                ima_pcrread(i, &d);
                /* now accumulate with current aggregate */
                rc = crypto_shash_update(shash, d.digest,
                                         crypto_shash_digestsize(tfm));
                if (rc != 0)
                        return rc;
        }
        /*
         * Extend cumulative digest over TPM registers 8-9, which contain
         * measurement for the kernel command line (reg. 8) and image (reg. 9)
         * in a typical PCR allocation. Registers 8-9 are only included in
         * non-SHA1 boot_aggregate digests to avoid ambiguity.
         */
        if (alg_id != TPM_ALG_SHA1) {
                for (i = TPM_PCR8; i < TPM_PCR10; i++) {
                        ima_pcrread(i, &d);
                        rc = crypto_shash_update(shash, d.digest,
                                                crypto_shash_digestsize(tfm));
                }
        }
        if (!rc)
                crypto_shash_final(shash, digest);
        return rc;
}

int ima_calc_boot_aggregate(struct ima_digest_data *hash)
{
        struct crypto_shash *tfm;
        u16 crypto_id, alg_id;
        int rc, i, bank_idx = -1;

        for (i = 0; i < ima_tpm_chip->nr_allocated_banks; i++) {
                crypto_id = ima_tpm_chip->allocated_banks[i].crypto_id;
                if (crypto_id == hash->algo) {
                        bank_idx = i;
                        break;
                }

                if (crypto_id == HASH_ALGO_SHA256)
                        bank_idx = i;

                if (bank_idx == -1 && crypto_id == HASH_ALGO_SHA1)
                        bank_idx = i;
        }

        if (bank_idx == -1) {
                pr_err("No suitable TPM algorithm for boot aggregate\n");
                return 0;
        }

        hash->algo = ima_tpm_chip->allocated_banks[bank_idx].crypto_id;

        tfm = ima_alloc_tfm(hash->algo);
        if (IS_ERR(tfm))
                return PTR_ERR(tfm);

        hash->length = crypto_shash_digestsize(tfm);
        alg_id = ima_tpm_chip->allocated_banks[bank_idx].alg_id;
        rc = ima_calc_boot_aggregate_tfm(hash->digest, alg_id, tfm);

        ima_free_tfm(tfm);

        return rc;
}




























  111 


   39 
  109 










   17 


   17 


    5 
    5 







   99 
   99 







   46 










   12 











   12 
   12 
    9 



    1 










    1 

   12 

















  169 


  169 
  170 
  169 

  168 










































    1 





    1 


















































    1 


    1 



  168 
    1 


    1 



  145 
   21 










  213 



  217 



   42 


   42 
   42 
   41 

   42 



   41 






  172 






  214 




    1 
    1 


    1 




    1 







   10 






   10 




    5 





























1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
/* License: GPL */

#include <linux/filter.h>
#include <linux/mutex.h>
#include <linux/socket.h>
#include <linux/skbuff.h>
#include <net/netlink.h>
#include <net/net_namespace.h>
#include <linux/module.h>
#include <net/sock.h>
#include <linux/kernel.h>
#include <linux/tcp.h>
#include <linux/workqueue.h>
#include <linux/nospec.h>
#include <linux/cookie.h>
#include <linux/inet_diag.h>
#include <linux/sock_diag.h>

static const struct sock_diag_handler __rcu *sock_diag_handlers[AF_MAX];

static const struct sock_diag_inet_compat __rcu *inet_rcv_compat;

static struct workqueue_struct *broadcast_wq;

DEFINE_COOKIE(sock_cookie);

u64 __sock_gen_cookie(struct sock *sk)
{
        u64 res = atomic64_read(&sk->sk_cookie);

        if (!res) {
                u64 new = gen_cookie_next(&sock_cookie);

                atomic64_cmpxchg(&sk->sk_cookie, res, new);

                /* Another thread might have changed sk_cookie before us. */
                res = atomic64_read(&sk->sk_cookie);
        }
        return res;
}

int sock_diag_check_cookie(struct sock *sk, const __u32 *cookie)
{
        u64 res;

        if (cookie[0] == INET_DIAG_NOCOOKIE && cookie[1] == INET_DIAG_NOCOOKIE)
                return 0;

        res = sock_gen_cookie(sk);
        if ((u32)res != cookie[0] || (u32)(res >> 32) != cookie[1])
                return -ESTALE;

        return 0;
}
EXPORT_SYMBOL_GPL(sock_diag_check_cookie);

void sock_diag_save_cookie(struct sock *sk, __u32 *cookie)
{
        u64 res = sock_gen_cookie(sk);

        cookie[0] = (u32)res;
        cookie[1] = (u32)(res >> 32);
}
EXPORT_SYMBOL_GPL(sock_diag_save_cookie);

int sock_diag_put_meminfo(struct sock *sk, struct sk_buff *skb, int attrtype)
{
        u32 mem[SK_MEMINFO_VARS];

        sk_get_meminfo(sk, mem);

        return nla_put(skb, attrtype, sizeof(mem), &mem);
}
EXPORT_SYMBOL_GPL(sock_diag_put_meminfo);

int sock_diag_put_filterinfo(bool may_report_filterinfo, struct sock *sk,
                             struct sk_buff *skb, int attrtype)
{
        struct sock_fprog_kern *fprog;
        struct sk_filter *filter;
        struct nlattr *attr;
        unsigned int flen;
        int err = 0;

        if (!may_report_filterinfo) {
                nla_reserve(skb, attrtype, 0);
                return 0;
        }

        rcu_read_lock();
        filter = rcu_dereference(sk->sk_filter);
        if (!filter)
                goto out;

        fprog = filter->prog->orig_prog;
        if (!fprog)
                goto out;

        flen = bpf_classic_proglen(fprog);

        attr = nla_reserve(skb, attrtype, flen);
        if (attr == NULL) {
                err = -EMSGSIZE;
                goto out;
        }

        memcpy(nla_data(attr), fprog->filter, flen);
out:
        rcu_read_unlock();
        return err;
}
EXPORT_SYMBOL(sock_diag_put_filterinfo);

struct broadcast_sk {
        struct sock *sk;
        struct work_struct work;
};

static size_t sock_diag_nlmsg_size(void)
{
        return NLMSG_ALIGN(sizeof(struct inet_diag_msg)
               + nla_total_size(sizeof(u8)) /* INET_DIAG_PROTOCOL */
               + nla_total_size_64bit(sizeof(struct tcp_info))); /* INET_DIAG_INFO */
}

static const struct sock_diag_handler *sock_diag_lock_handler(int family)
{
        const struct sock_diag_handler *handler;

        rcu_read_lock();
        handler = rcu_dereference(sock_diag_handlers[family]);
        if (handler && !try_module_get(handler->owner))
                handler = NULL;
        rcu_read_unlock();

        return handler;
}

static void sock_diag_unlock_handler(const struct sock_diag_handler *handler)
{
        module_put(handler->owner);
}

static void sock_diag_broadcast_destroy_work(struct work_struct *work)
{
        struct broadcast_sk *bsk =
                container_of(work, struct broadcast_sk, work);
        struct sock *sk = bsk->sk;
        const struct sock_diag_handler *hndl;
        struct sk_buff *skb;
        const enum sknetlink_groups group = sock_diag_destroy_group(sk);
        int err = -1;

        WARN_ON(group == SKNLGRP_NONE);

        skb = nlmsg_new(sock_diag_nlmsg_size(), GFP_KERNEL);
        if (!skb)
                goto out;

        hndl = sock_diag_lock_handler(sk->sk_family);
        if (hndl) {
                if (hndl->get_info)
                        err = hndl->get_info(skb, sk);
                sock_diag_unlock_handler(hndl);
        }
        if (!err)
                nlmsg_multicast(sock_net(sk)->diag_nlsk, skb, 0, group,
                                GFP_KERNEL);
        else
                kfree_skb(skb);
out:
        sk_destruct(sk);
        kfree(bsk);
}

void sock_diag_broadcast_destroy(struct sock *sk)
{
        /* Note, this function is often called from an interrupt context. */
        struct broadcast_sk *bsk =
                kmalloc(sizeof(struct broadcast_sk), GFP_ATOMIC);
        if (!bsk)
                return sk_destruct(sk);
        bsk->sk = sk;
        INIT_WORK(&bsk->work, sock_diag_broadcast_destroy_work);
        queue_work(broadcast_wq, &bsk->work);
}

void sock_diag_register_inet_compat(const struct sock_diag_inet_compat *ptr)
{
        xchg(&inet_rcv_compat, RCU_INITIALIZER(ptr));
}
EXPORT_SYMBOL_GPL(sock_diag_register_inet_compat);

void sock_diag_unregister_inet_compat(const struct sock_diag_inet_compat *ptr)
{
        const struct sock_diag_inet_compat *old;

        old = unrcu_pointer(xchg(&inet_rcv_compat, NULL));
        WARN_ON_ONCE(old != ptr);
}
EXPORT_SYMBOL_GPL(sock_diag_unregister_inet_compat);

int sock_diag_register(const struct sock_diag_handler *hndl)
{
        int family = hndl->family;

        if (family >= AF_MAX)
                return -EINVAL;

        return !cmpxchg((const struct sock_diag_handler **)
                                &sock_diag_handlers[family],
                        NULL, hndl) ? 0 : -EBUSY;
}
EXPORT_SYMBOL_GPL(sock_diag_register);

void sock_diag_unregister(const struct sock_diag_handler *hndl)
{
        int family = hndl->family;

        if (family >= AF_MAX)
                return;

        xchg((const struct sock_diag_handler **)&sock_diag_handlers[family],
             NULL);
}
EXPORT_SYMBOL_GPL(sock_diag_unregister);

static int __sock_diag_cmd(struct sk_buff *skb, struct nlmsghdr *nlh)
{
        int err;
        struct sock_diag_req *req = nlmsg_data(nlh);
        const struct sock_diag_handler *hndl;

        if (nlmsg_len(nlh) < sizeof(*req))
                return -EINVAL;

        if (req->sdiag_family >= AF_MAX)
                return -EINVAL;
        req->sdiag_family = array_index_nospec(req->sdiag_family, AF_MAX);

        if (!rcu_access_pointer(sock_diag_handlers[req->sdiag_family]))
                sock_load_diag_module(req->sdiag_family, 0);

        hndl = sock_diag_lock_handler(req->sdiag_family);
        if (hndl == NULL)
                return -ENOENT;

        if (nlh->nlmsg_type == SOCK_DIAG_BY_FAMILY)
                err = hndl->dump(skb, nlh);
        else if (nlh->nlmsg_type == SOCK_DESTROY && hndl->destroy)
                err = hndl->destroy(skb, nlh);
        else
                err = -EOPNOTSUPP;
        sock_diag_unlock_handler(hndl);

        return err;
}

static int sock_diag_rcv_msg(struct sk_buff *skb, struct nlmsghdr *nlh,
                             struct netlink_ext_ack *extack)
{
        const struct sock_diag_inet_compat *ptr;
        int ret;

        switch (nlh->nlmsg_type) {
        case TCPDIAG_GETSOCK:
        case DCCPDIAG_GETSOCK:

                if (!rcu_access_pointer(inet_rcv_compat))
                        sock_load_diag_module(AF_INET, 0);

                rcu_read_lock();
                ptr = rcu_dereference(inet_rcv_compat);
                if (ptr && !try_module_get(ptr->owner))
                        ptr = NULL;
                rcu_read_unlock();

                ret = -EOPNOTSUPP;
                if (ptr) {
                        ret = ptr->fn(skb, nlh);
                        module_put(ptr->owner);
                }

                return ret;
        case SOCK_DIAG_BY_FAMILY:
        case SOCK_DESTROY:
                return __sock_diag_cmd(skb, nlh);
        default:
                return -EINVAL;
        }
}

static void sock_diag_rcv(struct sk_buff *skb)
{
        netlink_rcv_skb(skb, &sock_diag_rcv_msg);
}

static int sock_diag_bind(struct net *net, int group)
{
        switch (group) {
        case SKNLGRP_INET_TCP_DESTROY:
        case SKNLGRP_INET_UDP_DESTROY:
                if (!rcu_access_pointer(sock_diag_handlers[AF_INET]))
                        sock_load_diag_module(AF_INET, 0);
                break;
        case SKNLGRP_INET6_TCP_DESTROY:
        case SKNLGRP_INET6_UDP_DESTROY:
                if (!rcu_access_pointer(sock_diag_handlers[AF_INET6]))
                        sock_load_diag_module(AF_INET6, 0);
                break;
        }
        return 0;
}

int sock_diag_destroy(struct sock *sk, int err)
{
        if (!ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN))
                return -EPERM;

        if (!sk->sk_prot->diag_destroy)
                return -EOPNOTSUPP;

        return sk->sk_prot->diag_destroy(sk, err);
}
EXPORT_SYMBOL_GPL(sock_diag_destroy);

static int __net_init diag_net_init(struct net *net)
{
        struct netlink_kernel_cfg cfg = {
                .groups        = SKNLGRP_MAX,
                .input        = sock_diag_rcv,
                .bind        = sock_diag_bind,
                .flags        = NL_CFG_F_NONROOT_RECV,
        };

        net->diag_nlsk = netlink_kernel_create(net, NETLINK_SOCK_DIAG, &cfg);
        return net->diag_nlsk == NULL ? -ENOMEM : 0;
}

static void __net_exit diag_net_exit(struct net *net)
{
        netlink_kernel_release(net->diag_nlsk);
        net->diag_nlsk = NULL;
}

static struct pernet_operations diag_net_ops = {
        .init = diag_net_init,
        .exit = diag_net_exit,
};

static int __init sock_diag_init(void)
{
        broadcast_wq = alloc_workqueue("sock_diag_events", 0, 0);
        BUG_ON(!broadcast_wq);
        return register_pernet_subsys(&diag_net_ops);
}
device_initcall(sock_diag_init);




































    5 






    5 
























































































































    5 











































































































































































    2 





    2 









































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright (C) ST-Ericsson AB 2010
 * Author:        Sjur Brendeland
 */

#define pr_fmt(fmt) KBUILD_MODNAME ":%s(): " fmt, __func__

#include <linux/stddef.h>
#include <linux/spinlock.h>
#include <linux/slab.h>
#include <linux/pkt_sched.h>
#include <net/caif/caif_layer.h>
#include <net/caif/cfpkt.h>
#include <net/caif/cfctrl.h>

#define container_obj(layr) container_of(layr, struct cfctrl, serv.layer)
#define UTILITY_NAME_LENGTH 16
#define CFPKT_CTRL_PKT_LEN 20

#ifdef CAIF_NO_LOOP
static int handle_loop(struct cfctrl *ctrl,
                       int cmd, struct cfpkt *pkt){
        return -1;
}
#else
static int handle_loop(struct cfctrl *ctrl,
                       int cmd, struct cfpkt *pkt);
#endif
static int cfctrl_recv(struct cflayer *layr, struct cfpkt *pkt);
static void cfctrl_ctrlcmd(struct cflayer *layr, enum caif_ctrlcmd ctrl,
                           int phyid);


struct cflayer *cfctrl_create(void)
{
        struct dev_info dev_info;
        struct cfctrl *this =
                kzalloc(sizeof(struct cfctrl), GFP_ATOMIC);
        if (!this)
                return NULL;
        caif_assert(offsetof(struct cfctrl, serv.layer) == 0);
        memset(&dev_info, 0, sizeof(dev_info));
        dev_info.id = 0xff;
        cfsrvl_init(&this->serv, 0, &dev_info, false);
        atomic_set(&this->req_seq_no, 1);
        atomic_set(&this->rsp_seq_no, 1);
        this->serv.layer.receive = cfctrl_recv;
        sprintf(this->serv.layer.name, "ctrl");
        this->serv.layer.ctrlcmd = cfctrl_ctrlcmd;
#ifndef CAIF_NO_LOOP
        spin_lock_init(&this->loop_linkid_lock);
        this->loop_linkid = 1;
#endif
        spin_lock_init(&this->info_list_lock);
        INIT_LIST_HEAD(&this->list);
        return &this->serv.layer;
}

void cfctrl_remove(struct cflayer *layer)
{
        struct cfctrl_request_info *p, *tmp;
        struct cfctrl *ctrl = container_obj(layer);

        spin_lock_bh(&ctrl->info_list_lock);
        list_for_each_entry_safe(p, tmp, &ctrl->list, list) {
                list_del(&p->list);
                kfree(p);
        }
        spin_unlock_bh(&ctrl->info_list_lock);
        kfree(layer);
}

static bool param_eq(const struct cfctrl_link_param *p1,
                     const struct cfctrl_link_param *p2)
{
        bool eq =
            p1->linktype == p2->linktype &&
            p1->priority == p2->priority &&
            p1->phyid == p2->phyid &&
            p1->endpoint == p2->endpoint && p1->chtype == p2->chtype;

        if (!eq)
                return false;

        switch (p1->linktype) {
        case CFCTRL_SRV_VEI:
                return true;
        case CFCTRL_SRV_DATAGRAM:
                return p1->u.datagram.connid == p2->u.datagram.connid;
        case CFCTRL_SRV_RFM:
                return
                    p1->u.rfm.connid == p2->u.rfm.connid &&
                    strcmp(p1->u.rfm.volume, p2->u.rfm.volume) == 0;
        case CFCTRL_SRV_UTIL:
                return
                    p1->u.utility.fifosize_kb == p2->u.utility.fifosize_kb
                    && p1->u.utility.fifosize_bufs ==
                    p2->u.utility.fifosize_bufs
                    && strcmp(p1->u.utility.name, p2->u.utility.name) == 0
                    && p1->u.utility.paramlen == p2->u.utility.paramlen
                    && memcmp(p1->u.utility.params, p2->u.utility.params,
                              p1->u.utility.paramlen) == 0;

        case CFCTRL_SRV_VIDEO:
                return p1->u.video.connid == p2->u.video.connid;
        case CFCTRL_SRV_DBG:
                return true;
        case CFCTRL_SRV_DECM:
                return false;
        default:
                return false;
        }
        return false;
}

static bool cfctrl_req_eq(const struct cfctrl_request_info *r1,
                          const struct cfctrl_request_info *r2)
{
        if (r1->cmd != r2->cmd)
                return false;
        if (r1->cmd == CFCTRL_CMD_LINK_SETUP)
                return param_eq(&r1->param, &r2->param);
        else
                return r1->channel_id == r2->channel_id;
}

/* Insert request at the end */
static void cfctrl_insert_req(struct cfctrl *ctrl,
                              struct cfctrl_request_info *req)
{
        spin_lock_bh(&ctrl->info_list_lock);
        atomic_inc(&ctrl->req_seq_no);
        req->sequence_no = atomic_read(&ctrl->req_seq_no);
        list_add_tail(&req->list, &ctrl->list);
        spin_unlock_bh(&ctrl->info_list_lock);
}

/* Compare and remove request */
static struct cfctrl_request_info *cfctrl_remove_req(struct cfctrl *ctrl,
                                                struct cfctrl_request_info *req)
{
        struct cfctrl_request_info *p, *tmp, *first;

        first = list_first_entry(&ctrl->list, struct cfctrl_request_info, list);

        list_for_each_entry_safe(p, tmp, &ctrl->list, list) {
                if (cfctrl_req_eq(req, p)) {
                        if (p != first)
                                pr_warn("Requests are not received in order\n");

                        atomic_set(&ctrl->rsp_seq_no,
                                         p->sequence_no);
                        list_del(&p->list);
                        goto out;
                }
        }
        p = NULL;
out:
        return p;
}

struct cfctrl_rsp *cfctrl_get_respfuncs(struct cflayer *layer)
{
        struct cfctrl *this = container_obj(layer);
        return &this->res;
}

static void init_info(struct caif_payload_info *info, struct cfctrl *cfctrl)
{
        info->hdr_len = 0;
        info->channel_id = cfctrl->serv.layer.id;
        info->dev_info = &cfctrl->serv.dev_info;
}

void cfctrl_enum_req(struct cflayer *layer, u8 physlinkid)
{
        struct cfpkt *pkt;
        struct cfctrl *cfctrl = container_obj(layer);
        struct cflayer *dn = cfctrl->serv.layer.dn;

        if (!dn) {
                pr_debug("not able to send enum request\n");
                return;
        }
        pkt = cfpkt_create(CFPKT_CTRL_PKT_LEN);
        if (!pkt)
                return;
        caif_assert(offsetof(struct cfctrl, serv.layer) == 0);
        init_info(cfpkt_info(pkt), cfctrl);
        cfpkt_info(pkt)->dev_info->id = physlinkid;
        cfctrl->serv.dev_info.id = physlinkid;
        cfpkt_addbdy(pkt, CFCTRL_CMD_ENUM);
        cfpkt_addbdy(pkt, physlinkid);
        cfpkt_set_prio(pkt, TC_PRIO_CONTROL);
        dn->transmit(dn, pkt);
}

int cfctrl_linkup_request(struct cflayer *layer,
                          struct cfctrl_link_param *param,
                          struct cflayer *user_layer)
{
        struct cfctrl *cfctrl = container_obj(layer);
        struct cflayer *dn = cfctrl->serv.layer.dn;
        char utility_name[UTILITY_NAME_LENGTH];
        struct cfctrl_request_info *req;
        struct cfpkt *pkt;
        u32 tmp32;
        u16 tmp16;
        u8 tmp8;
        int ret;

        if (!dn) {
                pr_debug("not able to send linkup request\n");
                return -ENODEV;
        }

        if (cfctrl_cancel_req(layer, user_layer) > 0) {
                /* Slight Paranoia, check if already connecting */
                pr_err("Duplicate connect request for same client\n");
                WARN_ON(1);
                return -EALREADY;
        }

        pkt = cfpkt_create(CFPKT_CTRL_PKT_LEN);
        if (!pkt)
                return -ENOMEM;
        cfpkt_addbdy(pkt, CFCTRL_CMD_LINK_SETUP);
        cfpkt_addbdy(pkt, (param->chtype << 4) | param->linktype);
        cfpkt_addbdy(pkt, (param->priority << 3) | param->phyid);
        cfpkt_addbdy(pkt, param->endpoint & 0x03);

        switch (param->linktype) {
        case CFCTRL_SRV_VEI:
                break;
        case CFCTRL_SRV_VIDEO:
                cfpkt_addbdy(pkt, (u8) param->u.video.connid);
                break;
        case CFCTRL_SRV_DBG:
                break;
        case CFCTRL_SRV_DATAGRAM:
                tmp32 = cpu_to_le32(param->u.datagram.connid);
                cfpkt_add_body(pkt, &tmp32, 4);
                break;
        case CFCTRL_SRV_RFM:
                /* Construct a frame, convert DatagramConnectionID to network
                 * format long and copy it out...
                 */
                tmp32 = cpu_to_le32(param->u.rfm.connid);
                cfpkt_add_body(pkt, &tmp32, 4);
                /* Add volume name, including zero termination... */
                cfpkt_add_body(pkt, param->u.rfm.volume,
                               strlen(param->u.rfm.volume) + 1);
                break;
        case CFCTRL_SRV_UTIL:
                tmp16 = cpu_to_le16(param->u.utility.fifosize_kb);
                cfpkt_add_body(pkt, &tmp16, 2);
                tmp16 = cpu_to_le16(param->u.utility.fifosize_bufs);
                cfpkt_add_body(pkt, &tmp16, 2);
                memset(utility_name, 0, sizeof(utility_name));
                strscpy(utility_name, param->u.utility.name,
                        UTILITY_NAME_LENGTH);
                cfpkt_add_body(pkt, utility_name, UTILITY_NAME_LENGTH);
                tmp8 = param->u.utility.paramlen;
                cfpkt_add_body(pkt, &tmp8, 1);
                cfpkt_add_body(pkt, param->u.utility.params,
                               param->u.utility.paramlen);
                break;
        default:
                pr_warn("Request setup of bad link type = %d\n",
                        param->linktype);
                cfpkt_destroy(pkt);
                return -EINVAL;
        }
        req = kzalloc(sizeof(*req), GFP_KERNEL);
        if (!req) {
                cfpkt_destroy(pkt);
                return -ENOMEM;
        }

        req->client_layer = user_layer;
        req->cmd = CFCTRL_CMD_LINK_SETUP;
        req->param = *param;
        cfctrl_insert_req(cfctrl, req);
        init_info(cfpkt_info(pkt), cfctrl);
        /*
         * NOTE:Always send linkup and linkdown request on the same
         *        device as the payload. Otherwise old queued up payload
         *        might arrive with the newly allocated channel ID.
         */
        cfpkt_info(pkt)->dev_info->id = param->phyid;
        cfpkt_set_prio(pkt, TC_PRIO_CONTROL);
        ret =
            dn->transmit(dn, pkt);
        if (ret < 0) {
                int count;

                count = cfctrl_cancel_req(&cfctrl->serv.layer,
                                                user_layer);
                if (count != 1) {
                        pr_err("Could not remove request (%d)", count);
                        return -ENODEV;
                }
        }
        return 0;
}

int cfctrl_linkdown_req(struct cflayer *layer, u8 channelid,
                        struct cflayer *client)
{
        int ret;
        struct cfpkt *pkt;
        struct cfctrl *cfctrl = container_obj(layer);
        struct cflayer *dn = cfctrl->serv.layer.dn;

        if (!dn) {
                pr_debug("not able to send link-down request\n");
                return -ENODEV;
        }
        pkt = cfpkt_create(CFPKT_CTRL_PKT_LEN);
        if (!pkt)
                return -ENOMEM;
        cfpkt_addbdy(pkt, CFCTRL_CMD_LINK_DESTROY);
        cfpkt_addbdy(pkt, channelid);
        init_info(cfpkt_info(pkt), cfctrl);
        cfpkt_set_prio(pkt, TC_PRIO_CONTROL);
        ret =
            dn->transmit(dn, pkt);
#ifndef CAIF_NO_LOOP
        cfctrl->loop_linkused[channelid] = 0;
#endif
        return ret;
}

int cfctrl_cancel_req(struct cflayer *layr, struct cflayer *adap_layer)
{
        struct cfctrl_request_info *p, *tmp;
        struct cfctrl *ctrl = container_obj(layr);
        int found = 0;
        spin_lock_bh(&ctrl->info_list_lock);

        list_for_each_entry_safe(p, tmp, &ctrl->list, list) {
                if (p->client_layer == adap_layer) {
                        list_del(&p->list);
                        kfree(p);
                        found++;
                }
        }

        spin_unlock_bh(&ctrl->info_list_lock);
        return found;
}

static int cfctrl_recv(struct cflayer *layer, struct cfpkt *pkt)
{
        u8 cmdrsp;
        u8 cmd;
        int ret = -1;
        u8 len;
        u8 param[255];
        u8 linkid = 0;
        struct cfctrl *cfctrl = container_obj(layer);
        struct cfctrl_request_info rsp, *req;


        cmdrsp = cfpkt_extr_head_u8(pkt);
        cmd = cmdrsp & CFCTRL_CMD_MASK;
        if (cmd != CFCTRL_CMD_LINK_ERR
            && CFCTRL_RSP_BIT != (CFCTRL_RSP_BIT & cmdrsp)
                && CFCTRL_ERR_BIT != (CFCTRL_ERR_BIT & cmdrsp)) {
                if (handle_loop(cfctrl, cmd, pkt) != 0)
                        cmdrsp |= CFCTRL_ERR_BIT;
        }

        switch (cmd) {
        case CFCTRL_CMD_LINK_SETUP:
                {
                        enum cfctrl_srv serv;
                        enum cfctrl_srv servtype;
                        u8 endpoint;
                        u8 physlinkid;
                        u8 prio;
                        u8 tmp;
                        u8 *cp;
                        int i;
                        struct cfctrl_link_param linkparam;
                        memset(&linkparam, 0, sizeof(linkparam));

                        tmp = cfpkt_extr_head_u8(pkt);

                        serv = tmp & CFCTRL_SRV_MASK;
                        linkparam.linktype = serv;

                        servtype = tmp >> 4;
                        linkparam.chtype = servtype;

                        tmp = cfpkt_extr_head_u8(pkt);
                        physlinkid = tmp & 0x07;
                        prio = tmp >> 3;

                        linkparam.priority = prio;
                        linkparam.phyid = physlinkid;
                        endpoint = cfpkt_extr_head_u8(pkt);
                        linkparam.endpoint = endpoint & 0x03;

                        switch (serv) {
                        case CFCTRL_SRV_VEI:
                        case CFCTRL_SRV_DBG:
                                if (CFCTRL_ERR_BIT & cmdrsp)
                                        break;
                                /* Link ID */
                                linkid = cfpkt_extr_head_u8(pkt);
                                break;
                        case CFCTRL_SRV_VIDEO:
                                tmp = cfpkt_extr_head_u8(pkt);
                                linkparam.u.video.connid = tmp;
                                if (CFCTRL_ERR_BIT & cmdrsp)
                                        break;
                                /* Link ID */
                                linkid = cfpkt_extr_head_u8(pkt);
                                break;

                        case CFCTRL_SRV_DATAGRAM:
                                linkparam.u.datagram.connid =
                                    cfpkt_extr_head_u32(pkt);
                                if (CFCTRL_ERR_BIT & cmdrsp)
                                        break;
                                /* Link ID */
                                linkid = cfpkt_extr_head_u8(pkt);
                                break;
                        case CFCTRL_SRV_RFM:
                                /* Construct a frame, convert
                                 * DatagramConnectionID
                                 * to network format long and copy it out...
                                 */
                                linkparam.u.rfm.connid =
                                    cfpkt_extr_head_u32(pkt);
                                cp = (u8 *) linkparam.u.rfm.volume;
                                for (tmp = cfpkt_extr_head_u8(pkt);
                                     cfpkt_more(pkt) && tmp != '\0';
                                     tmp = cfpkt_extr_head_u8(pkt))
                                        *cp++ = tmp;
                                *cp = '\0';

                                if (CFCTRL_ERR_BIT & cmdrsp)
                                        break;
                                /* Link ID */
                                linkid = cfpkt_extr_head_u8(pkt);

                                break;
                        case CFCTRL_SRV_UTIL:
                                /* Construct a frame, convert
                                 * DatagramConnectionID
                                 * to network format long and copy it out...
                                 */
                                /* Fifosize KB */
                                linkparam.u.utility.fifosize_kb =
                                    cfpkt_extr_head_u16(pkt);
                                /* Fifosize bufs */
                                linkparam.u.utility.fifosize_bufs =
                                    cfpkt_extr_head_u16(pkt);
                                /* name */
                                cp = (u8 *) linkparam.u.utility.name;
                                caif_assert(sizeof(linkparam.u.utility.name)
                                             >= UTILITY_NAME_LENGTH);
                                for (i = 0;
                                     i < UTILITY_NAME_LENGTH
                                     && cfpkt_more(pkt); i++) {
                                        tmp = cfpkt_extr_head_u8(pkt);
                                        *cp++ = tmp;
                                }
                                /* Length */
                                len = cfpkt_extr_head_u8(pkt);
                                linkparam.u.utility.paramlen = len;
                                /* Param Data */
                                cp = linkparam.u.utility.params;
                                while (cfpkt_more(pkt) && len--) {
                                        tmp = cfpkt_extr_head_u8(pkt);
                                        *cp++ = tmp;
                                }
                                if (CFCTRL_ERR_BIT & cmdrsp)
                                        break;
                                /* Link ID */
                                linkid = cfpkt_extr_head_u8(pkt);
                                /* Length */
                                len = cfpkt_extr_head_u8(pkt);
                                /* Param Data */
                                cfpkt_extr_head(pkt, &param, len);
                                break;
                        default:
                                pr_warn("Request setup, invalid type (%d)\n",
                                        serv);
                                goto error;
                        }

                        rsp.cmd = cmd;
                        rsp.param = linkparam;
                        spin_lock_bh(&cfctrl->info_list_lock);
                        req = cfctrl_remove_req(cfctrl, &rsp);

                        if (CFCTRL_ERR_BIT == (CFCTRL_ERR_BIT & cmdrsp) ||
                                cfpkt_erroneous(pkt)) {
                                pr_err("Invalid O/E bit or parse error "
                                                "on CAIF control channel\n");
                                cfctrl->res.reject_rsp(cfctrl->serv.layer.up,
                                                       0,
                                                       req ? req->client_layer
                                                       : NULL);
                        } else {
                                cfctrl->res.linksetup_rsp(cfctrl->serv.
                                                          layer.up, linkid,
                                                          serv, physlinkid,
                                                          req ? req->
                                                          client_layer : NULL);
                        }

                        kfree(req);

                        spin_unlock_bh(&cfctrl->info_list_lock);
                }
                break;
        case CFCTRL_CMD_LINK_DESTROY:
                linkid = cfpkt_extr_head_u8(pkt);
                cfctrl->res.linkdestroy_rsp(cfctrl->serv.layer.up, linkid);
                break;
        case CFCTRL_CMD_LINK_ERR:
                pr_err("Frame Error Indication received\n");
                cfctrl->res.linkerror_ind();
                break;
        case CFCTRL_CMD_ENUM:
                cfctrl->res.enum_rsp();
                break;
        case CFCTRL_CMD_SLEEP:
                cfctrl->res.sleep_rsp();
                break;
        case CFCTRL_CMD_WAKE:
                cfctrl->res.wake_rsp();
                break;
        case CFCTRL_CMD_LINK_RECONF:
                cfctrl->res.restart_rsp();
                break;
        case CFCTRL_CMD_RADIO_SET:
                cfctrl->res.radioset_rsp();
                break;
        default:
                pr_err("Unrecognized Control Frame\n");
                goto error;
        }
        ret = 0;
error:
        cfpkt_destroy(pkt);
        return ret;
}

static void cfctrl_ctrlcmd(struct cflayer *layr, enum caif_ctrlcmd ctrl,
                           int phyid)
{
        struct cfctrl *this = container_obj(layr);
        switch (ctrl) {
        case _CAIF_CTRLCMD_PHYIF_FLOW_OFF_IND:
        case CAIF_CTRLCMD_FLOW_OFF_IND:
                spin_lock_bh(&this->info_list_lock);
                if (!list_empty(&this->list))
                        pr_debug("Received flow off in control layer\n");
                spin_unlock_bh(&this->info_list_lock);
                break;
        case _CAIF_CTRLCMD_PHYIF_DOWN_IND: {
                struct cfctrl_request_info *p, *tmp;

                /* Find all connect request and report failure */
                spin_lock_bh(&this->info_list_lock);
                list_for_each_entry_safe(p, tmp, &this->list, list) {
                        if (p->param.phyid == phyid) {
                                list_del(&p->list);
                                p->client_layer->ctrlcmd(p->client_layer,
                                                CAIF_CTRLCMD_INIT_FAIL_RSP,
                                                phyid);
                                kfree(p);
                        }
                }
                spin_unlock_bh(&this->info_list_lock);
                break;
        }
        default:
                break;
        }
}

#ifndef CAIF_NO_LOOP
static int handle_loop(struct cfctrl *ctrl, int cmd, struct cfpkt *pkt)
{
        static int last_linkid;
        static int dec;
        u8 linkid, linktype, tmp;
        switch (cmd) {
        case CFCTRL_CMD_LINK_SETUP:
                spin_lock_bh(&ctrl->loop_linkid_lock);
                if (!dec) {
                        for (linkid = last_linkid + 1; linkid < 254; linkid++)
                                if (!ctrl->loop_linkused[linkid])
                                        goto found;
                }
                dec = 1;
                for (linkid = last_linkid - 1; linkid > 1; linkid--)
                        if (!ctrl->loop_linkused[linkid])
                                goto found;
                spin_unlock_bh(&ctrl->loop_linkid_lock);
                return -1;
found:
                if (linkid < 10)
                        dec = 0;

                if (!ctrl->loop_linkused[linkid])
                        ctrl->loop_linkused[linkid] = 1;

                last_linkid = linkid;

                cfpkt_add_trail(pkt, &linkid, 1);
                spin_unlock_bh(&ctrl->loop_linkid_lock);
                cfpkt_peek_head(pkt, &linktype, 1);
                if (linktype ==  CFCTRL_SRV_UTIL) {
                        tmp = 0x01;
                        cfpkt_add_trail(pkt, &tmp, 1);
                        cfpkt_add_trail(pkt, &tmp, 1);
                }
                break;

        case CFCTRL_CMD_LINK_DESTROY:
                spin_lock_bh(&ctrl->loop_linkid_lock);
                cfpkt_peek_head(pkt, &linkid, 1);
                ctrl->loop_linkused[linkid] = 0;
                spin_unlock_bh(&ctrl->loop_linkid_lock);
                break;
        default:
                break;
        }
        return 0;
}
#endif


















































































 2610 























  573 





















 2366 




























 1301 

























   79 
  528 











































































































































































































































































































































  516 
    2 



















   20 
    4 





















































    2 




 1821 
 1006 



























    6 



    6 



   10 






















































   74 

























































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_RCULIST_H
#define _LINUX_RCULIST_H

#ifdef __KERNEL__

/*
 * RCU-protected list version
 */
#include <linux/list.h>
#include <linux/rcupdate.h>

/*
 * INIT_LIST_HEAD_RCU - Initialize a list_head visible to RCU readers
 * @list: list to be initialized
 *
 * You should instead use INIT_LIST_HEAD() for normal initialization and
 * cleanup tasks, when readers have no access to the list being initialized.
 * However, if the list being initialized is visible to readers, you
 * need to keep the compiler from being too mischievous.
 */
static inline void INIT_LIST_HEAD_RCU(struct list_head *list)
{
        WRITE_ONCE(list->next, list);
        WRITE_ONCE(list->prev, list);
}

/*
 * return the ->next pointer of a list_head in an rcu safe
 * way, we must not access it directly
 */
#define list_next_rcu(list)        (*((struct list_head __rcu **)(&(list)->next)))

/**
 * list_tail_rcu - returns the prev pointer of the head of the list
 * @head: the head of the list
 *
 * Note: This should only be used with the list header, and even then
 * only if list_del() and similar primitives are not also used on the
 * list header.
 */
#define list_tail_rcu(head)        (*((struct list_head __rcu **)(&(head)->prev)))

/*
 * Check during list traversal that we are within an RCU reader
 */

#define check_arg_count_one(dummy)

#ifdef CONFIG_PROVE_RCU_LIST
#define __list_check_rcu(dummy, cond, extra...)                                \
        ({                                                                \
        check_arg_count_one(extra);                                        \
        RCU_LOCKDEP_WARN(!(cond) && !rcu_read_lock_any_held(),                \
                         "RCU-list traversed in non-reader section!");        \
        })

#define __list_check_srcu(cond)                                         \
        ({                                                                 \
        RCU_LOCKDEP_WARN(!(cond),                                         \
                "RCU-list traversed without holding the required lock!");\
        })
#else
#define __list_check_rcu(dummy, cond, extra...)                                \
        ({ check_arg_count_one(extra); })

#define __list_check_srcu(cond) ({ })
#endif

/*
 * Insert a new entry between two known consecutive entries.
 *
 * This is only for internal list manipulation where we know
 * the prev/next entries already!
 */
static inline void __list_add_rcu(struct list_head *new,
                struct list_head *prev, struct list_head *next)
{
        if (!__list_add_valid(new, prev, next))
                return;

        new->next = next;
        new->prev = prev;
        rcu_assign_pointer(list_next_rcu(prev), new);
        next->prev = new;
}

/**
 * list_add_rcu - add a new entry to rcu-protected list
 * @new: new entry to be added
 * @head: list head to add it after
 *
 * Insert a new entry after the specified head.
 * This is good for implementing stacks.
 *
 * The caller must take whatever precautions are necessary
 * (such as holding appropriate locks) to avoid racing
 * with another list-mutation primitive, such as list_add_rcu()
 * or list_del_rcu(), running on this same list.
 * However, it is perfectly legal to run concurrently with
 * the _rcu list-traversal primitives, such as
 * list_for_each_entry_rcu().
 */
static inline void list_add_rcu(struct list_head *new, struct list_head *head)
{
        __list_add_rcu(new, head, head->next);
}

/**
 * list_add_tail_rcu - add a new entry to rcu-protected list
 * @new: new entry to be added
 * @head: list head to add it before
 *
 * Insert a new entry before the specified head.
 * This is useful for implementing queues.
 *
 * The caller must take whatever precautions are necessary
 * (such as holding appropriate locks) to avoid racing
 * with another list-mutation primitive, such as list_add_tail_rcu()
 * or list_del_rcu(), running on this same list.
 * However, it is perfectly legal to run concurrently with
 * the _rcu list-traversal primitives, such as
 * list_for_each_entry_rcu().
 */
static inline void list_add_tail_rcu(struct list_head *new,
                                        struct list_head *head)
{
        __list_add_rcu(new, head->prev, head);
}

/**
 * list_del_rcu - deletes entry from list without re-initialization
 * @entry: the element to delete from the list.
 *
 * Note: list_empty() on entry does not return true after this,
 * the entry is in an undefined state. It is useful for RCU based
 * lockfree traversal.
 *
 * In particular, it means that we can not poison the forward
 * pointers that may still be used for walking the list.
 *
 * The caller must take whatever precautions are necessary
 * (such as holding appropriate locks) to avoid racing
 * with another list-mutation primitive, such as list_del_rcu()
 * or list_add_rcu(), running on this same list.
 * However, it is perfectly legal to run concurrently with
 * the _rcu list-traversal primitives, such as
 * list_for_each_entry_rcu().
 *
 * Note that the caller is not permitted to immediately free
 * the newly deleted entry.  Instead, either synchronize_rcu()
 * or call_rcu() must be used to defer freeing until an RCU
 * grace period has elapsed.
 */
static inline void list_del_rcu(struct list_head *entry)
{
        __list_del_entry(entry);
        entry->prev = LIST_POISON2;
}

/**
 * hlist_del_init_rcu - deletes entry from hash list with re-initialization
 * @n: the element to delete from the hash list.
 *
 * Note: list_unhashed() on the node return true after this. It is
 * useful for RCU based read lockfree traversal if the writer side
 * must know if the list entry is still hashed or already unhashed.
 *
 * In particular, it means that we can not poison the forward pointers
 * that may still be used for walking the hash list and we can only
 * zero the pprev pointer so list_unhashed() will return true after
 * this.
 *
 * The caller must take whatever precautions are necessary (such as
 * holding appropriate locks) to avoid racing with another
 * list-mutation primitive, such as hlist_add_head_rcu() or
 * hlist_del_rcu(), running on this same list.  However, it is
 * perfectly legal to run concurrently with the _rcu list-traversal
 * primitives, such as hlist_for_each_entry_rcu().
 */
static inline void hlist_del_init_rcu(struct hlist_node *n)
{
        if (!hlist_unhashed(n)) {
                __hlist_del(n);
                WRITE_ONCE(n->pprev, NULL);
        }
}

/**
 * list_replace_rcu - replace old entry by new one
 * @old : the element to be replaced
 * @new : the new element to insert
 *
 * The @old entry will be replaced with the @new entry atomically from
 * the perspective of concurrent readers.  It is the caller's responsibility
 * to synchronize with concurrent updaters, if any.
 *
 * Note: @old should not be empty.
 */
static inline void list_replace_rcu(struct list_head *old,
                                struct list_head *new)
{
        new->next = old->next;
        new->prev = old->prev;
        rcu_assign_pointer(list_next_rcu(new->prev), new);
        new->next->prev = new;
        old->prev = LIST_POISON2;
}

/**
 * __list_splice_init_rcu - join an RCU-protected list into an existing list.
 * @list:        the RCU-protected list to splice
 * @prev:        points to the last element of the existing list
 * @next:        points to the first element of the existing list
 * @sync:        synchronize_rcu, synchronize_rcu_expedited, ...
 *
 * The list pointed to by @prev and @next can be RCU-read traversed
 * concurrently with this function.
 *
 * Note that this function blocks.
 *
 * Important note: the caller must take whatever action is necessary to prevent
 * any other updates to the existing list.  In principle, it is possible to
 * modify the list as soon as sync() begins execution. If this sort of thing
 * becomes necessary, an alternative version based on call_rcu() could be
 * created.  But only if -really- needed -- there is no shortage of RCU API
 * members.
 */
static inline void __list_splice_init_rcu(struct list_head *list,
                                          struct list_head *prev,
                                          struct list_head *next,
                                          void (*sync)(void))
{
        struct list_head *first = list->next;
        struct list_head *last = list->prev;

        /*
         * "first" and "last" tracking list, so initialize it.  RCU readers
         * have access to this list, so we must use INIT_LIST_HEAD_RCU()
         * instead of INIT_LIST_HEAD().
         */

        INIT_LIST_HEAD_RCU(list);

        /*
         * At this point, the list body still points to the source list.
         * Wait for any readers to finish using the list before splicing
         * the list body into the new list.  Any new readers will see
         * an empty list.
         */

        sync();
        ASSERT_EXCLUSIVE_ACCESS(*first);
        ASSERT_EXCLUSIVE_ACCESS(*last);

        /*
         * Readers are finished with the source list, so perform splice.
         * The order is important if the new list is global and accessible
         * to concurrent RCU readers.  Note that RCU readers are not
         * permitted to traverse the prev pointers without excluding
         * this function.
         */

        last->next = next;
        rcu_assign_pointer(list_next_rcu(prev), first);
        first->prev = prev;
        next->prev = last;
}

/**
 * list_splice_init_rcu - splice an RCU-protected list into an existing list,
 *                        designed for stacks.
 * @list:        the RCU-protected list to splice
 * @head:        the place in the existing list to splice the first list into
 * @sync:        synchronize_rcu, synchronize_rcu_expedited, ...
 */
static inline void list_splice_init_rcu(struct list_head *list,
                                        struct list_head *head,
                                        void (*sync)(void))
{
        if (!list_empty(list))
                __list_splice_init_rcu(list, head, head->next, sync);
}

/**
 * list_splice_tail_init_rcu - splice an RCU-protected list into an existing
 *                             list, designed for queues.
 * @list:        the RCU-protected list to splice
 * @head:        the place in the existing list to splice the first list into
 * @sync:        synchronize_rcu, synchronize_rcu_expedited, ...
 */
static inline void list_splice_tail_init_rcu(struct list_head *list,
                                             struct list_head *head,
                                             void (*sync)(void))
{
        if (!list_empty(list))
                __list_splice_init_rcu(list, head->prev, head, sync);
}

/**
 * list_entry_rcu - get the struct for this entry
 * @ptr:        the &struct list_head pointer.
 * @type:       the type of the struct this is embedded in.
 * @member:     the name of the list_head within the struct.
 *
 * This primitive may safely run concurrently with the _rcu list-mutation
 * primitives such as list_add_rcu() as long as it's guarded by rcu_read_lock().
 */
#define list_entry_rcu(ptr, type, member) \
        container_of(READ_ONCE(ptr), type, member)

/*
 * Where are list_empty_rcu() and list_first_entry_rcu()?
 *
 * They do not exist because they would lead to subtle race conditions:
 *
 * if (!list_empty_rcu(mylist)) {
 *        struct foo *bar = list_first_entry_rcu(mylist, struct foo, list_member);
 *        do_something(bar);
 * }
 *
 * The list might be non-empty when list_empty_rcu() checks it, but it
 * might have become empty by the time that list_first_entry_rcu() rereads
 * the ->next pointer, which would result in a SEGV.
 *
 * When not using RCU, it is OK for list_first_entry() to re-read that
 * pointer because both functions should be protected by some lock that
 * blocks writers.
 *
 * When using RCU, list_empty() uses READ_ONCE() to fetch the
 * RCU-protected ->next pointer and then compares it to the address of the
 * list head.  However, it neither dereferences this pointer nor provides
 * this pointer to its caller.  Thus, READ_ONCE() suffices (that is,
 * rcu_dereference() is not needed), which means that list_empty() can be
 * used anywhere you would want to use list_empty_rcu().  Just don't
 * expect anything useful to happen if you do a subsequent lockless
 * call to list_first_entry_rcu()!!!
 *
 * See list_first_or_null_rcu for an alternative.
 */

/**
 * list_first_or_null_rcu - get the first element from a list
 * @ptr:        the list head to take the element from.
 * @type:       the type of the struct this is embedded in.
 * @member:     the name of the list_head within the struct.
 *
 * Note that if the list is empty, it returns NULL.
 *
 * This primitive may safely run concurrently with the _rcu list-mutation
 * primitives such as list_add_rcu() as long as it's guarded by rcu_read_lock().
 */
#define list_first_or_null_rcu(ptr, type, member) \
({ \
        struct list_head *__ptr = (ptr); \
        struct list_head *__next = READ_ONCE(__ptr->next); \
        likely(__ptr != __next) ? list_entry_rcu(__next, type, member) : NULL; \
})

/**
 * list_next_or_null_rcu - get the next element from a list
 * @head:        the head for the list.
 * @ptr:        the list head to take the next element from.
 * @type:       the type of the struct this is embedded in.
 * @member:     the name of the list_head within the struct.
 *
 * Note that if the ptr is at the end of the list, NULL is returned.
 *
 * This primitive may safely run concurrently with the _rcu list-mutation
 * primitives such as list_add_rcu() as long as it's guarded by rcu_read_lock().
 */
#define list_next_or_null_rcu(head, ptr, type, member) \
({ \
        struct list_head *__head = (head); \
        struct list_head *__ptr = (ptr); \
        struct list_head *__next = READ_ONCE(__ptr->next); \
        likely(__next != __head) ? list_entry_rcu(__next, type, \
                                                  member) : NULL; \
})

/**
 * list_for_each_entry_rcu        -        iterate over rcu list of given type
 * @pos:        the type * to use as a loop cursor.
 * @head:        the head for your list.
 * @member:        the name of the list_head within the struct.
 * @cond:        optional lockdep expression if called from non-RCU protection.
 *
 * This list-traversal primitive may safely run concurrently with
 * the _rcu list-mutation primitives such as list_add_rcu()
 * as long as the traversal is guarded by rcu_read_lock().
 */
#define list_for_each_entry_rcu(pos, head, member, cond...)                \
        for (__list_check_rcu(dummy, ## cond, 0),                        \
             pos = list_entry_rcu((head)->next, typeof(*pos), member);        \
                &pos->member != (head);                                        \
                pos = list_entry_rcu(pos->member.next, typeof(*pos), member))

/**
 * list_for_each_entry_srcu        -        iterate over rcu list of given type
 * @pos:        the type * to use as a loop cursor.
 * @head:        the head for your list.
 * @member:        the name of the list_head within the struct.
 * @cond:        lockdep expression for the lock required to traverse the list.
 *
 * This list-traversal primitive may safely run concurrently with
 * the _rcu list-mutation primitives such as list_add_rcu()
 * as long as the traversal is guarded by srcu_read_lock().
 * The lockdep expression srcu_read_lock_held() can be passed as the
 * cond argument from read side.
 */
#define list_for_each_entry_srcu(pos, head, member, cond)                \
        for (__list_check_srcu(cond),                                        \
             pos = list_entry_rcu((head)->next, typeof(*pos), member);        \
                &pos->member != (head);                                        \
                pos = list_entry_rcu(pos->member.next, typeof(*pos), member))

/**
 * list_entry_lockless - get the struct for this entry
 * @ptr:        the &struct list_head pointer.
 * @type:       the type of the struct this is embedded in.
 * @member:     the name of the list_head within the struct.
 *
 * This primitive may safely run concurrently with the _rcu
 * list-mutation primitives such as list_add_rcu(), but requires some
 * implicit RCU read-side guarding.  One example is running within a special
 * exception-time environment where preemption is disabled and where lockdep
 * cannot be invoked.  Another example is when items are added to the list,
 * but never deleted.
 */
#define list_entry_lockless(ptr, type, member) \
        container_of((typeof(ptr))READ_ONCE(ptr), type, member)

/**
 * list_for_each_entry_lockless - iterate over rcu list of given type
 * @pos:        the type * to use as a loop cursor.
 * @head:        the head for your list.
 * @member:        the name of the list_struct within the struct.
 *
 * This primitive may safely run concurrently with the _rcu
 * list-mutation primitives such as list_add_rcu(), but requires some
 * implicit RCU read-side guarding.  One example is running within a special
 * exception-time environment where preemption is disabled and where lockdep
 * cannot be invoked.  Another example is when items are added to the list,
 * but never deleted.
 */
#define list_for_each_entry_lockless(pos, head, member) \
        for (pos = list_entry_lockless((head)->next, typeof(*pos), member); \
             &pos->member != (head); \
             pos = list_entry_lockless(pos->member.next, typeof(*pos), member))

/**
 * list_for_each_entry_continue_rcu - continue iteration over list of given type
 * @pos:        the type * to use as a loop cursor.
 * @head:        the head for your list.
 * @member:        the name of the list_head within the struct.
 *
 * Continue to iterate over list of given type, continuing after
 * the current position which must have been in the list when the RCU read
 * lock was taken.
 * This would typically require either that you obtained the node from a
 * previous walk of the list in the same RCU read-side critical section, or
 * that you held some sort of non-RCU reference (such as a reference count)
 * to keep the node alive *and* in the list.
 *
 * This iterator is similar to list_for_each_entry_from_rcu() except
 * this starts after the given position and that one starts at the given
 * position.
 */
#define list_for_each_entry_continue_rcu(pos, head, member)                 \
        for (pos = list_entry_rcu(pos->member.next, typeof(*pos), member); \
             &pos->member != (head);        \
             pos = list_entry_rcu(pos->member.next, typeof(*pos), member))

/**
 * list_for_each_entry_from_rcu - iterate over a list from current point
 * @pos:        the type * to use as a loop cursor.
 * @head:        the head for your list.
 * @member:        the name of the list_node within the struct.
 *
 * Iterate over the tail of a list starting from a given position,
 * which must have been in the list when the RCU read lock was taken.
 * This would typically require either that you obtained the node from a
 * previous walk of the list in the same RCU read-side critical section, or
 * that you held some sort of non-RCU reference (such as a reference count)
 * to keep the node alive *and* in the list.
 *
 * This iterator is similar to list_for_each_entry_continue_rcu() except
 * this starts from the given position and that one starts from the position
 * after the given position.
 */
#define list_for_each_entry_from_rcu(pos, head, member)                        \
        for (; &(pos)->member != (head);                                        \
                pos = list_entry_rcu(pos->member.next, typeof(*(pos)), member))

/**
 * hlist_del_rcu - deletes entry from hash list without re-initialization
 * @n: the element to delete from the hash list.
 *
 * Note: list_unhashed() on entry does not return true after this,
 * the entry is in an undefined state. It is useful for RCU based
 * lockfree traversal.
 *
 * In particular, it means that we can not poison the forward
 * pointers that may still be used for walking the hash list.
 *
 * The caller must take whatever precautions are necessary
 * (such as holding appropriate locks) to avoid racing
 * with another list-mutation primitive, such as hlist_add_head_rcu()
 * or hlist_del_rcu(), running on this same list.
 * However, it is perfectly legal to run concurrently with
 * the _rcu list-traversal primitives, such as
 * hlist_for_each_entry().
 */
static inline void hlist_del_rcu(struct hlist_node *n)
{
        __hlist_del(n);
        WRITE_ONCE(n->pprev, LIST_POISON2);
}

/**
 * hlist_replace_rcu - replace old entry by new one
 * @old : the element to be replaced
 * @new : the new element to insert
 *
 * The @old entry will be replaced with the @new entry atomically from
 * the perspective of concurrent readers.  It is the caller's responsibility
 * to synchronize with concurrent updaters, if any.
 */
static inline void hlist_replace_rcu(struct hlist_node *old,
                                        struct hlist_node *new)
{
        struct hlist_node *next = old->next;

        new->next = next;
        WRITE_ONCE(new->pprev, old->pprev);
        rcu_assign_pointer(*(struct hlist_node __rcu **)new->pprev, new);
        if (next)
                WRITE_ONCE(new->next->pprev, &new->next);
        WRITE_ONCE(old->pprev, LIST_POISON2);
}

/**
 * hlists_swap_heads_rcu - swap the lists the hlist heads point to
 * @left:  The hlist head on the left
 * @right: The hlist head on the right
 *
 * The lists start out as [@left  ][node1 ... ] and
 *                        [@right ][node2 ... ]
 * The lists end up as    [@left  ][node2 ... ]
 *                        [@right ][node1 ... ]
 */
static inline void hlists_swap_heads_rcu(struct hlist_head *left, struct hlist_head *right)
{
        struct hlist_node *node1 = left->first;
        struct hlist_node *node2 = right->first;

        rcu_assign_pointer(left->first, node2);
        rcu_assign_pointer(right->first, node1);
        WRITE_ONCE(node2->pprev, &left->first);
        WRITE_ONCE(node1->pprev, &right->first);
}

/*
 * return the first or the next element in an RCU protected hlist
 */
#define hlist_first_rcu(head)        (*((struct hlist_node __rcu **)(&(head)->first)))
#define hlist_next_rcu(node)        (*((struct hlist_node __rcu **)(&(node)->next)))
#define hlist_pprev_rcu(node)        (*((struct hlist_node __rcu **)((node)->pprev)))

/**
 * hlist_add_head_rcu
 * @n: the element to add to the hash list.
 * @h: the list to add to.
 *
 * Description:
 * Adds the specified element to the specified hlist,
 * while permitting racing traversals.
 *
 * The caller must take whatever precautions are necessary
 * (such as holding appropriate locks) to avoid racing
 * with another list-mutation primitive, such as hlist_add_head_rcu()
 * or hlist_del_rcu(), running on this same list.
 * However, it is perfectly legal to run concurrently with
 * the _rcu list-traversal primitives, such as
 * hlist_for_each_entry_rcu(), used to prevent memory-consistency
 * problems on Alpha CPUs.  Regardless of the type of CPU, the
 * list-traversal primitive must be guarded by rcu_read_lock().
 */
static inline void hlist_add_head_rcu(struct hlist_node *n,
                                        struct hlist_head *h)
{
        struct hlist_node *first = h->first;

        n->next = first;
        WRITE_ONCE(n->pprev, &h->first);
        rcu_assign_pointer(hlist_first_rcu(h), n);
        if (first)
                WRITE_ONCE(first->pprev, &n->next);
}

/**
 * hlist_add_tail_rcu
 * @n: the element to add to the hash list.
 * @h: the list to add to.
 *
 * Description:
 * Adds the specified element to the specified hlist,
 * while permitting racing traversals.
 *
 * The caller must take whatever precautions are necessary
 * (such as holding appropriate locks) to avoid racing
 * with another list-mutation primitive, such as hlist_add_head_rcu()
 * or hlist_del_rcu(), running on this same list.
 * However, it is perfectly legal to run concurrently with
 * the _rcu list-traversal primitives, such as
 * hlist_for_each_entry_rcu(), used to prevent memory-consistency
 * problems on Alpha CPUs.  Regardless of the type of CPU, the
 * list-traversal primitive must be guarded by rcu_read_lock().
 */
static inline void hlist_add_tail_rcu(struct hlist_node *n,
                                      struct hlist_head *h)
{
        struct hlist_node *i, *last = NULL;

        /* Note: write side code, so rcu accessors are not needed. */
        for (i = h->first; i; i = i->next)
                last = i;

        if (last) {
                n->next = last->next;
                WRITE_ONCE(n->pprev, &last->next);
                rcu_assign_pointer(hlist_next_rcu(last), n);
        } else {
                hlist_add_head_rcu(n, h);
        }
}

/**
 * hlist_add_before_rcu
 * @n: the new element to add to the hash list.
 * @next: the existing element to add the new element before.
 *
 * Description:
 * Adds the specified element to the specified hlist
 * before the specified node while permitting racing traversals.
 *
 * The caller must take whatever precautions are necessary
 * (such as holding appropriate locks) to avoid racing
 * with another list-mutation primitive, such as hlist_add_head_rcu()
 * or hlist_del_rcu(), running on this same list.
 * However, it is perfectly legal to run concurrently with
 * the _rcu list-traversal primitives, such as
 * hlist_for_each_entry_rcu(), used to prevent memory-consistency
 * problems on Alpha CPUs.
 */
static inline void hlist_add_before_rcu(struct hlist_node *n,
                                        struct hlist_node *next)
{
        WRITE_ONCE(n->pprev, next->pprev);
        n->next = next;
        rcu_assign_pointer(hlist_pprev_rcu(n), n);
        WRITE_ONCE(next->pprev, &n->next);
}

/**
 * hlist_add_behind_rcu
 * @n: the new element to add to the hash list.
 * @prev: the existing element to add the new element after.
 *
 * Description:
 * Adds the specified element to the specified hlist
 * after the specified node while permitting racing traversals.
 *
 * The caller must take whatever precautions are necessary
 * (such as holding appropriate locks) to avoid racing
 * with another list-mutation primitive, such as hlist_add_head_rcu()
 * or hlist_del_rcu(), running on this same list.
 * However, it is perfectly legal to run concurrently with
 * the _rcu list-traversal primitives, such as
 * hlist_for_each_entry_rcu(), used to prevent memory-consistency
 * problems on Alpha CPUs.
 */
static inline void hlist_add_behind_rcu(struct hlist_node *n,
                                        struct hlist_node *prev)
{
        n->next = prev->next;
        WRITE_ONCE(n->pprev, &prev->next);
        rcu_assign_pointer(hlist_next_rcu(prev), n);
        if (n->next)
                WRITE_ONCE(n->next->pprev, &n->next);
}

#define __hlist_for_each_rcu(pos, head)                                \
        for (pos = rcu_dereference(hlist_first_rcu(head));        \
             pos;                                                \
             pos = rcu_dereference(hlist_next_rcu(pos)))

/**
 * hlist_for_each_entry_rcu - iterate over rcu list of given type
 * @pos:        the type * to use as a loop cursor.
 * @head:        the head for your list.
 * @member:        the name of the hlist_node within the struct.
 * @cond:        optional lockdep expression if called from non-RCU protection.
 *
 * This list-traversal primitive may safely run concurrently with
 * the _rcu list-mutation primitives such as hlist_add_head_rcu()
 * as long as the traversal is guarded by rcu_read_lock().
 */
#define hlist_for_each_entry_rcu(pos, head, member, cond...)                \
        for (__list_check_rcu(dummy, ## cond, 0),                        \
             pos = hlist_entry_safe(rcu_dereference_raw(hlist_first_rcu(head)),\
                        typeof(*(pos)), member);                        \
                pos;                                                        \
                pos = hlist_entry_safe(rcu_dereference_raw(hlist_next_rcu(\
                        &(pos)->member)), typeof(*(pos)), member))

/**
 * hlist_for_each_entry_srcu - iterate over rcu list of given type
 * @pos:        the type * to use as a loop cursor.
 * @head:        the head for your list.
 * @member:        the name of the hlist_node within the struct.
 * @cond:        lockdep expression for the lock required to traverse the list.
 *
 * This list-traversal primitive may safely run concurrently with
 * the _rcu list-mutation primitives such as hlist_add_head_rcu()
 * as long as the traversal is guarded by srcu_read_lock().
 * The lockdep expression srcu_read_lock_held() can be passed as the
 * cond argument from read side.
 */
#define hlist_for_each_entry_srcu(pos, head, member, cond)                \
        for (__list_check_srcu(cond),                                        \
             pos = hlist_entry_safe(rcu_dereference_raw(hlist_first_rcu(head)),\
                        typeof(*(pos)), member);                        \
                pos;                                                        \
                pos = hlist_entry_safe(rcu_dereference_raw(hlist_next_rcu(\
                        &(pos)->member)), typeof(*(pos)), member))

/**
 * hlist_for_each_entry_rcu_notrace - iterate over rcu list of given type (for tracing)
 * @pos:        the type * to use as a loop cursor.
 * @head:        the head for your list.
 * @member:        the name of the hlist_node within the struct.
 *
 * This list-traversal primitive may safely run concurrently with
 * the _rcu list-mutation primitives such as hlist_add_head_rcu()
 * as long as the traversal is guarded by rcu_read_lock().
 *
 * This is the same as hlist_for_each_entry_rcu() except that it does
 * not do any RCU debugging or tracing.
 */
#define hlist_for_each_entry_rcu_notrace(pos, head, member)                        \
        for (pos = hlist_entry_safe(rcu_dereference_raw_check(hlist_first_rcu(head)),\
                        typeof(*(pos)), member);                        \
                pos;                                                        \
                pos = hlist_entry_safe(rcu_dereference_raw_check(hlist_next_rcu(\
                        &(pos)->member)), typeof(*(pos)), member))

/**
 * hlist_for_each_entry_rcu_bh - iterate over rcu list of given type
 * @pos:        the type * to use as a loop cursor.
 * @head:        the head for your list.
 * @member:        the name of the hlist_node within the struct.
 *
 * This list-traversal primitive may safely run concurrently with
 * the _rcu list-mutation primitives such as hlist_add_head_rcu()
 * as long as the traversal is guarded by rcu_read_lock().
 */
#define hlist_for_each_entry_rcu_bh(pos, head, member)                        \
        for (pos = hlist_entry_safe(rcu_dereference_bh(hlist_first_rcu(head)),\
                        typeof(*(pos)), member);                        \
                pos;                                                        \
                pos = hlist_entry_safe(rcu_dereference_bh(hlist_next_rcu(\
                        &(pos)->member)), typeof(*(pos)), member))

/**
 * hlist_for_each_entry_continue_rcu - iterate over a hlist continuing after current point
 * @pos:        the type * to use as a loop cursor.
 * @member:        the name of the hlist_node within the struct.
 */
#define hlist_for_each_entry_continue_rcu(pos, member)                        \
        for (pos = hlist_entry_safe(rcu_dereference_raw(hlist_next_rcu( \
                        &(pos)->member)), typeof(*(pos)), member);        \
             pos;                                                        \
             pos = hlist_entry_safe(rcu_dereference_raw(hlist_next_rcu(        \
                        &(pos)->member)), typeof(*(pos)), member))

/**
 * hlist_for_each_entry_continue_rcu_bh - iterate over a hlist continuing after current point
 * @pos:        the type * to use as a loop cursor.
 * @member:        the name of the hlist_node within the struct.
 */
#define hlist_for_each_entry_continue_rcu_bh(pos, member)                \
        for (pos = hlist_entry_safe(rcu_dereference_bh(hlist_next_rcu(  \
                        &(pos)->member)), typeof(*(pos)), member);        \
             pos;                                                        \
             pos = hlist_entry_safe(rcu_dereference_bh(hlist_next_rcu(        \
                        &(pos)->member)), typeof(*(pos)), member))

/**
 * hlist_for_each_entry_from_rcu - iterate over a hlist continuing from current point
 * @pos:        the type * to use as a loop cursor.
 * @member:        the name of the hlist_node within the struct.
 */
#define hlist_for_each_entry_from_rcu(pos, member)                        \
        for (; pos;                                                        \
             pos = hlist_entry_safe(rcu_dereference_raw(hlist_next_rcu(        \
                        &(pos)->member)), typeof(*(pos)), member))

#endif        /* __KERNEL__ */
#endif
































































    3 




    3 


    3 
    2 









    2 















































































































    1 


    1 
















































    3 







    3 






    3 



    2 




























    1 


































    7 








   15 







    4 


    4 





    3 
    1 










    4 






    4 





    1 


    2 













    4 





    4 











    6 





    6 
    1 



    2 






    3 




















    3 








    3 



























    1 

    1 











    1 




    1 

    1 






    2 





















    2 





    7 



    7 



    1 














    5 

    2 










































    2 









    2 





















    3 
    3 
    3 
    3 


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
/*
 * Copyright (C) 2017 Netronome Systems, Inc.
 *
 * This software is licensed under the GNU General License Version 2,
 * June 1991 as shown in the file COPYING in the top-level directory of this
 * source tree.
 *
 * THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS"
 * WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING,
 * BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
 * FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE
 * OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME
 * THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
 */

#include <linux/bpf.h>
#include <linux/bpf_verifier.h>
#include <linux/debugfs.h>
#include <linux/kernel.h>
#include <linux/mutex.h>
#include <linux/rtnetlink.h>
#include <net/pkt_cls.h>

#include "netdevsim.h"

#define pr_vlog(env, fmt, ...)        \
        bpf_verifier_log_write(env, "[netdevsim] " fmt, ##__VA_ARGS__)

struct nsim_bpf_bound_prog {
        struct nsim_dev *nsim_dev;
        struct bpf_prog *prog;
        struct dentry *ddir;
        const char *state;
        bool is_loaded;
        struct list_head l;
};

#define NSIM_BPF_MAX_KEYS                2

struct nsim_bpf_bound_map {
        struct netdevsim *ns;
        struct bpf_offloaded_map *map;
        struct mutex mutex;
        struct nsim_map_entry {
                void *key;
                void *value;
        } entry[NSIM_BPF_MAX_KEYS];
        struct list_head l;
};

static int nsim_bpf_string_show(struct seq_file *file, void *data)
{
        const char **str = file->private;

        if (*str)
                seq_printf(file, "%s\n", *str);

        return 0;
}
DEFINE_SHOW_ATTRIBUTE(nsim_bpf_string);

static int
nsim_bpf_verify_insn(struct bpf_verifier_env *env, int insn_idx, int prev_insn)
{
        struct nsim_bpf_bound_prog *state;
        int ret = 0;

        state = env->prog->aux->offload->dev_priv;
        if (state->nsim_dev->bpf_bind_verifier_delay && !insn_idx)
                msleep(state->nsim_dev->bpf_bind_verifier_delay);

        if (insn_idx == env->prog->len - 1) {
                pr_vlog(env, "Hello from netdevsim!\n");

                if (!state->nsim_dev->bpf_bind_verifier_accept)
                        ret = -EOPNOTSUPP;
        }

        return ret;
}

static int nsim_bpf_finalize(struct bpf_verifier_env *env)
{
        return 0;
}

static bool nsim_xdp_offload_active(struct netdevsim *ns)
{
        return ns->xdp_hw.prog;
}

static void nsim_prog_set_loaded(struct bpf_prog *prog, bool loaded)
{
        struct nsim_bpf_bound_prog *state;

        if (!prog || !bpf_prog_is_offloaded(prog->aux))
                return;

        state = prog->aux->offload->dev_priv;
        state->is_loaded = loaded;
}

static int
nsim_bpf_offload(struct netdevsim *ns, struct bpf_prog *prog, bool oldprog)
{
        nsim_prog_set_loaded(ns->bpf_offloaded, false);

        WARN(!!ns->bpf_offloaded != oldprog,
             "bad offload state, expected offload %sto be active",
             oldprog ? "" : "not ");
        ns->bpf_offloaded = prog;
        ns->bpf_offloaded_id = prog ? prog->aux->id : 0;
        nsim_prog_set_loaded(prog, true);

        return 0;
}

int nsim_bpf_setup_tc_block_cb(enum tc_setup_type type,
                               void *type_data, void *cb_priv)
{
        struct tc_cls_bpf_offload *cls_bpf = type_data;
        struct bpf_prog *prog = cls_bpf->prog;
        struct netdevsim *ns = cb_priv;
        struct bpf_prog *oldprog;

        if (type != TC_SETUP_CLSBPF) {
                NSIM_EA(cls_bpf->common.extack,
                        "only offload of BPF classifiers supported");
                return -EOPNOTSUPP;
        }

        if (!tc_cls_can_offload_and_chain0(ns->netdev, &cls_bpf->common))
                return -EOPNOTSUPP;

        if (cls_bpf->common.protocol != htons(ETH_P_ALL)) {
                NSIM_EA(cls_bpf->common.extack,
                        "only ETH_P_ALL supported as filter protocol");
                return -EOPNOTSUPP;
        }

        if (!ns->bpf_tc_accept) {
                NSIM_EA(cls_bpf->common.extack,
                        "netdevsim configured to reject BPF TC offload");
                return -EOPNOTSUPP;
        }
        /* Note: progs without skip_sw will probably not be dev bound */
        if (prog && !prog->aux->offload && !ns->bpf_tc_non_bound_accept) {
                NSIM_EA(cls_bpf->common.extack,
                        "netdevsim configured to reject unbound programs");
                return -EOPNOTSUPP;
        }

        if (cls_bpf->command != TC_CLSBPF_OFFLOAD)
                return -EOPNOTSUPP;

        oldprog = cls_bpf->oldprog;

        /* Don't remove if oldprog doesn't match driver's state */
        if (ns->bpf_offloaded != oldprog) {
                oldprog = NULL;
                if (!cls_bpf->prog)
                        return 0;
                if (ns->bpf_offloaded) {
                        NSIM_EA(cls_bpf->common.extack,
                                "driver and netdev offload states mismatch");
                        return -EBUSY;
                }
        }

        return nsim_bpf_offload(ns, cls_bpf->prog, oldprog);
}

int nsim_bpf_disable_tc(struct netdevsim *ns)
{
        if (ns->bpf_offloaded && !nsim_xdp_offload_active(ns))
                return -EBUSY;
        return 0;
}

static int nsim_xdp_offload_prog(struct netdevsim *ns, struct netdev_bpf *bpf)
{
        if (!nsim_xdp_offload_active(ns) && !bpf->prog)
                return 0;
        if (!nsim_xdp_offload_active(ns) && bpf->prog && ns->bpf_offloaded) {
                NSIM_EA(bpf->extack, "TC program is already loaded");
                return -EBUSY;
        }

        return nsim_bpf_offload(ns, bpf->prog, nsim_xdp_offload_active(ns));
}

static int
nsim_xdp_set_prog(struct netdevsim *ns, struct netdev_bpf *bpf,
                  struct xdp_attachment_info *xdp)
{
        int err;

        if (bpf->command == XDP_SETUP_PROG && !ns->bpf_xdpdrv_accept) {
                NSIM_EA(bpf->extack, "driver XDP disabled in DebugFS");
                return -EOPNOTSUPP;
        }
        if (bpf->command == XDP_SETUP_PROG_HW && !ns->bpf_xdpoffload_accept) {
                NSIM_EA(bpf->extack, "XDP offload disabled in DebugFS");
                return -EOPNOTSUPP;
        }

        if (bpf->command == XDP_SETUP_PROG_HW) {
                err = nsim_xdp_offload_prog(ns, bpf);
                if (err)
                        return err;
        }

        xdp_attachment_setup(xdp, bpf);

        return 0;
}

static int nsim_bpf_create_prog(struct nsim_dev *nsim_dev,
                                struct bpf_prog *prog)
{
        struct nsim_bpf_bound_prog *state;
        char name[16];
        int ret;

        state = kzalloc(sizeof(*state), GFP_KERNEL);
        if (!state)
                return -ENOMEM;

        state->nsim_dev = nsim_dev;
        state->prog = prog;
        state->state = "verify";

        /* Program id is not populated yet when we create the state. */
        sprintf(name, "%u", nsim_dev->prog_id_gen++);
        state->ddir = debugfs_create_dir(name, nsim_dev->ddir_bpf_bound_progs);
        if (IS_ERR(state->ddir)) {
                ret = PTR_ERR(state->ddir);
                kfree(state);
                return ret;
        }

        debugfs_create_u32("id", 0400, state->ddir, &prog->aux->id);
        debugfs_create_file("state", 0400, state->ddir,
                            &state->state, &nsim_bpf_string_fops);
        debugfs_create_bool("loaded", 0400, state->ddir, &state->is_loaded);

        list_add_tail(&state->l, &nsim_dev->bpf_bound_progs);

        prog->aux->offload->dev_priv = state;

        return 0;
}

static int nsim_bpf_verifier_prep(struct bpf_prog *prog)
{
        struct nsim_dev *nsim_dev =
                        bpf_offload_dev_priv(prog->aux->offload->offdev);

        if (!nsim_dev->bpf_bind_accept)
                return -EOPNOTSUPP;

        return nsim_bpf_create_prog(nsim_dev, prog);
}

static int nsim_bpf_translate(struct bpf_prog *prog)
{
        struct nsim_bpf_bound_prog *state = prog->aux->offload->dev_priv;

        state->state = "xlated";
        return 0;
}

static void nsim_bpf_destroy_prog(struct bpf_prog *prog)
{
        struct nsim_bpf_bound_prog *state;

        state = prog->aux->offload->dev_priv;
        WARN(state->is_loaded,
             "offload state destroyed while program still bound");
        debugfs_remove_recursive(state->ddir);
        list_del(&state->l);
        kfree(state);
}

static const struct bpf_prog_offload_ops nsim_bpf_dev_ops = {
        .insn_hook        = nsim_bpf_verify_insn,
        .finalize        = nsim_bpf_finalize,
        .prepare        = nsim_bpf_verifier_prep,
        .translate        = nsim_bpf_translate,
        .destroy        = nsim_bpf_destroy_prog,
};

static int nsim_setup_prog_checks(struct netdevsim *ns, struct netdev_bpf *bpf)
{
        if (bpf->prog && bpf->prog->aux->offload) {
                NSIM_EA(bpf->extack, "attempt to load offloaded prog to drv");
                return -EINVAL;
        }
        if (ns->netdev->mtu > NSIM_XDP_MAX_MTU) {
                NSIM_EA(bpf->extack, "MTU too large w/ XDP enabled");
                return -EINVAL;
        }
        return 0;
}

static int
nsim_setup_prog_hw_checks(struct netdevsim *ns, struct netdev_bpf *bpf)
{
        struct nsim_bpf_bound_prog *state;

        if (!bpf->prog)
                return 0;

        if (!bpf_prog_is_offloaded(bpf->prog->aux)) {
                NSIM_EA(bpf->extack, "xdpoffload of non-bound program");
                return -EINVAL;
        }

        state = bpf->prog->aux->offload->dev_priv;
        if (WARN_ON(strcmp(state->state, "xlated"))) {
                NSIM_EA(bpf->extack, "offloading program in bad state");
                return -EINVAL;
        }
        return 0;
}

static bool
nsim_map_key_match(struct bpf_map *map, struct nsim_map_entry *e, void *key)
{
        return e->key && !memcmp(key, e->key, map->key_size);
}

static int nsim_map_key_find(struct bpf_offloaded_map *offmap, void *key)
{
        struct nsim_bpf_bound_map *nmap = offmap->dev_priv;
        unsigned int i;

        for (i = 0; i < ARRAY_SIZE(nmap->entry); i++)
                if (nsim_map_key_match(&offmap->map, &nmap->entry[i], key))
                        return i;

        return -ENOENT;
}

static int
nsim_map_alloc_elem(struct bpf_offloaded_map *offmap, unsigned int idx)
{
        struct nsim_bpf_bound_map *nmap = offmap->dev_priv;

        nmap->entry[idx].key = kmalloc(offmap->map.key_size,
                                       GFP_KERNEL_ACCOUNT | __GFP_NOWARN);
        if (!nmap->entry[idx].key)
                return -ENOMEM;
        nmap->entry[idx].value = kmalloc(offmap->map.value_size,
                                         GFP_KERNEL_ACCOUNT | __GFP_NOWARN);
        if (!nmap->entry[idx].value) {
                kfree(nmap->entry[idx].key);
                nmap->entry[idx].key = NULL;
                return -ENOMEM;
        }

        return 0;
}

static int
nsim_map_get_next_key(struct bpf_offloaded_map *offmap,
                      void *key, void *next_key)
{
        struct nsim_bpf_bound_map *nmap = offmap->dev_priv;
        int idx = -ENOENT;

        mutex_lock(&nmap->mutex);

        if (key)
                idx = nsim_map_key_find(offmap, key);
        if (idx == -ENOENT)
                idx = 0;
        else
                idx++;

        for (; idx < ARRAY_SIZE(nmap->entry); idx++) {
                if (nmap->entry[idx].key) {
                        memcpy(next_key, nmap->entry[idx].key,
                               offmap->map.key_size);
                        break;
                }
        }

        mutex_unlock(&nmap->mutex);

        if (idx == ARRAY_SIZE(nmap->entry))
                return -ENOENT;
        return 0;
}

static int
nsim_map_lookup_elem(struct bpf_offloaded_map *offmap, void *key, void *value)
{
        struct nsim_bpf_bound_map *nmap = offmap->dev_priv;
        int idx;

        mutex_lock(&nmap->mutex);

        idx = nsim_map_key_find(offmap, key);
        if (idx >= 0)
                memcpy(value, nmap->entry[idx].value, offmap->map.value_size);

        mutex_unlock(&nmap->mutex);

        return idx < 0 ? idx : 0;
}

static int
nsim_map_update_elem(struct bpf_offloaded_map *offmap,
                     void *key, void *value, u64 flags)
{
        struct nsim_bpf_bound_map *nmap = offmap->dev_priv;
        int idx, err = 0;

        mutex_lock(&nmap->mutex);

        idx = nsim_map_key_find(offmap, key);
        if (idx < 0 && flags == BPF_EXIST) {
                err = idx;
                goto exit_unlock;
        }
        if (idx >= 0 && flags == BPF_NOEXIST) {
                err = -EEXIST;
                goto exit_unlock;
        }

        if (idx < 0) {
                for (idx = 0; idx < ARRAY_SIZE(nmap->entry); idx++)
                        if (!nmap->entry[idx].key)
                                break;
                if (idx == ARRAY_SIZE(nmap->entry)) {
                        err = -E2BIG;
                        goto exit_unlock;
                }

                err = nsim_map_alloc_elem(offmap, idx);
                if (err)
                        goto exit_unlock;
        }

        memcpy(nmap->entry[idx].key, key, offmap->map.key_size);
        memcpy(nmap->entry[idx].value, value, offmap->map.value_size);
exit_unlock:
        mutex_unlock(&nmap->mutex);

        return err;
}

static int nsim_map_delete_elem(struct bpf_offloaded_map *offmap, void *key)
{
        struct nsim_bpf_bound_map *nmap = offmap->dev_priv;
        int idx;

        if (offmap->map.map_type == BPF_MAP_TYPE_ARRAY)
                return -EINVAL;

        mutex_lock(&nmap->mutex);

        idx = nsim_map_key_find(offmap, key);
        if (idx >= 0) {
                kfree(nmap->entry[idx].key);
                kfree(nmap->entry[idx].value);
                memset(&nmap->entry[idx], 0, sizeof(nmap->entry[idx]));
        }

        mutex_unlock(&nmap->mutex);

        return idx < 0 ? idx : 0;
}

static const struct bpf_map_dev_ops nsim_bpf_map_ops = {
        .map_get_next_key        = nsim_map_get_next_key,
        .map_lookup_elem        = nsim_map_lookup_elem,
        .map_update_elem        = nsim_map_update_elem,
        .map_delete_elem        = nsim_map_delete_elem,
};

static int
nsim_bpf_map_alloc(struct netdevsim *ns, struct bpf_offloaded_map *offmap)
{
        struct nsim_bpf_bound_map *nmap;
        int i, err;

        if (WARN_ON(offmap->map.map_type != BPF_MAP_TYPE_ARRAY &&
                    offmap->map.map_type != BPF_MAP_TYPE_HASH))
                return -EINVAL;
        if (offmap->map.max_entries > NSIM_BPF_MAX_KEYS)
                return -ENOMEM;
        if (offmap->map.map_flags)
                return -EINVAL;

        nmap = kzalloc(sizeof(*nmap), GFP_KERNEL_ACCOUNT);
        if (!nmap)
                return -ENOMEM;

        offmap->dev_priv = nmap;
        nmap->ns = ns;
        nmap->map = offmap;
        mutex_init(&nmap->mutex);

        if (offmap->map.map_type == BPF_MAP_TYPE_ARRAY) {
                for (i = 0; i < ARRAY_SIZE(nmap->entry); i++) {
                        u32 *key;

                        err = nsim_map_alloc_elem(offmap, i);
                        if (err)
                                goto err_free;
                        key = nmap->entry[i].key;
                        *key = i;
                        memset(nmap->entry[i].value, 0, offmap->map.value_size);
                }
        }

        offmap->dev_ops = &nsim_bpf_map_ops;
        list_add_tail(&nmap->l, &ns->nsim_dev->bpf_bound_maps);

        return 0;

err_free:
        while (--i >= 0) {
                kfree(nmap->entry[i].key);
                kfree(nmap->entry[i].value);
        }
        kfree(nmap);
        return err;
}

static void nsim_bpf_map_free(struct bpf_offloaded_map *offmap)
{
        struct nsim_bpf_bound_map *nmap = offmap->dev_priv;
        unsigned int i;

        for (i = 0; i < ARRAY_SIZE(nmap->entry); i++) {
                kfree(nmap->entry[i].key);
                kfree(nmap->entry[i].value);
        }
        list_del_init(&nmap->l);
        mutex_destroy(&nmap->mutex);
        kfree(nmap);
}

int nsim_bpf(struct net_device *dev, struct netdev_bpf *bpf)
{
        struct netdevsim *ns = netdev_priv(dev);
        int err;

        ASSERT_RTNL();

        switch (bpf->command) {
        case XDP_SETUP_PROG:
                err = nsim_setup_prog_checks(ns, bpf);
                if (err)
                        return err;

                return nsim_xdp_set_prog(ns, bpf, &ns->xdp);
        case XDP_SETUP_PROG_HW:
                err = nsim_setup_prog_hw_checks(ns, bpf);
                if (err)
                        return err;

                return nsim_xdp_set_prog(ns, bpf, &ns->xdp_hw);
        case BPF_OFFLOAD_MAP_ALLOC:
                if (!ns->bpf_map_accept)
                        return -EOPNOTSUPP;

                return nsim_bpf_map_alloc(ns, bpf->offmap);
        case BPF_OFFLOAD_MAP_FREE:
                nsim_bpf_map_free(bpf->offmap);
                return 0;
        default:
                return -EINVAL;
        }
}

int nsim_bpf_dev_init(struct nsim_dev *nsim_dev)
{
        int err;

        INIT_LIST_HEAD(&nsim_dev->bpf_bound_progs);
        INIT_LIST_HEAD(&nsim_dev->bpf_bound_maps);

        nsim_dev->ddir_bpf_bound_progs = debugfs_create_dir("bpf_bound_progs",
                                                            nsim_dev->ddir);
        if (IS_ERR(nsim_dev->ddir_bpf_bound_progs))
                return PTR_ERR(nsim_dev->ddir_bpf_bound_progs);

        nsim_dev->bpf_dev = bpf_offload_dev_create(&nsim_bpf_dev_ops, nsim_dev);
        err = PTR_ERR_OR_ZERO(nsim_dev->bpf_dev);
        if (err)
                return err;

        nsim_dev->bpf_bind_accept = true;
        debugfs_create_bool("bpf_bind_accept", 0600, nsim_dev->ddir,
                            &nsim_dev->bpf_bind_accept);
        debugfs_create_u32("bpf_bind_verifier_delay", 0600, nsim_dev->ddir,
                           &nsim_dev->bpf_bind_verifier_delay);
        nsim_dev->bpf_bind_verifier_accept = true;
        debugfs_create_bool("bpf_bind_verifier_accept", 0600, nsim_dev->ddir,
                            &nsim_dev->bpf_bind_verifier_accept);
        return 0;
}

void nsim_bpf_dev_exit(struct nsim_dev *nsim_dev)
{
        WARN_ON(!list_empty(&nsim_dev->bpf_bound_progs));
        WARN_ON(!list_empty(&nsim_dev->bpf_bound_maps));
        bpf_offload_dev_destroy(nsim_dev->bpf_dev);
}

int nsim_bpf_init(struct netdevsim *ns)
{
        struct dentry *ddir = ns->nsim_dev_port->ddir;
        int err;

        err = bpf_offload_dev_netdev_register(ns->nsim_dev->bpf_dev,
                                              ns->netdev);
        if (err)
                return err;

        debugfs_create_u32("bpf_offloaded_id", 0400, ddir,
                           &ns->bpf_offloaded_id);

        ns->bpf_tc_accept = true;
        debugfs_create_bool("bpf_tc_accept", 0600, ddir,
                            &ns->bpf_tc_accept);
        debugfs_create_bool("bpf_tc_non_bound_accept", 0600, ddir,
                            &ns->bpf_tc_non_bound_accept);
        ns->bpf_xdpdrv_accept = true;
        debugfs_create_bool("bpf_xdpdrv_accept", 0600, ddir,
                            &ns->bpf_xdpdrv_accept);
        ns->bpf_xdpoffload_accept = true;
        debugfs_create_bool("bpf_xdpoffload_accept", 0600, ddir,
                            &ns->bpf_xdpoffload_accept);

        ns->bpf_map_accept = true;
        debugfs_create_bool("bpf_map_accept", 0600, ddir,
                            &ns->bpf_map_accept);

        return 0;
}

void nsim_bpf_uninit(struct netdevsim *ns)
{
        WARN_ON(ns->xdp.prog);
        WARN_ON(ns->xdp_hw.prog);
        WARN_ON(ns->bpf_offloaded);
        bpf_offload_dev_netdev_unregister(ns->nsim_dev->bpf_dev, ns->netdev);
}
















    3 
    3 


    2 



    2 

    1 


    3 
















    3 


    2 




    1 












    3 







    1 
    2 


    1 
    3 
























    1 







    1 















































    1 

















    2 









    2 

    1 


    3 






    1 








    1 
    3 









    9 
    9 











    9 























    4 






    4 




    1 







    2 











    2 
    1 









    3 











    3 





1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
// SPDX-License-Identifier: GPL-2.0
#include <linux/cgroup.h>
#include <linux/sched.h>
#include <linux/sched/task.h>
#include <linux/sched/signal.h>

#include "cgroup-internal.h"

#include <trace/events/cgroup.h>

/*
 * Update CGRP_FROZEN of cgroup.flag
 * Return true if flags is updated; false if flags has no change
 */
static bool cgroup_update_frozen_flag(struct cgroup *cgrp, bool frozen)
{
        lockdep_assert_held(&css_set_lock);

        /* Already there? */
        if (test_bit(CGRP_FROZEN, &cgrp->flags) == frozen)
                return false;

        if (frozen)
                set_bit(CGRP_FROZEN, &cgrp->flags);
        else
                clear_bit(CGRP_FROZEN, &cgrp->flags);

        cgroup_file_notify(&cgrp->events_file);
        TRACE_CGROUP_PATH(notify_frozen, cgrp, frozen);
        return true;
}

/*
 * Propagate the cgroup frozen state upwards by the cgroup tree.
 */
static void cgroup_propagate_frozen(struct cgroup *cgrp, bool frozen)
{
        int desc = 1;

        /*
         * If the new state is frozen, some freezing ancestor cgroups may change
         * their state too, depending on if all their descendants are frozen.
         *
         * Otherwise, all ancestor cgroups are forced into the non-frozen state.
         */
        while ((cgrp = cgroup_parent(cgrp))) {
                if (frozen) {
                        cgrp->freezer.nr_frozen_descendants += desc;
                        if (!test_bit(CGRP_FREEZE, &cgrp->flags) ||
                            (cgrp->freezer.nr_frozen_descendants !=
                            cgrp->nr_descendants))
                                continue;
                } else {
                        cgrp->freezer.nr_frozen_descendants -= desc;
                }

                if (cgroup_update_frozen_flag(cgrp, frozen))
                        desc++;
        }
}

/*
 * Revisit the cgroup frozen state.
 * Checks if the cgroup is really frozen and perform all state transitions.
 */
void cgroup_update_frozen(struct cgroup *cgrp)
{
        bool frozen;

        /*
         * If the cgroup has to be frozen (CGRP_FREEZE bit set),
         * and all tasks are frozen and/or stopped, let's consider
         * the cgroup frozen. Otherwise it's not frozen.
         */
        frozen = test_bit(CGRP_FREEZE, &cgrp->flags) &&
                cgrp->freezer.nr_frozen_tasks == __cgroup_task_count(cgrp);

        /* If flags is updated, update the state of ancestor cgroups. */
        if (cgroup_update_frozen_flag(cgrp, frozen))
                cgroup_propagate_frozen(cgrp, frozen);
}

/*
 * Increment cgroup's nr_frozen_tasks.
 */
static void cgroup_inc_frozen_cnt(struct cgroup *cgrp)
{
        cgrp->freezer.nr_frozen_tasks++;
}

/*
 * Decrement cgroup's nr_frozen_tasks.
 */
static void cgroup_dec_frozen_cnt(struct cgroup *cgrp)
{
        cgrp->freezer.nr_frozen_tasks--;
        WARN_ON_ONCE(cgrp->freezer.nr_frozen_tasks < 0);
}

/*
 * Enter frozen/stopped state, if not yet there. Update cgroup's counters,
 * and revisit the state of the cgroup, if necessary.
 */
void cgroup_enter_frozen(void)
{
        struct cgroup *cgrp;

        if (current->frozen)
                return;

        spin_lock_irq(&css_set_lock);
        current->frozen = true;
        cgrp = task_dfl_cgroup(current);
        cgroup_inc_frozen_cnt(cgrp);
        cgroup_update_frozen(cgrp);
        spin_unlock_irq(&css_set_lock);
}

/*
 * Conditionally leave frozen/stopped state. Update cgroup's counters,
 * and revisit the state of the cgroup, if necessary.
 *
 * If always_leave is not set, and the cgroup is freezing,
 * we're racing with the cgroup freezing. In this case, we don't
 * drop the frozen counter to avoid a transient switch to
 * the unfrozen state.
 */
void cgroup_leave_frozen(bool always_leave)
{
        struct cgroup *cgrp;

        spin_lock_irq(&css_set_lock);
        cgrp = task_dfl_cgroup(current);
        if (always_leave || !test_bit(CGRP_FREEZE, &cgrp->flags)) {
                cgroup_dec_frozen_cnt(cgrp);
                cgroup_update_frozen(cgrp);
                WARN_ON_ONCE(!current->frozen);
                current->frozen = false;
        } else if (!(current->jobctl & JOBCTL_TRAP_FREEZE)) {
                spin_lock(&current->sighand->siglock);
                current->jobctl |= JOBCTL_TRAP_FREEZE;
                set_thread_flag(TIF_SIGPENDING);
                spin_unlock(&current->sighand->siglock);
        }
        spin_unlock_irq(&css_set_lock);
}

/*
 * Freeze or unfreeze the task by setting or clearing the JOBCTL_TRAP_FREEZE
 * jobctl bit.
 */
static void cgroup_freeze_task(struct task_struct *task, bool freeze)
{
        unsigned long flags;

        /* If the task is about to die, don't bother with freezing it. */
        if (!lock_task_sighand(task, &flags))
                return;

        if (freeze) {
                task->jobctl |= JOBCTL_TRAP_FREEZE;
                signal_wake_up(task, false);
        } else {
                task->jobctl &= ~JOBCTL_TRAP_FREEZE;
                wake_up_process(task);
        }

        unlock_task_sighand(task, &flags);
}

/*
 * Freeze or unfreeze all tasks in the given cgroup.
 */
static void cgroup_do_freeze(struct cgroup *cgrp, bool freeze)
{
        struct css_task_iter it;
        struct task_struct *task;

        lockdep_assert_held(&cgroup_mutex);

        spin_lock_irq(&css_set_lock);
        if (freeze)
                set_bit(CGRP_FREEZE, &cgrp->flags);
        else
                clear_bit(CGRP_FREEZE, &cgrp->flags);
        spin_unlock_irq(&css_set_lock);

        if (freeze)
                TRACE_CGROUP_PATH(freeze, cgrp);
        else
                TRACE_CGROUP_PATH(unfreeze, cgrp);

        css_task_iter_start(&cgrp->self, 0, &it);
        while ((task = css_task_iter_next(&it))) {
                /*
                 * Ignore kernel threads here. Freezing cgroups containing
                 * kthreads isn't supported.
                 */
                if (task->flags & PF_KTHREAD)
                        continue;
                cgroup_freeze_task(task, freeze);
        }
        css_task_iter_end(&it);

        /*
         * Cgroup state should be revisited here to cover empty leaf cgroups
         * and cgroups which descendants are already in the desired state.
         */
        spin_lock_irq(&css_set_lock);
        if (cgrp->nr_descendants == cgrp->freezer.nr_frozen_descendants)
                cgroup_update_frozen(cgrp);
        spin_unlock_irq(&css_set_lock);
}

/*
 * Adjust the task state (freeze or unfreeze) and revisit the state of
 * source and destination cgroups.
 */
void cgroup_freezer_migrate_task(struct task_struct *task,
                                 struct cgroup *src, struct cgroup *dst)
{
        lockdep_assert_held(&css_set_lock);

        /*
         * Kernel threads are not supposed to be frozen at all.
         */
        if (task->flags & PF_KTHREAD)
                return;

        /*
         * It's not necessary to do changes if both of the src and dst cgroups
         * are not freezing and task is not frozen.
         */
        if (!test_bit(CGRP_FREEZE, &src->flags) &&
            !test_bit(CGRP_FREEZE, &dst->flags) &&
            !task->frozen)
                return;

        /*
         * Adjust counters of freezing and frozen tasks.
         * Note, that if the task is frozen, but the destination cgroup is not
         * frozen, we bump both counters to keep them balanced.
         */
        if (task->frozen) {
                cgroup_inc_frozen_cnt(dst);
                cgroup_dec_frozen_cnt(src);
        }
        cgroup_update_frozen(dst);
        cgroup_update_frozen(src);

        /*
         * Force the task to the desired state.
         */
        cgroup_freeze_task(task, test_bit(CGRP_FREEZE, &dst->flags));
}

void cgroup_freeze(struct cgroup *cgrp, bool freeze)
{
        struct cgroup_subsys_state *css;
        struct cgroup *parent;
        struct cgroup *dsct;
        bool applied = false;
        bool old_e;

        lockdep_assert_held(&cgroup_mutex);

        /*
         * Nothing changed? Just exit.
         */
        if (cgrp->freezer.freeze == freeze)
                return;

        cgrp->freezer.freeze = freeze;

        /*
         * Propagate changes downwards the cgroup tree.
         */
        css_for_each_descendant_pre(css, &cgrp->self) {
                dsct = css->cgroup;

                if (cgroup_is_dead(dsct))
                        continue;

                /*
                 * e_freeze is affected by parent's e_freeze and dst's freeze.
                 * If old e_freeze eq new e_freeze, no change, its children
                 * will not be affected. So do nothing and skip the subtree
                 */
                old_e = dsct->freezer.e_freeze;
                parent = cgroup_parent(dsct);
                dsct->freezer.e_freeze = (dsct->freezer.freeze ||
                                          parent->freezer.e_freeze);
                if (dsct->freezer.e_freeze == old_e) {
                        css = css_rightmost_descendant(css);
                        continue;
                }

                /*
                 * Do change actual state: freeze or unfreeze.
                 */
                cgroup_do_freeze(dsct, freeze);
                applied = true;
        }

        /*
         * Even if the actual state hasn't changed, let's notify a user.
         * The state can be enforced by an ancestor cgroup: the cgroup
         * can already be in the desired state or it can be locked in the
         * opposite state, so that the transition will never happen.
         * In both cases it's better to notify a user, that there is
         * nothing to wait for.
         */
        if (!applied) {
                TRACE_CGROUP_PATH(notify_frozen, cgrp,
                                  test_bit(CGRP_FROZEN, &cgrp->flags));
                cgroup_file_notify(&cgrp->events_file);
        }
}



















  108 


   40 
    1 

  103 









    1 
   40 









  111 
  110 







  111 
    1 






































































































  262 





   20 
   46 






















  410 



  271 


    2 

    4 

    2 




    1 




    1 


   41 

    4 
    2 
   51 






  259 







  118 



   15 








    7 













    1 



    1 



    1 









   13 








  231 






  227 
    5 





    2 




    2 





























   40 




  104 






   73 

    1 

    3 


    9 


  137 
   37 




   54 















1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
/* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com
 * Copyright (c) 2016 Facebook
 */

#include <linux/bpf.h>

#include "disasm.h"

#define __BPF_FUNC_STR_FN(x) [BPF_FUNC_ ## x] = __stringify(bpf_ ## x)
static const char * const func_id_str[] = {
        __BPF_FUNC_MAPPER(__BPF_FUNC_STR_FN)
};
#undef __BPF_FUNC_STR_FN

static const char *__func_get_name(const struct bpf_insn_cbs *cbs,
                                   const struct bpf_insn *insn,
                                   char *buff, size_t len)
{
        BUILD_BUG_ON(ARRAY_SIZE(func_id_str) != __BPF_FUNC_MAX_ID);

        if (!insn->src_reg &&
            insn->imm >= 0 && insn->imm < __BPF_FUNC_MAX_ID &&
            func_id_str[insn->imm])
                return func_id_str[insn->imm];

        if (cbs && cbs->cb_call) {
                const char *res;

                res = cbs->cb_call(cbs->private_data, insn);
                if (res)
                        return res;
        }

        if (insn->src_reg == BPF_PSEUDO_CALL)
                snprintf(buff, len, "%+d", insn->imm);
        else if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL)
                snprintf(buff, len, "kernel-function");

        return buff;
}

static const char *__func_imm_name(const struct bpf_insn_cbs *cbs,
                                   const struct bpf_insn *insn,
                                   u64 full_imm, char *buff, size_t len)
{
        if (cbs && cbs->cb_imm)
                return cbs->cb_imm(cbs->private_data, insn, full_imm);

        snprintf(buff, len, "0x%llx", (unsigned long long)full_imm);
        return buff;
}

const char *func_id_name(int id)
{
        if (id >= 0 && id < __BPF_FUNC_MAX_ID && func_id_str[id])
                return func_id_str[id];
        else
                return "unknown";
}

const char *const bpf_class_string[8] = {
        [BPF_LD]    = "ld",
        [BPF_LDX]   = "ldx",
        [BPF_ST]    = "st",
        [BPF_STX]   = "stx",
        [BPF_ALU]   = "alu",
        [BPF_JMP]   = "jmp",
        [BPF_JMP32] = "jmp32",
        [BPF_ALU64] = "alu64",
};

const char *const bpf_alu_string[16] = {
        [BPF_ADD >> 4]  = "+=",
        [BPF_SUB >> 4]  = "-=",
        [BPF_MUL >> 4]  = "*=",
        [BPF_DIV >> 4]  = "/=",
        [BPF_OR  >> 4]  = "|=",
        [BPF_AND >> 4]  = "&=",
        [BPF_LSH >> 4]  = "<<=",
        [BPF_RSH >> 4]  = ">>=",
        [BPF_NEG >> 4]  = "neg",
        [BPF_MOD >> 4]  = "%=",
        [BPF_XOR >> 4]  = "^=",
        [BPF_MOV >> 4]  = "=",
        [BPF_ARSH >> 4] = "s>>=",
        [BPF_END >> 4]  = "endian",
};

static const char *const bpf_alu_sign_string[16] = {
        [BPF_DIV >> 4]  = "s/=",
        [BPF_MOD >> 4]  = "s%=",
};

static const char *const bpf_movsx_string[4] = {
        [0] = "(s8)",
        [1] = "(s16)",
        [3] = "(s32)",
};

static const char *const bpf_atomic_alu_string[16] = {
        [BPF_ADD >> 4]  = "add",
        [BPF_AND >> 4]  = "and",
        [BPF_OR >> 4]  = "or",
        [BPF_XOR >> 4]  = "xor",
};

static const char *const bpf_ldst_string[] = {
        [BPF_W >> 3]  = "u32",
        [BPF_H >> 3]  = "u16",
        [BPF_B >> 3]  = "u8",
        [BPF_DW >> 3] = "u64",
};

static const char *const bpf_ldsx_string[] = {
        [BPF_W >> 3]  = "s32",
        [BPF_H >> 3]  = "s16",
        [BPF_B >> 3]  = "s8",
};

static const char *const bpf_jmp_string[16] = {
        [BPF_JA >> 4]   = "jmp",
        [BPF_JEQ >> 4]  = "==",
        [BPF_JGT >> 4]  = ">",
        [BPF_JLT >> 4]  = "<",
        [BPF_JGE >> 4]  = ">=",
        [BPF_JLE >> 4]  = "<=",
        [BPF_JSET >> 4] = "&",
        [BPF_JNE >> 4]  = "!=",
        [BPF_JSGT >> 4] = "s>",
        [BPF_JSLT >> 4] = "s<",
        [BPF_JSGE >> 4] = "s>=",
        [BPF_JSLE >> 4] = "s<=",
        [BPF_CALL >> 4] = "call",
        [BPF_EXIT >> 4] = "exit",
};

static void print_bpf_end_insn(bpf_insn_print_t verbose,
                               void *private_data,
                               const struct bpf_insn *insn)
{
        verbose(private_data, "(%02x) r%d = %s%d r%d\n",
                insn->code, insn->dst_reg,
                BPF_SRC(insn->code) == BPF_TO_BE ? "be" : "le",
                insn->imm, insn->dst_reg);
}

static void print_bpf_bswap_insn(bpf_insn_print_t verbose,
                               void *private_data,
                               const struct bpf_insn *insn)
{
        verbose(private_data, "(%02x) r%d = bswap%d r%d\n",
                insn->code, insn->dst_reg,
                insn->imm, insn->dst_reg);
}

static bool is_sdiv_smod(const struct bpf_insn *insn)
{
        return (BPF_OP(insn->code)  == BPF_DIV || BPF_OP(insn->code) == BPF_MOD) &&
               insn->off == 1;
}

static bool is_movsx(const struct bpf_insn *insn)
{
        return BPF_OP(insn->code) == BPF_MOV &&
               (insn->off == 8 || insn->off == 16 || insn->off == 32);
}

static bool is_addr_space_cast(const struct bpf_insn *insn)
{
        return insn->code == (BPF_ALU64 | BPF_MOV | BPF_X) &&
                insn->off == BPF_ADDR_SPACE_CAST;
}

/* Special (internal-only) form of mov, used to resolve per-CPU addrs:
 * dst_reg = src_reg + <percpu_base_off>
 * BPF_ADDR_PERCPU is used as a special insn->off value.
 */
#define BPF_ADDR_PERCPU        (-1)

static inline bool is_mov_percpu_addr(const struct bpf_insn *insn)
{
        return insn->code == (BPF_ALU64 | BPF_MOV | BPF_X) && insn->off == BPF_ADDR_PERCPU;
}

void print_bpf_insn(const struct bpf_insn_cbs *cbs,
                    const struct bpf_insn *insn,
                    bool allow_ptr_leaks)
{
        const bpf_insn_print_t verbose = cbs->cb_print;
        u8 class = BPF_CLASS(insn->code);

        if (class == BPF_ALU || class == BPF_ALU64) {
                if (BPF_OP(insn->code) == BPF_END) {
                        if (class == BPF_ALU64)
                                print_bpf_bswap_insn(verbose, cbs->private_data, insn);
                        else
                                print_bpf_end_insn(verbose, cbs->private_data, insn);
                } else if (BPF_OP(insn->code) == BPF_NEG) {
                        verbose(cbs->private_data, "(%02x) %c%d = -%c%d\n",
                                insn->code, class == BPF_ALU ? 'w' : 'r',
                                insn->dst_reg, class == BPF_ALU ? 'w' : 'r',
                                insn->dst_reg);
                } else if (is_addr_space_cast(insn)) {
                        verbose(cbs->private_data, "(%02x) r%d = addr_space_cast(r%d, %d, %d)\n",
                                insn->code, insn->dst_reg,
                                insn->src_reg, ((u32)insn->imm) >> 16, (u16)insn->imm);
                } else if (is_mov_percpu_addr(insn)) {
                        verbose(cbs->private_data, "(%02x) r%d = &(void __percpu *)(r%d)\n",
                                insn->code, insn->dst_reg, insn->src_reg);
                } else if (BPF_SRC(insn->code) == BPF_X) {
                        verbose(cbs->private_data, "(%02x) %c%d %s %s%c%d\n",
                                insn->code, class == BPF_ALU ? 'w' : 'r',
                                insn->dst_reg,
                                is_sdiv_smod(insn) ? bpf_alu_sign_string[BPF_OP(insn->code) >> 4]
                                                   : bpf_alu_string[BPF_OP(insn->code) >> 4],
                                is_movsx(insn) ? bpf_movsx_string[(insn->off >> 3) - 1] : "",
                                class == BPF_ALU ? 'w' : 'r',
                                insn->src_reg);
                } else {
                        verbose(cbs->private_data, "(%02x) %c%d %s %d\n",
                                insn->code, class == BPF_ALU ? 'w' : 'r',
                                insn->dst_reg,
                                is_sdiv_smod(insn) ? bpf_alu_sign_string[BPF_OP(insn->code) >> 4]
                                                   : bpf_alu_string[BPF_OP(insn->code) >> 4],
                                insn->imm);
                }
        } else if (class == BPF_STX) {
                if (BPF_MODE(insn->code) == BPF_MEM)
                        verbose(cbs->private_data, "(%02x) *(%s *)(r%d %+d) = r%d\n",
                                insn->code,
                                bpf_ldst_string[BPF_SIZE(insn->code) >> 3],
                                insn->dst_reg,
                                insn->off, insn->src_reg);
                else if (BPF_MODE(insn->code) == BPF_ATOMIC &&
                         (insn->imm == BPF_ADD || insn->imm == BPF_AND ||
                          insn->imm == BPF_OR || insn->imm == BPF_XOR)) {
                        verbose(cbs->private_data, "(%02x) lock *(%s *)(r%d %+d) %s r%d\n",
                                insn->code,
                                bpf_ldst_string[BPF_SIZE(insn->code) >> 3],
                                insn->dst_reg, insn->off,
                                bpf_alu_string[BPF_OP(insn->imm) >> 4],
                                insn->src_reg);
                } else if (BPF_MODE(insn->code) == BPF_ATOMIC &&
                           (insn->imm == (BPF_ADD | BPF_FETCH) ||
                            insn->imm == (BPF_AND | BPF_FETCH) ||
                            insn->imm == (BPF_OR | BPF_FETCH) ||
                            insn->imm == (BPF_XOR | BPF_FETCH))) {
                        verbose(cbs->private_data, "(%02x) r%d = atomic%s_fetch_%s((%s *)(r%d %+d), r%d)\n",
                                insn->code, insn->src_reg,
                                BPF_SIZE(insn->code) == BPF_DW ? "64" : "",
                                bpf_atomic_alu_string[BPF_OP(insn->imm) >> 4],
                                bpf_ldst_string[BPF_SIZE(insn->code) >> 3],
                                insn->dst_reg, insn->off, insn->src_reg);
                } else if (BPF_MODE(insn->code) == BPF_ATOMIC &&
                           insn->imm == BPF_CMPXCHG) {
                        verbose(cbs->private_data, "(%02x) r0 = atomic%s_cmpxchg((%s *)(r%d %+d), r0, r%d)\n",
                                insn->code,
                                BPF_SIZE(insn->code) == BPF_DW ? "64" : "",
                                bpf_ldst_string[BPF_SIZE(insn->code) >> 3],
                                insn->dst_reg, insn->off,
                                insn->src_reg);
                } else if (BPF_MODE(insn->code) == BPF_ATOMIC &&
                           insn->imm == BPF_XCHG) {
                        verbose(cbs->private_data, "(%02x) r%d = atomic%s_xchg((%s *)(r%d %+d), r%d)\n",
                                insn->code, insn->src_reg,
                                BPF_SIZE(insn->code) == BPF_DW ? "64" : "",
                                bpf_ldst_string[BPF_SIZE(insn->code) >> 3],
                                insn->dst_reg, insn->off, insn->src_reg);
                } else {
                        verbose(cbs->private_data, "BUG_%02x\n", insn->code);
                }
        } else if (class == BPF_ST) {
                if (BPF_MODE(insn->code) == BPF_MEM) {
                        verbose(cbs->private_data, "(%02x) *(%s *)(r%d %+d) = %d\n",
                                insn->code,
                                bpf_ldst_string[BPF_SIZE(insn->code) >> 3],
                                insn->dst_reg,
                                insn->off, insn->imm);
                } else if (BPF_MODE(insn->code) == 0xc0 /* BPF_NOSPEC, no UAPI */) {
                        verbose(cbs->private_data, "(%02x) nospec\n", insn->code);
                } else {
                        verbose(cbs->private_data, "BUG_st_%02x\n", insn->code);
                }
        } else if (class == BPF_LDX) {
                if (BPF_MODE(insn->code) != BPF_MEM && BPF_MODE(insn->code) != BPF_MEMSX) {
                        verbose(cbs->private_data, "BUG_ldx_%02x\n", insn->code);
                        return;
                }
                verbose(cbs->private_data, "(%02x) r%d = *(%s *)(r%d %+d)\n",
                        insn->code, insn->dst_reg,
                        BPF_MODE(insn->code) == BPF_MEM ?
                                 bpf_ldst_string[BPF_SIZE(insn->code) >> 3] :
                                 bpf_ldsx_string[BPF_SIZE(insn->code) >> 3],
                        insn->src_reg, insn->off);
        } else if (class == BPF_LD) {
                if (BPF_MODE(insn->code) == BPF_ABS) {
                        verbose(cbs->private_data, "(%02x) r0 = *(%s *)skb[%d]\n",
                                insn->code,
                                bpf_ldst_string[BPF_SIZE(insn->code) >> 3],
                                insn->imm);
                } else if (BPF_MODE(insn->code) == BPF_IND) {
                        verbose(cbs->private_data, "(%02x) r0 = *(%s *)skb[r%d + %d]\n",
                                insn->code,
                                bpf_ldst_string[BPF_SIZE(insn->code) >> 3],
                                insn->src_reg, insn->imm);
                } else if (BPF_MODE(insn->code) == BPF_IMM &&
                           BPF_SIZE(insn->code) == BPF_DW) {
                        /* At this point, we already made sure that the second
                         * part of the ldimm64 insn is accessible.
                         */
                        u64 imm = ((u64)(insn + 1)->imm << 32) | (u32)insn->imm;
                        bool is_ptr = insn->src_reg == BPF_PSEUDO_MAP_FD ||
                                      insn->src_reg == BPF_PSEUDO_MAP_VALUE;
                        char tmp[64];

                        if (is_ptr && !allow_ptr_leaks)
                                imm = 0;

                        verbose(cbs->private_data, "(%02x) r%d = %s\n",
                                insn->code, insn->dst_reg,
                                __func_imm_name(cbs, insn, imm,
                                                tmp, sizeof(tmp)));
                } else {
                        verbose(cbs->private_data, "BUG_ld_%02x\n", insn->code);
                        return;
                }
        } else if (class == BPF_JMP32 || class == BPF_JMP) {
                u8 opcode = BPF_OP(insn->code);

                if (opcode == BPF_CALL) {
                        char tmp[64];

                        if (insn->src_reg == BPF_PSEUDO_CALL) {
                                verbose(cbs->private_data, "(%02x) call pc%s\n",
                                        insn->code,
                                        __func_get_name(cbs, insn,
                                                        tmp, sizeof(tmp)));
                        } else {
                                strcpy(tmp, "unknown");
                                verbose(cbs->private_data, "(%02x) call %s#%d\n", insn->code,
                                        __func_get_name(cbs, insn,
                                                        tmp, sizeof(tmp)),
                                        insn->imm);
                        }
                } else if (insn->code == (BPF_JMP | BPF_JA)) {
                        verbose(cbs->private_data, "(%02x) goto pc%+d\n",
                                insn->code, insn->off);
                } else if (insn->code == (BPF_JMP | BPF_JCOND) &&
                           insn->src_reg == BPF_MAY_GOTO) {
                        verbose(cbs->private_data, "(%02x) may_goto pc%+d\n",
                                insn->code, insn->off);
                } else if (insn->code == (BPF_JMP32 | BPF_JA)) {
                        verbose(cbs->private_data, "(%02x) gotol pc%+d\n",
                                insn->code, insn->imm);
                } else if (insn->code == (BPF_JMP | BPF_EXIT)) {
                        verbose(cbs->private_data, "(%02x) exit\n", insn->code);
                } else if (BPF_SRC(insn->code) == BPF_X) {
                        verbose(cbs->private_data,
                                "(%02x) if %c%d %s %c%d goto pc%+d\n",
                                insn->code, class == BPF_JMP32 ? 'w' : 'r',
                                insn->dst_reg,
                                bpf_jmp_string[BPF_OP(insn->code) >> 4],
                                class == BPF_JMP32 ? 'w' : 'r',
                                insn->src_reg, insn->off);
                } else {
                        verbose(cbs->private_data,
                                "(%02x) if %c%d %s 0x%x goto pc%+d\n",
                                insn->code, class == BPF_JMP32 ? 'w' : 'r',
                                insn->dst_reg,
                                bpf_jmp_string[BPF_OP(insn->code) >> 4],
                                insn->imm, insn->off);
                }
        } else {
                verbose(cbs->private_data, "(%02x) %s\n",
                        insn->code, bpf_class_string[class]);
        }
}



























































































































































































































































































































































































































































































































































































































































    1 



    1 









    1 






    1 




































































































    1 











    1 







    1 








    1 













    1 













    1 


































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
// SPDX-License-Identifier: GPL-2.0
/*
 * Functions related to sysfs handling
 */
#include <linux/kernel.h>
#include <linux/slab.h>
#include <linux/module.h>
#include <linux/bio.h>
#include <linux/blkdev.h>
#include <linux/backing-dev.h>
#include <linux/blktrace_api.h>
#include <linux/debugfs.h>

#include "blk.h"
#include "blk-mq.h"
#include "blk-mq-debugfs.h"
#include "blk-mq-sched.h"
#include "blk-rq-qos.h"
#include "blk-wbt.h"
#include "blk-cgroup.h"
#include "blk-throttle.h"

struct queue_sysfs_entry {
        struct attribute attr;
        ssize_t (*show)(struct gendisk *disk, char *page);
        ssize_t (*store)(struct gendisk *disk, const char *page, size_t count);
        void (*load_module)(struct gendisk *disk, const char *page, size_t count);
};

static ssize_t
queue_var_show(unsigned long var, char *page)
{
        return sysfs_emit(page, "%lu\n", var);
}

static ssize_t
queue_var_store(unsigned long *var, const char *page, size_t count)
{
        int err;
        unsigned long v;

        err = kstrtoul(page, 10, &v);
        if (err || v > UINT_MAX)
                return -EINVAL;

        *var = v;

        return count;
}

static ssize_t queue_requests_show(struct gendisk *disk, char *page)
{
        return queue_var_show(disk->queue->nr_requests, page);
}

static ssize_t
queue_requests_store(struct gendisk *disk, const char *page, size_t count)
{
        unsigned long nr;
        int ret, err;

        if (!queue_is_mq(disk->queue))
                return -EINVAL;

        ret = queue_var_store(&nr, page, count);
        if (ret < 0)
                return ret;

        if (nr < BLKDEV_MIN_RQ)
                nr = BLKDEV_MIN_RQ;

        err = blk_mq_update_nr_requests(disk->queue, nr);
        if (err)
                return err;

        return ret;
}

static ssize_t queue_ra_show(struct gendisk *disk, char *page)
{
        return queue_var_show(disk->bdi->ra_pages << (PAGE_SHIFT - 10), page);
}

static ssize_t
queue_ra_store(struct gendisk *disk, const char *page, size_t count)
{
        unsigned long ra_kb;
        ssize_t ret;

        ret = queue_var_store(&ra_kb, page, count);
        if (ret < 0)
                return ret;
        disk->bdi->ra_pages = ra_kb >> (PAGE_SHIFT - 10);
        return ret;
}

#define QUEUE_SYSFS_LIMIT_SHOW(_field)                                        \
static ssize_t queue_##_field##_show(struct gendisk *disk, char *page)        \
{                                                                        \
        return queue_var_show(disk->queue->limits._field, page);        \
}

QUEUE_SYSFS_LIMIT_SHOW(max_segments)
QUEUE_SYSFS_LIMIT_SHOW(max_discard_segments)
QUEUE_SYSFS_LIMIT_SHOW(max_integrity_segments)
QUEUE_SYSFS_LIMIT_SHOW(max_segment_size)
QUEUE_SYSFS_LIMIT_SHOW(logical_block_size)
QUEUE_SYSFS_LIMIT_SHOW(physical_block_size)
QUEUE_SYSFS_LIMIT_SHOW(chunk_sectors)
QUEUE_SYSFS_LIMIT_SHOW(io_min)
QUEUE_SYSFS_LIMIT_SHOW(io_opt)
QUEUE_SYSFS_LIMIT_SHOW(discard_granularity)
QUEUE_SYSFS_LIMIT_SHOW(zone_write_granularity)
QUEUE_SYSFS_LIMIT_SHOW(virt_boundary_mask)
QUEUE_SYSFS_LIMIT_SHOW(dma_alignment)
QUEUE_SYSFS_LIMIT_SHOW(max_open_zones)
QUEUE_SYSFS_LIMIT_SHOW(max_active_zones)
QUEUE_SYSFS_LIMIT_SHOW(atomic_write_unit_min)
QUEUE_SYSFS_LIMIT_SHOW(atomic_write_unit_max)

#define QUEUE_SYSFS_LIMIT_SHOW_SECTORS_TO_BYTES(_field)                        \
static ssize_t queue_##_field##_show(struct gendisk *disk, char *page)        \
{                                                                        \
        return sysfs_emit(page, "%llu\n",                                \
                (unsigned long long)disk->queue->limits._field <<        \
                        SECTOR_SHIFT);                                        \
}

QUEUE_SYSFS_LIMIT_SHOW_SECTORS_TO_BYTES(max_discard_sectors)
QUEUE_SYSFS_LIMIT_SHOW_SECTORS_TO_BYTES(max_hw_discard_sectors)
QUEUE_SYSFS_LIMIT_SHOW_SECTORS_TO_BYTES(max_write_zeroes_sectors)
QUEUE_SYSFS_LIMIT_SHOW_SECTORS_TO_BYTES(atomic_write_max_sectors)
QUEUE_SYSFS_LIMIT_SHOW_SECTORS_TO_BYTES(atomic_write_boundary_sectors)
QUEUE_SYSFS_LIMIT_SHOW_SECTORS_TO_BYTES(max_zone_append_sectors)

#define QUEUE_SYSFS_LIMIT_SHOW_SECTORS_TO_KB(_field)                        \
static ssize_t queue_##_field##_show(struct gendisk *disk, char *page)        \
{                                                                        \
        return queue_var_show(disk->queue->limits._field >> 1, page);        \
}

QUEUE_SYSFS_LIMIT_SHOW_SECTORS_TO_KB(max_sectors)
QUEUE_SYSFS_LIMIT_SHOW_SECTORS_TO_KB(max_hw_sectors)

#define QUEUE_SYSFS_SHOW_CONST(_name, _val)                                \
static ssize_t queue_##_name##_show(struct gendisk *disk, char *page)        \
{                                                                        \
        return sysfs_emit(page, "%d\n", _val);                                \
}

/* deprecated fields */
QUEUE_SYSFS_SHOW_CONST(discard_zeroes_data, 0)
QUEUE_SYSFS_SHOW_CONST(write_same_max, 0)
QUEUE_SYSFS_SHOW_CONST(poll_delay, -1)

static ssize_t queue_max_discard_sectors_store(struct gendisk *disk,
                const char *page, size_t count)
{
        unsigned long max_discard_bytes;
        struct queue_limits lim;
        ssize_t ret;
        int err;

        ret = queue_var_store(&max_discard_bytes, page, count);
        if (ret < 0)
                return ret;

        if (max_discard_bytes & (disk->queue->limits.discard_granularity - 1))
                return -EINVAL;

        if ((max_discard_bytes >> SECTOR_SHIFT) > UINT_MAX)
                return -EINVAL;

        lim = queue_limits_start_update(disk->queue);
        lim.max_user_discard_sectors = max_discard_bytes >> SECTOR_SHIFT;
        err = queue_limits_commit_update(disk->queue, &lim);
        if (err)
                return err;
        return ret;
}

static ssize_t
queue_max_sectors_store(struct gendisk *disk, const char *page, size_t count)
{
        unsigned long max_sectors_kb;
        struct queue_limits lim;
        ssize_t ret;
        int err;

        ret = queue_var_store(&max_sectors_kb, page, count);
        if (ret < 0)
                return ret;

        lim = queue_limits_start_update(disk->queue);
        lim.max_user_sectors = max_sectors_kb << 1;
        err = queue_limits_commit_update(disk->queue, &lim);
        if (err)
                return err;
        return ret;
}

static ssize_t queue_feature_store(struct gendisk *disk, const char *page,
                size_t count, blk_features_t feature)
{
        struct queue_limits lim;
        unsigned long val;
        ssize_t ret;

        ret = queue_var_store(&val, page, count);
        if (ret < 0)
                return ret;

        lim = queue_limits_start_update(disk->queue);
        if (val)
                lim.features |= feature;
        else
                lim.features &= ~feature;
        ret = queue_limits_commit_update(disk->queue, &lim);
        if (ret)
                return ret;
        return count;
}

#define QUEUE_SYSFS_FEATURE(_name, _feature)                                \
static ssize_t queue_##_name##_show(struct gendisk *disk, char *page)        \
{                                                                        \
        return sysfs_emit(page, "%u\n",                                        \
                !!(disk->queue->limits.features & _feature));                \
}                                                                        \
static ssize_t queue_##_name##_store(struct gendisk *disk,                \
                const char *page, size_t count)                                \
{                                                                        \
        return queue_feature_store(disk, page, count, _feature);        \
}

QUEUE_SYSFS_FEATURE(rotational, BLK_FEAT_ROTATIONAL)
QUEUE_SYSFS_FEATURE(add_random, BLK_FEAT_ADD_RANDOM)
QUEUE_SYSFS_FEATURE(iostats, BLK_FEAT_IO_STAT)
QUEUE_SYSFS_FEATURE(stable_writes, BLK_FEAT_STABLE_WRITES);

#define QUEUE_SYSFS_FEATURE_SHOW(_name, _feature)                        \
static ssize_t queue_##_name##_show(struct gendisk *disk, char *page)        \
{                                                                        \
        return sysfs_emit(page, "%u\n",                                        \
                !!(disk->queue->limits.features & _feature));                \
}

QUEUE_SYSFS_FEATURE_SHOW(poll, BLK_FEAT_POLL);
QUEUE_SYSFS_FEATURE_SHOW(fua, BLK_FEAT_FUA);
QUEUE_SYSFS_FEATURE_SHOW(dax, BLK_FEAT_DAX);

static ssize_t queue_zoned_show(struct gendisk *disk, char *page)
{
        if (blk_queue_is_zoned(disk->queue))
                return sysfs_emit(page, "host-managed\n");
        return sysfs_emit(page, "none\n");
}

static ssize_t queue_nr_zones_show(struct gendisk *disk, char *page)
{
        return queue_var_show(disk_nr_zones(disk), page);
}

static ssize_t queue_iostats_passthrough_show(struct gendisk *disk, char *page)
{
        return queue_var_show(blk_queue_passthrough_stat(disk->queue), page);
}

static ssize_t queue_iostats_passthrough_store(struct gendisk *disk,
                                               const char *page, size_t count)
{
        struct queue_limits lim;
        unsigned long ios;
        ssize_t ret;

        ret = queue_var_store(&ios, page, count);
        if (ret < 0)
                return ret;

        lim = queue_limits_start_update(disk->queue);
        if (ios)
                lim.flags |= BLK_FLAG_IOSTATS_PASSTHROUGH;
        else
                lim.flags &= ~BLK_FLAG_IOSTATS_PASSTHROUGH;

        ret = queue_limits_commit_update(disk->queue, &lim);
        if (ret)
                return ret;

        return count;
}
static ssize_t queue_nomerges_show(struct gendisk *disk, char *page)
{
        return queue_var_show((blk_queue_nomerges(disk->queue) << 1) |
                               blk_queue_noxmerges(disk->queue), page);
}

static ssize_t queue_nomerges_store(struct gendisk *disk, const char *page,
                                    size_t count)
{
        unsigned long nm;
        ssize_t ret = queue_var_store(&nm, page, count);

        if (ret < 0)
                return ret;

        blk_queue_flag_clear(QUEUE_FLAG_NOMERGES, disk->queue);
        blk_queue_flag_clear(QUEUE_FLAG_NOXMERGES, disk->queue);
        if (nm == 2)
                blk_queue_flag_set(QUEUE_FLAG_NOMERGES, disk->queue);
        else if (nm)
                blk_queue_flag_set(QUEUE_FLAG_NOXMERGES, disk->queue);

        return ret;
}

static ssize_t queue_rq_affinity_show(struct gendisk *disk, char *page)
{
        bool set = test_bit(QUEUE_FLAG_SAME_COMP, &disk->queue->queue_flags);
        bool force = test_bit(QUEUE_FLAG_SAME_FORCE, &disk->queue->queue_flags);

        return queue_var_show(set << force, page);
}

static ssize_t
queue_rq_affinity_store(struct gendisk *disk, const char *page, size_t count)
{
        ssize_t ret = -EINVAL;
#ifdef CONFIG_SMP
        struct request_queue *q = disk->queue;
        unsigned long val;

        ret = queue_var_store(&val, page, count);
        if (ret < 0)
                return ret;

        if (val == 2) {
                blk_queue_flag_set(QUEUE_FLAG_SAME_COMP, q);
                blk_queue_flag_set(QUEUE_FLAG_SAME_FORCE, q);
        } else if (val == 1) {
                blk_queue_flag_set(QUEUE_FLAG_SAME_COMP, q);
                blk_queue_flag_clear(QUEUE_FLAG_SAME_FORCE, q);
        } else if (val == 0) {
                blk_queue_flag_clear(QUEUE_FLAG_SAME_COMP, q);
                blk_queue_flag_clear(QUEUE_FLAG_SAME_FORCE, q);
        }
#endif
        return ret;
}

static ssize_t queue_poll_delay_store(struct gendisk *disk, const char *page,
                                size_t count)
{
        return count;
}

static ssize_t queue_poll_store(struct gendisk *disk, const char *page,
                                size_t count)
{
        if (!(disk->queue->limits.features & BLK_FEAT_POLL))
                return -EINVAL;
        pr_info_ratelimited("writes to the poll attribute are ignored.\n");
        pr_info_ratelimited("please use driver specific parameters instead.\n");
        return count;
}

static ssize_t queue_io_timeout_show(struct gendisk *disk, char *page)
{
        return sysfs_emit(page, "%u\n", jiffies_to_msecs(disk->queue->rq_timeout));
}

static ssize_t queue_io_timeout_store(struct gendisk *disk, const char *page,
                                  size_t count)
{
        unsigned int val;
        int err;

        err = kstrtou32(page, 10, &val);
        if (err || val == 0)
                return -EINVAL;

        blk_queue_rq_timeout(disk->queue, msecs_to_jiffies(val));

        return count;
}

static ssize_t queue_wc_show(struct gendisk *disk, char *page)
{
        if (blk_queue_write_cache(disk->queue))
                return sysfs_emit(page, "write back\n");
        return sysfs_emit(page, "write through\n");
}

static ssize_t queue_wc_store(struct gendisk *disk, const char *page,
                              size_t count)
{
        struct queue_limits lim;
        bool disable;
        int err;

        if (!strncmp(page, "write back", 10)) {
                disable = false;
        } else if (!strncmp(page, "write through", 13) ||
                   !strncmp(page, "none", 4)) {
                disable = true;
        } else {
                return -EINVAL;
        }

        lim = queue_limits_start_update(disk->queue);
        if (disable)
                lim.flags |= BLK_FLAG_WRITE_CACHE_DISABLED;
        else
                lim.flags &= ~BLK_FLAG_WRITE_CACHE_DISABLED;
        err = queue_limits_commit_update(disk->queue, &lim);
        if (err)
                return err;
        return count;
}

#define QUEUE_RO_ENTRY(_prefix, _name)                        \
static struct queue_sysfs_entry _prefix##_entry = {        \
        .attr        = { .name = _name, .mode = 0444 },        \
        .show        = _prefix##_show,                        \
};

#define QUEUE_RW_ENTRY(_prefix, _name)                        \
static struct queue_sysfs_entry _prefix##_entry = {        \
        .attr        = { .name = _name, .mode = 0644 },        \
        .show        = _prefix##_show,                        \
        .store        = _prefix##_store,                        \
};

#define QUEUE_RW_LOAD_MODULE_ENTRY(_prefix, _name)                \
static struct queue_sysfs_entry _prefix##_entry = {                \
        .attr                = { .name = _name, .mode = 0644 },        \
        .show                = _prefix##_show,                        \
        .load_module        = _prefix##_load_module,                \
        .store                = _prefix##_store,                        \
}

QUEUE_RW_ENTRY(queue_requests, "nr_requests");
QUEUE_RW_ENTRY(queue_ra, "read_ahead_kb");
QUEUE_RW_ENTRY(queue_max_sectors, "max_sectors_kb");
QUEUE_RO_ENTRY(queue_max_hw_sectors, "max_hw_sectors_kb");
QUEUE_RO_ENTRY(queue_max_segments, "max_segments");
QUEUE_RO_ENTRY(queue_max_integrity_segments, "max_integrity_segments");
QUEUE_RO_ENTRY(queue_max_segment_size, "max_segment_size");
QUEUE_RW_LOAD_MODULE_ENTRY(elv_iosched, "scheduler");

QUEUE_RO_ENTRY(queue_logical_block_size, "logical_block_size");
QUEUE_RO_ENTRY(queue_physical_block_size, "physical_block_size");
QUEUE_RO_ENTRY(queue_chunk_sectors, "chunk_sectors");
QUEUE_RO_ENTRY(queue_io_min, "minimum_io_size");
QUEUE_RO_ENTRY(queue_io_opt, "optimal_io_size");

QUEUE_RO_ENTRY(queue_max_discard_segments, "max_discard_segments");
QUEUE_RO_ENTRY(queue_discard_granularity, "discard_granularity");
QUEUE_RO_ENTRY(queue_max_hw_discard_sectors, "discard_max_hw_bytes");
QUEUE_RW_ENTRY(queue_max_discard_sectors, "discard_max_bytes");
QUEUE_RO_ENTRY(queue_discard_zeroes_data, "discard_zeroes_data");

QUEUE_RO_ENTRY(queue_atomic_write_max_sectors, "atomic_write_max_bytes");
QUEUE_RO_ENTRY(queue_atomic_write_boundary_sectors,
                "atomic_write_boundary_bytes");
QUEUE_RO_ENTRY(queue_atomic_write_unit_max, "atomic_write_unit_max_bytes");
QUEUE_RO_ENTRY(queue_atomic_write_unit_min, "atomic_write_unit_min_bytes");

QUEUE_RO_ENTRY(queue_write_same_max, "write_same_max_bytes");
QUEUE_RO_ENTRY(queue_max_write_zeroes_sectors, "write_zeroes_max_bytes");
QUEUE_RO_ENTRY(queue_max_zone_append_sectors, "zone_append_max_bytes");
QUEUE_RO_ENTRY(queue_zone_write_granularity, "zone_write_granularity");

QUEUE_RO_ENTRY(queue_zoned, "zoned");
QUEUE_RO_ENTRY(queue_nr_zones, "nr_zones");
QUEUE_RO_ENTRY(queue_max_open_zones, "max_open_zones");
QUEUE_RO_ENTRY(queue_max_active_zones, "max_active_zones");

QUEUE_RW_ENTRY(queue_nomerges, "nomerges");
QUEUE_RW_ENTRY(queue_iostats_passthrough, "iostats_passthrough");
QUEUE_RW_ENTRY(queue_rq_affinity, "rq_affinity");
QUEUE_RW_ENTRY(queue_poll, "io_poll");
QUEUE_RW_ENTRY(queue_poll_delay, "io_poll_delay");
QUEUE_RW_ENTRY(queue_wc, "write_cache");
QUEUE_RO_ENTRY(queue_fua, "fua");
QUEUE_RO_ENTRY(queue_dax, "dax");
QUEUE_RW_ENTRY(queue_io_timeout, "io_timeout");
QUEUE_RO_ENTRY(queue_virt_boundary_mask, "virt_boundary_mask");
QUEUE_RO_ENTRY(queue_dma_alignment, "dma_alignment");

/* legacy alias for logical_block_size: */
static struct queue_sysfs_entry queue_hw_sector_size_entry = {
        .attr = {.name = "hw_sector_size", .mode = 0444 },
        .show = queue_logical_block_size_show,
};

QUEUE_RW_ENTRY(queue_rotational, "rotational");
QUEUE_RW_ENTRY(queue_iostats, "iostats");
QUEUE_RW_ENTRY(queue_add_random, "add_random");
QUEUE_RW_ENTRY(queue_stable_writes, "stable_writes");

#ifdef CONFIG_BLK_WBT
static ssize_t queue_var_store64(s64 *var, const char *page)
{
        int err;
        s64 v;

        err = kstrtos64(page, 10, &v);
        if (err < 0)
                return err;

        *var = v;
        return 0;
}

static ssize_t queue_wb_lat_show(struct gendisk *disk, char *page)
{
        if (!wbt_rq_qos(disk->queue))
                return -EINVAL;

        if (wbt_disabled(disk->queue))
                return sysfs_emit(page, "0\n");

        return sysfs_emit(page, "%llu\n",
                div_u64(wbt_get_min_lat(disk->queue), 1000));
}

static ssize_t queue_wb_lat_store(struct gendisk *disk, const char *page,
                                  size_t count)
{
        struct request_queue *q = disk->queue;
        struct rq_qos *rqos;
        ssize_t ret;
        s64 val;

        ret = queue_var_store64(&val, page);
        if (ret < 0)
                return ret;
        if (val < -1)
                return -EINVAL;

        rqos = wbt_rq_qos(q);
        if (!rqos) {
                ret = wbt_init(disk);
                if (ret)
                        return ret;
        }

        if (val == -1)
                val = wbt_default_latency_nsec(q);
        else if (val >= 0)
                val *= 1000ULL;

        if (wbt_get_min_lat(q) == val)
                return count;

        /*
         * Ensure that the queue is idled, in case the latency update
         * ends up either enabling or disabling wbt completely. We can't
         * have IO inflight if that happens.
         */
        blk_mq_quiesce_queue(q);

        wbt_set_min_lat(q, val);

        blk_mq_unquiesce_queue(q);

        return count;
}

QUEUE_RW_ENTRY(queue_wb_lat, "wbt_lat_usec");
#endif

/* Common attributes for bio-based and request-based queues. */
static struct attribute *queue_attrs[] = {
        &queue_ra_entry.attr,
        &queue_max_hw_sectors_entry.attr,
        &queue_max_sectors_entry.attr,
        &queue_max_segments_entry.attr,
        &queue_max_discard_segments_entry.attr,
        &queue_max_integrity_segments_entry.attr,
        &queue_max_segment_size_entry.attr,
        &queue_hw_sector_size_entry.attr,
        &queue_logical_block_size_entry.attr,
        &queue_physical_block_size_entry.attr,
        &queue_chunk_sectors_entry.attr,
        &queue_io_min_entry.attr,
        &queue_io_opt_entry.attr,
        &queue_discard_granularity_entry.attr,
        &queue_max_discard_sectors_entry.attr,
        &queue_max_hw_discard_sectors_entry.attr,
        &queue_discard_zeroes_data_entry.attr,
        &queue_atomic_write_max_sectors_entry.attr,
        &queue_atomic_write_boundary_sectors_entry.attr,
        &queue_atomic_write_unit_min_entry.attr,
        &queue_atomic_write_unit_max_entry.attr,
        &queue_write_same_max_entry.attr,
        &queue_max_write_zeroes_sectors_entry.attr,
        &queue_max_zone_append_sectors_entry.attr,
        &queue_zone_write_granularity_entry.attr,
        &queue_rotational_entry.attr,
        &queue_zoned_entry.attr,
        &queue_nr_zones_entry.attr,
        &queue_max_open_zones_entry.attr,
        &queue_max_active_zones_entry.attr,
        &queue_nomerges_entry.attr,
        &queue_iostats_passthrough_entry.attr,
        &queue_iostats_entry.attr,
        &queue_stable_writes_entry.attr,
        &queue_add_random_entry.attr,
        &queue_poll_entry.attr,
        &queue_wc_entry.attr,
        &queue_fua_entry.attr,
        &queue_dax_entry.attr,
        &queue_poll_delay_entry.attr,
        &queue_virt_boundary_mask_entry.attr,
        &queue_dma_alignment_entry.attr,
        NULL,
};

/* Request-based queue attributes that are not relevant for bio-based queues. */
static struct attribute *blk_mq_queue_attrs[] = {
        &queue_requests_entry.attr,
        &elv_iosched_entry.attr,
        &queue_rq_affinity_entry.attr,
        &queue_io_timeout_entry.attr,
#ifdef CONFIG_BLK_WBT
        &queue_wb_lat_entry.attr,
#endif
        NULL,
};

static umode_t queue_attr_visible(struct kobject *kobj, struct attribute *attr,
                                int n)
{
        struct gendisk *disk = container_of(kobj, struct gendisk, queue_kobj);
        struct request_queue *q = disk->queue;

        if ((attr == &queue_max_open_zones_entry.attr ||
             attr == &queue_max_active_zones_entry.attr) &&
            !blk_queue_is_zoned(q))
                return 0;

        return attr->mode;
}

static umode_t blk_mq_queue_attr_visible(struct kobject *kobj,
                                         struct attribute *attr, int n)
{
        struct gendisk *disk = container_of(kobj, struct gendisk, queue_kobj);
        struct request_queue *q = disk->queue;

        if (!queue_is_mq(q))
                return 0;

        if (attr == &queue_io_timeout_entry.attr && !q->mq_ops->timeout)
                return 0;

        return attr->mode;
}

static struct attribute_group queue_attr_group = {
        .attrs = queue_attrs,
        .is_visible = queue_attr_visible,
};

static struct attribute_group blk_mq_queue_attr_group = {
        .attrs = blk_mq_queue_attrs,
        .is_visible = blk_mq_queue_attr_visible,
};

#define to_queue(atr) container_of((atr), struct queue_sysfs_entry, attr)

static ssize_t
queue_attr_show(struct kobject *kobj, struct attribute *attr, char *page)
{
        struct queue_sysfs_entry *entry = to_queue(attr);
        struct gendisk *disk = container_of(kobj, struct gendisk, queue_kobj);
        ssize_t res;

        if (!entry->show)
                return -EIO;
        mutex_lock(&disk->queue->sysfs_lock);
        res = entry->show(disk, page);
        mutex_unlock(&disk->queue->sysfs_lock);
        return res;
}

static ssize_t
queue_attr_store(struct kobject *kobj, struct attribute *attr,
                    const char *page, size_t length)
{
        struct queue_sysfs_entry *entry = to_queue(attr);
        struct gendisk *disk = container_of(kobj, struct gendisk, queue_kobj);
        struct request_queue *q = disk->queue;
        ssize_t res;

        if (!entry->store)
                return -EIO;

        /*
         * If the attribute needs to load a module, do it before freezing the
         * queue to ensure that the module file can be read when the request
         * queue is the one for the device storing the module file.
         */
        if (entry->load_module)
                entry->load_module(disk, page, length);

        blk_mq_freeze_queue(q);
        mutex_lock(&q->sysfs_lock);
        res = entry->store(disk, page, length);
        mutex_unlock(&q->sysfs_lock);
        blk_mq_unfreeze_queue(q);
        return res;
}

static const struct sysfs_ops queue_sysfs_ops = {
        .show        = queue_attr_show,
        .store        = queue_attr_store,
};

static const struct attribute_group *blk_queue_attr_groups[] = {
        &queue_attr_group,
        &blk_mq_queue_attr_group,
        NULL
};

static void blk_queue_release(struct kobject *kobj)
{
        /* nothing to do here, all data is associated with the parent gendisk */
}

static const struct kobj_type blk_queue_ktype = {
        .default_groups = blk_queue_attr_groups,
        .sysfs_ops        = &queue_sysfs_ops,
        .release        = blk_queue_release,
};

static void blk_debugfs_remove(struct gendisk *disk)
{
        struct request_queue *q = disk->queue;

        mutex_lock(&q->debugfs_mutex);
        blk_trace_shutdown(q);
        debugfs_remove_recursive(q->debugfs_dir);
        q->debugfs_dir = NULL;
        q->sched_debugfs_dir = NULL;
        q->rqos_debugfs_dir = NULL;
        mutex_unlock(&q->debugfs_mutex);
}

/**
 * blk_register_queue - register a block layer queue with sysfs
 * @disk: Disk of which the request queue should be registered with sysfs.
 */
int blk_register_queue(struct gendisk *disk)
{
        struct request_queue *q = disk->queue;
        int ret;

        mutex_lock(&q->sysfs_dir_lock);
        kobject_init(&disk->queue_kobj, &blk_queue_ktype);
        ret = kobject_add(&disk->queue_kobj, &disk_to_dev(disk)->kobj, "queue");
        if (ret < 0)
                goto out_put_queue_kobj;

        if (queue_is_mq(q)) {
                ret = blk_mq_sysfs_register(disk);
                if (ret)
                        goto out_put_queue_kobj;
        }
        mutex_lock(&q->sysfs_lock);

        mutex_lock(&q->debugfs_mutex);
        q->debugfs_dir = debugfs_create_dir(disk->disk_name, blk_debugfs_root);
        if (queue_is_mq(q))
                blk_mq_debugfs_register(q);
        mutex_unlock(&q->debugfs_mutex);

        ret = disk_register_independent_access_ranges(disk);
        if (ret)
                goto out_debugfs_remove;

        if (q->elevator) {
                ret = elv_register_queue(q, false);
                if (ret)
                        goto out_unregister_ia_ranges;
        }

        ret = blk_crypto_sysfs_register(disk);
        if (ret)
                goto out_elv_unregister;

        blk_queue_flag_set(QUEUE_FLAG_REGISTERED, q);
        wbt_enable_default(disk);

        /* Now everything is ready and send out KOBJ_ADD uevent */
        kobject_uevent(&disk->queue_kobj, KOBJ_ADD);
        if (q->elevator)
                kobject_uevent(&q->elevator->kobj, KOBJ_ADD);
        mutex_unlock(&q->sysfs_lock);
        mutex_unlock(&q->sysfs_dir_lock);

        /*
         * SCSI probing may synchronously create and destroy a lot of
         * request_queues for non-existent devices.  Shutting down a fully
         * functional queue takes measureable wallclock time as RCU grace
         * periods are involved.  To avoid excessive latency in these
         * cases, a request_queue starts out in a degraded mode which is
         * faster to shut down and is made fully functional here as
         * request_queues for non-existent devices never get registered.
         */
        if (!blk_queue_init_done(q)) {
                blk_queue_flag_set(QUEUE_FLAG_INIT_DONE, q);
                percpu_ref_switch_to_percpu(&q->q_usage_counter);
        }

        return ret;

out_elv_unregister:
        elv_unregister_queue(q);
out_unregister_ia_ranges:
        disk_unregister_independent_access_ranges(disk);
out_debugfs_remove:
        blk_debugfs_remove(disk);
        mutex_unlock(&q->sysfs_lock);
out_put_queue_kobj:
        kobject_put(&disk->queue_kobj);
        mutex_unlock(&q->sysfs_dir_lock);
        return ret;
}

/**
 * blk_unregister_queue - counterpart of blk_register_queue()
 * @disk: Disk of which the request queue should be unregistered from sysfs.
 *
 * Note: the caller is responsible for guaranteeing that this function is called
 * after blk_register_queue() has finished.
 */
void blk_unregister_queue(struct gendisk *disk)
{
        struct request_queue *q = disk->queue;

        if (WARN_ON(!q))
                return;

        /* Return early if disk->queue was never registered. */
        if (!blk_queue_registered(q))
                return;

        /*
         * Since sysfs_remove_dir() prevents adding new directory entries
         * before removal of existing entries starts, protect against
         * concurrent elv_iosched_store() calls.
         */
        mutex_lock(&q->sysfs_lock);
        blk_queue_flag_clear(QUEUE_FLAG_REGISTERED, q);
        mutex_unlock(&q->sysfs_lock);

        mutex_lock(&q->sysfs_dir_lock);
        /*
         * Remove the sysfs attributes before unregistering the queue data
         * structures that can be modified through sysfs.
         */
        if (queue_is_mq(q))
                blk_mq_sysfs_unregister(disk);
        blk_crypto_sysfs_unregister(disk);

        mutex_lock(&q->sysfs_lock);
        elv_unregister_queue(q);
        disk_unregister_independent_access_ranges(disk);
        mutex_unlock(&q->sysfs_lock);

        /* Now that we've deleted all child objects, we can delete the queue. */
        kobject_uevent(&disk->queue_kobj, KOBJ_REMOVE);
        kobject_del(&disk->queue_kobj);
        mutex_unlock(&q->sysfs_dir_lock);

        blk_debugfs_remove(disk);
}






















    2 








    1 


    1 







    1 


















    5 


















1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
// SPDX-License-Identifier: GPL-2.0-only
/* (C) 1999-2001 Paul `Rusty' Russell
 * (C) 2002-2004 Netfilter Core Team <coreteam@netfilter.org>
 */

#include <linux/types.h>
#include <linux/jiffies.h>
#include <linux/timer.h>
#include <linux/netfilter.h>
#include <net/netfilter/nf_conntrack_l4proto.h>
#include <net/netfilter/nf_conntrack_timeout.h>

static const unsigned int nf_ct_generic_timeout = 600*HZ;

#ifdef CONFIG_NF_CONNTRACK_TIMEOUT

#include <linux/netfilter/nfnetlink.h>
#include <linux/netfilter/nfnetlink_cttimeout.h>

static int generic_timeout_nlattr_to_obj(struct nlattr *tb[],
                                         struct net *net, void *data)
{
        struct nf_generic_net *gn = nf_generic_pernet(net);
        unsigned int *timeout = data;

        if (!timeout)
                timeout = &gn->timeout;

        if (tb[CTA_TIMEOUT_GENERIC_TIMEOUT])
                *timeout =
                    ntohl(nla_get_be32(tb[CTA_TIMEOUT_GENERIC_TIMEOUT])) * HZ;
        else {
                /* Set default generic timeout. */
                *timeout = gn->timeout;
        }

        return 0;
}

static int
generic_timeout_obj_to_nlattr(struct sk_buff *skb, const void *data)
{
        const unsigned int *timeout = data;

        if (nla_put_be32(skb, CTA_TIMEOUT_GENERIC_TIMEOUT, htonl(*timeout / HZ)))
                goto nla_put_failure;

        return 0;

nla_put_failure:
        return -ENOSPC;
}

static const struct nla_policy
generic_timeout_nla_policy[CTA_TIMEOUT_GENERIC_MAX+1] = {
        [CTA_TIMEOUT_GENERIC_TIMEOUT]        = { .type = NLA_U32 },
};
#endif /* CONFIG_NF_CONNTRACK_TIMEOUT */

void nf_conntrack_generic_init_net(struct net *net)
{
        struct nf_generic_net *gn = nf_generic_pernet(net);

        gn->timeout = nf_ct_generic_timeout;
}

const struct nf_conntrack_l4proto nf_conntrack_l4proto_generic =
{
        .l4proto                = 255,
#ifdef CONFIG_NF_CONNTRACK_TIMEOUT
        .ctnl_timeout                = {
                .nlattr_to_obj        = generic_timeout_nlattr_to_obj,
                .obj_to_nlattr        = generic_timeout_obj_to_nlattr,
                .nlattr_max        = CTA_TIMEOUT_GENERIC_MAX,
                .obj_size        = sizeof(unsigned int),
                .nla_policy        = generic_timeout_nla_policy,
        },
#endif /* CONFIG_NF_CONNTRACK_TIMEOUT */
};























































































































































































































































































































































































  130 





















































































  104 



   48 
   49 





   31 















   18 























































  131 







  124 













   11 



























































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
/* SPDX-License-Identifier: GPL-2.0-or-later */
/* include/asm-generic/tlb.h
 *
 *        Generic TLB shootdown code
 *
 * Copyright 2001 Red Hat, Inc.
 * Based on code from mm/memory.c Copyright Linus Torvalds and others.
 *
 * Copyright 2011 Red Hat, Inc., Peter Zijlstra
 */
#ifndef _ASM_GENERIC__TLB_H
#define _ASM_GENERIC__TLB_H

#include <linux/mmu_notifier.h>
#include <linux/swap.h>
#include <linux/hugetlb_inline.h>
#include <asm/tlbflush.h>
#include <asm/cacheflush.h>

/*
 * Blindly accessing user memory from NMI context can be dangerous
 * if we're in the middle of switching the current user task or switching
 * the loaded mm.
 */
#ifndef nmi_uaccess_okay
# define nmi_uaccess_okay() true
#endif

#ifdef CONFIG_MMU

/*
 * Generic MMU-gather implementation.
 *
 * The mmu_gather data structure is used by the mm code to implement the
 * correct and efficient ordering of freeing pages and TLB invalidations.
 *
 * This correct ordering is:
 *
 *  1) unhook page
 *  2) TLB invalidate page
 *  3) free page
 *
 * That is, we must never free a page before we have ensured there are no live
 * translations left to it. Otherwise it might be possible to observe (or
 * worse, change) the page content after it has been reused.
 *
 * The mmu_gather API consists of:
 *
 *  - tlb_gather_mmu() / tlb_gather_mmu_fullmm() / tlb_finish_mmu()
 *
 *    start and finish a mmu_gather
 *
 *    Finish in particular will issue a (final) TLB invalidate and free
 *    all (remaining) queued pages.
 *
 *  - tlb_start_vma() / tlb_end_vma(); marks the start / end of a VMA
 *
 *    Defaults to flushing at tlb_end_vma() to reset the range; helps when
 *    there's large holes between the VMAs.
 *
 *  - tlb_remove_table()
 *
 *    tlb_remove_table() is the basic primitive to free page-table directories
 *    (__p*_free_tlb()).  In it's most primitive form it is an alias for
 *    tlb_remove_page() below, for when page directories are pages and have no
 *    additional constraints.
 *
 *    See also MMU_GATHER_TABLE_FREE and MMU_GATHER_RCU_TABLE_FREE.
 *
 *  - tlb_remove_page() / __tlb_remove_page()
 *  - tlb_remove_page_size() / __tlb_remove_page_size()
 *  - __tlb_remove_folio_pages()
 *
 *    __tlb_remove_page_size() is the basic primitive that queues a page for
 *    freeing. __tlb_remove_page() assumes PAGE_SIZE. Both will return a
 *    boolean indicating if the queue is (now) full and a call to
 *    tlb_flush_mmu() is required.
 *
 *    tlb_remove_page() and tlb_remove_page_size() imply the call to
 *    tlb_flush_mmu() when required and has no return value.
 *
 *    __tlb_remove_folio_pages() is similar to __tlb_remove_page(), however,
 *    instead of removing a single page, remove the given number of consecutive
 *    pages that are all part of the same (large) folio: just like calling
 *    __tlb_remove_page() on each page individually.
 *
 *  - tlb_change_page_size()
 *
 *    call before __tlb_remove_page*() to set the current page-size; implies a
 *    possible tlb_flush_mmu() call.
 *
 *  - tlb_flush_mmu() / tlb_flush_mmu_tlbonly()
 *
 *    tlb_flush_mmu_tlbonly() - does the TLB invalidate (and resets
 *                              related state, like the range)
 *
 *    tlb_flush_mmu() - in addition to the above TLB invalidate, also frees
 *                        whatever pages are still batched.
 *
 *  - mmu_gather::fullmm
 *
 *    A flag set by tlb_gather_mmu_fullmm() to indicate we're going to free
 *    the entire mm; this allows a number of optimizations.
 *
 *    - We can ignore tlb_{start,end}_vma(); because we don't
 *      care about ranges. Everything will be shot down.
 *
 *    - (RISC) architectures that use ASIDs can cycle to a new ASID
 *      and delay the invalidation until ASID space runs out.
 *
 *  - mmu_gather::need_flush_all
 *
 *    A flag that can be set by the arch code if it wants to force
 *    flush the entire TLB irrespective of the range. For instance
 *    x86-PAE needs this when changing top-level entries.
 *
 * And allows the architecture to provide and implement tlb_flush():
 *
 * tlb_flush() may, in addition to the above mentioned mmu_gather fields, make
 * use of:
 *
 *  - mmu_gather::start / mmu_gather::end
 *
 *    which provides the range that needs to be flushed to cover the pages to
 *    be freed.
 *
 *  - mmu_gather::freed_tables
 *
 *    set when we freed page table pages
 *
 *  - tlb_get_unmap_shift() / tlb_get_unmap_size()
 *
 *    returns the smallest TLB entry size unmapped in this range.
 *
 * If an architecture does not provide tlb_flush() a default implementation
 * based on flush_tlb_range() will be used, unless MMU_GATHER_NO_RANGE is
 * specified, in which case we'll default to flush_tlb_mm().
 *
 * Additionally there are a few opt-in features:
 *
 *  MMU_GATHER_PAGE_SIZE
 *
 *  This ensures we call tlb_flush() every time tlb_change_page_size() actually
 *  changes the size and provides mmu_gather::page_size to tlb_flush().
 *
 *  This might be useful if your architecture has size specific TLB
 *  invalidation instructions.
 *
 *  MMU_GATHER_TABLE_FREE
 *
 *  This provides tlb_remove_table(), to be used instead of tlb_remove_page()
 *  for page directores (__p*_free_tlb()).
 *
 *  Useful if your architecture has non-page page directories.
 *
 *  When used, an architecture is expected to provide __tlb_remove_table()
 *  which does the actual freeing of these pages.
 *
 *  MMU_GATHER_RCU_TABLE_FREE
 *
 *  Like MMU_GATHER_TABLE_FREE, and adds semi-RCU semantics to the free (see
 *  comment below).
 *
 *  Useful if your architecture doesn't use IPIs for remote TLB invalidates
 *  and therefore doesn't naturally serialize with software page-table walkers.
 *
 *  MMU_GATHER_NO_FLUSH_CACHE
 *
 *  Indicates the architecture has flush_cache_range() but it needs *NOT* be called
 *  before unmapping a VMA.
 *
 *  NOTE: strictly speaking we shouldn't have this knob and instead rely on
 *          flush_cache_range() being a NOP, except Sparc64 seems to be
 *          different here.
 *
 *  MMU_GATHER_MERGE_VMAS
 *
 *  Indicates the architecture wants to merge ranges over VMAs; typical when
 *  multiple range invalidates are more expensive than a full invalidate.
 *
 *  MMU_GATHER_NO_RANGE
 *
 *  Use this if your architecture lacks an efficient flush_tlb_range(). This
 *  option implies MMU_GATHER_MERGE_VMAS above.
 *
 *  MMU_GATHER_NO_GATHER
 *
 *  If the option is set the mmu_gather will not track individual pages for
 *  delayed page free anymore. A platform that enables the option needs to
 *  provide its own implementation of the __tlb_remove_page_size() function to
 *  free pages.
 *
 *  This is useful if your architecture already flushes TLB entries in the
 *  various ptep_get_and_clear() functions.
 */

#ifdef CONFIG_MMU_GATHER_TABLE_FREE

struct mmu_table_batch {
#ifdef CONFIG_MMU_GATHER_RCU_TABLE_FREE
        struct rcu_head                rcu;
#endif
        unsigned int                nr;
        void                        *tables[];
};

#define MAX_TABLE_BATCH                \
        ((PAGE_SIZE - sizeof(struct mmu_table_batch)) / sizeof(void *))

extern void tlb_remove_table(struct mmu_gather *tlb, void *table);

#else /* !CONFIG_MMU_GATHER_HAVE_TABLE_FREE */

/*
 * Without MMU_GATHER_TABLE_FREE the architecture is assumed to have page based
 * page directories and we can use the normal page batching to free them.
 */
#define tlb_remove_table(tlb, page) tlb_remove_page((tlb), (page))

#endif /* CONFIG_MMU_GATHER_TABLE_FREE */

#ifdef CONFIG_MMU_GATHER_RCU_TABLE_FREE
/*
 * This allows an architecture that does not use the linux page-tables for
 * hardware to skip the TLBI when freeing page tables.
 */
#ifndef tlb_needs_table_invalidate
#define tlb_needs_table_invalidate() (true)
#endif

void tlb_remove_table_sync_one(void);

#else

#ifdef tlb_needs_table_invalidate
#error tlb_needs_table_invalidate() requires MMU_GATHER_RCU_TABLE_FREE
#endif

static inline void tlb_remove_table_sync_one(void) { }

#endif /* CONFIG_MMU_GATHER_RCU_TABLE_FREE */


#ifndef CONFIG_MMU_GATHER_NO_GATHER
/*
 * If we can't allocate a page to make a big batch of page pointers
 * to work on, then just handle a few from the on-stack structure.
 */
#define MMU_GATHER_BUNDLE        8

struct mmu_gather_batch {
        struct mmu_gather_batch        *next;
        unsigned int                nr;
        unsigned int                max;
        struct encoded_page        *encoded_pages[];
};

#define MAX_GATHER_BATCH        \
        ((PAGE_SIZE - sizeof(struct mmu_gather_batch)) / sizeof(void *))

/*
 * Limit the maximum number of mmu_gather batches to reduce a risk of soft
 * lockups for non-preemptible kernels on huge machines when a lot of memory
 * is zapped during unmapping.
 * 10K pages freed at once should be safe even without a preemption point.
 */
#define MAX_GATHER_BATCH_COUNT        (10000UL/MAX_GATHER_BATCH)

extern bool __tlb_remove_page_size(struct mmu_gather *tlb, struct page *page,
                bool delay_rmap, int page_size);
bool __tlb_remove_folio_pages(struct mmu_gather *tlb, struct page *page,
                unsigned int nr_pages, bool delay_rmap);

#ifdef CONFIG_SMP
/*
 * This both sets 'delayed_rmap', and returns true. It would be an inline
 * function, except we define it before the 'struct mmu_gather'.
 */
#define tlb_delay_rmap(tlb) (((tlb)->delayed_rmap = 1), true)
extern void tlb_flush_rmaps(struct mmu_gather *tlb, struct vm_area_struct *vma);
#endif

#endif

/*
 * We have a no-op version of the rmap removal that doesn't
 * delay anything. That is used on S390, which flushes remote
 * TLBs synchronously, and on UP, which doesn't have any
 * remote TLBs to flush and is not preemptible due to this
 * all happening under the page table lock.
 */
#ifndef tlb_delay_rmap
#define tlb_delay_rmap(tlb) (false)
static inline void tlb_flush_rmaps(struct mmu_gather *tlb, struct vm_area_struct *vma) { }
#endif

/*
 * struct mmu_gather is an opaque type used by the mm code for passing around
 * any data needed by arch specific code for tlb_remove_page.
 */
struct mmu_gather {
        struct mm_struct        *mm;

#ifdef CONFIG_MMU_GATHER_TABLE_FREE
        struct mmu_table_batch        *batch;
#endif

        unsigned long                start;
        unsigned long                end;
        /*
         * we are in the middle of an operation to clear
         * a full mm and can make some optimizations
         */
        unsigned int                fullmm : 1;

        /*
         * we have performed an operation which
         * requires a complete flush of the tlb
         */
        unsigned int                need_flush_all : 1;

        /*
         * we have removed page directories
         */
        unsigned int                freed_tables : 1;

        /*
         * Do we have pending delayed rmap removals?
         */
        unsigned int                delayed_rmap : 1;

        /*
         * at which levels have we cleared entries?
         */
        unsigned int                cleared_ptes : 1;
        unsigned int                cleared_pmds : 1;
        unsigned int                cleared_puds : 1;
        unsigned int                cleared_p4ds : 1;

        /*
         * tracks VM_EXEC | VM_HUGETLB in tlb_start_vma
         */
        unsigned int                vma_exec : 1;
        unsigned int                vma_huge : 1;
        unsigned int                vma_pfn  : 1;

        unsigned int                batch_count;

#ifndef CONFIG_MMU_GATHER_NO_GATHER
        struct mmu_gather_batch *active;
        struct mmu_gather_batch        local;
        struct page                *__pages[MMU_GATHER_BUNDLE];

#ifdef CONFIG_MMU_GATHER_PAGE_SIZE
        unsigned int page_size;
#endif
#endif
};

void tlb_flush_mmu(struct mmu_gather *tlb);

static inline void __tlb_adjust_range(struct mmu_gather *tlb,
                                      unsigned long address,
                                      unsigned int range_size)
{
        tlb->start = min(tlb->start, address);
        tlb->end = max(tlb->end, address + range_size);
}

static inline void __tlb_reset_range(struct mmu_gather *tlb)
{
        if (tlb->fullmm) {
                tlb->start = tlb->end = ~0;
        } else {
                tlb->start = TASK_SIZE;
                tlb->end = 0;
        }
        tlb->freed_tables = 0;
        tlb->cleared_ptes = 0;
        tlb->cleared_pmds = 0;
        tlb->cleared_puds = 0;
        tlb->cleared_p4ds = 0;
        /*
         * Do not reset mmu_gather::vma_* fields here, we do not
         * call into tlb_start_vma() again to set them if there is an
         * intermediate flush.
         */
}

#ifdef CONFIG_MMU_GATHER_NO_RANGE

#if defined(tlb_flush)
#error MMU_GATHER_NO_RANGE relies on default tlb_flush()
#endif

/*
 * When an architecture does not have efficient means of range flushing TLBs
 * there is no point in doing intermediate flushes on tlb_end_vma() to keep the
 * range small. We equally don't have to worry about page granularity or other
 * things.
 *
 * All we need to do is issue a full flush for any !0 range.
 */
static inline void tlb_flush(struct mmu_gather *tlb)
{
        if (tlb->end)
                flush_tlb_mm(tlb->mm);
}

#else /* CONFIG_MMU_GATHER_NO_RANGE */

#ifndef tlb_flush
/*
 * When an architecture does not provide its own tlb_flush() implementation
 * but does have a reasonably efficient flush_vma_range() implementation
 * use that.
 */
static inline void tlb_flush(struct mmu_gather *tlb)
{
        if (tlb->fullmm || tlb->need_flush_all) {
                flush_tlb_mm(tlb->mm);
        } else if (tlb->end) {
                struct vm_area_struct vma = {
                        .vm_mm = tlb->mm,
                        .vm_flags = (tlb->vma_exec ? VM_EXEC    : 0) |
                                    (tlb->vma_huge ? VM_HUGETLB : 0),
                };

                flush_tlb_range(&vma, tlb->start, tlb->end);
        }
}
#endif

#endif /* CONFIG_MMU_GATHER_NO_RANGE */

static inline void
tlb_update_vma_flags(struct mmu_gather *tlb, struct vm_area_struct *vma)
{
        /*
         * flush_tlb_range() implementations that look at VM_HUGETLB (tile,
         * mips-4k) flush only large pages.
         *
         * flush_tlb_range() implementations that flush I-TLB also flush D-TLB
         * (tile, xtensa, arm), so it's ok to just add VM_EXEC to an existing
         * range.
         *
         * We rely on tlb_end_vma() to issue a flush, such that when we reset
         * these values the batch is empty.
         */
        tlb->vma_huge = is_vm_hugetlb_page(vma);
        tlb->vma_exec = !!(vma->vm_flags & VM_EXEC);
        tlb->vma_pfn  = !!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP));
}

static inline void tlb_flush_mmu_tlbonly(struct mmu_gather *tlb)
{
        /*
         * Anything calling __tlb_adjust_range() also sets at least one of
         * these bits.
         */
        if (!(tlb->freed_tables || tlb->cleared_ptes || tlb->cleared_pmds ||
              tlb->cleared_puds || tlb->cleared_p4ds))
                return;

        tlb_flush(tlb);
        __tlb_reset_range(tlb);
}

static inline void tlb_remove_page_size(struct mmu_gather *tlb,
                                        struct page *page, int page_size)
{
        if (__tlb_remove_page_size(tlb, page, false, page_size))
                tlb_flush_mmu(tlb);
}

static __always_inline bool __tlb_remove_page(struct mmu_gather *tlb,
                struct page *page, bool delay_rmap)
{
        return __tlb_remove_page_size(tlb, page, delay_rmap, PAGE_SIZE);
}

/* tlb_remove_page
 *        Similar to __tlb_remove_page but will call tlb_flush_mmu() itself when
 *        required.
 */
static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page)
{
        return tlb_remove_page_size(tlb, page, PAGE_SIZE);
}

static inline void tlb_remove_ptdesc(struct mmu_gather *tlb, void *pt)
{
        tlb_remove_table(tlb, pt);
}

/* Like tlb_remove_ptdesc, but for page-like page directories. */
static inline void tlb_remove_page_ptdesc(struct mmu_gather *tlb, struct ptdesc *pt)
{
        tlb_remove_page(tlb, ptdesc_page(pt));
}

static inline void tlb_change_page_size(struct mmu_gather *tlb,
                                                     unsigned int page_size)
{
#ifdef CONFIG_MMU_GATHER_PAGE_SIZE
        if (tlb->page_size && tlb->page_size != page_size) {
                if (!tlb->fullmm && !tlb->need_flush_all)
                        tlb_flush_mmu(tlb);
        }

        tlb->page_size = page_size;
#endif
}

static inline unsigned long tlb_get_unmap_shift(struct mmu_gather *tlb)
{
        if (tlb->cleared_ptes)
                return PAGE_SHIFT;
        if (tlb->cleared_pmds)
                return PMD_SHIFT;
        if (tlb->cleared_puds)
                return PUD_SHIFT;
        if (tlb->cleared_p4ds)
                return P4D_SHIFT;

        return PAGE_SHIFT;
}

static inline unsigned long tlb_get_unmap_size(struct mmu_gather *tlb)
{
        return 1UL << tlb_get_unmap_shift(tlb);
}

/*
 * In the case of tlb vma handling, we can optimise these away in the
 * case where we're doing a full MM flush.  When we're doing a munmap,
 * the vmas are adjusted to only cover the region to be torn down.
 */
static inline void tlb_start_vma(struct mmu_gather *tlb, struct vm_area_struct *vma)
{
        if (tlb->fullmm)
                return;

        tlb_update_vma_flags(tlb, vma);
#ifndef CONFIG_MMU_GATHER_NO_FLUSH_CACHE
        flush_cache_range(vma, vma->vm_start, vma->vm_end);
#endif
}

static inline void tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vma)
{
        if (tlb->fullmm)
                return;

        /*
         * VM_PFNMAP is more fragile because the core mm will not track the
         * page mapcount -- there might not be page-frames for these PFNs after
         * all. Force flush TLBs for such ranges to avoid munmap() vs
         * unmap_mapping_range() races.
         */
        if (tlb->vma_pfn || !IS_ENABLED(CONFIG_MMU_GATHER_MERGE_VMAS)) {
                /*
                 * Do a TLB flush and reset the range at VMA boundaries; this avoids
                 * the ranges growing with the unused space between consecutive VMAs.
                 */
                tlb_flush_mmu_tlbonly(tlb);
        }
}

/*
 * tlb_flush_{pte|pmd|pud|p4d}_range() adjust the tlb->start and tlb->end,
 * and set corresponding cleared_*.
 */
static inline void tlb_flush_pte_range(struct mmu_gather *tlb,
                                     unsigned long address, unsigned long size)
{
        __tlb_adjust_range(tlb, address, size);
        tlb->cleared_ptes = 1;
}

static inline void tlb_flush_pmd_range(struct mmu_gather *tlb,
                                     unsigned long address, unsigned long size)
{
        __tlb_adjust_range(tlb, address, size);
        tlb->cleared_pmds = 1;
}

static inline void tlb_flush_pud_range(struct mmu_gather *tlb,
                                     unsigned long address, unsigned long size)
{
        __tlb_adjust_range(tlb, address, size);
        tlb->cleared_puds = 1;
}

static inline void tlb_flush_p4d_range(struct mmu_gather *tlb,
                                     unsigned long address, unsigned long size)
{
        __tlb_adjust_range(tlb, address, size);
        tlb->cleared_p4ds = 1;
}

#ifndef __tlb_remove_tlb_entry
static inline void __tlb_remove_tlb_entry(struct mmu_gather *tlb, pte_t *ptep, unsigned long address)
{
}
#endif

/**
 * tlb_remove_tlb_entry - remember a pte unmapping for later tlb invalidation.
 *
 * Record the fact that pte's were really unmapped by updating the range,
 * so we can later optimise away the tlb invalidate.   This helps when
 * userspace is unmapping already-unmapped pages, which happens quite a lot.
 */
#define tlb_remove_tlb_entry(tlb, ptep, address)                \
        do {                                                        \
                tlb_flush_pte_range(tlb, address, PAGE_SIZE);        \
                __tlb_remove_tlb_entry(tlb, ptep, address);        \
        } while (0)

/**
 * tlb_remove_tlb_entries - remember unmapping of multiple consecutive ptes for
 *                            later tlb invalidation.
 *
 * Similar to tlb_remove_tlb_entry(), but remember unmapping of multiple
 * consecutive ptes instead of only a single one.
 */
static inline void tlb_remove_tlb_entries(struct mmu_gather *tlb,
                pte_t *ptep, unsigned int nr, unsigned long address)
{
        tlb_flush_pte_range(tlb, address, PAGE_SIZE * nr);
        for (;;) {
                __tlb_remove_tlb_entry(tlb, ptep, address);
                if (--nr == 0)
                        break;
                ptep++;
                address += PAGE_SIZE;
        }
}

#define tlb_remove_huge_tlb_entry(h, tlb, ptep, address)        \
        do {                                                        \
                unsigned long _sz = huge_page_size(h);                \
                if (_sz >= P4D_SIZE)                                \
                        tlb_flush_p4d_range(tlb, address, _sz);        \
                else if (_sz >= PUD_SIZE)                        \
                        tlb_flush_pud_range(tlb, address, _sz);        \
                else if (_sz >= PMD_SIZE)                        \
                        tlb_flush_pmd_range(tlb, address, _sz);        \
                else                                                \
                        tlb_flush_pte_range(tlb, address, _sz);        \
                __tlb_remove_tlb_entry(tlb, ptep, address);        \
        } while (0)

/**
 * tlb_remove_pmd_tlb_entry - remember a pmd mapping for later tlb invalidation
 * This is a nop so far, because only x86 needs it.
 */
#ifndef __tlb_remove_pmd_tlb_entry
#define __tlb_remove_pmd_tlb_entry(tlb, pmdp, address) do {} while (0)
#endif

#define tlb_remove_pmd_tlb_entry(tlb, pmdp, address)                        \
        do {                                                                \
                tlb_flush_pmd_range(tlb, address, HPAGE_PMD_SIZE);        \
                __tlb_remove_pmd_tlb_entry(tlb, pmdp, address);                \
        } while (0)

/**
 * tlb_remove_pud_tlb_entry - remember a pud mapping for later tlb
 * invalidation. This is a nop so far, because only x86 needs it.
 */
#ifndef __tlb_remove_pud_tlb_entry
#define __tlb_remove_pud_tlb_entry(tlb, pudp, address) do {} while (0)
#endif

#define tlb_remove_pud_tlb_entry(tlb, pudp, address)                        \
        do {                                                                \
                tlb_flush_pud_range(tlb, address, HPAGE_PUD_SIZE);        \
                __tlb_remove_pud_tlb_entry(tlb, pudp, address);                \
        } while (0)

/*
 * For things like page tables caches (ie caching addresses "inside" the
 * page tables, like x86 does), for legacy reasons, flushing an
 * individual page had better flush the page table caches behind it. This
 * is definitely how x86 works, for example. And if you have an
 * architected non-legacy page table cache (which I'm not aware of
 * anybody actually doing), you're going to have some architecturally
 * explicit flushing for that, likely *separate* from a regular TLB entry
 * flush, and thus you'd need more than just some range expansion..
 *
 * So if we ever find an architecture
 * that would want something that odd, I think it is up to that
 * architecture to do its own odd thing, not cause pain for others
 * http://lkml.kernel.org/r/CA+55aFzBggoXtNXQeng5d_mRoDnaMBE5Y+URs+PHR67nUpMtaw@mail.gmail.com
 *
 * For now w.r.t page table cache, mark the range_size as PAGE_SIZE
 */

#ifndef pte_free_tlb
#define pte_free_tlb(tlb, ptep, address)                        \
        do {                                                        \
                tlb_flush_pmd_range(tlb, address, PAGE_SIZE);        \
                tlb->freed_tables = 1;                                \
                __pte_free_tlb(tlb, ptep, address);                \
        } while (0)
#endif

#ifndef pmd_free_tlb
#define pmd_free_tlb(tlb, pmdp, address)                        \
        do {                                                        \
                tlb_flush_pud_range(tlb, address, PAGE_SIZE);        \
                tlb->freed_tables = 1;                                \
                __pmd_free_tlb(tlb, pmdp, address);                \
        } while (0)
#endif

#ifndef pud_free_tlb
#define pud_free_tlb(tlb, pudp, address)                        \
        do {                                                        \
                tlb_flush_p4d_range(tlb, address, PAGE_SIZE);        \
                tlb->freed_tables = 1;                                \
                __pud_free_tlb(tlb, pudp, address);                \
        } while (0)
#endif

#ifndef p4d_free_tlb
#define p4d_free_tlb(tlb, pudp, address)                        \
        do {                                                        \
                __tlb_adjust_range(tlb, address, PAGE_SIZE);        \
                tlb->freed_tables = 1;                                \
                __p4d_free_tlb(tlb, pudp, address);                \
        } while (0)
#endif

#ifndef pte_needs_flush
static inline bool pte_needs_flush(pte_t oldpte, pte_t newpte)
{
        return true;
}
#endif

#ifndef huge_pmd_needs_flush
static inline bool huge_pmd_needs_flush(pmd_t oldpmd, pmd_t newpmd)
{
        return true;
}
#endif

#endif /* CONFIG_MMU */

#endif /* _ASM_GENERIC__TLB_H */




















































































    4 
















    4 

    4 












    4 













    4 




    4 
    4 


















    4 


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
/*
 * llc_if.c - Defines LLC interface to upper layer
 *
 * Copyright (c) 1997 by Procom Technology, Inc.
 *                  2001-2003 by Arnaldo Carvalho de Melo <acme@conectiva.com.br>
 *
 * This program can be redistributed or modified under the terms of the
 * GNU General Public License as published by the Free Software Foundation.
 * This program is distributed without any warranty or implied warranty
 * of merchantability or fitness for a particular purpose.
 *
 * See the GNU General Public License for more details.
 */
#include <linux/gfp.h>
#include <linux/module.h>
#include <linux/kernel.h>
#include <linux/netdevice.h>
#include <linux/errno.h>
#include <net/llc_if.h>
#include <net/llc_sap.h>
#include <net/llc_s_ev.h>
#include <net/llc_conn.h>
#include <net/sock.h>
#include <net/llc_c_ev.h>
#include <net/llc_c_ac.h>
#include <net/llc_c_st.h>
#include <net/tcp_states.h>

/**
 *        llc_build_and_send_pkt - Connection data sending for upper layers.
 *        @sk: connection
 *        @skb: packet to send
 *
 *        This function is called when upper layer wants to send data using
 *        connection oriented communication mode. During sending data, connection
 *        will be locked and received frames and expired timers will be queued.
 *        Returns 0 for success, -ECONNABORTED when the connection already
 *        closed and -EBUSY when sending data is not permitted in this state or
 *        LLC has send an I pdu with p bit set to 1 and is waiting for it's
 *        response.
 *
 *        This function always consumes a reference to the skb.
 */
int llc_build_and_send_pkt(struct sock *sk, struct sk_buff *skb)
{
        struct llc_conn_state_ev *ev;
        int rc = -ECONNABORTED;
        struct llc_sock *llc = llc_sk(sk);

        if (unlikely(llc->state == LLC_CONN_STATE_ADM))
                goto out_free;
        rc = -EBUSY;
        if (unlikely(llc_data_accept_state(llc->state) || /* data_conn_refuse */
                     llc->p_flag)) {
                llc->failed_data_req = 1;
                goto out_free;
        }
        ev = llc_conn_ev(skb);
        ev->type      = LLC_CONN_EV_TYPE_PRIM;
        ev->prim      = LLC_DATA_PRIM;
        ev->prim_type = LLC_PRIM_TYPE_REQ;
        skb->dev      = llc->dev;
        return llc_conn_state_process(sk, skb);

out_free:
        kfree_skb(skb);
        return rc;
}

/**
 *        llc_establish_connection - Called by upper layer to establish a conn
 *        @sk: connection
 *        @lmac: local mac address
 *        @dmac: destination mac address
 *        @dsap: destination sap
 *
 *        Upper layer calls this to establish an LLC connection with a remote
 *        machine. This function packages a proper event and sends it connection
 *        component state machine. Success or failure of connection
 *        establishment will inform to upper layer via calling it's confirm
 *        function and passing proper information.
 */
int llc_establish_connection(struct sock *sk, const u8 *lmac, u8 *dmac, u8 dsap)
{
        int rc = -EISCONN;
        struct llc_addr laddr, daddr;
        struct sk_buff *skb;
        struct llc_sock *llc = llc_sk(sk);
        struct sock *existing;

        laddr.lsap = llc->sap->laddr.lsap;
        daddr.lsap = dsap;
        memcpy(daddr.mac, dmac, sizeof(daddr.mac));
        memcpy(laddr.mac, lmac, sizeof(laddr.mac));
        existing = llc_lookup_established(llc->sap, &daddr, &laddr, sock_net(sk));
        if (existing) {
                if (existing->sk_state == TCP_ESTABLISHED) {
                        sk = existing;
                        goto out_put;
                } else
                        sock_put(existing);
        }
        sock_hold(sk);
        rc = -ENOMEM;
        skb = alloc_skb(0, GFP_ATOMIC);
        if (skb) {
                struct llc_conn_state_ev *ev = llc_conn_ev(skb);

                ev->type      = LLC_CONN_EV_TYPE_PRIM;
                ev->prim      = LLC_CONN_PRIM;
                ev->prim_type = LLC_PRIM_TYPE_REQ;
                skb_set_owner_w(skb, sk);
                rc = llc_conn_state_process(sk, skb);
        }
out_put:
        sock_put(sk);
        return rc;
}

/**
 *        llc_send_disc - Called by upper layer to close a connection
 *        @sk: connection to be closed
 *
 *        Upper layer calls this when it wants to close an established LLC
 *        connection with a remote machine. This function packages a proper event
 *        and sends it to connection component state machine. Returns 0 for
 *        success, 1 otherwise.
 */
int llc_send_disc(struct sock *sk)
{
        u16 rc = 1;
        struct llc_conn_state_ev *ev;
        struct sk_buff *skb;

        sock_hold(sk);
        if (sk->sk_type != SOCK_STREAM || sk->sk_state != TCP_ESTABLISHED ||
            llc_sk(sk)->state == LLC_CONN_STATE_ADM ||
            llc_sk(sk)->state == LLC_CONN_OUT_OF_SVC)
                goto out;
        /*
         * Postpone unassigning the connection from its SAP and returning the
         * connection until all ACTIONs have been completely executed
         */
        skb = alloc_skb(0, GFP_ATOMIC);
        if (!skb)
                goto out;
        skb_set_owner_w(skb, sk);
        sk->sk_state  = TCP_CLOSING;
        ev              = llc_conn_ev(skb);
        ev->type      = LLC_CONN_EV_TYPE_PRIM;
        ev->prim      = LLC_DISC_PRIM;
        ev->prim_type = LLC_PRIM_TYPE_REQ;
        rc = llc_conn_state_process(sk, skb);
out:
        sock_put(sk);
        return rc;
}




























































































   10 






   11 


   11 






   18 



   18 
   16 






    7 


   42 
    4 
    3 















   26 

   24 










    1 










  350 

  355 




  350 
  352 
   36 





    2 




   34 






   30 


  353 




  352 


  355 






   20 






    5 
   13 
    2 









   18 


    1 















    2 







   13 



   12 
    1 



    1 






   86 


   88 








   87 
   87 

   23 


   20 

   88 









   28 
    2 







   30 
   30 




   30 



   30 







   12 










    1 



    1 


    1 







    1 






   10 



   10 





   10 

    8 
    2 



   10 












    1 


    4 
    4 
    1 


    2 
    3 



    4 


    1 







    9 








    4 








   27 





    1 











   14 


   11 







    5 


















   51 















    1 









    1 









    1 

   33 
    6 


    6 









   13 









   33 


    5 


   15 






    9 


   39 
   39 
   36 

    2 


   39 


   39 




    1 

    8 
    1 

    1 


   46 
   46 



   23 

   16 
    8 

   23 
    3 














   46 
   46 






   13 



   32 













   45 


   37 





   12 


   28 
    2 





    2 
    1 

   29 





   39 
    9 



   50 




    3 
    2 
    5 






    4 









    4 







    5 








    1 


    3 
    1 





    4 




    3 















    9 






    1 



    2 




    2 



    1 
    5 




    1 


    5 


    1 
    4 





    3 
    2 
    4 
    1 









    3 


    2 
    1 
















    1 


    1 




    1 
    1 








    1 
    1 


    2 






  189 

  185 
    4 





    1 
    1 


    4 






   46 

   40 
    6 



    8 


    1 







    2 
    1 








    5 
























    4 

























































































































































    5 





































    5 














1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * INET                An implementation of the TCP/IP protocol suite for the LINUX
 *                operating system.  INET is implemented using the  BSD Socket
 *                interface as the means of communication with the user level.
 *
 *                RAW - implementation of IP "raw" sockets.
 *
 * Authors:        Ross Biro
 *                Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
 *
 * Fixes:
 *                Alan Cox        :        verify_area() fixed up
 *                Alan Cox        :        ICMP error handling
 *                Alan Cox        :        EMSGSIZE if you send too big a packet
 *                Alan Cox        :         Now uses generic datagrams and shared
 *                                        skbuff library. No more peek crashes,
 *                                        no more backlogs
 *                Alan Cox        :        Checks sk->broadcast.
 *                Alan Cox        :        Uses skb_free_datagram/skb_copy_datagram
 *                Alan Cox        :        Raw passes ip options too
 *                Alan Cox        :        Setsocketopt added
 *                Alan Cox        :        Fixed error return for broadcasts
 *                Alan Cox        :        Removed wake_up calls
 *                Alan Cox        :        Use ttl/tos
 *                Alan Cox        :        Cleaned up old debugging
 *                Alan Cox        :        Use new kernel side addresses
 *        Arnt Gulbrandsen        :        Fixed MSG_DONTROUTE in raw sockets.
 *                Alan Cox        :        BSD style RAW socket demultiplexing.
 *                Alan Cox        :        Beginnings of mrouted support.
 *                Alan Cox        :        Added IP_HDRINCL option.
 *                Alan Cox        :        Skip broadcast check if BSDism set.
 *                David S. Miller        :        New socket lookup architecture.
 */

#include <linux/types.h>
#include <linux/atomic.h>
#include <asm/byteorder.h>
#include <asm/current.h>
#include <linux/uaccess.h>
#include <asm/ioctls.h>
#include <linux/stddef.h>
#include <linux/slab.h>
#include <linux/errno.h>
#include <linux/kernel.h>
#include <linux/export.h>
#include <linux/spinlock.h>
#include <linux/sockios.h>
#include <linux/socket.h>
#include <linux/in.h>
#include <linux/mroute.h>
#include <linux/netdevice.h>
#include <linux/in_route.h>
#include <linux/route.h>
#include <linux/skbuff.h>
#include <linux/igmp.h>
#include <net/net_namespace.h>
#include <net/dst.h>
#include <net/sock.h>
#include <linux/ip.h>
#include <linux/net.h>
#include <net/ip.h>
#include <net/icmp.h>
#include <net/udp.h>
#include <net/raw.h>
#include <net/snmp.h>
#include <net/tcp_states.h>
#include <net/inet_common.h>
#include <net/checksum.h>
#include <net/xfrm.h>
#include <linux/rtnetlink.h>
#include <linux/proc_fs.h>
#include <linux/seq_file.h>
#include <linux/netfilter.h>
#include <linux/netfilter_ipv4.h>
#include <linux/compat.h>
#include <linux/uio.h>

struct raw_frag_vec {
        struct msghdr *msg;
        union {
                struct icmphdr icmph;
                char c[1];
        } hdr;
        int hlen;
};

struct raw_hashinfo raw_v4_hashinfo;
EXPORT_SYMBOL_GPL(raw_v4_hashinfo);

int raw_hash_sk(struct sock *sk)
{
        struct raw_hashinfo *h = sk->sk_prot->h.raw_hash;
        struct hlist_head *hlist;

        hlist = &h->ht[raw_hashfunc(sock_net(sk), inet_sk(sk)->inet_num)];

        spin_lock(&h->lock);
        sk_add_node_rcu(sk, hlist);
        sock_set_flag(sk, SOCK_RCU_FREE);
        spin_unlock(&h->lock);
        sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);

        return 0;
}
EXPORT_SYMBOL_GPL(raw_hash_sk);

void raw_unhash_sk(struct sock *sk)
{
        struct raw_hashinfo *h = sk->sk_prot->h.raw_hash;

        spin_lock(&h->lock);
        if (sk_del_node_init_rcu(sk))
                sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1);
        spin_unlock(&h->lock);
}
EXPORT_SYMBOL_GPL(raw_unhash_sk);

bool raw_v4_match(struct net *net, const struct sock *sk, unsigned short num,
                  __be32 raddr, __be32 laddr, int dif, int sdif)
{
        const struct inet_sock *inet = inet_sk(sk);

        if (net_eq(sock_net(sk), net) && inet->inet_num == num        &&
            !(inet->inet_daddr && inet->inet_daddr != raddr)         &&
            !(inet->inet_rcv_saddr && inet->inet_rcv_saddr != laddr) &&
            raw_sk_bound_dev_eq(net, sk->sk_bound_dev_if, dif, sdif))
                return true;
        return false;
}
EXPORT_SYMBOL_GPL(raw_v4_match);

/*
 *        0 - deliver
 *        1 - block
 */
static int icmp_filter(const struct sock *sk, const struct sk_buff *skb)
{
        struct icmphdr _hdr;
        const struct icmphdr *hdr;

        hdr = skb_header_pointer(skb, skb_transport_offset(skb),
                                 sizeof(_hdr), &_hdr);
        if (!hdr)
                return 1;

        if (hdr->type < 32) {
                __u32 data = raw_sk(sk)->filter.data;

                return ((1U << hdr->type) & data) != 0;
        }

        /* Do not block unknown ICMP types */
        return 0;
}

/* IP input processing comes here for RAW socket delivery.
 * Caller owns SKB, so we must make clones.
 *
 * RFC 1122: SHOULD pass TOS value up to the transport layer.
 * -> It does. And not only TOS, but all IP header.
 */
static int raw_v4_input(struct net *net, struct sk_buff *skb,
                        const struct iphdr *iph, int hash)
{
        int sdif = inet_sdif(skb);
        struct hlist_head *hlist;
        int dif = inet_iif(skb);
        int delivered = 0;
        struct sock *sk;

        hlist = &raw_v4_hashinfo.ht[hash];
        rcu_read_lock();
        sk_for_each_rcu(sk, hlist) {
                if (!raw_v4_match(net, sk, iph->protocol,
                                  iph->saddr, iph->daddr, dif, sdif))
                        continue;

                if (atomic_read(&sk->sk_rmem_alloc) >=
                    READ_ONCE(sk->sk_rcvbuf)) {
                        atomic_inc(&sk->sk_drops);
                        continue;
                }

                delivered = 1;
                if ((iph->protocol != IPPROTO_ICMP || !icmp_filter(sk, skb)) &&
                    ip_mc_sf_allow(sk, iph->daddr, iph->saddr,
                                   skb->dev->ifindex, sdif)) {
                        struct sk_buff *clone = skb_clone(skb, GFP_ATOMIC);

                        /* Not releasing hash table! */
                        if (clone)
                                raw_rcv(sk, clone);
                }
        }
        rcu_read_unlock();
        return delivered;
}

int raw_local_deliver(struct sk_buff *skb, int protocol)
{
        struct net *net = dev_net(skb->dev);

        return raw_v4_input(net, skb, ip_hdr(skb),
                            raw_hashfunc(net, protocol));
}

static void raw_err(struct sock *sk, struct sk_buff *skb, u32 info)
{
        struct inet_sock *inet = inet_sk(sk);
        const int type = icmp_hdr(skb)->type;
        const int code = icmp_hdr(skb)->code;
        int harderr = 0;
        bool recverr;
        int err = 0;

        if (type == ICMP_DEST_UNREACH && code == ICMP_FRAG_NEEDED)
                ipv4_sk_update_pmtu(skb, sk, info);
        else if (type == ICMP_REDIRECT) {
                ipv4_sk_redirect(skb, sk);
                return;
        }

        /* Report error on raw socket, if:
           1. User requested ip_recverr.
           2. Socket is connected (otherwise the error indication
              is useless without ip_recverr and error is hard.
         */
        recverr = inet_test_bit(RECVERR, sk);
        if (!recverr && sk->sk_state != TCP_ESTABLISHED)
                return;

        switch (type) {
        default:
        case ICMP_TIME_EXCEEDED:
                err = EHOSTUNREACH;
                break;
        case ICMP_SOURCE_QUENCH:
                return;
        case ICMP_PARAMETERPROB:
                err = EPROTO;
                harderr = 1;
                break;
        case ICMP_DEST_UNREACH:
                err = EHOSTUNREACH;
                if (code > NR_ICMP_UNREACH)
                        break;
                if (code == ICMP_FRAG_NEEDED) {
                        harderr = READ_ONCE(inet->pmtudisc) != IP_PMTUDISC_DONT;
                        err = EMSGSIZE;
                } else {
                        err = icmp_err_convert[code].errno;
                        harderr = icmp_err_convert[code].fatal;
                }
        }

        if (recverr) {
                const struct iphdr *iph = (const struct iphdr *)skb->data;
                u8 *payload = skb->data + (iph->ihl << 2);

                if (inet_test_bit(HDRINCL, sk))
                        payload = skb->data;
                ip_icmp_error(sk, skb, err, 0, info, payload);
        }

        if (recverr || harderr) {
                sk->sk_err = err;
                sk_error_report(sk);
        }
}

void raw_icmp_error(struct sk_buff *skb, int protocol, u32 info)
{
        struct net *net = dev_net(skb->dev);
        int dif = skb->dev->ifindex;
        int sdif = inet_sdif(skb);
        struct hlist_head *hlist;
        const struct iphdr *iph;
        struct sock *sk;
        int hash;

        hash = raw_hashfunc(net, protocol);
        hlist = &raw_v4_hashinfo.ht[hash];

        rcu_read_lock();
        sk_for_each_rcu(sk, hlist) {
                iph = (const struct iphdr *)skb->data;
                if (!raw_v4_match(net, sk, iph->protocol,
                                  iph->daddr, iph->saddr, dif, sdif))
                        continue;
                raw_err(sk, skb, info);
        }
        rcu_read_unlock();
}

static int raw_rcv_skb(struct sock *sk, struct sk_buff *skb)
{
        enum skb_drop_reason reason;

        /* Charge it to the socket. */

        ipv4_pktinfo_prepare(sk, skb, true);
        if (sock_queue_rcv_skb_reason(sk, skb, &reason) < 0) {
                sk_skb_reason_drop(sk, skb, reason);
                return NET_RX_DROP;
        }

        return NET_RX_SUCCESS;
}

int raw_rcv(struct sock *sk, struct sk_buff *skb)
{
        if (!xfrm4_policy_check(sk, XFRM_POLICY_IN, skb)) {
                atomic_inc(&sk->sk_drops);
                sk_skb_reason_drop(sk, skb, SKB_DROP_REASON_XFRM_POLICY);
                return NET_RX_DROP;
        }
        nf_reset_ct(skb);

        skb_push(skb, -skb_network_offset(skb));

        raw_rcv_skb(sk, skb);
        return 0;
}

static int raw_send_hdrinc(struct sock *sk, struct flowi4 *fl4,
                           struct msghdr *msg, size_t length,
                           struct rtable **rtp, unsigned int flags,
                           const struct sockcm_cookie *sockc)
{
        struct inet_sock *inet = inet_sk(sk);
        struct net *net = sock_net(sk);
        struct iphdr *iph;
        struct sk_buff *skb;
        unsigned int iphlen;
        int err;
        struct rtable *rt = *rtp;
        int hlen, tlen;

        if (length > rt->dst.dev->mtu) {
                ip_local_error(sk, EMSGSIZE, fl4->daddr, inet->inet_dport,
                               rt->dst.dev->mtu);
                return -EMSGSIZE;
        }
        if (length < sizeof(struct iphdr))
                return -EINVAL;

        if (flags&MSG_PROBE)
                goto out;

        hlen = LL_RESERVED_SPACE(rt->dst.dev);
        tlen = rt->dst.dev->needed_tailroom;
        skb = sock_alloc_send_skb(sk,
                                  length + hlen + tlen + 15,
                                  flags & MSG_DONTWAIT, &err);
        if (!skb)
                goto error;
        skb_reserve(skb, hlen);

        skb->protocol = htons(ETH_P_IP);
        skb->priority = READ_ONCE(sk->sk_priority);
        skb->mark = sockc->mark;
        skb_set_delivery_type_by_clockid(skb, sockc->transmit_time, sk->sk_clockid);
        skb_dst_set(skb, &rt->dst);
        *rtp = NULL;

        skb_reset_network_header(skb);
        iph = ip_hdr(skb);
        skb_put(skb, length);

        skb->ip_summed = CHECKSUM_NONE;

        skb_setup_tx_timestamp(skb, sockc);

        if (flags & MSG_CONFIRM)
                skb_set_dst_pending_confirm(skb, 1);

        skb->transport_header = skb->network_header;
        err = -EFAULT;
        if (memcpy_from_msg(iph, msg, length))
                goto error_free;

        iphlen = iph->ihl * 4;

        /*
         * We don't want to modify the ip header, but we do need to
         * be sure that it won't cause problems later along the network
         * stack.  Specifically we want to make sure that iph->ihl is a
         * sane value.  If ihl points beyond the length of the buffer passed
         * in, reject the frame as invalid
         */
        err = -EINVAL;
        if (iphlen > length)
                goto error_free;

        if (iphlen >= sizeof(*iph)) {
                if (!iph->saddr)
                        iph->saddr = fl4->saddr;
                iph->check   = 0;
                iph->tot_len = htons(length);
                if (!iph->id)
                        ip_select_ident(net, skb, NULL);

                iph->check = ip_fast_csum((unsigned char *)iph, iph->ihl);
                skb->transport_header += iphlen;
                if (iph->protocol == IPPROTO_ICMP &&
                    length >= iphlen + sizeof(struct icmphdr))
                        icmp_out_count(net, ((struct icmphdr *)
                                skb_transport_header(skb))->type);
        }

        err = NF_HOOK(NFPROTO_IPV4, NF_INET_LOCAL_OUT,
                      net, sk, skb, NULL, rt->dst.dev,
                      dst_output);
        if (err > 0)
                err = net_xmit_errno(err);
        if (err)
                goto error;
out:
        return 0;

error_free:
        kfree_skb(skb);
error:
        IP_INC_STATS(net, IPSTATS_MIB_OUTDISCARDS);
        if (err == -ENOBUFS && !inet_test_bit(RECVERR, sk))
                err = 0;
        return err;
}

static int raw_probe_proto_opt(struct raw_frag_vec *rfv, struct flowi4 *fl4)
{
        int err;

        if (fl4->flowi4_proto != IPPROTO_ICMP)
                return 0;

        /* We only need the first two bytes. */
        rfv->hlen = 2;

        err = memcpy_from_msg(rfv->hdr.c, rfv->msg, rfv->hlen);
        if (err)
                return err;

        fl4->fl4_icmp_type = rfv->hdr.icmph.type;
        fl4->fl4_icmp_code = rfv->hdr.icmph.code;

        return 0;
}

static int raw_getfrag(void *from, char *to, int offset, int len, int odd,
                       struct sk_buff *skb)
{
        struct raw_frag_vec *rfv = from;

        if (offset < rfv->hlen) {
                int copy = min(rfv->hlen - offset, len);

                if (skb->ip_summed == CHECKSUM_PARTIAL)
                        memcpy(to, rfv->hdr.c + offset, copy);
                else
                        skb->csum = csum_block_add(
                                skb->csum,
                                csum_partial_copy_nocheck(rfv->hdr.c + offset,
                                                          to, copy),
                                odd);

                odd = 0;
                offset += copy;
                to += copy;
                len -= copy;

                if (!len)
                        return 0;
        }

        offset -= rfv->hlen;

        return ip_generic_getfrag(rfv->msg, to, offset, len, odd, skb);
}

static int raw_sendmsg(struct sock *sk, struct msghdr *msg, size_t len)
{
        struct inet_sock *inet = inet_sk(sk);
        struct net *net = sock_net(sk);
        struct ipcm_cookie ipc;
        struct rtable *rt = NULL;
        struct flowi4 fl4;
        u8 tos, scope;
        int free = 0;
        __be32 daddr;
        __be32 saddr;
        int uc_index, err;
        struct ip_options_data opt_copy;
        struct raw_frag_vec rfv;
        int hdrincl;

        err = -EMSGSIZE;
        if (len > 0xFFFF)
                goto out;

        hdrincl = inet_test_bit(HDRINCL, sk);

        /*
         *        Check the flags.
         */

        err = -EOPNOTSUPP;
        if (msg->msg_flags & MSG_OOB)        /* Mirror BSD error message */
                goto out;               /* compatibility */

        /*
         *        Get and verify the address.
         */

        if (msg->msg_namelen) {
                DECLARE_SOCKADDR(struct sockaddr_in *, usin, msg->msg_name);
                err = -EINVAL;
                if (msg->msg_namelen < sizeof(*usin))
                        goto out;
                if (usin->sin_family != AF_INET) {
                        pr_info_once("%s: %s forgot to set AF_INET. Fix it!\n",
                                     __func__, current->comm);
                        err = -EAFNOSUPPORT;
                        if (usin->sin_family)
                                goto out;
                }
                daddr = usin->sin_addr.s_addr;
                /* ANK: I did not forget to get protocol from port field.
                 * I just do not know, who uses this weirdness.
                 * IP_HDRINCL is much more convenient.
                 */
        } else {
                err = -EDESTADDRREQ;
                if (sk->sk_state != TCP_ESTABLISHED)
                        goto out;
                daddr = inet->inet_daddr;
        }

        ipcm_init_sk(&ipc, inet);
        /* Keep backward compat */
        if (hdrincl)
                ipc.protocol = IPPROTO_RAW;

        if (msg->msg_controllen) {
                err = ip_cmsg_send(sk, msg, &ipc, false);
                if (unlikely(err)) {
                        kfree(ipc.opt);
                        goto out;
                }
                if (ipc.opt)
                        free = 1;
        }

        saddr = ipc.addr;
        ipc.addr = daddr;

        if (!ipc.opt) {
                struct ip_options_rcu *inet_opt;

                rcu_read_lock();
                inet_opt = rcu_dereference(inet->inet_opt);
                if (inet_opt) {
                        memcpy(&opt_copy, inet_opt,
                               sizeof(*inet_opt) + inet_opt->opt.optlen);
                        ipc.opt = &opt_copy.opt;
                }
                rcu_read_unlock();
        }

        if (ipc.opt) {
                err = -EINVAL;
                /* Linux does not mangle headers on raw sockets,
                 * so that IP options + IP_HDRINCL is non-sense.
                 */
                if (hdrincl)
                        goto done;
                if (ipc.opt->opt.srr) {
                        if (!daddr)
                                goto done;
                        daddr = ipc.opt->opt.faddr;
                }
        }
        tos = get_rttos(&ipc, inet);
        scope = ip_sendmsg_scope(inet, &ipc, msg);

        uc_index = READ_ONCE(inet->uc_index);
        if (ipv4_is_multicast(daddr)) {
                if (!ipc.oif || netif_index_is_l3_master(sock_net(sk), ipc.oif))
                        ipc.oif = READ_ONCE(inet->mc_index);
                if (!saddr)
                        saddr = READ_ONCE(inet->mc_addr);
        } else if (!ipc.oif) {
                ipc.oif = uc_index;
        } else if (ipv4_is_lbcast(daddr) && uc_index) {
                /* oif is set, packet is to local broadcast
                 * and uc_index is set. oif is most likely set
                 * by sk_bound_dev_if. If uc_index != oif check if the
                 * oif is an L3 master and uc_index is an L3 slave.
                 * If so, we want to allow the send using the uc_index.
                 */
                if (ipc.oif != uc_index &&
                    ipc.oif == l3mdev_master_ifindex_by_index(sock_net(sk),
                                                              uc_index)) {
                        ipc.oif = uc_index;
                }
        }

        flowi4_init_output(&fl4, ipc.oif, ipc.sockc.mark, tos, scope,
                           hdrincl ? ipc.protocol : sk->sk_protocol,
                           inet_sk_flowi_flags(sk) |
                            (hdrincl ? FLOWI_FLAG_KNOWN_NH : 0),
                           daddr, saddr, 0, 0, sk->sk_uid);

        fl4.fl4_icmp_type = 0;
        fl4.fl4_icmp_code = 0;

        if (!hdrincl) {
                rfv.msg = msg;
                rfv.hlen = 0;

                err = raw_probe_proto_opt(&rfv, &fl4);
                if (err)
                        goto done;
        }

        security_sk_classify_flow(sk, flowi4_to_flowi_common(&fl4));
        rt = ip_route_output_flow(net, &fl4, sk);
        if (IS_ERR(rt)) {
                err = PTR_ERR(rt);
                rt = NULL;
                goto done;
        }

        err = -EACCES;
        if (rt->rt_flags & RTCF_BROADCAST && !sock_flag(sk, SOCK_BROADCAST))
                goto done;

        if (msg->msg_flags & MSG_CONFIRM)
                goto do_confirm;
back_from_confirm:

        if (hdrincl)
                err = raw_send_hdrinc(sk, &fl4, msg, len,
                                      &rt, msg->msg_flags, &ipc.sockc);

         else {
                if (!ipc.addr)
                        ipc.addr = fl4.daddr;
                lock_sock(sk);
                err = ip_append_data(sk, &fl4, raw_getfrag,
                                     &rfv, len, 0,
                                     &ipc, &rt, msg->msg_flags);
                if (err)
                        ip_flush_pending_frames(sk);
                else if (!(msg->msg_flags & MSG_MORE)) {
                        err = ip_push_pending_frames(sk, &fl4);
                        if (err == -ENOBUFS && !inet_test_bit(RECVERR, sk))
                                err = 0;
                }
                release_sock(sk);
        }
done:
        if (free)
                kfree(ipc.opt);
        ip_rt_put(rt);

out:
        if (err < 0)
                return err;
        return len;

do_confirm:
        if (msg->msg_flags & MSG_PROBE)
                dst_confirm_neigh(&rt->dst, &fl4.daddr);
        if (!(msg->msg_flags & MSG_PROBE) || len)
                goto back_from_confirm;
        err = 0;
        goto done;
}

static void raw_close(struct sock *sk, long timeout)
{
        /*
         * Raw sockets may have direct kernel references. Kill them.
         */
        ip_ra_control(sk, 0, NULL);

        sk_common_release(sk);
}

static void raw_destroy(struct sock *sk)
{
        lock_sock(sk);
        ip_flush_pending_frames(sk);
        release_sock(sk);
}

/* This gets rid of all the nasties in af_inet. -DaveM */
static int raw_bind(struct sock *sk, struct sockaddr *uaddr, int addr_len)
{
        struct inet_sock *inet = inet_sk(sk);
        struct sockaddr_in *addr = (struct sockaddr_in *) uaddr;
        struct net *net = sock_net(sk);
        u32 tb_id = RT_TABLE_LOCAL;
        int ret = -EINVAL;
        int chk_addr_ret;

        lock_sock(sk);
        if (sk->sk_state != TCP_CLOSE || addr_len < sizeof(struct sockaddr_in))
                goto out;

        if (sk->sk_bound_dev_if)
                tb_id = l3mdev_fib_table_by_index(net,
                                                  sk->sk_bound_dev_if) ? : tb_id;

        chk_addr_ret = inet_addr_type_table(net, addr->sin_addr.s_addr, tb_id);

        ret = -EADDRNOTAVAIL;
        if (!inet_addr_valid_or_nonlocal(net, inet, addr->sin_addr.s_addr,
                                         chk_addr_ret))
                goto out;

        inet->inet_rcv_saddr = inet->inet_saddr = addr->sin_addr.s_addr;
        if (chk_addr_ret == RTN_MULTICAST || chk_addr_ret == RTN_BROADCAST)
                inet->inet_saddr = 0;  /* Use device */
        sk_dst_reset(sk);
        ret = 0;
out:
        release_sock(sk);
        return ret;
}

/*
 *        This should be easy, if there is something there
 *        we return it, otherwise we block.
 */

static int raw_recvmsg(struct sock *sk, struct msghdr *msg, size_t len,
                       int flags, int *addr_len)
{
        struct inet_sock *inet = inet_sk(sk);
        size_t copied = 0;
        int err = -EOPNOTSUPP;
        DECLARE_SOCKADDR(struct sockaddr_in *, sin, msg->msg_name);
        struct sk_buff *skb;

        if (flags & MSG_OOB)
                goto out;

        if (flags & MSG_ERRQUEUE) {
                err = ip_recv_error(sk, msg, len, addr_len);
                goto out;
        }

        skb = skb_recv_datagram(sk, flags, &err);
        if (!skb)
                goto out;

        copied = skb->len;
        if (len < copied) {
                msg->msg_flags |= MSG_TRUNC;
                copied = len;
        }

        err = skb_copy_datagram_msg(skb, 0, msg, copied);
        if (err)
                goto done;

        sock_recv_cmsgs(msg, sk, skb);

        /* Copy the address. */
        if (sin) {
                sin->sin_family = AF_INET;
                sin->sin_addr.s_addr = ip_hdr(skb)->saddr;
                sin->sin_port = 0;
                memset(&sin->sin_zero, 0, sizeof(sin->sin_zero));
                *addr_len = sizeof(*sin);
        }
        if (inet_cmsg_flags(inet))
                ip_cmsg_recv(msg, skb);
        if (flags & MSG_TRUNC)
                copied = skb->len;
done:
        skb_free_datagram(sk, skb);
out:
        if (err)
                return err;
        return copied;
}

static int raw_sk_init(struct sock *sk)
{
        struct raw_sock *rp = raw_sk(sk);

        if (inet_sk(sk)->inet_num == IPPROTO_ICMP)
                memset(&rp->filter, 0, sizeof(rp->filter));
        return 0;
}

static int raw_seticmpfilter(struct sock *sk, sockptr_t optval, int optlen)
{
        if (optlen > sizeof(struct icmp_filter))
                optlen = sizeof(struct icmp_filter);
        if (copy_from_sockptr(&raw_sk(sk)->filter, optval, optlen))
                return -EFAULT;
        return 0;
}

static int raw_geticmpfilter(struct sock *sk, char __user *optval, int __user *optlen)
{
        int len, ret = -EFAULT;

        if (get_user(len, optlen))
                goto out;
        ret = -EINVAL;
        if (len < 0)
                goto out;
        if (len > sizeof(struct icmp_filter))
                len = sizeof(struct icmp_filter);
        ret = -EFAULT;
        if (put_user(len, optlen) ||
            copy_to_user(optval, &raw_sk(sk)->filter, len))
                goto out;
        ret = 0;
out:        return ret;
}

static int do_raw_setsockopt(struct sock *sk, int optname,
                             sockptr_t optval, unsigned int optlen)
{
        if (optname == ICMP_FILTER) {
                if (inet_sk(sk)->inet_num != IPPROTO_ICMP)
                        return -EOPNOTSUPP;
                else
                        return raw_seticmpfilter(sk, optval, optlen);
        }
        return -ENOPROTOOPT;
}

static int raw_setsockopt(struct sock *sk, int level, int optname,
                          sockptr_t optval, unsigned int optlen)
{
        if (level != SOL_RAW)
                return ip_setsockopt(sk, level, optname, optval, optlen);
        return do_raw_setsockopt(sk, optname, optval, optlen);
}

static int do_raw_getsockopt(struct sock *sk, int optname,
                             char __user *optval, int __user *optlen)
{
        if (optname == ICMP_FILTER) {
                if (inet_sk(sk)->inet_num != IPPROTO_ICMP)
                        return -EOPNOTSUPP;
                else
                        return raw_geticmpfilter(sk, optval, optlen);
        }
        return -ENOPROTOOPT;
}

static int raw_getsockopt(struct sock *sk, int level, int optname,
                          char __user *optval, int __user *optlen)
{
        if (level != SOL_RAW)
                return ip_getsockopt(sk, level, optname, optval, optlen);
        return do_raw_getsockopt(sk, optname, optval, optlen);
}

static int raw_ioctl(struct sock *sk, int cmd, int *karg)
{
        switch (cmd) {
        case SIOCOUTQ: {
                *karg = sk_wmem_alloc_get(sk);
                return 0;
        }
        case SIOCINQ: {
                struct sk_buff *skb;

                spin_lock_bh(&sk->sk_receive_queue.lock);
                skb = skb_peek(&sk->sk_receive_queue);
                if (skb)
                        *karg = skb->len;
                else
                        *karg = 0;
                spin_unlock_bh(&sk->sk_receive_queue.lock);
                return 0;
        }

        default:
#ifdef CONFIG_IP_MROUTE
                return ipmr_ioctl(sk, cmd, karg);
#else
                return -ENOIOCTLCMD;
#endif
        }
}

#ifdef CONFIG_COMPAT
static int compat_raw_ioctl(struct sock *sk, unsigned int cmd, unsigned long arg)
{
        switch (cmd) {
        case SIOCOUTQ:
        case SIOCINQ:
                return -ENOIOCTLCMD;
        default:
#ifdef CONFIG_IP_MROUTE
                return ipmr_compat_ioctl(sk, cmd, compat_ptr(arg));
#else
                return -ENOIOCTLCMD;
#endif
        }
}
#endif

int raw_abort(struct sock *sk, int err)
{
        lock_sock(sk);

        sk->sk_err = err;
        sk_error_report(sk);
        __udp_disconnect(sk, 0);

        release_sock(sk);

        return 0;
}
EXPORT_SYMBOL_GPL(raw_abort);

struct proto raw_prot = {
        .name                   = "RAW",
        .owner                   = THIS_MODULE,
        .close                   = raw_close,
        .destroy           = raw_destroy,
        .connect           = ip4_datagram_connect,
        .disconnect           = __udp_disconnect,
        .ioctl                   = raw_ioctl,
        .init                   = raw_sk_init,
        .setsockopt           = raw_setsockopt,
        .getsockopt           = raw_getsockopt,
        .sendmsg           = raw_sendmsg,
        .recvmsg           = raw_recvmsg,
        .bind                   = raw_bind,
        .backlog_rcv           = raw_rcv_skb,
        .release_cb           = ip4_datagram_release_cb,
        .hash                   = raw_hash_sk,
        .unhash                   = raw_unhash_sk,
        .obj_size           = sizeof(struct raw_sock),
        .useroffset           = offsetof(struct raw_sock, filter),
        .usersize           = sizeof_field(struct raw_sock, filter),
        .h.raw_hash           = &raw_v4_hashinfo,
#ifdef CONFIG_COMPAT
        .compat_ioctl           = compat_raw_ioctl,
#endif
        .diag_destroy           = raw_abort,
};

#ifdef CONFIG_PROC_FS
static struct sock *raw_get_first(struct seq_file *seq, int bucket)
{
        struct raw_hashinfo *h = pde_data(file_inode(seq->file));
        struct raw_iter_state *state = raw_seq_private(seq);
        struct hlist_head *hlist;
        struct sock *sk;

        for (state->bucket = bucket; state->bucket < RAW_HTABLE_SIZE;
                        ++state->bucket) {
                hlist = &h->ht[state->bucket];
                sk_for_each(sk, hlist) {
                        if (sock_net(sk) == seq_file_net(seq))
                                return sk;
                }
        }
        return NULL;
}

static struct sock *raw_get_next(struct seq_file *seq, struct sock *sk)
{
        struct raw_iter_state *state = raw_seq_private(seq);

        do {
                sk = sk_next(sk);
        } while (sk && sock_net(sk) != seq_file_net(seq));

        if (!sk)
                return raw_get_first(seq, state->bucket + 1);
        return sk;
}

static struct sock *raw_get_idx(struct seq_file *seq, loff_t pos)
{
        struct sock *sk = raw_get_first(seq, 0);

        if (sk)
                while (pos && (sk = raw_get_next(seq, sk)) != NULL)
                        --pos;
        return pos ? NULL : sk;
}

void *raw_seq_start(struct seq_file *seq, loff_t *pos)
        __acquires(&h->lock)
{
        struct raw_hashinfo *h = pde_data(file_inode(seq->file));

        spin_lock(&h->lock);

        return *pos ? raw_get_idx(seq, *pos - 1) : SEQ_START_TOKEN;
}
EXPORT_SYMBOL_GPL(raw_seq_start);

void *raw_seq_next(struct seq_file *seq, void *v, loff_t *pos)
{
        struct sock *sk;

        if (v == SEQ_START_TOKEN)
                sk = raw_get_first(seq, 0);
        else
                sk = raw_get_next(seq, v);
        ++*pos;
        return sk;
}
EXPORT_SYMBOL_GPL(raw_seq_next);

void raw_seq_stop(struct seq_file *seq, void *v)
        __releases(&h->lock)
{
        struct raw_hashinfo *h = pde_data(file_inode(seq->file));

        spin_unlock(&h->lock);
}
EXPORT_SYMBOL_GPL(raw_seq_stop);

static void raw_sock_seq_show(struct seq_file *seq, struct sock *sp, int i)
{
        struct inet_sock *inet = inet_sk(sp);
        __be32 dest = inet->inet_daddr,
               src = inet->inet_rcv_saddr;
        __u16 destp = 0,
              srcp  = inet->inet_num;

        seq_printf(seq, "%4d: %08X:%04X %08X:%04X"
                " %02X %08X:%08X %02X:%08lX %08X %5u %8d %lu %d %pK %u\n",
                i, src, srcp, dest, destp, sp->sk_state,
                sk_wmem_alloc_get(sp),
                sk_rmem_alloc_get(sp),
                0, 0L, 0,
                from_kuid_munged(seq_user_ns(seq), sock_i_uid(sp)),
                0, sock_i_ino(sp),
                refcount_read(&sp->sk_refcnt), sp, atomic_read(&sp->sk_drops));
}

static int raw_seq_show(struct seq_file *seq, void *v)
{
        if (v == SEQ_START_TOKEN)
                seq_printf(seq, "  sl  local_address rem_address   st tx_queue "
                                "rx_queue tr tm->when retrnsmt   uid  timeout "
                                "inode ref pointer drops\n");
        else
                raw_sock_seq_show(seq, v, raw_seq_private(seq)->bucket);
        return 0;
}

static const struct seq_operations raw_seq_ops = {
        .start = raw_seq_start,
        .next  = raw_seq_next,
        .stop  = raw_seq_stop,
        .show  = raw_seq_show,
};

static __net_init int raw_init_net(struct net *net)
{
        if (!proc_create_net_data("raw", 0444, net->proc_net, &raw_seq_ops,
                        sizeof(struct raw_iter_state), &raw_v4_hashinfo))
                return -ENOMEM;

        return 0;
}

static __net_exit void raw_exit_net(struct net *net)
{
        remove_proc_entry("raw", net->proc_net);
}

static __net_initdata struct pernet_operations raw_net_ops = {
        .init = raw_init_net,
        .exit = raw_exit_net,
};

int __init raw_proc_init(void)
{

        return register_pernet_subsys(&raw_net_ops);
}

void __init raw_proc_exit(void)
{
        unregister_pernet_subsys(&raw_net_ops);
}
#endif /* CONFIG_PROC_FS */

static void raw_sysctl_init_net(struct net *net)
{
#ifdef CONFIG_NET_L3_MASTER_DEV
        net->ipv4.sysctl_raw_l3mdev_accept = 1;
#endif
}

static int __net_init raw_sysctl_init(struct net *net)
{
        raw_sysctl_init_net(net);
        return 0;
}

static struct pernet_operations __net_initdata raw_sysctl_ops = {
        .init        = raw_sysctl_init,
};

void __init raw_init(void)
{
        raw_sysctl_init_net(&init_net);
        if (register_pernet_subsys(&raw_sysctl_ops))
                panic("RAW: failed to init sysctl parameters.\n");
}



















































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
/* SPDX-License-Identifier: GPL-2.0 */
/*
 * PHY device list allow maintaining a list of PHY devices that are
 * part of a netdevice's link topology. PHYs can for example be chained,
 * as is the case when using a PHY that exposes an SFP module, on which an
 * SFP transceiver that embeds a PHY is connected.
 *
 * This list can then be used by userspace to leverage individual PHY
 * capabilities.
 */
#ifndef __PHY_LINK_TOPOLOGY_H
#define __PHY_LINK_TOPOLOGY_H

#include <linux/ethtool.h>
#include <linux/netdevice.h>

struct xarray;
struct phy_device;
struct sfp_bus;

struct phy_link_topology {
        struct xarray phys;
        u32 next_phy_index;
};

struct phy_device_node {
        enum phy_upstream upstream_type;

        union {
                struct net_device        *netdev;
                struct phy_device        *phydev;
        } upstream;

        struct sfp_bus *parent_sfp_bus;

        struct phy_device *phy;
};

#if IS_ENABLED(CONFIG_PHYLIB)
int phy_link_topo_add_phy(struct net_device *dev,
                          struct phy_device *phy,
                          enum phy_upstream upt, void *upstream);

void phy_link_topo_del_phy(struct net_device *dev, struct phy_device *phy);

static inline struct phy_device *
phy_link_topo_get_phy(struct net_device *dev, u32 phyindex)
{
        struct phy_link_topology *topo = dev->link_topo;
        struct phy_device_node *pdn;

        if (!topo)
                return NULL;

        pdn = xa_load(&topo->phys, phyindex);
        if (pdn)
                return pdn->phy;

        return NULL;
}

#else
static inline int phy_link_topo_add_phy(struct net_device *dev,
                                        struct phy_device *phy,
                                        enum phy_upstream upt, void *upstream)
{
        return 0;
}

static inline void phy_link_topo_del_phy(struct net_device *dev,
                                         struct phy_device *phy)
{
}

static inline struct phy_device *
phy_link_topo_get_phy(struct net_device *dev, u32 phyindex)
{
        return NULL;
}
#endif

#endif /* __PHY_LINK_TOPOLOGY_H */





































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __LINUX_BOOTMEM_INFO_H
#define __LINUX_BOOTMEM_INFO_H

#include <linux/mm.h>
#include <linux/kmemleak.h>

/*
 * Types for free bootmem stored in page->lru.next. These have to be in
 * some random range in unsigned long space for debugging purposes.
 */
enum {
        MEMORY_HOTPLUG_MIN_BOOTMEM_TYPE = 12,
        SECTION_INFO = MEMORY_HOTPLUG_MIN_BOOTMEM_TYPE,
        MIX_SECTION_INFO,
        NODE_INFO,
        MEMORY_HOTPLUG_MAX_BOOTMEM_TYPE = NODE_INFO,
};

#ifdef CONFIG_HAVE_BOOTMEM_INFO_NODE
void __init register_page_bootmem_info_node(struct pglist_data *pgdat);

void get_page_bootmem(unsigned long info, struct page *page,
                      unsigned long type);
void put_page_bootmem(struct page *page);

/*
 * Any memory allocated via the memblock allocator and not via the
 * buddy will be marked reserved already in the memmap. For those
 * pages, we can call this function to free it to buddy allocator.
 */
static inline void free_bootmem_page(struct page *page)
{
        unsigned long magic = page->index;

        /*
         * The reserve_bootmem_region sets the reserved flag on bootmem
         * pages.
         */
        VM_BUG_ON_PAGE(page_ref_count(page) != 2, page);

        if (magic == SECTION_INFO || magic == MIX_SECTION_INFO)
                put_page_bootmem(page);
        else
                VM_BUG_ON_PAGE(1, page);
}
#else
static inline void register_page_bootmem_info_node(struct pglist_data *pgdat)
{
}

static inline void put_page_bootmem(struct page *page)
{
}

static inline void get_page_bootmem(unsigned long info, struct page *page,
                                    unsigned long type)
{
}

static inline void free_bootmem_page(struct page *page)
{
        kmemleak_free_part_phys(PFN_PHYS(page_to_pfn(page)), PAGE_SIZE);
        free_reserved_page(page);
}
#endif

#endif /* __LINUX_BOOTMEM_INFO_H */














  474 



  474 
































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
// SPDX-License-Identifier: GPL-2.0
/*
 * Device physical location support
 *
 * Author: Won Chung <wonchung@google.com>
 */

#include <linux/acpi.h>
#include <linux/sysfs.h>

#include "physical_location.h"

bool dev_add_physical_location(struct device *dev)
{
        struct acpi_pld_info *pld;
        acpi_status status;

        if (!has_acpi_companion(dev))
                return false;

        status = acpi_get_physical_device_location(ACPI_HANDLE(dev), &pld);
        if (ACPI_FAILURE(status))
                return false;

        dev->physical_location =
                kzalloc(sizeof(*dev->physical_location), GFP_KERNEL);
        if (!dev->physical_location) {
                ACPI_FREE(pld);
                return false;
        }

        dev->physical_location->panel = pld->panel;
        dev->physical_location->vertical_position = pld->vertical_position;
        dev->physical_location->horizontal_position = pld->horizontal_position;
        dev->physical_location->dock = pld->dock;
        dev->physical_location->lid = pld->lid;

        ACPI_FREE(pld);
        return true;
}

static ssize_t panel_show(struct device *dev, struct device_attribute *attr,
        char *buf)
{
        const char *panel;

        switch (dev->physical_location->panel) {
        case DEVICE_PANEL_TOP:
                panel = "top";
                break;
        case DEVICE_PANEL_BOTTOM:
                panel = "bottom";
                break;
        case DEVICE_PANEL_LEFT:
                panel = "left";
                break;
        case DEVICE_PANEL_RIGHT:
                panel = "right";
                break;
        case DEVICE_PANEL_FRONT:
                panel = "front";
                break;
        case DEVICE_PANEL_BACK:
                panel = "back";
                break;
        default:
                panel = "unknown";
        }
        return sysfs_emit(buf, "%s\n", panel);
}
static DEVICE_ATTR_RO(panel);

static ssize_t vertical_position_show(struct device *dev,
        struct device_attribute *attr, char *buf)
{
        const char *vertical_position;

        switch (dev->physical_location->vertical_position) {
        case DEVICE_VERT_POS_UPPER:
                vertical_position = "upper";
                break;
        case DEVICE_VERT_POS_CENTER:
                vertical_position = "center";
                break;
        case DEVICE_VERT_POS_LOWER:
                vertical_position = "lower";
                break;
        default:
                vertical_position = "unknown";
        }
        return sysfs_emit(buf, "%s\n", vertical_position);
}
static DEVICE_ATTR_RO(vertical_position);

static ssize_t horizontal_position_show(struct device *dev,
        struct device_attribute *attr, char *buf)
{
        const char *horizontal_position;

        switch (dev->physical_location->horizontal_position) {
        case DEVICE_HORI_POS_LEFT:
                horizontal_position = "left";
                break;
        case DEVICE_HORI_POS_CENTER:
                horizontal_position = "center";
                break;
        case DEVICE_HORI_POS_RIGHT:
                horizontal_position = "right";
                break;
        default:
                horizontal_position = "unknown";
        }
        return sysfs_emit(buf, "%s\n", horizontal_position);
}
static DEVICE_ATTR_RO(horizontal_position);

static ssize_t dock_show(struct device *dev, struct device_attribute *attr,
        char *buf)
{
        return sysfs_emit(buf, "%s\n",
                dev->physical_location->dock ? "yes" : "no");
}
static DEVICE_ATTR_RO(dock);

static ssize_t lid_show(struct device *dev, struct device_attribute *attr,
        char *buf)
{
        return sysfs_emit(buf, "%s\n",
                dev->physical_location->lid ? "yes" : "no");
}
static DEVICE_ATTR_RO(lid);

static struct attribute *dev_attr_physical_location[] = {
        &dev_attr_panel.attr,
        &dev_attr_vertical_position.attr,
        &dev_attr_horizontal_position.attr,
        &dev_attr_dock.attr,
        &dev_attr_lid.attr,
        NULL,
};

const struct attribute_group dev_attr_physical_location_group = {
        .name = "physical_location",
        .attrs = dev_attr_physical_location,
};






















































































   20 
    1 








1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
 *  SR-IPv6 implementation
 *
 *  Author:
 *  David Lebrun <david.lebrun@uclouvain.be>
 */

#ifndef _NET_SEG6_H
#define _NET_SEG6_H

#include <linux/net.h>
#include <linux/ipv6.h>
#include <linux/seg6.h>
#include <linux/rhashtable-types.h>

static inline void update_csum_diff4(struct sk_buff *skb, __be32 from,
                                     __be32 to)
{
        __be32 diff[] = { ~from, to };

        skb->csum = ~csum_partial((char *)diff, sizeof(diff), ~skb->csum);
}

static inline void update_csum_diff16(struct sk_buff *skb, __be32 *from,
                                      __be32 *to)
{
        __be32 diff[] = {
                ~from[0], ~from[1], ~from[2], ~from[3],
                to[0], to[1], to[2], to[3],
        };

        skb->csum = ~csum_partial((char *)diff, sizeof(diff), ~skb->csum);
}

struct seg6_pernet_data {
        struct mutex lock;
        struct in6_addr __rcu *tun_src;
#ifdef CONFIG_IPV6_SEG6_HMAC
        struct rhashtable hmac_infos;
#endif
};

static inline struct seg6_pernet_data *seg6_pernet(struct net *net)
{
#if IS_ENABLED(CONFIG_IPV6)
        return net->ipv6.seg6_data;
#else
        return NULL;
#endif
}

extern int seg6_init(void);
extern void seg6_exit(void);
#ifdef CONFIG_IPV6_SEG6_LWTUNNEL
extern int seg6_iptunnel_init(void);
extern void seg6_iptunnel_exit(void);
extern int seg6_local_init(void);
extern void seg6_local_exit(void);
#else
static inline int seg6_iptunnel_init(void) { return 0; }
static inline void seg6_iptunnel_exit(void) {}
static inline int seg6_local_init(void) { return 0; }
static inline void seg6_local_exit(void) {}
#endif

extern bool seg6_validate_srh(struct ipv6_sr_hdr *srh, int len, bool reduced);
extern struct ipv6_sr_hdr *seg6_get_srh(struct sk_buff *skb, int flags);
extern void seg6_icmp_srh(struct sk_buff *skb, struct inet6_skb_parm *opt);
extern int seg6_do_srh_encap(struct sk_buff *skb, struct ipv6_sr_hdr *osrh,
                             int proto);
extern int seg6_do_srh_inline(struct sk_buff *skb, struct ipv6_sr_hdr *osrh);
extern int seg6_lookup_nexthop(struct sk_buff *skb, struct in6_addr *nhaddr,
                               u32 tbl_id);

/* If the packet which invoked an ICMP error contains an SRH return
 * the true destination address from within the SRH, otherwise use the
 * destination address in the IP header.
 */
static inline const struct in6_addr *seg6_get_daddr(struct sk_buff *skb,
                                                    struct inet6_skb_parm *opt)
{
        struct ipv6_sr_hdr *srh;

        if (opt->flags & IP6SKB_SEG6) {
                srh = (struct ipv6_sr_hdr *)(skb->data + opt->srhoff);
                return  &srh->segments[0];
        }

        return NULL;
}


#endif








































    4 



























    4 

























    4 













































    4 



    4 



    4 


























































































































































































































































































































































































































































































































































































    4 









    4 










    4 






    4 
















































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
// SPDX-License-Identifier: GPL-2.0-or-later
/* Maintain an RxRPC server socket to do AFS communications through
 *
 * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved.
 * Written by David Howells (dhowells@redhat.com)
 */

#include <linux/slab.h>
#include <linux/sched/signal.h>

#include <net/sock.h>
#include <net/af_rxrpc.h>
#include "internal.h"
#include "afs_cm.h"
#include "protocol_yfs.h"
#define RXRPC_TRACE_ONLY_DEFINE_ENUMS
#include <trace/events/rxrpc.h>

struct workqueue_struct *afs_async_calls;

static void afs_deferred_free_worker(struct work_struct *work);
static void afs_wake_up_call_waiter(struct sock *, struct rxrpc_call *, unsigned long);
static void afs_wake_up_async_call(struct sock *, struct rxrpc_call *, unsigned long);
static void afs_process_async_call(struct work_struct *);
static void afs_rx_new_call(struct sock *, struct rxrpc_call *, unsigned long);
static void afs_rx_discard_new_call(struct rxrpc_call *, unsigned long);
static int afs_deliver_cm_op_id(struct afs_call *);

/* asynchronous incoming call initial processing */
static const struct afs_call_type afs_RXCMxxxx = {
        .name                = "CB.xxxx",
        .deliver        = afs_deliver_cm_op_id,
};

/*
 * open an RxRPC socket and bind it to be a server for callback notifications
 * - the socket is left in blocking mode and non-blocking ops use MSG_DONTWAIT
 */
int afs_open_socket(struct afs_net *net)
{
        struct sockaddr_rxrpc srx;
        struct socket *socket;
        int ret;

        _enter("");

        ret = sock_create_kern(net->net, AF_RXRPC, SOCK_DGRAM, PF_INET6, &socket);
        if (ret < 0)
                goto error_1;

        socket->sk->sk_allocation = GFP_NOFS;

        /* bind the callback manager's address to make this a server socket */
        memset(&srx, 0, sizeof(srx));
        srx.srx_family                        = AF_RXRPC;
        srx.srx_service                        = CM_SERVICE;
        srx.transport_type                = SOCK_DGRAM;
        srx.transport_len                = sizeof(srx.transport.sin6);
        srx.transport.sin6.sin6_family        = AF_INET6;
        srx.transport.sin6.sin6_port        = htons(AFS_CM_PORT);

        ret = rxrpc_sock_set_min_security_level(socket->sk,
                                                RXRPC_SECURITY_ENCRYPT);
        if (ret < 0)
                goto error_2;

        ret = kernel_bind(socket, (struct sockaddr *) &srx, sizeof(srx));
        if (ret == -EADDRINUSE) {
                srx.transport.sin6.sin6_port = 0;
                ret = kernel_bind(socket, (struct sockaddr *) &srx, sizeof(srx));
        }
        if (ret < 0)
                goto error_2;

        srx.srx_service = YFS_CM_SERVICE;
        ret = kernel_bind(socket, (struct sockaddr *) &srx, sizeof(srx));
        if (ret < 0)
                goto error_2;

        /* Ideally, we'd turn on service upgrade here, but we can't because
         * OpenAFS is buggy and leaks the userStatus field from packet to
         * packet and between FS packets and CB packets - so if we try to do an
         * upgrade on an FS packet, OpenAFS will leak that into the CB packet
         * it sends back to us.
         */

        rxrpc_kernel_new_call_notification(socket, afs_rx_new_call,
                                           afs_rx_discard_new_call);

        ret = kernel_listen(socket, INT_MAX);
        if (ret < 0)
                goto error_2;

        net->socket = socket;
        afs_charge_preallocation(&net->charge_preallocation_work);
        _leave(" = 0");
        return 0;

error_2:
        sock_release(socket);
error_1:
        _leave(" = %d", ret);
        return ret;
}

/*
 * close the RxRPC socket AFS was using
 */
void afs_close_socket(struct afs_net *net)
{
        _enter("");

        kernel_listen(net->socket, 0);
        flush_workqueue(afs_async_calls);

        if (net->spare_incoming_call) {
                afs_put_call(net->spare_incoming_call);
                net->spare_incoming_call = NULL;
        }

        _debug("outstanding %u", atomic_read(&net->nr_outstanding_calls));
        wait_var_event(&net->nr_outstanding_calls,
                       !atomic_read(&net->nr_outstanding_calls));
        _debug("no outstanding calls");

        kernel_sock_shutdown(net->socket, SHUT_RDWR);
        flush_workqueue(afs_async_calls);
        sock_release(net->socket);

        _debug("dework");
        _leave("");
}

/*
 * Allocate a call.
 */
static struct afs_call *afs_alloc_call(struct afs_net *net,
                                       const struct afs_call_type *type,
                                       gfp_t gfp)
{
        struct afs_call *call;
        int o;

        call = kzalloc(sizeof(*call), gfp);
        if (!call)
                return NULL;

        call->type = type;
        call->net = net;
        call->debug_id = atomic_inc_return(&rxrpc_debug_id);
        refcount_set(&call->ref, 1);
        INIT_WORK(&call->async_work, afs_process_async_call);
        INIT_WORK(&call->free_work, afs_deferred_free_worker);
        init_waitqueue_head(&call->waitq);
        spin_lock_init(&call->state_lock);
        call->iter = &call->def_iter;

        o = atomic_inc_return(&net->nr_outstanding_calls);
        trace_afs_call(call->debug_id, afs_call_trace_alloc, 1, o,
                       __builtin_return_address(0));
        return call;
}

static void afs_free_call(struct afs_call *call)
{
        struct afs_net *net = call->net;
        int o;

        ASSERT(!work_pending(&call->async_work));

        rxrpc_kernel_put_peer(call->peer);

        if (call->rxcall) {
                rxrpc_kernel_shutdown_call(net->socket, call->rxcall);
                rxrpc_kernel_put_call(net->socket, call->rxcall);
                call->rxcall = NULL;
        }
        if (call->type->destructor)
                call->type->destructor(call);

        afs_unuse_server_notime(call->net, call->server, afs_server_trace_put_call);
        kfree(call->request);

        o = atomic_read(&net->nr_outstanding_calls);
        trace_afs_call(call->debug_id, afs_call_trace_free, 0, o,
                       __builtin_return_address(0));
        kfree(call);

        o = atomic_dec_return(&net->nr_outstanding_calls);
        if (o == 0)
                wake_up_var(&net->nr_outstanding_calls);
}

/*
 * Dispose of a reference on a call.
 */
void afs_put_call(struct afs_call *call)
{
        struct afs_net *net = call->net;
        unsigned int debug_id = call->debug_id;
        bool zero;
        int r, o;

        zero = __refcount_dec_and_test(&call->ref, &r);
        o = atomic_read(&net->nr_outstanding_calls);
        trace_afs_call(debug_id, afs_call_trace_put, r - 1, o,
                       __builtin_return_address(0));
        if (zero)
                afs_free_call(call);
}

static void afs_deferred_free_worker(struct work_struct *work)
{
        struct afs_call *call = container_of(work, struct afs_call, free_work);

        afs_free_call(call);
}

/*
 * Dispose of a reference on a call, deferring the cleanup to a workqueue
 * to avoid lock recursion.
 */
void afs_deferred_put_call(struct afs_call *call)
{
        struct afs_net *net = call->net;
        unsigned int debug_id = call->debug_id;
        bool zero;
        int r, o;

        zero = __refcount_dec_and_test(&call->ref, &r);
        o = atomic_read(&net->nr_outstanding_calls);
        trace_afs_call(debug_id, afs_call_trace_put, r - 1, o,
                       __builtin_return_address(0));
        if (zero)
                schedule_work(&call->free_work);
}

static struct afs_call *afs_get_call(struct afs_call *call,
                                     enum afs_call_trace why)
{
        int r;

        __refcount_inc(&call->ref, &r);

        trace_afs_call(call->debug_id, why, r + 1,
                       atomic_read(&call->net->nr_outstanding_calls),
                       __builtin_return_address(0));
        return call;
}

/*
 * Queue the call for actual work.
 */
static void afs_queue_call_work(struct afs_call *call)
{
        if (call->type->work) {
                INIT_WORK(&call->work, call->type->work);

                afs_get_call(call, afs_call_trace_work);
                if (!queue_work(afs_wq, &call->work))
                        afs_put_call(call);
        }
}

/*
 * allocate a call with flat request and reply buffers
 */
struct afs_call *afs_alloc_flat_call(struct afs_net *net,
                                     const struct afs_call_type *type,
                                     size_t request_size, size_t reply_max)
{
        struct afs_call *call;

        call = afs_alloc_call(net, type, GFP_NOFS);
        if (!call)
                goto nomem_call;

        if (request_size) {
                call->request_size = request_size;
                call->request = kmalloc(request_size, GFP_NOFS);
                if (!call->request)
                        goto nomem_free;
        }

        if (reply_max) {
                call->reply_max = reply_max;
                call->buffer = kmalloc(reply_max, GFP_NOFS);
                if (!call->buffer)
                        goto nomem_free;
        }

        afs_extract_to_buf(call, call->reply_max);
        call->operation_ID = type->op;
        init_waitqueue_head(&call->waitq);
        return call;

nomem_free:
        afs_put_call(call);
nomem_call:
        return NULL;
}

/*
 * clean up a call with flat buffer
 */
void afs_flat_call_destructor(struct afs_call *call)
{
        _enter("");

        kfree(call->request);
        call->request = NULL;
        kfree(call->buffer);
        call->buffer = NULL;
}

/*
 * Advance the AFS call state when the RxRPC call ends the transmit phase.
 */
static void afs_notify_end_request_tx(struct sock *sock,
                                      struct rxrpc_call *rxcall,
                                      unsigned long call_user_ID)
{
        struct afs_call *call = (struct afs_call *)call_user_ID;

        afs_set_call_state(call, AFS_CALL_CL_REQUESTING, AFS_CALL_CL_AWAIT_REPLY);
}

/*
 * Initiate a call and synchronously queue up the parameters for dispatch.  Any
 * error is stored into the call struct, which the caller must check for.
 */
void afs_make_call(struct afs_call *call, gfp_t gfp)
{
        struct rxrpc_call *rxcall;
        struct msghdr msg;
        struct kvec iov[1];
        size_t len;
        s64 tx_total_len;
        int ret;

        _enter(",{%pISp+%u},", rxrpc_kernel_remote_addr(call->peer), call->service_id);

        ASSERT(call->type != NULL);
        ASSERT(call->type->name != NULL);

        _debug("____MAKE %p{%s,%x} [%d]____",
               call, call->type->name, key_serial(call->key),
               atomic_read(&call->net->nr_outstanding_calls));

        trace_afs_make_call(call);

        /* Work out the length we're going to transmit.  This is awkward for
         * calls such as FS.StoreData where there's an extra injection of data
         * after the initial fixed part.
         */
        tx_total_len = call->request_size;
        if (call->write_iter)
                tx_total_len += iov_iter_count(call->write_iter);

        /* If the call is going to be asynchronous, we need an extra ref for
         * the call to hold itself so the caller need not hang on to its ref.
         */
        if (call->async) {
                afs_get_call(call, afs_call_trace_get);
                call->drop_ref = true;
        }

        /* create a call */
        rxcall = rxrpc_kernel_begin_call(call->net->socket, call->peer, call->key,
                                         (unsigned long)call,
                                         tx_total_len,
                                         call->max_lifespan,
                                         gfp,
                                         (call->async ?
                                          afs_wake_up_async_call :
                                          afs_wake_up_call_waiter),
                                         call->service_id,
                                         call->upgrade,
                                         (call->intr ? RXRPC_PREINTERRUPTIBLE :
                                          RXRPC_UNINTERRUPTIBLE),
                                         call->debug_id);
        if (IS_ERR(rxcall)) {
                ret = PTR_ERR(rxcall);
                call->error = ret;
                goto error_kill_call;
        }

        call->rxcall = rxcall;
        call->issue_time = ktime_get_real();

        /* send the request */
        iov[0].iov_base        = call->request;
        iov[0].iov_len        = call->request_size;

        msg.msg_name                = NULL;
        msg.msg_namelen                = 0;
        iov_iter_kvec(&msg.msg_iter, ITER_SOURCE, iov, 1, call->request_size);
        msg.msg_control                = NULL;
        msg.msg_controllen        = 0;
        msg.msg_flags                = MSG_WAITALL | (call->write_iter ? MSG_MORE : 0);

        ret = rxrpc_kernel_send_data(call->net->socket, rxcall,
                                     &msg, call->request_size,
                                     afs_notify_end_request_tx);
        if (ret < 0)
                goto error_do_abort;

        if (call->write_iter) {
                msg.msg_iter = *call->write_iter;
                msg.msg_flags &= ~MSG_MORE;
                trace_afs_send_data(call, &msg);

                ret = rxrpc_kernel_send_data(call->net->socket,
                                             call->rxcall, &msg,
                                             iov_iter_count(&msg.msg_iter),
                                             afs_notify_end_request_tx);
                *call->write_iter = msg.msg_iter;

                trace_afs_sent_data(call, &msg, ret);
                if (ret < 0)
                        goto error_do_abort;
        }

        /* Note that at this point, we may have received the reply or an abort
         * - and an asynchronous call may already have completed.
         *
         * afs_wait_for_call_to_complete(call)
         * must be called to synchronously clean up.
         */
        return;

error_do_abort:
        if (ret != -ECONNABORTED) {
                rxrpc_kernel_abort_call(call->net->socket, rxcall,
                                        RX_USER_ABORT, ret,
                                        afs_abort_send_data_error);
        } else {
                len = 0;
                iov_iter_kvec(&msg.msg_iter, ITER_DEST, NULL, 0, 0);
                rxrpc_kernel_recv_data(call->net->socket, rxcall,
                                       &msg.msg_iter, &len, false,
                                       &call->abort_code, &call->service_id);
                call->responded = true;
        }
        call->error = ret;
        trace_afs_call_done(call);
error_kill_call:
        if (call->type->done)
                call->type->done(call);

        /* We need to dispose of the extra ref we grabbed for an async call.
         * The call, however, might be queued on afs_async_calls and we need to
         * make sure we don't get any more notifications that might requeue it.
         */
        if (call->rxcall)
                rxrpc_kernel_shutdown_call(call->net->socket, call->rxcall);
        if (call->async) {
                if (cancel_work_sync(&call->async_work))
                        afs_put_call(call);
                afs_set_call_complete(call, ret, 0);
        }

        call->error = ret;
        call->state = AFS_CALL_COMPLETE;
        _leave(" = %d", ret);
}

/*
 * Log remote abort codes that indicate that we have a protocol disagreement
 * with the server.
 */
static void afs_log_error(struct afs_call *call, s32 remote_abort)
{
        static int max = 0;
        const char *msg;
        int m;

        switch (remote_abort) {
        case RX_EOF:                 msg = "unexpected EOF";        break;
        case RXGEN_CC_MARSHAL:         msg = "client marshalling";        break;
        case RXGEN_CC_UNMARSHAL: msg = "client unmarshalling";        break;
        case RXGEN_SS_MARSHAL:         msg = "server marshalling";        break;
        case RXGEN_SS_UNMARSHAL: msg = "server unmarshalling";        break;
        case RXGEN_DECODE:         msg = "opcode decode";                break;
        case RXGEN_SS_XDRFREE:         msg = "server XDR cleanup";        break;
        case RXGEN_CC_XDRFREE:         msg = "client XDR cleanup";        break;
        case -32:                 msg = "insufficient data";        break;
        default:
                return;
        }

        m = max;
        if (m < 3) {
                max = m + 1;
                pr_notice("kAFS: Peer reported %s failure on %s [%pISp]\n",
                          msg, call->type->name,
                          rxrpc_kernel_remote_addr(call->peer));
        }
}

/*
 * deliver messages to a call
 */
static void afs_deliver_to_call(struct afs_call *call)
{
        enum afs_call_state state;
        size_t len;
        u32 abort_code, remote_abort = 0;
        int ret;

        _enter("%s", call->type->name);

        while (state = READ_ONCE(call->state),
               state == AFS_CALL_CL_AWAIT_REPLY ||
               state == AFS_CALL_SV_AWAIT_OP_ID ||
               state == AFS_CALL_SV_AWAIT_REQUEST ||
               state == AFS_CALL_SV_AWAIT_ACK
               ) {
                if (state == AFS_CALL_SV_AWAIT_ACK) {
                        len = 0;
                        iov_iter_kvec(&call->def_iter, ITER_DEST, NULL, 0, 0);
                        ret = rxrpc_kernel_recv_data(call->net->socket,
                                                     call->rxcall, &call->def_iter,
                                                     &len, false, &remote_abort,
                                                     &call->service_id);
                        trace_afs_receive_data(call, &call->def_iter, false, ret);

                        if (ret == -EINPROGRESS || ret == -EAGAIN)
                                return;
                        if (ret < 0 || ret == 1) {
                                if (ret == 1)
                                        ret = 0;
                                goto call_complete;
                        }
                        return;
                }

                ret = call->type->deliver(call);
                state = READ_ONCE(call->state);
                if (ret == 0 && call->unmarshalling_error)
                        ret = -EBADMSG;
                switch (ret) {
                case 0:
                        call->responded = true;
                        afs_queue_call_work(call);
                        if (state == AFS_CALL_CL_PROC_REPLY) {
                                if (call->op)
                                        set_bit(AFS_SERVER_FL_MAY_HAVE_CB,
                                                &call->op->server->flags);
                                goto call_complete;
                        }
                        ASSERTCMP(state, >, AFS_CALL_CL_PROC_REPLY);
                        goto done;
                case -EINPROGRESS:
                case -EAGAIN:
                        goto out;
                case -ECONNABORTED:
                        ASSERTCMP(state, ==, AFS_CALL_COMPLETE);
                        call->responded = true;
                        afs_log_error(call, call->abort_code);
                        goto done;
                case -ENOTSUPP:
                        call->responded = true;
                        abort_code = RXGEN_OPCODE;
                        rxrpc_kernel_abort_call(call->net->socket, call->rxcall,
                                                abort_code, ret,
                                                afs_abort_op_not_supported);
                        goto local_abort;
                case -EIO:
                        pr_err("kAFS: Call %u in bad state %u\n",
                               call->debug_id, state);
                        fallthrough;
                case -ENODATA:
                case -EBADMSG:
                case -EMSGSIZE:
                case -ENOMEM:
                case -EFAULT:
                        abort_code = RXGEN_CC_UNMARSHAL;
                        if (state != AFS_CALL_CL_AWAIT_REPLY)
                                abort_code = RXGEN_SS_UNMARSHAL;
                        rxrpc_kernel_abort_call(call->net->socket, call->rxcall,
                                                abort_code, ret,
                                                afs_abort_unmarshal_error);
                        goto local_abort;
                default:
                        abort_code = RX_CALL_DEAD;
                        rxrpc_kernel_abort_call(call->net->socket, call->rxcall,
                                                abort_code, ret,
                                                afs_abort_general_error);
                        goto local_abort;
                }
        }

done:
        if (call->type->done)
                call->type->done(call);
out:
        _leave("");
        return;

local_abort:
        abort_code = 0;
call_complete:
        afs_set_call_complete(call, ret, remote_abort);
        state = AFS_CALL_COMPLETE;
        goto done;
}

/*
 * Wait synchronously for a call to complete.
 */
void afs_wait_for_call_to_complete(struct afs_call *call)
{
        bool rxrpc_complete = false;

        _enter("");

        if (!afs_check_call_state(call, AFS_CALL_COMPLETE)) {
                DECLARE_WAITQUEUE(myself, current);

                add_wait_queue(&call->waitq, &myself);
                for (;;) {
                        set_current_state(TASK_UNINTERRUPTIBLE);

                        /* deliver any messages that are in the queue */
                        if (!afs_check_call_state(call, AFS_CALL_COMPLETE) &&
                            call->need_attention) {
                                call->need_attention = false;
                                __set_current_state(TASK_RUNNING);
                                afs_deliver_to_call(call);
                                continue;
                        }

                        if (afs_check_call_state(call, AFS_CALL_COMPLETE))
                                break;

                        if (!rxrpc_kernel_check_life(call->net->socket, call->rxcall)) {
                                /* rxrpc terminated the call. */
                                rxrpc_complete = true;
                                break;
                        }

                        schedule();
                }

                remove_wait_queue(&call->waitq, &myself);
                __set_current_state(TASK_RUNNING);
        }

        if (!afs_check_call_state(call, AFS_CALL_COMPLETE)) {
                if (rxrpc_complete) {
                        afs_set_call_complete(call, call->error, call->abort_code);
                } else {
                        /* Kill off the call if it's still live. */
                        _debug("call interrupted");
                        if (rxrpc_kernel_abort_call(call->net->socket, call->rxcall,
                                                    RX_USER_ABORT, -EINTR,
                                                    afs_abort_interrupted))
                                afs_set_call_complete(call, -EINTR, 0);
                }
        }
}

/*
 * wake up a waiting call
 */
static void afs_wake_up_call_waiter(struct sock *sk, struct rxrpc_call *rxcall,
                                    unsigned long call_user_ID)
{
        struct afs_call *call = (struct afs_call *)call_user_ID;

        call->need_attention = true;
        wake_up(&call->waitq);
}

/*
 * Wake up an asynchronous call.  The caller is holding the call notify
 * spinlock around this, so we can't call afs_put_call().
 */
static void afs_wake_up_async_call(struct sock *sk, struct rxrpc_call *rxcall,
                                   unsigned long call_user_ID)
{
        struct afs_call *call = (struct afs_call *)call_user_ID;
        int r;

        trace_afs_notify_call(rxcall, call);
        call->need_attention = true;

        if (__refcount_inc_not_zero(&call->ref, &r)) {
                trace_afs_call(call->debug_id, afs_call_trace_wake, r + 1,
                               atomic_read(&call->net->nr_outstanding_calls),
                               __builtin_return_address(0));

                if (!queue_work(afs_async_calls, &call->async_work))
                        afs_deferred_put_call(call);
        }
}

/*
 * Perform I/O processing on an asynchronous call.  The work item carries a ref
 * to the call struct that we either need to release or to pass on.
 */
static void afs_process_async_call(struct work_struct *work)
{
        struct afs_call *call = container_of(work, struct afs_call, async_work);

        _enter("");

        if (call->state < AFS_CALL_COMPLETE && call->need_attention) {
                call->need_attention = false;
                afs_deliver_to_call(call);
        }

        afs_put_call(call);
        _leave("");
}

static void afs_rx_attach(struct rxrpc_call *rxcall, unsigned long user_call_ID)
{
        struct afs_call *call = (struct afs_call *)user_call_ID;

        call->rxcall = rxcall;
}

/*
 * Charge the incoming call preallocation.
 */
void afs_charge_preallocation(struct work_struct *work)
{
        struct afs_net *net =
                container_of(work, struct afs_net, charge_preallocation_work);
        struct afs_call *call = net->spare_incoming_call;

        for (;;) {
                if (!call) {
                        call = afs_alloc_call(net, &afs_RXCMxxxx, GFP_KERNEL);
                        if (!call)
                                break;

                        call->drop_ref = true;
                        call->async = true;
                        call->state = AFS_CALL_SV_AWAIT_OP_ID;
                        init_waitqueue_head(&call->waitq);
                        afs_extract_to_tmp(call);
                }

                if (rxrpc_kernel_charge_accept(net->socket,
                                               afs_wake_up_async_call,
                                               afs_rx_attach,
                                               (unsigned long)call,
                                               GFP_KERNEL,
                                               call->debug_id) < 0)
                        break;
                call = NULL;
        }
        net->spare_incoming_call = call;
}

/*
 * Discard a preallocated call when a socket is shut down.
 */
static void afs_rx_discard_new_call(struct rxrpc_call *rxcall,
                                    unsigned long user_call_ID)
{
        struct afs_call *call = (struct afs_call *)user_call_ID;

        call->rxcall = NULL;
        afs_put_call(call);
}

/*
 * Notification of an incoming call.
 */
static void afs_rx_new_call(struct sock *sk, struct rxrpc_call *rxcall,
                            unsigned long user_call_ID)
{
        struct afs_net *net = afs_sock2net(sk);

        queue_work(afs_wq, &net->charge_preallocation_work);
}

/*
 * Grab the operation ID from an incoming cache manager call.  The socket
 * buffer is discarded on error or if we don't yet have sufficient data.
 */
static int afs_deliver_cm_op_id(struct afs_call *call)
{
        int ret;

        _enter("{%zu}", iov_iter_count(call->iter));

        /* the operation ID forms the first four bytes of the request data */
        ret = afs_extract_data(call, true);
        if (ret < 0)
                return ret;

        call->operation_ID = ntohl(call->tmp);
        afs_set_call_state(call, AFS_CALL_SV_AWAIT_OP_ID, AFS_CALL_SV_AWAIT_REQUEST);

        /* ask the cache manager to route the call (it'll change the call type
         * if successful) */
        if (!afs_cm_incoming_call(call))
                return -ENOTSUPP;

        trace_afs_cb_call(call);

        /* pass responsibility for the remainer of this message off to the
         * cache manager op */
        return call->type->deliver(call);
}

/*
 * Advance the AFS call state when an RxRPC service call ends the transmit
 * phase.
 */
static void afs_notify_end_reply_tx(struct sock *sock,
                                    struct rxrpc_call *rxcall,
                                    unsigned long call_user_ID)
{
        struct afs_call *call = (struct afs_call *)call_user_ID;

        afs_set_call_state(call, AFS_CALL_SV_REPLYING, AFS_CALL_SV_AWAIT_ACK);
}

/*
 * send an empty reply
 */
void afs_send_empty_reply(struct afs_call *call)
{
        struct afs_net *net = call->net;
        struct msghdr msg;

        _enter("");

        rxrpc_kernel_set_tx_length(net->socket, call->rxcall, 0);

        msg.msg_name                = NULL;
        msg.msg_namelen                = 0;
        iov_iter_kvec(&msg.msg_iter, ITER_SOURCE, NULL, 0, 0);
        msg.msg_control                = NULL;
        msg.msg_controllen        = 0;
        msg.msg_flags                = 0;

        switch (rxrpc_kernel_send_data(net->socket, call->rxcall, &msg, 0,
                                       afs_notify_end_reply_tx)) {
        case 0:
                _leave(" [replied]");
                return;

        case -ENOMEM:
                _debug("oom");
                rxrpc_kernel_abort_call(net->socket, call->rxcall,
                                        RXGEN_SS_MARSHAL, -ENOMEM,
                                        afs_abort_oom);
                fallthrough;
        default:
                _leave(" [error]");
                return;
        }
}

/*
 * send a simple reply
 */
void afs_send_simple_reply(struct afs_call *call, const void *buf, size_t len)
{
        struct afs_net *net = call->net;
        struct msghdr msg;
        struct kvec iov[1];
        int n;

        _enter("");

        rxrpc_kernel_set_tx_length(net->socket, call->rxcall, len);

        iov[0].iov_base                = (void *) buf;
        iov[0].iov_len                = len;
        msg.msg_name                = NULL;
        msg.msg_namelen                = 0;
        iov_iter_kvec(&msg.msg_iter, ITER_SOURCE, iov, 1, len);
        msg.msg_control                = NULL;
        msg.msg_controllen        = 0;
        msg.msg_flags                = 0;

        n = rxrpc_kernel_send_data(net->socket, call->rxcall, &msg, len,
                                   afs_notify_end_reply_tx);
        if (n >= 0) {
                /* Success */
                _leave(" [replied]");
                return;
        }

        if (n == -ENOMEM) {
                _debug("oom");
                rxrpc_kernel_abort_call(net->socket, call->rxcall,
                                        RXGEN_SS_MARSHAL, -ENOMEM,
                                        afs_abort_oom);
        }
        _leave(" [error]");
}

/*
 * Extract a piece of data from the received data socket buffers.
 */
int afs_extract_data(struct afs_call *call, bool want_more)
{
        struct afs_net *net = call->net;
        struct iov_iter *iter = call->iter;
        enum afs_call_state state;
        u32 remote_abort = 0;
        int ret;

        _enter("{%s,%zu,%zu},%d",
               call->type->name, call->iov_len, iov_iter_count(iter), want_more);

        ret = rxrpc_kernel_recv_data(net->socket, call->rxcall, iter,
                                     &call->iov_len, want_more, &remote_abort,
                                     &call->service_id);
        trace_afs_receive_data(call, call->iter, want_more, ret);
        if (ret == 0 || ret == -EAGAIN)
                return ret;

        state = READ_ONCE(call->state);
        if (ret == 1) {
                switch (state) {
                case AFS_CALL_CL_AWAIT_REPLY:
                        afs_set_call_state(call, state, AFS_CALL_CL_PROC_REPLY);
                        break;
                case AFS_CALL_SV_AWAIT_REQUEST:
                        afs_set_call_state(call, state, AFS_CALL_SV_REPLYING);
                        break;
                case AFS_CALL_COMPLETE:
                        kdebug("prem complete %d", call->error);
                        return afs_io_error(call, afs_io_error_extract);
                default:
                        break;
                }
                return 0;
        }

        afs_set_call_complete(call, ret, remote_abort);
        return ret;
}

/*
 * Log protocol error production.
 */
noinline int afs_protocol_error(struct afs_call *call,
                                enum afs_eproto_cause cause)
{
        trace_afs_protocol_error(call, cause);
        if (call)
                call->unmarshalling_error = true;
        return -EBADMSG;
}

























    1 










1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _IP_SET_GETPORT_H
#define _IP_SET_GETPORT_H

#include <linux/skbuff.h>
#include <linux/types.h>
#include <uapi/linux/in.h>

extern bool ip_set_get_ip4_port(const struct sk_buff *skb, bool src,
                                __be16 *port, u8 *proto);

#if IS_ENABLED(CONFIG_IP6_NF_IPTABLES)
extern bool ip_set_get_ip6_port(const struct sk_buff *skb, bool src,
                                __be16 *port, u8 *proto);
#else
static inline bool ip_set_get_ip6_port(const struct sk_buff *skb, bool src,
                                       __be16 *port, u8 *proto)
{
        return false;
}
#endif

static inline bool ip_set_proto_with_ports(u8 proto)
{
        switch (proto) {
        case IPPROTO_TCP:
        case IPPROTO_SCTP:
        case IPPROTO_UDP:
        case IPPROTO_UDPLITE:
                return true;
        }
        return false;
}

#endif /*_IP_SET_GETPORT_H*/


































































































































































































   19 
    3 

    1 






























    3 


    3 
















































    1 

    1 

















   33 







    3 







    1 



















    6 


    4 



    4 















    1 















   27 

















    6 



   10 




































    1 











   36 












   35 



   36 














   36 






   28 
   16 







   36 
    5 
    1 








    5 
    4 



    5 














   16 
    8 








   13 
    8 











    6 
    4 
    5 



   10 




    9 




    1 









    1 








    1 










    1 




   10 








    5 











    1 


    1 









    1 





    1 
   13 

















































    5 
    9 






   36 




    6 



   12 



   36 





   10 


    2 
    9 









    9 







    6 






    3 









    1 

    1 








   33 

    3 






   27 


   27 
   27 
   25 
    2 




















    2 







    4 











    2 
    2 
    1 
    3 






    1 

    3 






































    1 

    1 







    5 
    1 



    1 


    1 















    3 


    1 









































    1 








    2 






   21 










    3 
   18 


   16 
    3 




   14 
    4 




   17 





    7 






   10 


   13 

   13 














    7 
    7 

   13 
    1 

   13 
    1 

   12 
    2 

   11 
    2 

   12 



   13 
    1 

   12 
    2 

   11 
    3 

   10 
    4 





    1 

   13 



   17 









   22 





    1 




   14 
    7 




   31 



   28 
    2 






   20 









   18 






































   20 











































    2 




   18 







   20 





   16 


    2 
    2 
    1 
    3 
    4 







   20 








    1 


    1 










    1 


    1 




    2 





    4 




    1 

    1 










































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
// SPDX-License-Identifier: GPL-2.0-only
/*
 * net/sched/sch_netem.c        Network emulator
 *
 *                  Many of the algorithms and ideas for this came from
 *                NIST Net which is not copyrighted.
 *
 * Authors:        Stephen Hemminger <shemminger@osdl.org>
 *                Catalin(ux aka Dino) BOIE <catab at umbrella dot ro>
 */

#include <linux/mm.h>
#include <linux/module.h>
#include <linux/slab.h>
#include <linux/types.h>
#include <linux/kernel.h>
#include <linux/errno.h>
#include <linux/skbuff.h>
#include <linux/vmalloc.h>
#include <linux/prandom.h>
#include <linux/rtnetlink.h>
#include <linux/reciprocal_div.h>
#include <linux/rbtree.h>

#include <net/gso.h>
#include <net/netlink.h>
#include <net/pkt_sched.h>
#include <net/inet_ecn.h>

#define VERSION "1.3"

/*        Network Emulation Queuing algorithm.
        ====================================

        Sources: [1] Mark Carson, Darrin Santay, "NIST Net - A Linux-based
                 Network Emulation Tool
                 [2] Luigi Rizzo, DummyNet for FreeBSD

         ----------------------------------------------------------------

         This started out as a simple way to delay outgoing packets to
         test TCP but has grown to include most of the functionality
         of a full blown network emulator like NISTnet. It can delay
         packets and add random jitter (and correlation). The random
         distribution can be loaded from a table as well to provide
         normal, Pareto, or experimental curves. Packet loss,
         duplication, and reordering can also be emulated.

         This qdisc does not do classification that can be handled in
         layering other disciplines.  It does not need to do bandwidth
         control either since that can be handled by using token
         bucket or other rate control.

     Correlated Loss Generator models

        Added generation of correlated loss according to the
        "Gilbert-Elliot" model, a 4-state markov model.

        References:
        [1] NetemCLG Home http://netgroup.uniroma2.it/NetemCLG
        [2] S. Salsano, F. Ludovici, A. Ordine, "Definition of a general
        and intuitive loss model for packet networks and its implementation
        in the Netem module in the Linux kernel", available in [1]

        Authors: Stefano Salsano <stefano.salsano at uniroma2.it
                 Fabio Ludovici <fabio.ludovici at yahoo.it>
*/

struct disttable {
        u32  size;
        s16 table[] __counted_by(size);
};

struct netem_sched_data {
        /* internal t(ime)fifo qdisc uses t_root and sch->limit */
        struct rb_root t_root;

        /* a linear queue; reduces rbtree rebalancing when jitter is low */
        struct sk_buff        *t_head;
        struct sk_buff        *t_tail;

        /* optional qdisc for classful handling (NULL at netem init) */
        struct Qdisc        *qdisc;

        struct qdisc_watchdog watchdog;

        s64 latency;
        s64 jitter;

        u32 loss;
        u32 ecn;
        u32 limit;
        u32 counter;
        u32 gap;
        u32 duplicate;
        u32 reorder;
        u32 corrupt;
        u64 rate;
        s32 packet_overhead;
        u32 cell_size;
        struct reciprocal_value cell_size_reciprocal;
        s32 cell_overhead;

        struct crndstate {
                u32 last;
                u32 rho;
        } delay_cor, loss_cor, dup_cor, reorder_cor, corrupt_cor;

        struct prng  {
                u64 seed;
                struct rnd_state prng_state;
        } prng;

        struct disttable *delay_dist;

        enum  {
                CLG_RANDOM,
                CLG_4_STATES,
                CLG_GILB_ELL,
        } loss_model;

        enum {
                TX_IN_GAP_PERIOD = 1,
                TX_IN_BURST_PERIOD,
                LOST_IN_GAP_PERIOD,
                LOST_IN_BURST_PERIOD,
        } _4_state_model;

        enum {
                GOOD_STATE = 1,
                BAD_STATE,
        } GE_state_model;

        /* Correlated Loss Generation models */
        struct clgstate {
                /* state of the Markov chain */
                u8 state;

                /* 4-states and Gilbert-Elliot models */
                u32 a1;        /* p13 for 4-states or p for GE */
                u32 a2;        /* p31 for 4-states or r for GE */
                u32 a3;        /* p32 for 4-states or h for GE */
                u32 a4;        /* p14 for 4-states or 1-k for GE */
                u32 a5; /* p23 used only in 4-states */
        } clg;

        struct tc_netem_slot slot_config;
        struct slotstate {
                u64 slot_next;
                s32 packets_left;
                s32 bytes_left;
        } slot;

        struct disttable *slot_dist;
};

/* Time stamp put into socket buffer control block
 * Only valid when skbs are in our internal t(ime)fifo queue.
 *
 * As skb->rbnode uses same storage than skb->next, skb->prev and skb->tstamp,
 * and skb->next & skb->prev are scratch space for a qdisc,
 * we save skb->tstamp value in skb->cb[] before destroying it.
 */
struct netem_skb_cb {
        u64                time_to_send;
};

static inline struct netem_skb_cb *netem_skb_cb(struct sk_buff *skb)
{
        /* we assume we can use skb next/prev/tstamp as storage for rb_node */
        qdisc_cb_private_validate(skb, sizeof(struct netem_skb_cb));
        return (struct netem_skb_cb *)qdisc_skb_cb(skb)->data;
}

/* init_crandom - initialize correlated random number generator
 * Use entropy source for initial seed.
 */
static void init_crandom(struct crndstate *state, unsigned long rho)
{
        state->rho = rho;
        state->last = get_random_u32();
}

/* get_crandom - correlated random number generator
 * Next number depends on last value.
 * rho is scaled to avoid floating point.
 */
static u32 get_crandom(struct crndstate *state, struct prng *p)
{
        u64 value, rho;
        unsigned long answer;
        struct rnd_state *s = &p->prng_state;

        if (!state || state->rho == 0)        /* no correlation */
                return prandom_u32_state(s);

        value = prandom_u32_state(s);
        rho = (u64)state->rho + 1;
        answer = (value * ((1ull<<32) - rho) + state->last * rho) >> 32;
        state->last = answer;
        return answer;
}

/* loss_4state - 4-state model loss generator
 * Generates losses according to the 4-state Markov chain adopted in
 * the GI (General and Intuitive) loss model.
 */
static bool loss_4state(struct netem_sched_data *q)
{
        struct clgstate *clg = &q->clg;
        u32 rnd = prandom_u32_state(&q->prng.prng_state);

        /*
         * Makes a comparison between rnd and the transition
         * probabilities outgoing from the current state, then decides the
         * next state and if the next packet has to be transmitted or lost.
         * The four states correspond to:
         *   TX_IN_GAP_PERIOD => successfully transmitted packets within a gap period
         *   LOST_IN_GAP_PERIOD => isolated losses within a gap period
         *   LOST_IN_BURST_PERIOD => lost packets within a burst period
         *   TX_IN_BURST_PERIOD => successfully transmitted packets within a burst period
         */
        switch (clg->state) {
        case TX_IN_GAP_PERIOD:
                if (rnd < clg->a4) {
                        clg->state = LOST_IN_GAP_PERIOD;
                        return true;
                } else if (clg->a4 < rnd && rnd < clg->a1 + clg->a4) {
                        clg->state = LOST_IN_BURST_PERIOD;
                        return true;
                } else if (clg->a1 + clg->a4 < rnd) {
                        clg->state = TX_IN_GAP_PERIOD;
                }

                break;
        case TX_IN_BURST_PERIOD:
                if (rnd < clg->a5) {
                        clg->state = LOST_IN_BURST_PERIOD;
                        return true;
                } else {
                        clg->state = TX_IN_BURST_PERIOD;
                }

                break;
        case LOST_IN_BURST_PERIOD:
                if (rnd < clg->a3)
                        clg->state = TX_IN_BURST_PERIOD;
                else if (clg->a3 < rnd && rnd < clg->a2 + clg->a3) {
                        clg->state = TX_IN_GAP_PERIOD;
                } else if (clg->a2 + clg->a3 < rnd) {
                        clg->state = LOST_IN_BURST_PERIOD;
                        return true;
                }
                break;
        case LOST_IN_GAP_PERIOD:
                clg->state = TX_IN_GAP_PERIOD;
                break;
        }

        return false;
}

/* loss_gilb_ell - Gilbert-Elliot model loss generator
 * Generates losses according to the Gilbert-Elliot loss model or
 * its special cases  (Gilbert or Simple Gilbert)
 *
 * Makes a comparison between random number and the transition
 * probabilities outgoing from the current state, then decides the
 * next state. A second random number is extracted and the comparison
 * with the loss probability of the current state decides if the next
 * packet will be transmitted or lost.
 */
static bool loss_gilb_ell(struct netem_sched_data *q)
{
        struct clgstate *clg = &q->clg;
        struct rnd_state *s = &q->prng.prng_state;

        switch (clg->state) {
        case GOOD_STATE:
                if (prandom_u32_state(s) < clg->a1)
                        clg->state = BAD_STATE;
                if (prandom_u32_state(s) < clg->a4)
                        return true;
                break;
        case BAD_STATE:
                if (prandom_u32_state(s) < clg->a2)
                        clg->state = GOOD_STATE;
                if (prandom_u32_state(s) > clg->a3)
                        return true;
        }

        return false;
}

static bool loss_event(struct netem_sched_data *q)
{
        switch (q->loss_model) {
        case CLG_RANDOM:
                /* Random packet drop 0 => none, ~0 => all */
                return q->loss && q->loss >= get_crandom(&q->loss_cor, &q->prng);

        case CLG_4_STATES:
                /* 4state loss model algorithm (used also for GI model)
                * Extracts a value from the markov 4 state loss generator,
                * if it is 1 drops a packet and if needed writes the event in
                * the kernel logs
                */
                return loss_4state(q);

        case CLG_GILB_ELL:
                /* Gilbert-Elliot loss model algorithm
                * Extracts a value from the Gilbert-Elliot loss generator,
                * if it is 1 drops a packet and if needed writes the event in
                * the kernel logs
                */
                return loss_gilb_ell(q);
        }

        return false;        /* not reached */
}


/* tabledist - return a pseudo-randomly distributed value with mean mu and
 * std deviation sigma.  Uses table lookup to approximate the desired
 * distribution, and a uniformly-distributed pseudo-random source.
 */
static s64 tabledist(s64 mu, s32 sigma,
                     struct crndstate *state,
                     struct prng *prng,
                     const struct disttable *dist)
{
        s64 x;
        long t;
        u32 rnd;

        if (sigma == 0)
                return mu;

        rnd = get_crandom(state, prng);

        /* default uniform distribution */
        if (dist == NULL)
                return ((rnd % (2 * (u32)sigma)) + mu) - sigma;

        t = dist->table[rnd % dist->size];
        x = (sigma % NETEM_DIST_SCALE) * t;
        if (x >= 0)
                x += NETEM_DIST_SCALE/2;
        else
                x -= NETEM_DIST_SCALE/2;

        return  x / NETEM_DIST_SCALE + (sigma / NETEM_DIST_SCALE) * t + mu;
}

static u64 packet_time_ns(u64 len, const struct netem_sched_data *q)
{
        len += q->packet_overhead;

        if (q->cell_size) {
                u32 cells = reciprocal_divide(len, q->cell_size_reciprocal);

                if (len > cells * q->cell_size)        /* extra cell needed for remainder */
                        cells++;
                len = cells * (q->cell_size + q->cell_overhead);
        }

        return div64_u64(len * NSEC_PER_SEC, q->rate);
}

static void tfifo_reset(struct Qdisc *sch)
{
        struct netem_sched_data *q = qdisc_priv(sch);
        struct rb_node *p = rb_first(&q->t_root);

        while (p) {
                struct sk_buff *skb = rb_to_skb(p);

                p = rb_next(p);
                rb_erase(&skb->rbnode, &q->t_root);
                rtnl_kfree_skbs(skb, skb);
        }

        rtnl_kfree_skbs(q->t_head, q->t_tail);
        q->t_head = NULL;
        q->t_tail = NULL;
}

static void tfifo_enqueue(struct sk_buff *nskb, struct Qdisc *sch)
{
        struct netem_sched_data *q = qdisc_priv(sch);
        u64 tnext = netem_skb_cb(nskb)->time_to_send;

        if (!q->t_tail || tnext >= netem_skb_cb(q->t_tail)->time_to_send) {
                if (q->t_tail)
                        q->t_tail->next = nskb;
                else
                        q->t_head = nskb;
                q->t_tail = nskb;
        } else {
                struct rb_node **p = &q->t_root.rb_node, *parent = NULL;

                while (*p) {
                        struct sk_buff *skb;

                        parent = *p;
                        skb = rb_to_skb(parent);
                        if (tnext >= netem_skb_cb(skb)->time_to_send)
                                p = &parent->rb_right;
                        else
                                p = &parent->rb_left;
                }
                rb_link_node(&nskb->rbnode, parent, p);
                rb_insert_color(&nskb->rbnode, &q->t_root);
        }
        sch->q.qlen++;
}

/* netem can't properly corrupt a megapacket (like we get from GSO), so instead
 * when we statistically choose to corrupt one, we instead segment it, returning
 * the first packet to be corrupted, and re-enqueue the remaining frames
 */
static struct sk_buff *netem_segment(struct sk_buff *skb, struct Qdisc *sch,
                                     struct sk_buff **to_free)
{
        struct sk_buff *segs;
        netdev_features_t features = netif_skb_features(skb);

        segs = skb_gso_segment(skb, features & ~NETIF_F_GSO_MASK);

        if (IS_ERR_OR_NULL(segs)) {
                qdisc_drop(skb, sch, to_free);
                return NULL;
        }
        consume_skb(skb);
        return segs;
}

/*
 * Insert one skb into qdisc.
 * Note: parent depends on return value to account for queue length.
 *         NET_XMIT_DROP: queue length didn't change.
 *      NET_XMIT_SUCCESS: one skb was queued.
 */
static int netem_enqueue(struct sk_buff *skb, struct Qdisc *sch,
                         struct sk_buff **to_free)
{
        struct netem_sched_data *q = qdisc_priv(sch);
        /* We don't fill cb now as skb_unshare() may invalidate it */
        struct netem_skb_cb *cb;
        struct sk_buff *skb2 = NULL;
        struct sk_buff *segs = NULL;
        unsigned int prev_len = qdisc_pkt_len(skb);
        int count = 1;

        /* Do not fool qdisc_drop_all() */
        skb->prev = NULL;

        /* Random duplication */
        if (q->duplicate && q->duplicate >= get_crandom(&q->dup_cor, &q->prng))
                ++count;

        /* Drop packet? */
        if (loss_event(q)) {
                if (q->ecn && INET_ECN_set_ce(skb))
                        qdisc_qstats_drop(sch); /* mark packet */
                else
                        --count;
        }
        if (count == 0) {
                qdisc_qstats_drop(sch);
                __qdisc_drop(skb, to_free);
                return NET_XMIT_SUCCESS | __NET_XMIT_BYPASS;
        }

        /* If a delay is expected, orphan the skb. (orphaning usually takes
         * place at TX completion time, so _before_ the link transit delay)
         */
        if (q->latency || q->jitter || q->rate)
                skb_orphan_partial(skb);

        /*
         * If we need to duplicate packet, then clone it before
         * original is modified.
         */
        if (count > 1)
                skb2 = skb_clone(skb, GFP_ATOMIC);

        /*
         * Randomized packet corruption.
         * Make copy if needed since we are modifying
         * If packet is going to be hardware checksummed, then
         * do it now in software before we mangle it.
         */
        if (q->corrupt && q->corrupt >= get_crandom(&q->corrupt_cor, &q->prng)) {
                if (skb_is_gso(skb)) {
                        skb = netem_segment(skb, sch, to_free);
                        if (!skb)
                                goto finish_segs;

                        segs = skb->next;
                        skb_mark_not_on_list(skb);
                        qdisc_skb_cb(skb)->pkt_len = skb->len;
                }

                skb = skb_unshare(skb, GFP_ATOMIC);
                if (unlikely(!skb)) {
                        qdisc_qstats_drop(sch);
                        goto finish_segs;
                }
                if (skb->ip_summed == CHECKSUM_PARTIAL &&
                    skb_checksum_help(skb)) {
                        qdisc_drop(skb, sch, to_free);
                        skb = NULL;
                        goto finish_segs;
                }

                skb->data[get_random_u32_below(skb_headlen(skb))] ^=
                        1<<get_random_u32_below(8);
        }

        if (unlikely(sch->q.qlen >= sch->limit)) {
                /* re-link segs, so that qdisc_drop_all() frees them all */
                skb->next = segs;
                qdisc_drop_all(skb, sch, to_free);
                if (skb2)
                        __qdisc_drop(skb2, to_free);
                return NET_XMIT_DROP;
        }

        /*
         * If doing duplication then re-insert at top of the
         * qdisc tree, since parent queuer expects that only one
         * skb will be queued.
         */
        if (skb2) {
                struct Qdisc *rootq = qdisc_root_bh(sch);
                u32 dupsave = q->duplicate; /* prevent duplicating a dup... */

                q->duplicate = 0;
                rootq->enqueue(skb2, rootq, to_free);
                q->duplicate = dupsave;
                skb2 = NULL;
        }

        qdisc_qstats_backlog_inc(sch, skb);

        cb = netem_skb_cb(skb);
        if (q->gap == 0 ||                /* not doing reordering */
            q->counter < q->gap - 1 ||        /* inside last reordering gap */
            q->reorder < get_crandom(&q->reorder_cor, &q->prng)) {
                u64 now;
                s64 delay;

                delay = tabledist(q->latency, q->jitter,
                                  &q->delay_cor, &q->prng, q->delay_dist);

                now = ktime_get_ns();

                if (q->rate) {
                        struct netem_skb_cb *last = NULL;

                        if (sch->q.tail)
                                last = netem_skb_cb(sch->q.tail);
                        if (q->t_root.rb_node) {
                                struct sk_buff *t_skb;
                                struct netem_skb_cb *t_last;

                                t_skb = skb_rb_last(&q->t_root);
                                t_last = netem_skb_cb(t_skb);
                                if (!last ||
                                    t_last->time_to_send > last->time_to_send)
                                        last = t_last;
                        }
                        if (q->t_tail) {
                                struct netem_skb_cb *t_last =
                                        netem_skb_cb(q->t_tail);

                                if (!last ||
                                    t_last->time_to_send > last->time_to_send)
                                        last = t_last;
                        }

                        if (last) {
                                /*
                                 * Last packet in queue is reference point (now),
                                 * calculate this time bonus and subtract
                                 * from delay.
                                 */
                                delay -= last->time_to_send - now;
                                delay = max_t(s64, 0, delay);
                                now = last->time_to_send;
                        }

                        delay += packet_time_ns(qdisc_pkt_len(skb), q);
                }

                cb->time_to_send = now + delay;
                ++q->counter;
                tfifo_enqueue(skb, sch);
        } else {
                /*
                 * Do re-ordering by putting one out of N packets at the front
                 * of the queue.
                 */
                cb->time_to_send = ktime_get_ns();
                q->counter = 0;

                __qdisc_enqueue_head(skb, &sch->q);
                sch->qstats.requeues++;
        }

finish_segs:
        if (skb2)
                __qdisc_drop(skb2, to_free);

        if (segs) {
                unsigned int len, last_len;
                int rc, nb;

                len = skb ? skb->len : 0;
                nb = skb ? 1 : 0;

                while (segs) {
                        skb2 = segs->next;
                        skb_mark_not_on_list(segs);
                        qdisc_skb_cb(segs)->pkt_len = segs->len;
                        last_len = segs->len;
                        rc = qdisc_enqueue(segs, sch, to_free);
                        if (rc != NET_XMIT_SUCCESS) {
                                if (net_xmit_drop_count(rc))
                                        qdisc_qstats_drop(sch);
                        } else {
                                nb++;
                                len += last_len;
                        }
                        segs = skb2;
                }
                /* Parent qdiscs accounted for 1 skb of size @prev_len */
                qdisc_tree_reduce_backlog(sch, -(nb - 1), -(len - prev_len));
        } else if (!skb) {
                return NET_XMIT_DROP;
        }
        return NET_XMIT_SUCCESS;
}

/* Delay the next round with a new future slot with a
 * correct number of bytes and packets.
 */

static void get_slot_next(struct netem_sched_data *q, u64 now)
{
        s64 next_delay;

        if (!q->slot_dist)
                next_delay = q->slot_config.min_delay +
                                (get_random_u32() *
                                 (q->slot_config.max_delay -
                                  q->slot_config.min_delay) >> 32);
        else
                next_delay = tabledist(q->slot_config.dist_delay,
                                       (s32)(q->slot_config.dist_jitter),
                                       NULL, &q->prng, q->slot_dist);

        q->slot.slot_next = now + next_delay;
        q->slot.packets_left = q->slot_config.max_packets;
        q->slot.bytes_left = q->slot_config.max_bytes;
}

static struct sk_buff *netem_peek(struct netem_sched_data *q)
{
        struct sk_buff *skb = skb_rb_first(&q->t_root);
        u64 t1, t2;

        if (!skb)
                return q->t_head;
        if (!q->t_head)
                return skb;

        t1 = netem_skb_cb(skb)->time_to_send;
        t2 = netem_skb_cb(q->t_head)->time_to_send;
        if (t1 < t2)
                return skb;
        return q->t_head;
}

static void netem_erase_head(struct netem_sched_data *q, struct sk_buff *skb)
{
        if (skb == q->t_head) {
                q->t_head = skb->next;
                if (!q->t_head)
                        q->t_tail = NULL;
        } else {
                rb_erase(&skb->rbnode, &q->t_root);
        }
}

static struct sk_buff *netem_dequeue(struct Qdisc *sch)
{
        struct netem_sched_data *q = qdisc_priv(sch);
        struct sk_buff *skb;

tfifo_dequeue:
        skb = __qdisc_dequeue_head(&sch->q);
        if (skb) {
                qdisc_qstats_backlog_dec(sch, skb);
deliver:
                qdisc_bstats_update(sch, skb);
                return skb;
        }
        skb = netem_peek(q);
        if (skb) {
                u64 time_to_send;
                u64 now = ktime_get_ns();

                /* if more time remaining? */
                time_to_send = netem_skb_cb(skb)->time_to_send;
                if (q->slot.slot_next && q->slot.slot_next < time_to_send)
                        get_slot_next(q, now);

                if (time_to_send <= now && q->slot.slot_next <= now) {
                        netem_erase_head(q, skb);
                        sch->q.qlen--;
                        qdisc_qstats_backlog_dec(sch, skb);
                        skb->next = NULL;
                        skb->prev = NULL;
                        /* skb->dev shares skb->rbnode area,
                         * we need to restore its value.
                         */
                        skb->dev = qdisc_dev(sch);

                        if (q->slot.slot_next) {
                                q->slot.packets_left--;
                                q->slot.bytes_left -= qdisc_pkt_len(skb);
                                if (q->slot.packets_left <= 0 ||
                                    q->slot.bytes_left <= 0)
                                        get_slot_next(q, now);
                        }

                        if (q->qdisc) {
                                unsigned int pkt_len = qdisc_pkt_len(skb);
                                struct sk_buff *to_free = NULL;
                                int err;

                                err = qdisc_enqueue(skb, q->qdisc, &to_free);
                                kfree_skb_list(to_free);
                                if (err != NET_XMIT_SUCCESS) {
                                        if (net_xmit_drop_count(err))
                                                qdisc_qstats_drop(sch);
                                        qdisc_tree_reduce_backlog(sch, 1, pkt_len);
                                }
                                goto tfifo_dequeue;
                        }
                        goto deliver;
                }

                if (q->qdisc) {
                        skb = q->qdisc->ops->dequeue(q->qdisc);
                        if (skb)
                                goto deliver;
                }

                qdisc_watchdog_schedule_ns(&q->watchdog,
                                           max(time_to_send,
                                               q->slot.slot_next));
        }

        if (q->qdisc) {
                skb = q->qdisc->ops->dequeue(q->qdisc);
                if (skb)
                        goto deliver;
        }
        return NULL;
}

static void netem_reset(struct Qdisc *sch)
{
        struct netem_sched_data *q = qdisc_priv(sch);

        qdisc_reset_queue(sch);
        tfifo_reset(sch);
        if (q->qdisc)
                qdisc_reset(q->qdisc);
        qdisc_watchdog_cancel(&q->watchdog);
}

static void dist_free(struct disttable *d)
{
        kvfree(d);
}

/*
 * Distribution data is a variable size payload containing
 * signed 16 bit values.
 */

static int get_dist_table(struct disttable **tbl, const struct nlattr *attr)
{
        size_t n = nla_len(attr)/sizeof(__s16);
        const __s16 *data = nla_data(attr);
        struct disttable *d;
        int i;

        if (!n || n > NETEM_DIST_MAX)
                return -EINVAL;

        d = kvmalloc(struct_size(d, table, n), GFP_KERNEL);
        if (!d)
                return -ENOMEM;

        d->size = n;
        for (i = 0; i < n; i++)
                d->table[i] = data[i];

        *tbl = d;
        return 0;
}

static void get_slot(struct netem_sched_data *q, const struct nlattr *attr)
{
        const struct tc_netem_slot *c = nla_data(attr);

        q->slot_config = *c;
        if (q->slot_config.max_packets == 0)
                q->slot_config.max_packets = INT_MAX;
        if (q->slot_config.max_bytes == 0)
                q->slot_config.max_bytes = INT_MAX;

        /* capping dist_jitter to the range acceptable by tabledist() */
        q->slot_config.dist_jitter = min_t(__s64, INT_MAX, abs(q->slot_config.dist_jitter));

        q->slot.packets_left = q->slot_config.max_packets;
        q->slot.bytes_left = q->slot_config.max_bytes;
        if (q->slot_config.min_delay | q->slot_config.max_delay |
            q->slot_config.dist_jitter)
                q->slot.slot_next = ktime_get_ns();
        else
                q->slot.slot_next = 0;
}

static void get_correlation(struct netem_sched_data *q, const struct nlattr *attr)
{
        const struct tc_netem_corr *c = nla_data(attr);

        init_crandom(&q->delay_cor, c->delay_corr);
        init_crandom(&q->loss_cor, c->loss_corr);
        init_crandom(&q->dup_cor, c->dup_corr);
}

static void get_reorder(struct netem_sched_data *q, const struct nlattr *attr)
{
        const struct tc_netem_reorder *r = nla_data(attr);

        q->reorder = r->probability;
        init_crandom(&q->reorder_cor, r->correlation);
}

static void get_corrupt(struct netem_sched_data *q, const struct nlattr *attr)
{
        const struct tc_netem_corrupt *r = nla_data(attr);

        q->corrupt = r->probability;
        init_crandom(&q->corrupt_cor, r->correlation);
}

static void get_rate(struct netem_sched_data *q, const struct nlattr *attr)
{
        const struct tc_netem_rate *r = nla_data(attr);

        q->rate = r->rate;
        q->packet_overhead = r->packet_overhead;
        q->cell_size = r->cell_size;
        q->cell_overhead = r->cell_overhead;
        if (q->cell_size)
                q->cell_size_reciprocal = reciprocal_value(q->cell_size);
        else
                q->cell_size_reciprocal = (struct reciprocal_value) { 0 };
}

static int get_loss_clg(struct netem_sched_data *q, const struct nlattr *attr)
{
        const struct nlattr *la;
        int rem;

        nla_for_each_nested(la, attr, rem) {
                u16 type = nla_type(la);

                switch (type) {
                case NETEM_LOSS_GI: {
                        const struct tc_netem_gimodel *gi = nla_data(la);

                        if (nla_len(la) < sizeof(struct tc_netem_gimodel)) {
                                pr_info("netem: incorrect gi model size\n");
                                return -EINVAL;
                        }

                        q->loss_model = CLG_4_STATES;

                        q->clg.state = TX_IN_GAP_PERIOD;
                        q->clg.a1 = gi->p13;
                        q->clg.a2 = gi->p31;
                        q->clg.a3 = gi->p32;
                        q->clg.a4 = gi->p14;
                        q->clg.a5 = gi->p23;
                        break;
                }

                case NETEM_LOSS_GE: {
                        const struct tc_netem_gemodel *ge = nla_data(la);

                        if (nla_len(la) < sizeof(struct tc_netem_gemodel)) {
                                pr_info("netem: incorrect ge model size\n");
                                return -EINVAL;
                        }

                        q->loss_model = CLG_GILB_ELL;
                        q->clg.state = GOOD_STATE;
                        q->clg.a1 = ge->p;
                        q->clg.a2 = ge->r;
                        q->clg.a3 = ge->h;
                        q->clg.a4 = ge->k1;
                        break;
                }

                default:
                        pr_info("netem: unknown loss type %u\n", type);
                        return -EINVAL;
                }
        }

        return 0;
}

static const struct nla_policy netem_policy[TCA_NETEM_MAX + 1] = {
        [TCA_NETEM_CORR]        = { .len = sizeof(struct tc_netem_corr) },
        [TCA_NETEM_REORDER]        = { .len = sizeof(struct tc_netem_reorder) },
        [TCA_NETEM_CORRUPT]        = { .len = sizeof(struct tc_netem_corrupt) },
        [TCA_NETEM_RATE]        = { .len = sizeof(struct tc_netem_rate) },
        [TCA_NETEM_LOSS]        = { .type = NLA_NESTED },
        [TCA_NETEM_ECN]                = { .type = NLA_U32 },
        [TCA_NETEM_RATE64]        = { .type = NLA_U64 },
        [TCA_NETEM_LATENCY64]        = { .type = NLA_S64 },
        [TCA_NETEM_JITTER64]        = { .type = NLA_S64 },
        [TCA_NETEM_SLOT]        = { .len = sizeof(struct tc_netem_slot) },
        [TCA_NETEM_PRNG_SEED]        = { .type = NLA_U64 },
};

static int parse_attr(struct nlattr *tb[], int maxtype, struct nlattr *nla,
                      const struct nla_policy *policy, int len)
{
        int nested_len = nla_len(nla) - NLA_ALIGN(len);

        if (nested_len < 0) {
                pr_info("netem: invalid attributes len %d\n", nested_len);
                return -EINVAL;
        }

        if (nested_len >= nla_attr_size(0))
                return nla_parse_deprecated(tb, maxtype,
                                            nla_data(nla) + NLA_ALIGN(len),
                                            nested_len, policy, NULL);

        memset(tb, 0, sizeof(struct nlattr *) * (maxtype + 1));
        return 0;
}

/* Parse netlink message to set options */
static int netem_change(struct Qdisc *sch, struct nlattr *opt,
                        struct netlink_ext_ack *extack)
{
        struct netem_sched_data *q = qdisc_priv(sch);
        struct nlattr *tb[TCA_NETEM_MAX + 1];
        struct disttable *delay_dist = NULL;
        struct disttable *slot_dist = NULL;
        struct tc_netem_qopt *qopt;
        struct clgstate old_clg;
        int old_loss_model = CLG_RANDOM;
        int ret;

        qopt = nla_data(opt);
        ret = parse_attr(tb, TCA_NETEM_MAX, opt, netem_policy, sizeof(*qopt));
        if (ret < 0)
                return ret;

        if (tb[TCA_NETEM_DELAY_DIST]) {
                ret = get_dist_table(&delay_dist, tb[TCA_NETEM_DELAY_DIST]);
                if (ret)
                        goto table_free;
        }

        if (tb[TCA_NETEM_SLOT_DIST]) {
                ret = get_dist_table(&slot_dist, tb[TCA_NETEM_SLOT_DIST]);
                if (ret)
                        goto table_free;
        }

        sch_tree_lock(sch);
        /* backup q->clg and q->loss_model */
        old_clg = q->clg;
        old_loss_model = q->loss_model;

        if (tb[TCA_NETEM_LOSS]) {
                ret = get_loss_clg(q, tb[TCA_NETEM_LOSS]);
                if (ret) {
                        q->loss_model = old_loss_model;
                        q->clg = old_clg;
                        goto unlock;
                }
        } else {
                q->loss_model = CLG_RANDOM;
        }

        if (delay_dist)
                swap(q->delay_dist, delay_dist);
        if (slot_dist)
                swap(q->slot_dist, slot_dist);
        sch->limit = qopt->limit;

        q->latency = PSCHED_TICKS2NS(qopt->latency);
        q->jitter = PSCHED_TICKS2NS(qopt->jitter);
        q->limit = qopt->limit;
        q->gap = qopt->gap;
        q->counter = 0;
        q->loss = qopt->loss;
        q->duplicate = qopt->duplicate;

        /* for compatibility with earlier versions.
         * if gap is set, need to assume 100% probability
         */
        if (q->gap)
                q->reorder = ~0;

        if (tb[TCA_NETEM_CORR])
                get_correlation(q, tb[TCA_NETEM_CORR]);

        if (tb[TCA_NETEM_REORDER])
                get_reorder(q, tb[TCA_NETEM_REORDER]);

        if (tb[TCA_NETEM_CORRUPT])
                get_corrupt(q, tb[TCA_NETEM_CORRUPT]);

        if (tb[TCA_NETEM_RATE])
                get_rate(q, tb[TCA_NETEM_RATE]);

        if (tb[TCA_NETEM_RATE64])
                q->rate = max_t(u64, q->rate,
                                nla_get_u64(tb[TCA_NETEM_RATE64]));

        if (tb[TCA_NETEM_LATENCY64])
                q->latency = nla_get_s64(tb[TCA_NETEM_LATENCY64]);

        if (tb[TCA_NETEM_JITTER64])
                q->jitter = nla_get_s64(tb[TCA_NETEM_JITTER64]);

        if (tb[TCA_NETEM_ECN])
                q->ecn = nla_get_u32(tb[TCA_NETEM_ECN]);

        if (tb[TCA_NETEM_SLOT])
                get_slot(q, tb[TCA_NETEM_SLOT]);

        /* capping jitter to the range acceptable by tabledist() */
        q->jitter = min_t(s64, abs(q->jitter), INT_MAX);

        if (tb[TCA_NETEM_PRNG_SEED])
                q->prng.seed = nla_get_u64(tb[TCA_NETEM_PRNG_SEED]);
        else
                q->prng.seed = get_random_u64();
        prandom_seed_state(&q->prng.prng_state, q->prng.seed);

unlock:
        sch_tree_unlock(sch);

table_free:
        dist_free(delay_dist);
        dist_free(slot_dist);
        return ret;
}

static int netem_init(struct Qdisc *sch, struct nlattr *opt,
                      struct netlink_ext_ack *extack)
{
        struct netem_sched_data *q = qdisc_priv(sch);
        int ret;

        qdisc_watchdog_init(&q->watchdog, sch);

        if (!opt)
                return -EINVAL;

        q->loss_model = CLG_RANDOM;
        ret = netem_change(sch, opt, extack);
        if (ret)
                pr_info("netem: change failed\n");
        return ret;
}

static void netem_destroy(struct Qdisc *sch)
{
        struct netem_sched_data *q = qdisc_priv(sch);

        qdisc_watchdog_cancel(&q->watchdog);
        if (q->qdisc)
                qdisc_put(q->qdisc);
        dist_free(q->delay_dist);
        dist_free(q->slot_dist);
}

static int dump_loss_model(const struct netem_sched_data *q,
                           struct sk_buff *skb)
{
        struct nlattr *nest;

        nest = nla_nest_start_noflag(skb, TCA_NETEM_LOSS);
        if (nest == NULL)
                goto nla_put_failure;

        switch (q->loss_model) {
        case CLG_RANDOM:
                /* legacy loss model */
                nla_nest_cancel(skb, nest);
                return 0;        /* no data */

        case CLG_4_STATES: {
                struct tc_netem_gimodel gi = {
                        .p13 = q->clg.a1,
                        .p31 = q->clg.a2,
                        .p32 = q->clg.a3,
                        .p14 = q->clg.a4,
                        .p23 = q->clg.a5,
                };

                if (nla_put(skb, NETEM_LOSS_GI, sizeof(gi), &gi))
                        goto nla_put_failure;
                break;
        }
        case CLG_GILB_ELL: {
                struct tc_netem_gemodel ge = {
                        .p = q->clg.a1,
                        .r = q->clg.a2,
                        .h = q->clg.a3,
                        .k1 = q->clg.a4,
                };

                if (nla_put(skb, NETEM_LOSS_GE, sizeof(ge), &ge))
                        goto nla_put_failure;
                break;
        }
        }

        nla_nest_end(skb, nest);
        return 0;

nla_put_failure:
        nla_nest_cancel(skb, nest);
        return -1;
}

static int netem_dump(struct Qdisc *sch, struct sk_buff *skb)
{
        const struct netem_sched_data *q = qdisc_priv(sch);
        struct nlattr *nla = (struct nlattr *) skb_tail_pointer(skb);
        struct tc_netem_qopt qopt;
        struct tc_netem_corr cor;
        struct tc_netem_reorder reorder;
        struct tc_netem_corrupt corrupt;
        struct tc_netem_rate rate;
        struct tc_netem_slot slot;

        qopt.latency = min_t(psched_time_t, PSCHED_NS2TICKS(q->latency),
                             UINT_MAX);
        qopt.jitter = min_t(psched_time_t, PSCHED_NS2TICKS(q->jitter),
                            UINT_MAX);
        qopt.limit = q->limit;
        qopt.loss = q->loss;
        qopt.gap = q->gap;
        qopt.duplicate = q->duplicate;
        if (nla_put(skb, TCA_OPTIONS, sizeof(qopt), &qopt))
                goto nla_put_failure;

        if (nla_put(skb, TCA_NETEM_LATENCY64, sizeof(q->latency), &q->latency))
                goto nla_put_failure;

        if (nla_put(skb, TCA_NETEM_JITTER64, sizeof(q->jitter), &q->jitter))
                goto nla_put_failure;

        cor.delay_corr = q->delay_cor.rho;
        cor.loss_corr = q->loss_cor.rho;
        cor.dup_corr = q->dup_cor.rho;
        if (nla_put(skb, TCA_NETEM_CORR, sizeof(cor), &cor))
                goto nla_put_failure;

        reorder.probability = q->reorder;
        reorder.correlation = q->reorder_cor.rho;
        if (nla_put(skb, TCA_NETEM_REORDER, sizeof(reorder), &reorder))
                goto nla_put_failure;

        corrupt.probability = q->corrupt;
        corrupt.correlation = q->corrupt_cor.rho;
        if (nla_put(skb, TCA_NETEM_CORRUPT, sizeof(corrupt), &corrupt))
                goto nla_put_failure;

        if (q->rate >= (1ULL << 32)) {
                if (nla_put_u64_64bit(skb, TCA_NETEM_RATE64, q->rate,
                                      TCA_NETEM_PAD))
                        goto nla_put_failure;
                rate.rate = ~0U;
        } else {
                rate.rate = q->rate;
        }
        rate.packet_overhead = q->packet_overhead;
        rate.cell_size = q->cell_size;
        rate.cell_overhead = q->cell_overhead;
        if (nla_put(skb, TCA_NETEM_RATE, sizeof(rate), &rate))
                goto nla_put_failure;

        if (q->ecn && nla_put_u32(skb, TCA_NETEM_ECN, q->ecn))
                goto nla_put_failure;

        if (dump_loss_model(q, skb) != 0)
                goto nla_put_failure;

        if (q->slot_config.min_delay | q->slot_config.max_delay |
            q->slot_config.dist_jitter) {
                slot = q->slot_config;
                if (slot.max_packets == INT_MAX)
                        slot.max_packets = 0;
                if (slot.max_bytes == INT_MAX)
                        slot.max_bytes = 0;
                if (nla_put(skb, TCA_NETEM_SLOT, sizeof(slot), &slot))
                        goto nla_put_failure;
        }

        if (nla_put_u64_64bit(skb, TCA_NETEM_PRNG_SEED, q->prng.seed,
                              TCA_NETEM_PAD))
                goto nla_put_failure;

        return nla_nest_end(skb, nla);

nla_put_failure:
        nlmsg_trim(skb, nla);
        return -1;
}

static int netem_dump_class(struct Qdisc *sch, unsigned long cl,
                          struct sk_buff *skb, struct tcmsg *tcm)
{
        struct netem_sched_data *q = qdisc_priv(sch);

        if (cl != 1 || !q->qdisc)         /* only one class */
                return -ENOENT;

        tcm->tcm_handle |= TC_H_MIN(1);
        tcm->tcm_info = q->qdisc->handle;

        return 0;
}

static int netem_graft(struct Qdisc *sch, unsigned long arg, struct Qdisc *new,
                     struct Qdisc **old, struct netlink_ext_ack *extack)
{
        struct netem_sched_data *q = qdisc_priv(sch);

        *old = qdisc_replace(sch, new, &q->qdisc);
        return 0;
}

static struct Qdisc *netem_leaf(struct Qdisc *sch, unsigned long arg)
{
        struct netem_sched_data *q = qdisc_priv(sch);
        return q->qdisc;
}

static unsigned long netem_find(struct Qdisc *sch, u32 classid)
{
        return 1;
}

static void netem_walk(struct Qdisc *sch, struct qdisc_walker *walker)
{
        if (!walker->stop) {
                if (!tc_qdisc_stats_dump(sch, 1, walker))
                        return;
        }
}

static const struct Qdisc_class_ops netem_class_ops = {
        .graft                =        netem_graft,
        .leaf                =        netem_leaf,
        .find                =        netem_find,
        .walk                =        netem_walk,
        .dump                =        netem_dump_class,
};

static struct Qdisc_ops netem_qdisc_ops __read_mostly = {
        .id                =        "netem",
        .cl_ops                =        &netem_class_ops,
        .priv_size        =        sizeof(struct netem_sched_data),
        .enqueue        =        netem_enqueue,
        .dequeue        =        netem_dequeue,
        .peek                =        qdisc_peek_dequeued,
        .init                =        netem_init,
        .reset                =        netem_reset,
        .destroy        =        netem_destroy,
        .change                =        netem_change,
        .dump                =        netem_dump,
        .owner                =        THIS_MODULE,
};
MODULE_ALIAS_NET_SCH("netem");


static int __init netem_module_init(void)
{
        pr_info("netem: version " VERSION "\n");
        return register_qdisc(&netem_qdisc_ops);
}
static void __exit netem_module_exit(void)
{
        unregister_qdisc(&netem_qdisc_ops);
}
module_init(netem_module_init)
module_exit(netem_module_exit)
MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("Network characteristics emulator qdisc");














































    1 



    2 






    3 



    3 





















    3 



    3 



















    1 


















































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
// SPDX-License-Identifier: GPL-2.0-only
/*
 * ip_vs_proto_ah_esp.c:        AH/ESP IPSec load balancing support for IPVS
 *
 * Authors:        Julian Anastasov <ja@ssi.bg>, February 2002
 *                Wensong Zhang <wensong@linuxvirtualserver.org>
 */

#define KMSG_COMPONENT "IPVS"
#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt

#include <linux/in.h>
#include <linux/ip.h>
#include <linux/module.h>
#include <linux/kernel.h>
#include <linux/netfilter.h>
#include <linux/netfilter_ipv4.h>

#include <net/ip_vs.h>


/* TODO:

struct isakmp_hdr {
        __u8                icookie[8];
        __u8                rcookie[8];
        __u8                np;
        __u8                version;
        __u8                xchgtype;
        __u8                flags;
        __u32                msgid;
        __u32                length;
};

*/

#define PORT_ISAKMP        500

static void
ah_esp_conn_fill_param_proto(struct netns_ipvs *ipvs, int af,
                             const struct ip_vs_iphdr *iph,
                             struct ip_vs_conn_param *p)
{
        if (likely(!ip_vs_iph_inverse(iph)))
                ip_vs_conn_fill_param(ipvs, af, IPPROTO_UDP,
                                      &iph->saddr, htons(PORT_ISAKMP),
                                      &iph->daddr, htons(PORT_ISAKMP), p);
        else
                ip_vs_conn_fill_param(ipvs, af, IPPROTO_UDP,
                                      &iph->daddr, htons(PORT_ISAKMP),
                                      &iph->saddr, htons(PORT_ISAKMP), p);
}

static struct ip_vs_conn *
ah_esp_conn_in_get(struct netns_ipvs *ipvs, int af, const struct sk_buff *skb,
                   const struct ip_vs_iphdr *iph)
{
        struct ip_vs_conn *cp;
        struct ip_vs_conn_param p;

        ah_esp_conn_fill_param_proto(ipvs, af, iph, &p);
        cp = ip_vs_conn_in_get(&p);
        if (!cp) {
                /*
                 * We are not sure if the packet is from our
                 * service, so our conn_schedule hook should return NF_ACCEPT
                 */
                IP_VS_DBG_BUF(12, "Unknown ISAKMP entry for outin packet "
                              "%s%s %s->%s\n",
                              ip_vs_iph_icmp(iph) ? "ICMP+" : "",
                              ip_vs_proto_get(iph->protocol)->name,
                              IP_VS_DBG_ADDR(af, &iph->saddr),
                              IP_VS_DBG_ADDR(af, &iph->daddr));
        }

        return cp;
}


static struct ip_vs_conn *
ah_esp_conn_out_get(struct netns_ipvs *ipvs, int af, const struct sk_buff *skb,
                    const struct ip_vs_iphdr *iph)
{
        struct ip_vs_conn *cp;
        struct ip_vs_conn_param p;

        ah_esp_conn_fill_param_proto(ipvs, af, iph, &p);
        cp = ip_vs_conn_out_get(&p);
        if (!cp) {
                IP_VS_DBG_BUF(12, "Unknown ISAKMP entry for inout packet "
                              "%s%s %s->%s\n",
                              ip_vs_iph_icmp(iph) ? "ICMP+" : "",
                              ip_vs_proto_get(iph->protocol)->name,
                              IP_VS_DBG_ADDR(af, &iph->saddr),
                              IP_VS_DBG_ADDR(af, &iph->daddr));
        }

        return cp;
}


static int
ah_esp_conn_schedule(struct netns_ipvs *ipvs, int af, struct sk_buff *skb,
                     struct ip_vs_proto_data *pd,
                     int *verdict, struct ip_vs_conn **cpp,
                     struct ip_vs_iphdr *iph)
{
        /*
         * AH/ESP is only related traffic. Pass the packet to IP stack.
         */
        *verdict = NF_ACCEPT;
        return 0;
}

#ifdef CONFIG_IP_VS_PROTO_AH
struct ip_vs_protocol ip_vs_protocol_ah = {
        .name =                        "AH",
        .protocol =                IPPROTO_AH,
        .num_states =                1,
        .dont_defrag =                1,
        .init =                        NULL,
        .exit =                        NULL,
        .conn_schedule =        ah_esp_conn_schedule,
        .conn_in_get =                ah_esp_conn_in_get,
        .conn_out_get =                ah_esp_conn_out_get,
        .snat_handler =                NULL,
        .dnat_handler =                NULL,
        .state_transition =        NULL,
        .register_app =                NULL,
        .unregister_app =        NULL,
        .app_conn_bind =        NULL,
        .debug_packet =                ip_vs_tcpudp_debug_packet,
        .timeout_change =        NULL,                /* ISAKMP */
};
#endif

#ifdef CONFIG_IP_VS_PROTO_ESP
struct ip_vs_protocol ip_vs_protocol_esp = {
        .name =                        "ESP",
        .protocol =                IPPROTO_ESP,
        .num_states =                1,
        .dont_defrag =                1,
        .init =                        NULL,
        .exit =                        NULL,
        .conn_schedule =        ah_esp_conn_schedule,
        .conn_in_get =                ah_esp_conn_in_get,
        .conn_out_get =                ah_esp_conn_out_get,
        .snat_handler =                NULL,
        .dnat_handler =                NULL,
        .state_transition =        NULL,
        .register_app =                NULL,
        .unregister_app =        NULL,
        .app_conn_bind =        NULL,
        .debug_packet =                ip_vs_tcpudp_debug_packet,
        .timeout_change =        NULL,                /* ISAKMP */
};
#endif






































































    3 














































    3 

















































































































































































































































































































































































































































































    3 























































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
/* SPDX-License-Identifier: GPL-2.0 */
/*
 * Copyright (C) 2001 Jens Axboe <axboe@suse.de>
 */
#ifndef __LINUX_BIO_H
#define __LINUX_BIO_H

#include <linux/mempool.h>
/* struct bio, bio_vec and BIO_* flags are defined in blk_types.h */
#include <linux/blk_types.h>
#include <linux/uio.h>

#define BIO_MAX_VECS                256U

struct queue_limits;

static inline unsigned int bio_max_segs(unsigned int nr_segs)
{
        return min(nr_segs, BIO_MAX_VECS);
}

#define bio_prio(bio)                        (bio)->bi_ioprio
#define bio_set_prio(bio, prio)                ((bio)->bi_ioprio = prio)

#define bio_iter_iovec(bio, iter)                                \
        bvec_iter_bvec((bio)->bi_io_vec, (iter))

#define bio_iter_page(bio, iter)                                \
        bvec_iter_page((bio)->bi_io_vec, (iter))
#define bio_iter_len(bio, iter)                                        \
        bvec_iter_len((bio)->bi_io_vec, (iter))
#define bio_iter_offset(bio, iter)                                \
        bvec_iter_offset((bio)->bi_io_vec, (iter))

#define bio_page(bio)                bio_iter_page((bio), (bio)->bi_iter)
#define bio_offset(bio)                bio_iter_offset((bio), (bio)->bi_iter)
#define bio_iovec(bio)                bio_iter_iovec((bio), (bio)->bi_iter)

#define bvec_iter_sectors(iter)        ((iter).bi_size >> 9)
#define bvec_iter_end_sector(iter) ((iter).bi_sector + bvec_iter_sectors((iter)))

#define bio_sectors(bio)        bvec_iter_sectors((bio)->bi_iter)
#define bio_end_sector(bio)        bvec_iter_end_sector((bio)->bi_iter)

/*
 * Return the data direction, READ or WRITE.
 */
#define bio_data_dir(bio) \
        (op_is_write(bio_op(bio)) ? WRITE : READ)

/*
 * Check whether this bio carries any data or not. A NULL bio is allowed.
 */
static inline bool bio_has_data(struct bio *bio)
{
        if (bio &&
            bio->bi_iter.bi_size &&
            bio_op(bio) != REQ_OP_DISCARD &&
            bio_op(bio) != REQ_OP_SECURE_ERASE &&
            bio_op(bio) != REQ_OP_WRITE_ZEROES)
                return true;

        return false;
}

static inline bool bio_no_advance_iter(const struct bio *bio)
{
        return bio_op(bio) == REQ_OP_DISCARD ||
               bio_op(bio) == REQ_OP_SECURE_ERASE ||
               bio_op(bio) == REQ_OP_WRITE_ZEROES;
}

static inline void *bio_data(struct bio *bio)
{
        if (bio_has_data(bio))
                return page_address(bio_page(bio)) + bio_offset(bio);

        return NULL;
}

static inline bool bio_next_segment(const struct bio *bio,
                                    struct bvec_iter_all *iter)
{
        if (iter->idx >= bio->bi_vcnt)
                return false;

        bvec_advance(&bio->bi_io_vec[iter->idx], iter);
        return true;
}

/*
 * drivers should _never_ use the all version - the bio may have been split
 * before it got to the driver and the driver won't own all of it
 */
#define bio_for_each_segment_all(bvl, bio, iter) \
        for (bvl = bvec_init_iter_all(&iter); bio_next_segment((bio), &iter); )

static inline void bio_advance_iter(const struct bio *bio,
                                    struct bvec_iter *iter, unsigned int bytes)
{
        iter->bi_sector += bytes >> 9;

        if (bio_no_advance_iter(bio))
                iter->bi_size -= bytes;
        else
                bvec_iter_advance(bio->bi_io_vec, iter, bytes);
                /* TODO: It is reasonable to complete bio with error here. */
}

/* @bytes should be less or equal to bvec[i->bi_idx].bv_len */
static inline void bio_advance_iter_single(const struct bio *bio,
                                           struct bvec_iter *iter,
                                           unsigned int bytes)
{
        iter->bi_sector += bytes >> 9;

        if (bio_no_advance_iter(bio))
                iter->bi_size -= bytes;
        else
                bvec_iter_advance_single(bio->bi_io_vec, iter, bytes);
}

void __bio_advance(struct bio *, unsigned bytes);

/**
 * bio_advance - increment/complete a bio by some number of bytes
 * @bio:        bio to advance
 * @nbytes:        number of bytes to complete
 *
 * This updates bi_sector, bi_size and bi_idx; if the number of bytes to
 * complete doesn't align with a bvec boundary, then bv_len and bv_offset will
 * be updated on the last bvec as well.
 *
 * @bio will then represent the remaining, uncompleted portion of the io.
 */
static inline void bio_advance(struct bio *bio, unsigned int nbytes)
{
        if (nbytes == bio->bi_iter.bi_size) {
                bio->bi_iter.bi_size = 0;
                return;
        }
        __bio_advance(bio, nbytes);
}

#define __bio_for_each_segment(bvl, bio, iter, start)                        \
        for (iter = (start);                                                \
             (iter).bi_size &&                                                \
                ((bvl = bio_iter_iovec((bio), (iter))), 1);                \
             bio_advance_iter_single((bio), &(iter), (bvl).bv_len))

#define bio_for_each_segment(bvl, bio, iter)                                \
        __bio_for_each_segment(bvl, bio, iter, (bio)->bi_iter)

#define __bio_for_each_bvec(bvl, bio, iter, start)                \
        for (iter = (start);                                                \
             (iter).bi_size &&                                                \
                ((bvl = mp_bvec_iter_bvec((bio)->bi_io_vec, (iter))), 1); \
             bio_advance_iter_single((bio), &(iter), (bvl).bv_len))

/* iterate over multi-page bvec */
#define bio_for_each_bvec(bvl, bio, iter)                        \
        __bio_for_each_bvec(bvl, bio, iter, (bio)->bi_iter)

/*
 * Iterate over all multi-page bvecs. Drivers shouldn't use this version for the
 * same reasons as bio_for_each_segment_all().
 */
#define bio_for_each_bvec_all(bvl, bio, i)                \
        for (i = 0, bvl = bio_first_bvec_all(bio);        \
             i < (bio)->bi_vcnt; i++, bvl++)

#define bio_iter_last(bvec, iter) ((iter).bi_size == (bvec).bv_len)

static inline unsigned bio_segments(struct bio *bio)
{
        unsigned segs = 0;
        struct bio_vec bv;
        struct bvec_iter iter;

        /*
         * We special case discard/write same/write zeroes, because they
         * interpret bi_size differently:
         */

        switch (bio_op(bio)) {
        case REQ_OP_DISCARD:
        case REQ_OP_SECURE_ERASE:
        case REQ_OP_WRITE_ZEROES:
                return 0;
        default:
                break;
        }

        bio_for_each_segment(bv, bio, iter)
                segs++;

        return segs;
}

/*
 * get a reference to a bio, so it won't disappear. the intended use is
 * something like:
 *
 * bio_get(bio);
 * submit_bio(rw, bio);
 * if (bio->bi_flags ...)
 *        do_something
 * bio_put(bio);
 *
 * without the bio_get(), it could potentially complete I/O before submit_bio
 * returns. and then bio would be freed memory when if (bio->bi_flags ...)
 * runs
 */
static inline void bio_get(struct bio *bio)
{
        bio->bi_flags |= (1 << BIO_REFFED);
        smp_mb__before_atomic();
        atomic_inc(&bio->__bi_cnt);
}

static inline void bio_cnt_set(struct bio *bio, unsigned int count)
{
        if (count != 1) {
                bio->bi_flags |= (1 << BIO_REFFED);
                smp_mb();
        }
        atomic_set(&bio->__bi_cnt, count);
}

static inline bool bio_flagged(struct bio *bio, unsigned int bit)
{
        return bio->bi_flags & (1U << bit);
}

static inline void bio_set_flag(struct bio *bio, unsigned int bit)
{
        bio->bi_flags |= (1U << bit);
}

static inline void bio_clear_flag(struct bio *bio, unsigned int bit)
{
        bio->bi_flags &= ~(1U << bit);
}

static inline struct bio_vec *bio_first_bvec_all(struct bio *bio)
{
        WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED));
        return bio->bi_io_vec;
}

static inline struct page *bio_first_page_all(struct bio *bio)
{
        return bio_first_bvec_all(bio)->bv_page;
}

static inline struct folio *bio_first_folio_all(struct bio *bio)
{
        return page_folio(bio_first_page_all(bio));
}

static inline struct bio_vec *bio_last_bvec_all(struct bio *bio)
{
        WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED));
        return &bio->bi_io_vec[bio->bi_vcnt - 1];
}

/**
 * struct folio_iter - State for iterating all folios in a bio.
 * @folio: The current folio we're iterating.  NULL after the last folio.
 * @offset: The byte offset within the current folio.
 * @length: The number of bytes in this iteration (will not cross folio
 *        boundary).
 */
struct folio_iter {
        struct folio *folio;
        size_t offset;
        size_t length;
        /* private: for use by the iterator */
        struct folio *_next;
        size_t _seg_count;
        int _i;
};

static inline void bio_first_folio(struct folio_iter *fi, struct bio *bio,
                                   int i)
{
        struct bio_vec *bvec = bio_first_bvec_all(bio) + i;

        if (unlikely(i >= bio->bi_vcnt)) {
                fi->folio = NULL;
                return;
        }

        fi->folio = page_folio(bvec->bv_page);
        fi->offset = bvec->bv_offset +
                        PAGE_SIZE * (bvec->bv_page - &fi->folio->page);
        fi->_seg_count = bvec->bv_len;
        fi->length = min(folio_size(fi->folio) - fi->offset, fi->_seg_count);
        fi->_next = folio_next(fi->folio);
        fi->_i = i;
}

static inline void bio_next_folio(struct folio_iter *fi, struct bio *bio)
{
        fi->_seg_count -= fi->length;
        if (fi->_seg_count) {
                fi->folio = fi->_next;
                fi->offset = 0;
                fi->length = min(folio_size(fi->folio), fi->_seg_count);
                fi->_next = folio_next(fi->folio);
        } else {
                bio_first_folio(fi, bio, fi->_i + 1);
        }
}

/**
 * bio_for_each_folio_all - Iterate over each folio in a bio.
 * @fi: struct folio_iter which is updated for each folio.
 * @bio: struct bio to iterate over.
 */
#define bio_for_each_folio_all(fi, bio)                                \
        for (bio_first_folio(&fi, bio, 0); fi.folio; bio_next_folio(&fi, bio))

void bio_trim(struct bio *bio, sector_t offset, sector_t size);
extern struct bio *bio_split(struct bio *bio, int sectors,
                             gfp_t gfp, struct bio_set *bs);
int bio_split_rw_at(struct bio *bio, const struct queue_limits *lim,
                unsigned *segs, unsigned max_bytes);

/**
 * bio_next_split - get next @sectors from a bio, splitting if necessary
 * @bio:        bio to split
 * @sectors:        number of sectors to split from the front of @bio
 * @gfp:        gfp mask
 * @bs:                bio set to allocate from
 *
 * Return: a bio representing the next @sectors of @bio - if the bio is smaller
 * than @sectors, returns the original bio unchanged.
 */
static inline struct bio *bio_next_split(struct bio *bio, int sectors,
                                         gfp_t gfp, struct bio_set *bs)
{
        if (sectors >= bio_sectors(bio))
                return bio;

        return bio_split(bio, sectors, gfp, bs);
}

enum {
        BIOSET_NEED_BVECS = BIT(0),
        BIOSET_NEED_RESCUER = BIT(1),
        BIOSET_PERCPU_CACHE = BIT(2),
};
extern int bioset_init(struct bio_set *, unsigned int, unsigned int, int flags);
extern void bioset_exit(struct bio_set *);
extern int biovec_init_pool(mempool_t *pool, int pool_entries);

struct bio *bio_alloc_bioset(struct block_device *bdev, unsigned short nr_vecs,
                             blk_opf_t opf, gfp_t gfp_mask,
                             struct bio_set *bs);
struct bio *bio_kmalloc(unsigned short nr_vecs, gfp_t gfp_mask);
extern void bio_put(struct bio *);

struct bio *bio_alloc_clone(struct block_device *bdev, struct bio *bio_src,
                gfp_t gfp, struct bio_set *bs);
int bio_init_clone(struct block_device *bdev, struct bio *bio,
                struct bio *bio_src, gfp_t gfp);

extern struct bio_set fs_bio_set;

static inline struct bio *bio_alloc(struct block_device *bdev,
                unsigned short nr_vecs, blk_opf_t opf, gfp_t gfp_mask)
{
        return bio_alloc_bioset(bdev, nr_vecs, opf, gfp_mask, &fs_bio_set);
}

void submit_bio(struct bio *bio);

extern void bio_endio(struct bio *);

static inline void bio_io_error(struct bio *bio)
{
        bio->bi_status = BLK_STS_IOERR;
        bio_endio(bio);
}

static inline void bio_wouldblock_error(struct bio *bio)
{
        bio_set_flag(bio, BIO_QUIET);
        bio->bi_status = BLK_STS_AGAIN;
        bio_endio(bio);
}

/*
 * Calculate number of bvec segments that should be allocated to fit data
 * pointed by @iter. If @iter is backed by bvec it's going to be reused
 * instead of allocating a new one.
 */
static inline int bio_iov_vecs_to_alloc(struct iov_iter *iter, int max_segs)
{
        if (iov_iter_is_bvec(iter))
                return 0;
        return iov_iter_npages(iter, max_segs);
}

struct request_queue;

extern int submit_bio_wait(struct bio *bio);
void bio_init(struct bio *bio, struct block_device *bdev, struct bio_vec *table,
              unsigned short max_vecs, blk_opf_t opf);
extern void bio_uninit(struct bio *);
void bio_reset(struct bio *bio, struct block_device *bdev, blk_opf_t opf);
void bio_chain(struct bio *, struct bio *);

int __must_check bio_add_page(struct bio *bio, struct page *page, unsigned len,
                              unsigned off);
bool __must_check bio_add_folio(struct bio *bio, struct folio *folio,
                                size_t len, size_t off);
extern int bio_add_pc_page(struct request_queue *, struct bio *, struct page *,
                           unsigned int, unsigned int);
void __bio_add_page(struct bio *bio, struct page *page,
                unsigned int len, unsigned int off);
void bio_add_folio_nofail(struct bio *bio, struct folio *folio, size_t len,
                          size_t off);
int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter);
void bio_iov_bvec_set(struct bio *bio, struct iov_iter *iter);
void __bio_release_pages(struct bio *bio, bool mark_dirty);
extern void bio_set_pages_dirty(struct bio *bio);
extern void bio_check_pages_dirty(struct bio *bio);

extern void bio_copy_data_iter(struct bio *dst, struct bvec_iter *dst_iter,
                               struct bio *src, struct bvec_iter *src_iter);
extern void bio_copy_data(struct bio *dst, struct bio *src);
extern void bio_free_pages(struct bio *bio);
void guard_bio_eod(struct bio *bio);
void zero_fill_bio_iter(struct bio *bio, struct bvec_iter iter);

static inline void zero_fill_bio(struct bio *bio)
{
        zero_fill_bio_iter(bio, bio->bi_iter);
}

static inline void bio_release_pages(struct bio *bio, bool mark_dirty)
{
        if (bio_flagged(bio, BIO_PAGE_PINNED))
                __bio_release_pages(bio, mark_dirty);
}

#define bio_dev(bio) \
        disk_devt((bio)->bi_bdev->bd_disk)

#ifdef CONFIG_BLK_CGROUP
void bio_associate_blkg(struct bio *bio);
void bio_associate_blkg_from_css(struct bio *bio,
                                 struct cgroup_subsys_state *css);
void bio_clone_blkg_association(struct bio *dst, struct bio *src);
void blkcg_punt_bio_submit(struct bio *bio);
#else        /* CONFIG_BLK_CGROUP */
static inline void bio_associate_blkg(struct bio *bio) { }
static inline void bio_associate_blkg_from_css(struct bio *bio,
                                               struct cgroup_subsys_state *css)
{ }
static inline void bio_clone_blkg_association(struct bio *dst,
                                              struct bio *src) { }
static inline void blkcg_punt_bio_submit(struct bio *bio)
{
        submit_bio(bio);
}
#endif        /* CONFIG_BLK_CGROUP */

static inline void bio_set_dev(struct bio *bio, struct block_device *bdev)
{
        bio_clear_flag(bio, BIO_REMAPPED);
        if (bio->bi_bdev != bdev)
                bio_clear_flag(bio, BIO_BPS_THROTTLED);
        bio->bi_bdev = bdev;
        bio_associate_blkg(bio);
}

/*
 * BIO list management for use by remapping drivers (e.g. DM or MD) and loop.
 *
 * A bio_list anchors a singly-linked list of bios chained through the bi_next
 * member of the bio.  The bio_list also caches the last list member to allow
 * fast access to the tail.
 */
struct bio_list {
        struct bio *head;
        struct bio *tail;
};

static inline int bio_list_empty(const struct bio_list *bl)
{
        return bl->head == NULL;
}

static inline void bio_list_init(struct bio_list *bl)
{
        bl->head = bl->tail = NULL;
}

#define BIO_EMPTY_LIST        { NULL, NULL }

#define bio_list_for_each(bio, bl) \
        for (bio = (bl)->head; bio; bio = bio->bi_next)

static inline unsigned bio_list_size(const struct bio_list *bl)
{
        unsigned sz = 0;
        struct bio *bio;

        bio_list_for_each(bio, bl)
                sz++;

        return sz;
}

static inline void bio_list_add(struct bio_list *bl, struct bio *bio)
{
        bio->bi_next = NULL;

        if (bl->tail)
                bl->tail->bi_next = bio;
        else
                bl->head = bio;

        bl->tail = bio;
}

static inline void bio_list_add_head(struct bio_list *bl, struct bio *bio)
{
        bio->bi_next = bl->head;

        bl->head = bio;

        if (!bl->tail)
                bl->tail = bio;
}

static inline void bio_list_merge(struct bio_list *bl, struct bio_list *bl2)
{
        if (!bl2->head)
                return;

        if (bl->tail)
                bl->tail->bi_next = bl2->head;
        else
                bl->head = bl2->head;

        bl->tail = bl2->tail;
}

static inline void bio_list_merge_init(struct bio_list *bl,
                struct bio_list *bl2)
{
        bio_list_merge(bl, bl2);
        bio_list_init(bl2);
}

static inline void bio_list_merge_head(struct bio_list *bl,
                                       struct bio_list *bl2)
{
        if (!bl2->head)
                return;

        if (bl->head)
                bl2->tail->bi_next = bl->head;
        else
                bl->tail = bl2->tail;

        bl->head = bl2->head;
}

static inline struct bio *bio_list_peek(struct bio_list *bl)
{
        return bl->head;
}

static inline struct bio *bio_list_pop(struct bio_list *bl)
{
        struct bio *bio = bl->head;

        if (bio) {
                bl->head = bl->head->bi_next;
                if (!bl->head)
                        bl->tail = NULL;

                bio->bi_next = NULL;
        }

        return bio;
}

static inline struct bio *bio_list_get(struct bio_list *bl)
{
        struct bio *bio = bl->head;

        bl->head = bl->tail = NULL;

        return bio;
}

/*
 * Increment chain count for the bio. Make sure the CHAIN flag update
 * is visible before the raised count.
 */
static inline void bio_inc_remaining(struct bio *bio)
{
        bio_set_flag(bio, BIO_CHAIN);
        smp_mb__before_atomic();
        atomic_inc(&bio->__bi_remaining);
}

/*
 * bio_set is used to allow other portions of the IO system to
 * allocate their own private memory pools for bio and iovec structures.
 * These memory pools in turn all allocate from the bio_slab
 * and the bvec_slabs[].
 */
#define BIO_POOL_SIZE 2

struct bio_set {
        struct kmem_cache *bio_slab;
        unsigned int front_pad;

        /*
         * per-cpu bio alloc cache
         */
        struct bio_alloc_cache __percpu *cache;

        mempool_t bio_pool;
        mempool_t bvec_pool;
#if defined(CONFIG_BLK_DEV_INTEGRITY)
        mempool_t bio_integrity_pool;
        mempool_t bvec_integrity_pool;
#endif

        unsigned int back_pad;
        /*
         * Deadlock avoidance for stacking block drivers: see comments in
         * bio_alloc_bioset() for details
         */
        spinlock_t                rescue_lock;
        struct bio_list                rescue_list;
        struct work_struct        rescue_work;
        struct workqueue_struct        *rescue_workqueue;

        /*
         * Hot un-plug notifier for the per-cpu cache, if used
         */
        struct hlist_node cpuhp_dead;
};

static inline bool bioset_initialized(struct bio_set *bs)
{
        return bs->bio_slab != NULL;
}

/*
 * Mark a bio as polled. Note that for async polled IO, the caller must
 * expect -EWOULDBLOCK if we cannot allocate a request (or other resources).
 * We cannot block waiting for requests on polled IO, as those completions
 * must be found by the caller. This is different than IRQ driven IO, where
 * it's safe to wait for IO to complete.
 */
static inline void bio_set_polled(struct bio *bio, struct kiocb *kiocb)
{
        bio->bi_opf |= REQ_POLLED;
        if (kiocb->ki_flags & IOCB_NOWAIT)
                bio->bi_opf |= REQ_NOWAIT;
}

static inline void bio_clear_polled(struct bio *bio)
{
        bio->bi_opf &= ~REQ_POLLED;
}

/**
 * bio_is_zone_append - is this a zone append bio?
 * @bio:        bio to check
 *
 * Check if @bio is a zone append operation.  Core block layer code and end_io
 * handlers must use this instead of an open coded REQ_OP_ZONE_APPEND check
 * because the block layer can rewrite REQ_OP_ZONE_APPEND to REQ_OP_WRITE if
 * it is not natively supported.
 */
static inline bool bio_is_zone_append(struct bio *bio)
{
        if (!IS_ENABLED(CONFIG_BLK_DEV_ZONED))
                return false;
        return bio_op(bio) == REQ_OP_ZONE_APPEND ||
                bio_flagged(bio, BIO_EMULATES_ZONE_APPEND);
}

struct bio *blk_next_bio(struct bio *bio, struct block_device *bdev,
                unsigned int nr_pages, blk_opf_t opf, gfp_t gfp);
struct bio *bio_chain_and_submit(struct bio *prev, struct bio *new);

struct bio *blk_alloc_discard_bio(struct block_device *bdev,
                sector_t *sector, sector_t *nr_sects, gfp_t gfp_mask);

#endif /* __LINUX_BIO_H */














































































































  852 



  858 




































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































    4 






























































































































































































































































































































































































































































    3 












    3 
    3 










    3 






















































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
/*
 * Copyright (c) 2004 Topspin Communications.  All rights reserved.
 * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
 * Copyright (c) 2004 Voltaire, Inc. All rights reserved.
 *
 * This software is available to you under a choice of one of two
 * licenses.  You may choose to be licensed under the terms of the GNU
 * General Public License (GPL) Version 2, available from the file
 * COPYING in the main directory of this source tree, or the
 * OpenIB.org BSD license below:
 *
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
 *
 *      - Redistributions of source code must retain the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer.
 *
 *      - Redistributions in binary form must reproduce the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer in the documentation and/or other materials
 *        provided with the distribution.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

#include "ipoib.h"

#include <linux/module.h>

#include <linux/init.h>
#include <linux/slab.h>
#include <linux/kernel.h>
#include <linux/vmalloc.h>

#include <linux/if_arp.h>        /* For ARPHRD_xxx */

#include <linux/ip.h>
#include <linux/in.h>

#include <linux/jhash.h>
#include <net/arp.h>
#include <net/addrconf.h>
#include <linux/inetdevice.h>
#include <rdma/ib_cache.h>

MODULE_AUTHOR("Roland Dreier");
MODULE_DESCRIPTION("IP-over-InfiniBand net driver");
MODULE_LICENSE("Dual BSD/GPL");

int ipoib_sendq_size __read_mostly = IPOIB_TX_RING_SIZE;
int ipoib_recvq_size __read_mostly = IPOIB_RX_RING_SIZE;

module_param_named(send_queue_size, ipoib_sendq_size, int, 0444);
MODULE_PARM_DESC(send_queue_size, "Number of descriptors in send queue");
module_param_named(recv_queue_size, ipoib_recvq_size, int, 0444);
MODULE_PARM_DESC(recv_queue_size, "Number of descriptors in receive queue");

#ifdef CONFIG_INFINIBAND_IPOIB_DEBUG
int ipoib_debug_level;

module_param_named(debug_level, ipoib_debug_level, int, 0644);
MODULE_PARM_DESC(debug_level, "Enable debug tracing if > 0");
#endif

struct ipoib_path_iter {
        struct net_device *dev;
        struct ipoib_path  path;
};

static const u8 ipv4_bcast_addr[] = {
        0x00, 0xff, 0xff, 0xff,
        0xff, 0x12, 0x40, 0x1b,        0x00, 0x00, 0x00, 0x00,
        0x00, 0x00, 0x00, 0x00,        0xff, 0xff, 0xff, 0xff
};

struct workqueue_struct *ipoib_workqueue;

struct ib_sa_client ipoib_sa_client;

static int ipoib_add_one(struct ib_device *device);
static void ipoib_remove_one(struct ib_device *device, void *client_data);
static void ipoib_neigh_reclaim(struct rcu_head *rp);
static struct net_device *ipoib_get_net_dev_by_params(
                struct ib_device *dev, u32 port, u16 pkey,
                const union ib_gid *gid, const struct sockaddr *addr,
                void *client_data);
static int ipoib_set_mac(struct net_device *dev, void *addr);
static int ipoib_ioctl(struct net_device *dev, struct ifreq *ifr,
                       int cmd);

static struct ib_client ipoib_client = {
        .name   = "ipoib",
        .add    = ipoib_add_one,
        .remove = ipoib_remove_one,
        .get_net_dev_by_params = ipoib_get_net_dev_by_params,
};

#ifdef CONFIG_INFINIBAND_IPOIB_DEBUG
static int ipoib_netdev_event(struct notifier_block *this,
                              unsigned long event, void *ptr)
{
        struct netdev_notifier_info *ni = ptr;
        struct net_device *dev = ni->dev;

        if (dev->netdev_ops->ndo_open != ipoib_open)
                return NOTIFY_DONE;

        switch (event) {
        case NETDEV_REGISTER:
                ipoib_create_debug_files(dev);
                break;
        case NETDEV_CHANGENAME:
                ipoib_delete_debug_files(dev);
                ipoib_create_debug_files(dev);
                break;
        case NETDEV_UNREGISTER:
                ipoib_delete_debug_files(dev);
                break;
        }

        return NOTIFY_DONE;
}
#endif

int ipoib_open(struct net_device *dev)
{
        struct ipoib_dev_priv *priv = ipoib_priv(dev);

        ipoib_dbg(priv, "bringing up interface\n");

        netif_carrier_off(dev);

        set_bit(IPOIB_FLAG_ADMIN_UP, &priv->flags);

        if (ipoib_ib_dev_open(dev)) {
                if (!test_bit(IPOIB_PKEY_ASSIGNED, &priv->flags))
                        return 0;
                goto err_disable;
        }

        ipoib_ib_dev_up(dev);

        if (!test_bit(IPOIB_FLAG_SUBINTERFACE, &priv->flags)) {
                struct ipoib_dev_priv *cpriv;

                /* Bring up any child interfaces too */
                down_read(&priv->vlan_rwsem);
                list_for_each_entry(cpriv, &priv->child_intfs, list) {
                        int flags;

                        flags = cpriv->dev->flags;
                        if (flags & IFF_UP)
                                continue;

                        dev_change_flags(cpriv->dev, flags | IFF_UP, NULL);
                }
                up_read(&priv->vlan_rwsem);
        } else if (priv->parent) {
                struct ipoib_dev_priv *ppriv = ipoib_priv(priv->parent);

                if (!test_bit(IPOIB_FLAG_ADMIN_UP, &ppriv->flags))
                        ipoib_dbg(priv, "parent device %s is not up, so child device may be not functioning.\n",
                                  ppriv->dev->name);
        }
        netif_start_queue(dev);

        return 0;

err_disable:
        clear_bit(IPOIB_FLAG_ADMIN_UP, &priv->flags);

        return -EINVAL;
}

static int ipoib_stop(struct net_device *dev)
{
        struct ipoib_dev_priv *priv = ipoib_priv(dev);

        ipoib_dbg(priv, "stopping interface\n");

        clear_bit(IPOIB_FLAG_ADMIN_UP, &priv->flags);

        netif_stop_queue(dev);

        ipoib_ib_dev_down(dev);
        ipoib_ib_dev_stop(dev);

        if (!test_bit(IPOIB_FLAG_SUBINTERFACE, &priv->flags)) {
                struct ipoib_dev_priv *cpriv;

                /* Bring down any child interfaces too */
                down_read(&priv->vlan_rwsem);
                list_for_each_entry(cpriv, &priv->child_intfs, list) {
                        int flags;

                        flags = cpriv->dev->flags;
                        if (!(flags & IFF_UP))
                                continue;

                        dev_change_flags(cpriv->dev, flags & ~IFF_UP, NULL);
                }
                up_read(&priv->vlan_rwsem);
        }

        return 0;
}

static netdev_features_t ipoib_fix_features(struct net_device *dev, netdev_features_t features)
{
        struct ipoib_dev_priv *priv = ipoib_priv(dev);

        if (test_bit(IPOIB_FLAG_ADMIN_CM, &priv->flags))
                features &= ~(NETIF_F_IP_CSUM | NETIF_F_TSO);

        return features;
}

static int ipoib_change_mtu(struct net_device *dev, int new_mtu)
{
        struct ipoib_dev_priv *priv = ipoib_priv(dev);
        int ret = 0;

        /* dev->mtu > 2K ==> connected mode */
        if (ipoib_cm_admin_enabled(dev)) {
                if (new_mtu > ipoib_cm_max_mtu(dev))
                        return -EINVAL;

                if (new_mtu > priv->mcast_mtu)
                        ipoib_warn(priv, "mtu > %d will cause multicast packet drops.\n",
                                   priv->mcast_mtu);

                WRITE_ONCE(dev->mtu, new_mtu);
                return 0;
        }

        if (new_mtu < (ETH_MIN_MTU + IPOIB_ENCAP_LEN) ||
            new_mtu > IPOIB_UD_MTU(priv->max_ib_mtu))
                return -EINVAL;

        priv->admin_mtu = new_mtu;

        if (priv->mcast_mtu < priv->admin_mtu)
                ipoib_dbg(priv, "MTU must be smaller than the underlying "
                                "link layer MTU - 4 (%u)\n", priv->mcast_mtu);

        new_mtu = min(priv->mcast_mtu, priv->admin_mtu);

        if (priv->rn_ops->ndo_change_mtu) {
                bool carrier_status = netif_carrier_ok(dev);

                netif_carrier_off(dev);

                /* notify lower level on the real mtu */
                ret = priv->rn_ops->ndo_change_mtu(dev, new_mtu);

                if (carrier_status)
                        netif_carrier_on(dev);
        } else {
                WRITE_ONCE(dev->mtu, new_mtu);
        }

        return ret;
}

static void ipoib_get_stats(struct net_device *dev,
                            struct rtnl_link_stats64 *stats)
{
        struct ipoib_dev_priv *priv = ipoib_priv(dev);

        if (priv->rn_ops->ndo_get_stats64)
                priv->rn_ops->ndo_get_stats64(dev, stats);
        else
                netdev_stats_to_stats64(stats, &dev->stats);
}

/* Called with an RCU read lock taken */
static bool ipoib_is_dev_match_addr_rcu(const struct sockaddr *addr,
                                        struct net_device *dev)
{
        struct net *net = dev_net(dev);
        struct in_device *in_dev;
        struct sockaddr_in *addr_in = (struct sockaddr_in *)addr;
        struct sockaddr_in6 *addr_in6 = (struct sockaddr_in6 *)addr;
        __be32 ret_addr;

        switch (addr->sa_family) {
        case AF_INET:
                in_dev = in_dev_get(dev);
                if (!in_dev)
                        return false;

                ret_addr = inet_confirm_addr(net, in_dev, 0,
                                             addr_in->sin_addr.s_addr,
                                             RT_SCOPE_HOST);
                in_dev_put(in_dev);
                if (ret_addr)
                        return true;

                break;
        case AF_INET6:
                if (IS_ENABLED(CONFIG_IPV6) &&
                    ipv6_chk_addr(net, &addr_in6->sin6_addr, dev, 1))
                        return true;

                break;
        }
        return false;
}

/*
 * Find the master net_device on top of the given net_device.
 * @dev: base IPoIB net_device
 *
 * Returns the master net_device with a reference held, or the same net_device
 * if no master exists.
 */
static struct net_device *ipoib_get_master_net_dev(struct net_device *dev)
{
        struct net_device *master;

        rcu_read_lock();
        master = netdev_master_upper_dev_get_rcu(dev);
        dev_hold(master);
        rcu_read_unlock();

        if (master)
                return master;

        dev_hold(dev);
        return dev;
}

struct ipoib_walk_data {
        const struct sockaddr *addr;
        struct net_device *result;
};

static int ipoib_upper_walk(struct net_device *upper,
                            struct netdev_nested_priv *priv)
{
        struct ipoib_walk_data *data = (struct ipoib_walk_data *)priv->data;
        int ret = 0;

        if (ipoib_is_dev_match_addr_rcu(data->addr, upper)) {
                dev_hold(upper);
                data->result = upper;
                ret = 1;
        }

        return ret;
}

/**
 * ipoib_get_net_dev_match_addr - Find a net_device matching
 * the given address, which is an upper device of the given net_device.
 *
 * @addr: IP address to look for.
 * @dev: base IPoIB net_device
 *
 * If found, returns the net_device with a reference held. Otherwise return
 * NULL.
 */
static struct net_device *ipoib_get_net_dev_match_addr(
                const struct sockaddr *addr, struct net_device *dev)
{
        struct netdev_nested_priv priv;
        struct ipoib_walk_data data = {
                .addr = addr,
        };

        priv.data = (void *)&data;
        rcu_read_lock();
        if (ipoib_is_dev_match_addr_rcu(addr, dev)) {
                dev_hold(dev);
                data.result = dev;
                goto out;
        }

        netdev_walk_all_upper_dev_rcu(dev, ipoib_upper_walk, &priv);
out:
        rcu_read_unlock();
        return data.result;
}

/* returns the number of IPoIB netdevs on top a given ipoib device matching a
 * pkey_index and address, if one exists.
 *
 * @found_net_dev: contains a matching net_device if the return value >= 1,
 * with a reference held. */
static int ipoib_match_gid_pkey_addr(struct ipoib_dev_priv *priv,
                                     const union ib_gid *gid,
                                     u16 pkey_index,
                                     const struct sockaddr *addr,
                                     int nesting,
                                     struct net_device **found_net_dev)
{
        struct ipoib_dev_priv *child_priv;
        struct net_device *net_dev = NULL;
        int matches = 0;

        if (priv->pkey_index == pkey_index &&
            (!gid || !memcmp(gid, &priv->local_gid, sizeof(*gid)))) {
                if (!addr) {
                        net_dev = ipoib_get_master_net_dev(priv->dev);
                } else {
                        /* Verify the net_device matches the IP address, as
                         * IPoIB child devices currently share a GID. */
                        net_dev = ipoib_get_net_dev_match_addr(addr, priv->dev);
                }
                if (net_dev) {
                        if (!*found_net_dev)
                                *found_net_dev = net_dev;
                        else
                                dev_put(net_dev);
                        ++matches;
                }
        }

        /* Check child interfaces */
        down_read_nested(&priv->vlan_rwsem, nesting);
        list_for_each_entry(child_priv, &priv->child_intfs, list) {
                matches += ipoib_match_gid_pkey_addr(child_priv, gid,
                                                    pkey_index, addr,
                                                    nesting + 1,
                                                    found_net_dev);
                if (matches > 1)
                        break;
        }
        up_read(&priv->vlan_rwsem);

        return matches;
}

/* Returns the number of matching net_devs found (between 0 and 2). Also
 * return the matching net_device in the @net_dev parameter, holding a
 * reference to the net_device, if the number of matches >= 1 */
static int __ipoib_get_net_dev_by_params(struct list_head *dev_list, u32 port,
                                         u16 pkey_index,
                                         const union ib_gid *gid,
                                         const struct sockaddr *addr,
                                         struct net_device **net_dev)
{
        struct ipoib_dev_priv *priv;
        int matches = 0;

        *net_dev = NULL;

        list_for_each_entry(priv, dev_list, list) {
                if (priv->port != port)
                        continue;

                matches += ipoib_match_gid_pkey_addr(priv, gid, pkey_index,
                                                     addr, 0, net_dev);
                if (matches > 1)
                        break;
        }

        return matches;
}

static struct net_device *ipoib_get_net_dev_by_params(
                struct ib_device *dev, u32 port, u16 pkey,
                const union ib_gid *gid, const struct sockaddr *addr,
                void *client_data)
{
        struct net_device *net_dev;
        struct list_head *dev_list = client_data;
        u16 pkey_index;
        int matches;
        int ret;

        if (!rdma_protocol_ib(dev, port))
                return NULL;

        ret = ib_find_cached_pkey(dev, port, pkey, &pkey_index);
        if (ret)
                return NULL;

        /* See if we can find a unique device matching the L2 parameters */
        matches = __ipoib_get_net_dev_by_params(dev_list, port, pkey_index,
                                                gid, NULL, &net_dev);

        switch (matches) {
        case 0:
                return NULL;
        case 1:
                return net_dev;
        }

        dev_put(net_dev);

        /* Couldn't find a unique device with L2 parameters only. Use L3
         * address to uniquely match the net device */
        matches = __ipoib_get_net_dev_by_params(dev_list, port, pkey_index,
                                                gid, addr, &net_dev);
        switch (matches) {
        case 0:
                return NULL;
        default:
                dev_warn_ratelimited(&dev->dev,
                                     "duplicate IP address detected\n");
                fallthrough;
        case 1:
                return net_dev;
        }
}

int ipoib_set_mode(struct net_device *dev, const char *buf)
{
        struct ipoib_dev_priv *priv = ipoib_priv(dev);

        if ((test_bit(IPOIB_FLAG_ADMIN_CM, &priv->flags) &&
             !strcmp(buf, "connected\n")) ||
             (!test_bit(IPOIB_FLAG_ADMIN_CM, &priv->flags) &&
             !strcmp(buf, "datagram\n"))) {
                return 0;
        }

        /* flush paths if we switch modes so that connections are restarted */
        if (IPOIB_CM_SUPPORTED(dev->dev_addr) && !strcmp(buf, "connected\n")) {
                set_bit(IPOIB_FLAG_ADMIN_CM, &priv->flags);
                ipoib_warn(priv, "enabling connected mode "
                           "will cause multicast packet drops\n");
                netdev_update_features(dev);
                dev_set_mtu(dev, ipoib_cm_max_mtu(dev));
                netif_set_real_num_tx_queues(dev, 1);
                rtnl_unlock();
                priv->tx_wr.wr.send_flags &= ~IB_SEND_IP_CSUM;

                ipoib_flush_paths(dev);
                return (!rtnl_trylock()) ? -EBUSY : 0;
        }

        if (!strcmp(buf, "datagram\n")) {
                clear_bit(IPOIB_FLAG_ADMIN_CM, &priv->flags);
                netdev_update_features(dev);
                dev_set_mtu(dev, min(priv->mcast_mtu, dev->mtu));
                netif_set_real_num_tx_queues(dev, dev->num_tx_queues);
                rtnl_unlock();
                ipoib_flush_paths(dev);
                return (!rtnl_trylock()) ? -EBUSY : 0;
        }

        return -EINVAL;
}

struct ipoib_path *__path_find(struct net_device *dev, void *gid)
{
        struct ipoib_dev_priv *priv = ipoib_priv(dev);
        struct rb_node *n = priv->path_tree.rb_node;
        struct ipoib_path *path;
        int ret;

        while (n) {
                path = rb_entry(n, struct ipoib_path, rb_node);

                ret = memcmp(gid, path->pathrec.dgid.raw,
                             sizeof (union ib_gid));

                if (ret < 0)
                        n = n->rb_left;
                else if (ret > 0)
                        n = n->rb_right;
                else
                        return path;
        }

        return NULL;
}

static int __path_add(struct net_device *dev, struct ipoib_path *path)
{
        struct ipoib_dev_priv *priv = ipoib_priv(dev);
        struct rb_node **n = &priv->path_tree.rb_node;
        struct rb_node *pn = NULL;
        struct ipoib_path *tpath;
        int ret;

        while (*n) {
                pn = *n;
                tpath = rb_entry(pn, struct ipoib_path, rb_node);

                ret = memcmp(path->pathrec.dgid.raw, tpath->pathrec.dgid.raw,
                             sizeof (union ib_gid));
                if (ret < 0)
                        n = &pn->rb_left;
                else if (ret > 0)
                        n = &pn->rb_right;
                else
                        return -EEXIST;
        }

        rb_link_node(&path->rb_node, pn, n);
        rb_insert_color(&path->rb_node, &priv->path_tree);

        list_add_tail(&path->list, &priv->path_list);

        return 0;
}

static void path_free(struct net_device *dev, struct ipoib_path *path)
{
        struct sk_buff *skb;

        while ((skb = __skb_dequeue(&path->queue)))
                dev_kfree_skb_irq(skb);

        ipoib_dbg(ipoib_priv(dev), "%s\n", __func__);

        /* remove all neigh connected to this path */
        ipoib_del_neighs_by_gid(dev, path->pathrec.dgid.raw);

        if (path->ah)
                ipoib_put_ah(path->ah);

        kfree(path);
}

#ifdef CONFIG_INFINIBAND_IPOIB_DEBUG

struct ipoib_path_iter *ipoib_path_iter_init(struct net_device *dev)
{
        struct ipoib_path_iter *iter;

        iter = kmalloc(sizeof(*iter), GFP_KERNEL);
        if (!iter)
                return NULL;

        iter->dev = dev;
        memset(iter->path.pathrec.dgid.raw, 0, 16);

        if (ipoib_path_iter_next(iter)) {
                kfree(iter);
                return NULL;
        }

        return iter;
}

int ipoib_path_iter_next(struct ipoib_path_iter *iter)
{
        struct ipoib_dev_priv *priv = ipoib_priv(iter->dev);
        struct rb_node *n;
        struct ipoib_path *path;
        int ret = 1;

        spin_lock_irq(&priv->lock);

        n = rb_first(&priv->path_tree);

        while (n) {
                path = rb_entry(n, struct ipoib_path, rb_node);

                if (memcmp(iter->path.pathrec.dgid.raw, path->pathrec.dgid.raw,
                           sizeof (union ib_gid)) < 0) {
                        iter->path = *path;
                        ret = 0;
                        break;
                }

                n = rb_next(n);
        }

        spin_unlock_irq(&priv->lock);

        return ret;
}

void ipoib_path_iter_read(struct ipoib_path_iter *iter,
                          struct ipoib_path *path)
{
        *path = iter->path;
}

#endif /* CONFIG_INFINIBAND_IPOIB_DEBUG */

void ipoib_mark_paths_invalid(struct net_device *dev)
{
        struct ipoib_dev_priv *priv = ipoib_priv(dev);
        struct ipoib_path *path, *tp;

        spin_lock_irq(&priv->lock);

        list_for_each_entry_safe(path, tp, &priv->path_list, list) {
                ipoib_dbg(priv, "mark path LID 0x%08x GID %pI6 invalid\n",
                          be32_to_cpu(sa_path_get_dlid(&path->pathrec)),
                          path->pathrec.dgid.raw);
                if (path->ah)
                        path->ah->valid = 0;
        }

        spin_unlock_irq(&priv->lock);
}

static void push_pseudo_header(struct sk_buff *skb, const char *daddr)
{
        struct ipoib_pseudo_header *phdr;

        phdr = skb_push(skb, sizeof(*phdr));
        memcpy(phdr->hwaddr, daddr, INFINIBAND_ALEN);
}

void ipoib_flush_paths(struct net_device *dev)
{
        struct ipoib_dev_priv *priv = ipoib_priv(dev);
        struct ipoib_path *path, *tp;
        LIST_HEAD(remove_list);
        unsigned long flags;

        netif_tx_lock_bh(dev);
        spin_lock_irqsave(&priv->lock, flags);

        list_splice_init(&priv->path_list, &remove_list);

        list_for_each_entry(path, &remove_list, list)
                rb_erase(&path->rb_node, &priv->path_tree);

        list_for_each_entry_safe(path, tp, &remove_list, list) {
                if (path->query)
                        ib_sa_cancel_query(path->query_id, path->query);
                spin_unlock_irqrestore(&priv->lock, flags);
                netif_tx_unlock_bh(dev);
                wait_for_completion(&path->done);
                path_free(dev, path);
                netif_tx_lock_bh(dev);
                spin_lock_irqsave(&priv->lock, flags);
        }

        spin_unlock_irqrestore(&priv->lock, flags);
        netif_tx_unlock_bh(dev);
}

static void path_rec_completion(int status,
                                struct sa_path_rec *pathrec,
                                unsigned int num_prs, void *path_ptr)
{
        struct ipoib_path *path = path_ptr;
        struct net_device *dev = path->dev;
        struct ipoib_dev_priv *priv = ipoib_priv(dev);
        struct ipoib_ah *ah = NULL;
        struct ipoib_ah *old_ah = NULL;
        struct ipoib_neigh *neigh, *tn;
        struct sk_buff_head skqueue;
        struct sk_buff *skb;
        unsigned long flags;

        if (!status)
                ipoib_dbg(priv, "PathRec LID 0x%04x for GID %pI6\n",
                          be32_to_cpu(sa_path_get_dlid(pathrec)),
                          pathrec->dgid.raw);
        else
                ipoib_dbg(priv, "PathRec status %d for GID %pI6\n",
                          status, path->pathrec.dgid.raw);

        skb_queue_head_init(&skqueue);

        if (!status) {
                struct rdma_ah_attr av;

                if (!ib_init_ah_attr_from_path(priv->ca, priv->port,
                                               pathrec, &av, NULL)) {
                        ah = ipoib_create_ah(dev, priv->pd, &av);
                        rdma_destroy_ah_attr(&av);
                }
        }

        spin_lock_irqsave(&priv->lock, flags);

        if (!IS_ERR_OR_NULL(ah)) {
                /*
                 * pathrec.dgid is used as the database key from the LLADDR,
                 * it must remain unchanged even if the SA returns a different
                 * GID to use in the AH.
                 */
                if (memcmp(pathrec->dgid.raw, path->pathrec.dgid.raw,
                           sizeof(union ib_gid))) {
                        ipoib_dbg(
                                priv,
                                "%s got PathRec for gid %pI6 while asked for %pI6\n",
                                dev->name, pathrec->dgid.raw,
                                path->pathrec.dgid.raw);
                        memcpy(pathrec->dgid.raw, path->pathrec.dgid.raw,
                               sizeof(union ib_gid));
                }

                path->pathrec = *pathrec;

                old_ah   = path->ah;
                path->ah = ah;

                ipoib_dbg(priv, "created address handle %p for LID 0x%04x, SL %d\n",
                          ah, be32_to_cpu(sa_path_get_dlid(pathrec)),
                          pathrec->sl);

                while ((skb = __skb_dequeue(&path->queue)))
                        __skb_queue_tail(&skqueue, skb);

                list_for_each_entry_safe(neigh, tn, &path->neigh_list, list) {
                        if (neigh->ah) {
                                WARN_ON(neigh->ah != old_ah);
                                /*
                                 * Dropping the ah reference inside
                                 * priv->lock is safe here, because we
                                 * will hold one more reference from
                                 * the original value of path->ah (ie
                                 * old_ah).
                                 */
                                ipoib_put_ah(neigh->ah);
                        }
                        kref_get(&path->ah->ref);
                        neigh->ah = path->ah;

                        if (ipoib_cm_enabled(dev, neigh->daddr)) {
                                if (!ipoib_cm_get(neigh))
                                        ipoib_cm_set(neigh, ipoib_cm_create_tx(dev,
                                                                               path,
                                                                               neigh));
                                if (!ipoib_cm_get(neigh)) {
                                        ipoib_neigh_free(neigh);
                                        continue;
                                }
                        }

                        while ((skb = __skb_dequeue(&neigh->queue)))
                                __skb_queue_tail(&skqueue, skb);
                }
                path->ah->valid = 1;
        }

        path->query = NULL;
        complete(&path->done);

        spin_unlock_irqrestore(&priv->lock, flags);

        if (IS_ERR_OR_NULL(ah))
                ipoib_del_neighs_by_gid(dev, path->pathrec.dgid.raw);

        if (old_ah)
                ipoib_put_ah(old_ah);

        while ((skb = __skb_dequeue(&skqueue))) {
                int ret;
                skb->dev = dev;
                ret = dev_queue_xmit(skb);
                if (ret)
                        ipoib_warn(priv, "%s: dev_queue_xmit failed to re-queue packet, ret:%d\n",
                                   __func__, ret);
        }
}

static void init_path_rec(struct ipoib_dev_priv *priv, struct ipoib_path *path,
                          void *gid)
{
        path->dev = priv->dev;

        if (rdma_cap_opa_ah(priv->ca, priv->port))
                path->pathrec.rec_type = SA_PATH_REC_TYPE_OPA;
        else
                path->pathrec.rec_type = SA_PATH_REC_TYPE_IB;

        memcpy(path->pathrec.dgid.raw, gid, sizeof(union ib_gid));
        path->pathrec.sgid            = priv->local_gid;
        path->pathrec.pkey            = cpu_to_be16(priv->pkey);
        path->pathrec.numb_path     = 1;
        path->pathrec.traffic_class = priv->broadcast->mcmember.traffic_class;
}

static struct ipoib_path *path_rec_create(struct net_device *dev, void *gid)
{
        struct ipoib_dev_priv *priv = ipoib_priv(dev);
        struct ipoib_path *path;

        if (!priv->broadcast)
                return NULL;

        path = kzalloc(sizeof(*path), GFP_ATOMIC);
        if (!path)
                return NULL;

        skb_queue_head_init(&path->queue);

        INIT_LIST_HEAD(&path->neigh_list);

        init_path_rec(priv, path, gid);

        return path;
}

static int path_rec_start(struct net_device *dev,
                          struct ipoib_path *path)
{
        struct ipoib_dev_priv *priv = ipoib_priv(dev);

        ipoib_dbg(priv, "Start path record lookup for %pI6\n",
                  path->pathrec.dgid.raw);

        init_completion(&path->done);

        path->query_id =
                ib_sa_path_rec_get(&ipoib_sa_client, priv->ca, priv->port,
                                   &path->pathrec,
                                   IB_SA_PATH_REC_DGID                |
                                   IB_SA_PATH_REC_SGID                |
                                   IB_SA_PATH_REC_NUMB_PATH        |
                                   IB_SA_PATH_REC_TRAFFIC_CLASS |
                                   IB_SA_PATH_REC_PKEY,
                                   1000, GFP_ATOMIC,
                                   path_rec_completion,
                                   path, &path->query);
        if (path->query_id < 0) {
                ipoib_warn(priv, "ib_sa_path_rec_get failed: %d\n", path->query_id);
                path->query = NULL;
                complete(&path->done);
                return path->query_id;
        }

        return 0;
}

static void neigh_refresh_path(struct ipoib_neigh *neigh, u8 *daddr,
                               struct net_device *dev)
{
        struct ipoib_dev_priv *priv = ipoib_priv(dev);
        struct ipoib_path *path;
        unsigned long flags;

        spin_lock_irqsave(&priv->lock, flags);

        path = __path_find(dev, daddr + 4);
        if (!path)
                goto out;
        if (!path->query)
                path_rec_start(dev, path);
out:
        spin_unlock_irqrestore(&priv->lock, flags);
}

static struct ipoib_neigh *neigh_add_path(struct sk_buff *skb, u8 *daddr,
                                          struct net_device *dev)
{
        struct ipoib_dev_priv *priv = ipoib_priv(dev);
        struct rdma_netdev *rn = netdev_priv(dev);
        struct ipoib_path *path;
        struct ipoib_neigh *neigh;
        unsigned long flags;

        spin_lock_irqsave(&priv->lock, flags);
        neigh = ipoib_neigh_alloc(daddr, dev);
        if (!neigh) {
                spin_unlock_irqrestore(&priv->lock, flags);
                ++dev->stats.tx_dropped;
                dev_kfree_skb_any(skb);
                return NULL;
        }

        /* To avoid race condition, make sure that the
         * neigh will be added only once.
         */
        if (unlikely(!list_empty(&neigh->list))) {
                spin_unlock_irqrestore(&priv->lock, flags);
                return neigh;
        }

        path = __path_find(dev, daddr + 4);
        if (!path) {
                path = path_rec_create(dev, daddr + 4);
                if (!path)
                        goto err_path;

                __path_add(dev, path);
        }

        list_add_tail(&neigh->list, &path->neigh_list);

        if (path->ah && path->ah->valid) {
                kref_get(&path->ah->ref);
                neigh->ah = path->ah;

                if (ipoib_cm_enabled(dev, neigh->daddr)) {
                        if (!ipoib_cm_get(neigh))
                                ipoib_cm_set(neigh, ipoib_cm_create_tx(dev, path, neigh));
                        if (!ipoib_cm_get(neigh)) {
                                ipoib_neigh_free(neigh);
                                goto err_drop;
                        }
                        if (skb_queue_len(&neigh->queue) <
                            IPOIB_MAX_PATH_REC_QUEUE) {
                                push_pseudo_header(skb, neigh->daddr);
                                __skb_queue_tail(&neigh->queue, skb);
                        } else {
                                ipoib_warn(priv, "queue length limit %d. Packet drop.\n",
                                           skb_queue_len(&neigh->queue));
                                goto err_drop;
                        }
                } else {
                        spin_unlock_irqrestore(&priv->lock, flags);
                        path->ah->last_send = rn->send(dev, skb, path->ah->ah,
                                                       IPOIB_QPN(daddr));
                        ipoib_neigh_put(neigh);
                        return NULL;
                }
        } else {
                neigh->ah  = NULL;

                if (!path->query && path_rec_start(dev, path))
                        goto err_path;
                if (skb_queue_len(&neigh->queue) < IPOIB_MAX_PATH_REC_QUEUE) {
                        push_pseudo_header(skb, neigh->daddr);
                        __skb_queue_tail(&neigh->queue, skb);
                } else {
                        goto err_drop;
                }
        }

        spin_unlock_irqrestore(&priv->lock, flags);
        ipoib_neigh_put(neigh);
        return NULL;

err_path:
        ipoib_neigh_free(neigh);
err_drop:
        ++dev->stats.tx_dropped;
        dev_kfree_skb_any(skb);

        spin_unlock_irqrestore(&priv->lock, flags);
        ipoib_neigh_put(neigh);

        return NULL;
}

static void unicast_arp_send(struct sk_buff *skb, struct net_device *dev,
                             struct ipoib_pseudo_header *phdr)
{
        struct ipoib_dev_priv *priv = ipoib_priv(dev);
        struct rdma_netdev *rn = netdev_priv(dev);
        struct ipoib_path *path;
        unsigned long flags;

        spin_lock_irqsave(&priv->lock, flags);

        /* no broadcast means that all paths are (going to be) not valid */
        if (!priv->broadcast)
                goto drop_and_unlock;

        path = __path_find(dev, phdr->hwaddr + 4);
        if (!path || !path->ah || !path->ah->valid) {
                if (!path) {
                        path = path_rec_create(dev, phdr->hwaddr + 4);
                        if (!path)
                                goto drop_and_unlock;
                        __path_add(dev, path);
                } else {
                        /*
                         * make sure there are no changes in the existing
                         * path record
                         */
                        init_path_rec(priv, path, phdr->hwaddr + 4);
                }
                if (!path->query && path_rec_start(dev, path)) {
                        goto drop_and_unlock;
                }

                if (skb_queue_len(&path->queue) < IPOIB_MAX_PATH_REC_QUEUE) {
                        push_pseudo_header(skb, phdr->hwaddr);
                        __skb_queue_tail(&path->queue, skb);
                        goto unlock;
                } else {
                        goto drop_and_unlock;
                }
        }

        spin_unlock_irqrestore(&priv->lock, flags);
        ipoib_dbg(priv, "Send unicast ARP to %08x\n",
                  be32_to_cpu(sa_path_get_dlid(&path->pathrec)));
        path->ah->last_send = rn->send(dev, skb, path->ah->ah,
                                       IPOIB_QPN(phdr->hwaddr));
        return;

drop_and_unlock:
        ++dev->stats.tx_dropped;
        dev_kfree_skb_any(skb);
unlock:
        spin_unlock_irqrestore(&priv->lock, flags);
}

static netdev_tx_t ipoib_start_xmit(struct sk_buff *skb, struct net_device *dev)
{
        struct ipoib_dev_priv *priv = ipoib_priv(dev);
        struct rdma_netdev *rn = netdev_priv(dev);
        struct ipoib_neigh *neigh;
        struct ipoib_pseudo_header *phdr;
        struct ipoib_header *header;
        unsigned long flags;

        phdr = (struct ipoib_pseudo_header *) skb->data;
        skb_pull(skb, sizeof(*phdr));
        header = (struct ipoib_header *) skb->data;

        if (unlikely(phdr->hwaddr[4] == 0xff)) {
                /* multicast, arrange "if" according to probability */
                if ((header->proto != htons(ETH_P_IP)) &&
                    (header->proto != htons(ETH_P_IPV6)) &&
                    (header->proto != htons(ETH_P_ARP)) &&
                    (header->proto != htons(ETH_P_RARP)) &&
                    (header->proto != htons(ETH_P_TIPC))) {
                        /* ethertype not supported by IPoIB */
                        ++dev->stats.tx_dropped;
                        dev_kfree_skb_any(skb);
                        return NETDEV_TX_OK;
                }
                /* Add in the P_Key for multicast*/
                phdr->hwaddr[8] = (priv->pkey >> 8) & 0xff;
                phdr->hwaddr[9] = priv->pkey & 0xff;

                neigh = ipoib_neigh_get(dev, phdr->hwaddr);
                if (likely(neigh))
                        goto send_using_neigh;
                ipoib_mcast_send(dev, phdr->hwaddr, skb);
                return NETDEV_TX_OK;
        }

        /* unicast, arrange "switch" according to probability */
        switch (header->proto) {
        case htons(ETH_P_IP):
        case htons(ETH_P_IPV6):
        case htons(ETH_P_TIPC):
                neigh = ipoib_neigh_get(dev, phdr->hwaddr);
                if (unlikely(!neigh)) {
                        neigh = neigh_add_path(skb, phdr->hwaddr, dev);
                        if (likely(!neigh))
                                return NETDEV_TX_OK;
                }
                break;
        case htons(ETH_P_ARP):
        case htons(ETH_P_RARP):
                /* for unicast ARP and RARP should always perform path find */
                unicast_arp_send(skb, dev, phdr);
                return NETDEV_TX_OK;
        default:
                /* ethertype not supported by IPoIB */
                ++dev->stats.tx_dropped;
                dev_kfree_skb_any(skb);
                return NETDEV_TX_OK;
        }

send_using_neigh:
        /* note we now hold a ref to neigh */
        if (ipoib_cm_get(neigh)) {
                if (ipoib_cm_up(neigh)) {
                        ipoib_cm_send(dev, skb, ipoib_cm_get(neigh));
                        goto unref;
                }
        } else if (neigh->ah && neigh->ah->valid) {
                neigh->ah->last_send = rn->send(dev, skb, neigh->ah->ah,
                                                IPOIB_QPN(phdr->hwaddr));
                goto unref;
        } else if (neigh->ah) {
                neigh_refresh_path(neigh, phdr->hwaddr, dev);
        }

        if (skb_queue_len(&neigh->queue) < IPOIB_MAX_PATH_REC_QUEUE) {
                push_pseudo_header(skb, phdr->hwaddr);
                spin_lock_irqsave(&priv->lock, flags);
                __skb_queue_tail(&neigh->queue, skb);
                spin_unlock_irqrestore(&priv->lock, flags);
        } else {
                ++dev->stats.tx_dropped;
                dev_kfree_skb_any(skb);
        }

unref:
        ipoib_neigh_put(neigh);

        return NETDEV_TX_OK;
}

static void ipoib_timeout(struct net_device *dev, unsigned int txqueue)
{
        struct ipoib_dev_priv *priv = ipoib_priv(dev);
        struct rdma_netdev *rn = netdev_priv(dev);

        if (rn->tx_timeout) {
                rn->tx_timeout(dev, txqueue);
                return;
        }
        ipoib_warn(priv, "transmit timeout: latency %d msecs\n",
                   jiffies_to_msecs(jiffies - dev_trans_start(dev)));
        ipoib_warn(priv,
                   "queue stopped %d, tx_head %u, tx_tail %u, global_tx_head %u, global_tx_tail %u\n",
                   netif_queue_stopped(dev), priv->tx_head, priv->tx_tail,
                   priv->global_tx_head, priv->global_tx_tail);


        schedule_work(&priv->tx_timeout_work);
}

void ipoib_ib_tx_timeout_work(struct work_struct *work)
{
        struct ipoib_dev_priv *priv = container_of(work,
                                                   struct ipoib_dev_priv,
                                                   tx_timeout_work);
        int err;

        rtnl_lock();

        if (!test_bit(IPOIB_FLAG_ADMIN_UP, &priv->flags))
                goto unlock;

        ipoib_stop(priv->dev);
        err = ipoib_open(priv->dev);
        if (err) {
                ipoib_warn(priv, "ipoib_open failed recovering from a tx_timeout, err(%d).\n",
                                err);
                goto unlock;
        }

        netif_tx_wake_all_queues(priv->dev);
unlock:
        rtnl_unlock();

}

static int ipoib_hard_header(struct sk_buff *skb,
                             struct net_device *dev,
                             unsigned short type,
                             const void *daddr,
                             const void *saddr,
                             unsigned int len)
{
        struct ipoib_header *header;

        header = skb_push(skb, sizeof(*header));

        header->proto = htons(type);
        header->reserved = 0;

        /*
         * we don't rely on dst_entry structure,  always stuff the
         * destination address into skb hard header so we can figure out where
         * to send the packet later.
         */
        push_pseudo_header(skb, daddr);

        return IPOIB_HARD_LEN;
}

static void ipoib_set_mcast_list(struct net_device *dev)
{
        struct ipoib_dev_priv *priv = ipoib_priv(dev);

        if (!test_bit(IPOIB_FLAG_OPER_UP, &priv->flags)) {
                ipoib_dbg(priv, "IPOIB_FLAG_OPER_UP not set");
                return;
        }

        queue_work(priv->wq, &priv->restart_task);
}

static int ipoib_get_iflink(const struct net_device *dev)
{
        struct ipoib_dev_priv *priv = ipoib_priv(dev);

        /* parent interface */
        if (!test_bit(IPOIB_FLAG_SUBINTERFACE, &priv->flags))
                return READ_ONCE(dev->ifindex);

        /* child/vlan interface */
        return READ_ONCE(priv->parent->ifindex);
}

static u32 ipoib_addr_hash(struct ipoib_neigh_hash *htbl, u8 *daddr)
{
        /*
         * Use only the address parts that contributes to spreading
         * The subnet prefix is not used as one can not connect to
         * same remote port (GUID) using the same remote QPN via two
         * different subnets.
         */
         /* qpn octets[1:4) & port GUID octets[12:20) */
        u32 *d32 = (u32 *) daddr;
        u32 hv;

        hv = jhash_3words(d32[3], d32[4], IPOIB_QPN_MASK & d32[0], 0);
        return hv & htbl->mask;
}

struct ipoib_neigh *ipoib_neigh_get(struct net_device *dev, u8 *daddr)
{
        struct ipoib_dev_priv *priv = ipoib_priv(dev);
        struct ipoib_neigh_table *ntbl = &priv->ntbl;
        struct ipoib_neigh_hash *htbl;
        struct ipoib_neigh *neigh = NULL;
        u32 hash_val;

        rcu_read_lock_bh();

        htbl = rcu_dereference_bh(ntbl->htbl);

        if (!htbl)
                goto out_unlock;

        hash_val = ipoib_addr_hash(htbl, daddr);
        for (neigh = rcu_dereference_bh(htbl->buckets[hash_val]);
             neigh != NULL;
             neigh = rcu_dereference_bh(neigh->hnext)) {
                if (memcmp(daddr, neigh->daddr, INFINIBAND_ALEN) == 0) {
                        /* found, take one ref on behalf of the caller */
                        if (!refcount_inc_not_zero(&neigh->refcnt)) {
                                /* deleted */
                                neigh = NULL;
                                goto out_unlock;
                        }

                        if (likely(skb_queue_len(&neigh->queue) < IPOIB_MAX_PATH_REC_QUEUE))
                                neigh->alive = jiffies;
                        goto out_unlock;
                }
        }

out_unlock:
        rcu_read_unlock_bh();
        return neigh;
}

static void __ipoib_reap_neigh(struct ipoib_dev_priv *priv)
{
        struct ipoib_neigh_table *ntbl = &priv->ntbl;
        struct ipoib_neigh_hash *htbl;
        unsigned long neigh_obsolete;
        unsigned long dt;
        unsigned long flags;
        int i;
        LIST_HEAD(remove_list);

        spin_lock_irqsave(&priv->lock, flags);

        htbl = rcu_dereference_protected(ntbl->htbl,
                                         lockdep_is_held(&priv->lock));

        if (!htbl)
                goto out_unlock;

        /* neigh is obsolete if it was idle for two GC periods */
        dt = 2 * arp_tbl.gc_interval;
        neigh_obsolete = jiffies - dt;

        for (i = 0; i < htbl->size; i++) {
                struct ipoib_neigh *neigh;
                struct ipoib_neigh __rcu **np = &htbl->buckets[i];

                while ((neigh = rcu_dereference_protected(*np,
                                                          lockdep_is_held(&priv->lock))) != NULL) {
                        /* was the neigh idle for two GC periods */
                        if (time_after(neigh_obsolete, neigh->alive)) {

                                ipoib_check_and_add_mcast_sendonly(priv, neigh->daddr + 4, &remove_list);

                                rcu_assign_pointer(*np,
                                                   rcu_dereference_protected(neigh->hnext,
                                                                             lockdep_is_held(&priv->lock)));
                                /* remove from path/mc list */
                                list_del_init(&neigh->list);
                                call_rcu(&neigh->rcu, ipoib_neigh_reclaim);
                        } else {
                                np = &neigh->hnext;
                        }

                }
        }

out_unlock:
        spin_unlock_irqrestore(&priv->lock, flags);
        ipoib_mcast_remove_list(&remove_list);
}

static void ipoib_reap_neigh(struct work_struct *work)
{
        struct ipoib_dev_priv *priv =
                container_of(work, struct ipoib_dev_priv, neigh_reap_task.work);

        __ipoib_reap_neigh(priv);

        queue_delayed_work(priv->wq, &priv->neigh_reap_task,
                           arp_tbl.gc_interval);
}


static struct ipoib_neigh *ipoib_neigh_ctor(u8 *daddr,
                                      struct net_device *dev)
{
        struct ipoib_neigh *neigh;

        neigh = kzalloc(sizeof(*neigh), GFP_ATOMIC);
        if (!neigh)
                return NULL;

        neigh->dev = dev;
        memcpy(&neigh->daddr, daddr, sizeof(neigh->daddr));
        skb_queue_head_init(&neigh->queue);
        INIT_LIST_HEAD(&neigh->list);
        ipoib_cm_set(neigh, NULL);
        /* one ref on behalf of the caller */
        refcount_set(&neigh->refcnt, 1);

        return neigh;
}

struct ipoib_neigh *ipoib_neigh_alloc(u8 *daddr,
                                      struct net_device *dev)
{
        struct ipoib_dev_priv *priv = ipoib_priv(dev);
        struct ipoib_neigh_table *ntbl = &priv->ntbl;
        struct ipoib_neigh_hash *htbl;
        struct ipoib_neigh *neigh;
        u32 hash_val;

        htbl = rcu_dereference_protected(ntbl->htbl,
                                         lockdep_is_held(&priv->lock));
        if (!htbl) {
                neigh = NULL;
                goto out_unlock;
        }

        /* need to add a new neigh, but maybe some other thread succeeded?
         * recalc hash, maybe hash resize took place so we do a search
         */
        hash_val = ipoib_addr_hash(htbl, daddr);
        for (neigh = rcu_dereference_protected(htbl->buckets[hash_val],
                                               lockdep_is_held(&priv->lock));
             neigh != NULL;
             neigh = rcu_dereference_protected(neigh->hnext,
                                               lockdep_is_held(&priv->lock))) {
                if (memcmp(daddr, neigh->daddr, INFINIBAND_ALEN) == 0) {
                        /* found, take one ref on behalf of the caller */
                        if (!refcount_inc_not_zero(&neigh->refcnt)) {
                                /* deleted */
                                neigh = NULL;
                                break;
                        }
                        neigh->alive = jiffies;
                        goto out_unlock;
                }
        }

        neigh = ipoib_neigh_ctor(daddr, dev);
        if (!neigh)
                goto out_unlock;

        /* one ref on behalf of the hash table */
        refcount_inc(&neigh->refcnt);
        neigh->alive = jiffies;
        /* put in hash */
        rcu_assign_pointer(neigh->hnext,
                           rcu_dereference_protected(htbl->buckets[hash_val],
                                                     lockdep_is_held(&priv->lock)));
        rcu_assign_pointer(htbl->buckets[hash_val], neigh);
        atomic_inc(&ntbl->entries);

out_unlock:

        return neigh;
}

void ipoib_neigh_dtor(struct ipoib_neigh *neigh)
{
        /* neigh reference count was dropprd to zero */
        struct net_device *dev = neigh->dev;
        struct ipoib_dev_priv *priv = ipoib_priv(dev);
        struct sk_buff *skb;
        if (neigh->ah)
                ipoib_put_ah(neigh->ah);
        while ((skb = __skb_dequeue(&neigh->queue))) {
                ++dev->stats.tx_dropped;
                dev_kfree_skb_any(skb);
        }
        if (ipoib_cm_get(neigh))
                ipoib_cm_destroy_tx(ipoib_cm_get(neigh));
        ipoib_dbg(ipoib_priv(dev),
                  "neigh free for %06x %pI6\n",
                  IPOIB_QPN(neigh->daddr),
                  neigh->daddr + 4);
        kfree(neigh);
        if (atomic_dec_and_test(&priv->ntbl.entries)) {
                if (test_bit(IPOIB_NEIGH_TBL_FLUSH, &priv->flags))
                        complete(&priv->ntbl.flushed);
        }
}

static void ipoib_neigh_reclaim(struct rcu_head *rp)
{
        /* Called as a result of removal from hash table */
        struct ipoib_neigh *neigh = container_of(rp, struct ipoib_neigh, rcu);
        /* note TX context may hold another ref */
        ipoib_neigh_put(neigh);
}

void ipoib_neigh_free(struct ipoib_neigh *neigh)
{
        struct net_device *dev = neigh->dev;
        struct ipoib_dev_priv *priv = ipoib_priv(dev);
        struct ipoib_neigh_table *ntbl = &priv->ntbl;
        struct ipoib_neigh_hash *htbl;
        struct ipoib_neigh __rcu **np;
        struct ipoib_neigh *n;
        u32 hash_val;

        htbl = rcu_dereference_protected(ntbl->htbl,
                                        lockdep_is_held(&priv->lock));
        if (!htbl)
                return;

        hash_val = ipoib_addr_hash(htbl, neigh->daddr);
        np = &htbl->buckets[hash_val];
        for (n = rcu_dereference_protected(*np,
                                            lockdep_is_held(&priv->lock));
             n != NULL;
             n = rcu_dereference_protected(*np,
                                        lockdep_is_held(&priv->lock))) {
                if (n == neigh) {
                        /* found */
                        rcu_assign_pointer(*np,
                                           rcu_dereference_protected(neigh->hnext,
                                                                     lockdep_is_held(&priv->lock)));
                        /* remove from parent list */
                        list_del_init(&neigh->list);
                        call_rcu(&neigh->rcu, ipoib_neigh_reclaim);
                        return;
                } else {
                        np = &n->hnext;
                }
        }
}

static int ipoib_neigh_hash_init(struct ipoib_dev_priv *priv)
{
        struct ipoib_neigh_table *ntbl = &priv->ntbl;
        struct ipoib_neigh_hash *htbl;
        struct ipoib_neigh __rcu **buckets;
        u32 size;

        clear_bit(IPOIB_NEIGH_TBL_FLUSH, &priv->flags);
        ntbl->htbl = NULL;
        htbl = kzalloc(sizeof(*htbl), GFP_KERNEL);
        if (!htbl)
                return -ENOMEM;
        size = roundup_pow_of_two(arp_tbl.gc_thresh3);
        buckets = kvcalloc(size, sizeof(*buckets), GFP_KERNEL);
        if (!buckets) {
                kfree(htbl);
                return -ENOMEM;
        }
        htbl->size = size;
        htbl->mask = (size - 1);
        htbl->buckets = buckets;
        RCU_INIT_POINTER(ntbl->htbl, htbl);
        htbl->ntbl = ntbl;
        atomic_set(&ntbl->entries, 0);

        /* start garbage collection */
        queue_delayed_work(priv->wq, &priv->neigh_reap_task,
                           arp_tbl.gc_interval);

        return 0;
}

static void neigh_hash_free_rcu(struct rcu_head *head)
{
        struct ipoib_neigh_hash *htbl = container_of(head,
                                                    struct ipoib_neigh_hash,
                                                    rcu);
        struct ipoib_neigh __rcu **buckets = htbl->buckets;
        struct ipoib_neigh_table *ntbl = htbl->ntbl;

        kvfree(buckets);
        kfree(htbl);
        complete(&ntbl->deleted);
}

void ipoib_del_neighs_by_gid(struct net_device *dev, u8 *gid)
{
        struct ipoib_dev_priv *priv = ipoib_priv(dev);
        struct ipoib_neigh_table *ntbl = &priv->ntbl;
        struct ipoib_neigh_hash *htbl;
        unsigned long flags;
        int i;

        /* remove all neigh connected to a given path or mcast */
        spin_lock_irqsave(&priv->lock, flags);

        htbl = rcu_dereference_protected(ntbl->htbl,
                                         lockdep_is_held(&priv->lock));

        if (!htbl)
                goto out_unlock;

        for (i = 0; i < htbl->size; i++) {
                struct ipoib_neigh *neigh;
                struct ipoib_neigh __rcu **np = &htbl->buckets[i];

                while ((neigh = rcu_dereference_protected(*np,
                                                          lockdep_is_held(&priv->lock))) != NULL) {
                        /* delete neighs belong to this parent */
                        if (!memcmp(gid, neigh->daddr + 4, sizeof (union ib_gid))) {
                                rcu_assign_pointer(*np,
                                                   rcu_dereference_protected(neigh->hnext,
                                                                             lockdep_is_held(&priv->lock)));
                                /* remove from parent list */
                                list_del_init(&neigh->list);
                                call_rcu(&neigh->rcu, ipoib_neigh_reclaim);
                        } else {
                                np = &neigh->hnext;
                        }

                }
        }
out_unlock:
        spin_unlock_irqrestore(&priv->lock, flags);
}

static void ipoib_flush_neighs(struct ipoib_dev_priv *priv)
{
        struct ipoib_neigh_table *ntbl = &priv->ntbl;
        struct ipoib_neigh_hash *htbl;
        unsigned long flags;
        int i, wait_flushed = 0;

        init_completion(&priv->ntbl.flushed);
        set_bit(IPOIB_NEIGH_TBL_FLUSH, &priv->flags);

        spin_lock_irqsave(&priv->lock, flags);

        htbl = rcu_dereference_protected(ntbl->htbl,
                                        lockdep_is_held(&priv->lock));
        if (!htbl)
                goto out_unlock;

        wait_flushed = atomic_read(&priv->ntbl.entries);
        if (!wait_flushed)
                goto free_htbl;

        for (i = 0; i < htbl->size; i++) {
                struct ipoib_neigh *neigh;
                struct ipoib_neigh __rcu **np = &htbl->buckets[i];

                while ((neigh = rcu_dereference_protected(*np,
                                       lockdep_is_held(&priv->lock))) != NULL) {
                        rcu_assign_pointer(*np,
                                           rcu_dereference_protected(neigh->hnext,
                                                                     lockdep_is_held(&priv->lock)));
                        /* remove from path/mc list */
                        list_del_init(&neigh->list);
                        call_rcu(&neigh->rcu, ipoib_neigh_reclaim);
                }
        }

free_htbl:
        rcu_assign_pointer(ntbl->htbl, NULL);
        call_rcu(&htbl->rcu, neigh_hash_free_rcu);

out_unlock:
        spin_unlock_irqrestore(&priv->lock, flags);
        if (wait_flushed)
                wait_for_completion(&priv->ntbl.flushed);
}

static void ipoib_neigh_hash_uninit(struct net_device *dev)
{
        struct ipoib_dev_priv *priv = ipoib_priv(dev);

        ipoib_dbg(priv, "%s\n", __func__);
        init_completion(&priv->ntbl.deleted);

        cancel_delayed_work_sync(&priv->neigh_reap_task);

        ipoib_flush_neighs(priv);

        wait_for_completion(&priv->ntbl.deleted);
}

static void ipoib_napi_add(struct net_device *dev)
{
        struct ipoib_dev_priv *priv = ipoib_priv(dev);

        netif_napi_add_weight(dev, &priv->recv_napi, ipoib_rx_poll,
                              IPOIB_NUM_WC);
        netif_napi_add_weight(dev, &priv->send_napi, ipoib_tx_poll,
                              MAX_SEND_CQE);
}

static void ipoib_napi_del(struct net_device *dev)
{
        struct ipoib_dev_priv *priv = ipoib_priv(dev);

        netif_napi_del(&priv->recv_napi);
        netif_napi_del(&priv->send_napi);
}

static void ipoib_dev_uninit_default(struct net_device *dev)
{
        struct ipoib_dev_priv *priv = ipoib_priv(dev);

        ipoib_transport_dev_cleanup(dev);

        ipoib_napi_del(dev);

        ipoib_cm_dev_cleanup(dev);

        kfree(priv->rx_ring);
        vfree(priv->tx_ring);

        priv->rx_ring = NULL;
        priv->tx_ring = NULL;
}

static int ipoib_dev_init_default(struct net_device *dev)
{
        struct ipoib_dev_priv *priv = ipoib_priv(dev);
        u8 addr_mod[3];

        ipoib_napi_add(dev);

        /* Allocate RX/TX "rings" to hold queued skbs */
        priv->rx_ring =        kcalloc(ipoib_recvq_size,
                                       sizeof(*priv->rx_ring),
                                       GFP_KERNEL);
        if (!priv->rx_ring)
                goto out;

        priv->tx_ring = vzalloc(array_size(ipoib_sendq_size,
                                           sizeof(*priv->tx_ring)));
        if (!priv->tx_ring) {
                pr_warn("%s: failed to allocate TX ring (%d entries)\n",
                        priv->ca->name, ipoib_sendq_size);
                goto out_rx_ring_cleanup;
        }

        /* priv->tx_head, tx_tail and global_tx_tail/head are already 0 */

        if (ipoib_transport_dev_init(dev, priv->ca)) {
                pr_warn("%s: ipoib_transport_dev_init failed\n",
                        priv->ca->name);
                goto out_tx_ring_cleanup;
        }

        /* after qp created set dev address */
        addr_mod[0] = (priv->qp->qp_num >> 16) & 0xff;
        addr_mod[1] = (priv->qp->qp_num >>  8) & 0xff;
        addr_mod[2] = (priv->qp->qp_num) & 0xff;
        dev_addr_mod(priv->dev, 1, addr_mod, sizeof(addr_mod));

        return 0;

out_tx_ring_cleanup:
        vfree(priv->tx_ring);

out_rx_ring_cleanup:
        kfree(priv->rx_ring);

out:
        ipoib_napi_del(dev);
        return -ENOMEM;
}

static int ipoib_ioctl(struct net_device *dev, struct ifreq *ifr,
                       int cmd)
{
        struct ipoib_dev_priv *priv = ipoib_priv(dev);

        if (!priv->rn_ops->ndo_eth_ioctl)
                return -EOPNOTSUPP;

        return priv->rn_ops->ndo_eth_ioctl(dev, ifr, cmd);
}

static int ipoib_dev_init(struct net_device *dev)
{
        struct ipoib_dev_priv *priv = ipoib_priv(dev);
        int ret = -ENOMEM;

        priv->qp = NULL;

        /*
         * the various IPoIB tasks assume they will never race against
         * themselves, so always use a single thread workqueue
         */
        priv->wq = alloc_ordered_workqueue("ipoib_wq", WQ_MEM_RECLAIM);
        if (!priv->wq) {
                pr_warn("%s: failed to allocate device WQ\n", dev->name);
                goto out;
        }

        /* create pd, which used both for control and datapath*/
        priv->pd = ib_alloc_pd(priv->ca, 0);
        if (IS_ERR(priv->pd)) {
                pr_warn("%s: failed to allocate PD\n", priv->ca->name);
                goto clean_wq;
        }

        ret = priv->rn_ops->ndo_init(dev);
        if (ret) {
                pr_warn("%s failed to init HW resource\n", dev->name);
                goto out_free_pd;
        }

        ret = ipoib_neigh_hash_init(priv);
        if (ret) {
                pr_warn("%s failed to init neigh hash\n", dev->name);
                goto out_dev_uninit;
        }

        if (dev->flags & IFF_UP) {
                if (ipoib_ib_dev_open(dev)) {
                        pr_warn("%s failed to open device\n", dev->name);
                        ret = -ENODEV;
                        goto out_hash_uninit;
                }
        }

        return 0;

out_hash_uninit:
        ipoib_neigh_hash_uninit(dev);

out_dev_uninit:
        ipoib_ib_dev_cleanup(dev);

out_free_pd:
        if (priv->pd) {
                ib_dealloc_pd(priv->pd);
                priv->pd = NULL;
        }

clean_wq:
        if (priv->wq) {
                destroy_workqueue(priv->wq);
                priv->wq = NULL;
        }

out:
        return ret;
}

/*
 * This must be called before doing an unregister_netdev on a parent device to
 * shutdown the IB event handler.
 */
static void ipoib_parent_unregister_pre(struct net_device *ndev)
{
        struct ipoib_dev_priv *priv = ipoib_priv(ndev);

        /*
         * ipoib_set_mac checks netif_running before pushing work, clearing
         * running ensures the it will not add more work.
         */
        rtnl_lock();
        dev_change_flags(priv->dev, priv->dev->flags & ~IFF_UP, NULL);
        rtnl_unlock();

        /* ipoib_event() cannot be running once this returns */
        ib_unregister_event_handler(&priv->event_handler);

        /*
         * Work on the queue grabs the rtnl lock, so this cannot be done while
         * also holding it.
         */
        flush_workqueue(ipoib_workqueue);
}

static void ipoib_set_dev_features(struct ipoib_dev_priv *priv)
{
        priv->hca_caps = priv->ca->attrs.device_cap_flags;
        priv->kernel_caps = priv->ca->attrs.kernel_cap_flags;

        if (priv->hca_caps & IB_DEVICE_UD_IP_CSUM) {
                priv->dev->hw_features |= NETIF_F_IP_CSUM | NETIF_F_RXCSUM;

                if (priv->kernel_caps & IBK_UD_TSO)
                        priv->dev->hw_features |= NETIF_F_TSO;

                priv->dev->features |= priv->dev->hw_features;
        }
}

static int ipoib_parent_init(struct net_device *ndev)
{
        struct ipoib_dev_priv *priv = ipoib_priv(ndev);
        struct ib_port_attr attr;
        int result;

        result = ib_query_port(priv->ca, priv->port, &attr);
        if (result) {
                pr_warn("%s: ib_query_port %d failed\n", priv->ca->name,
                        priv->port);
                return result;
        }
        priv->max_ib_mtu = rdma_mtu_from_attr(priv->ca, priv->port, &attr);

        result = ib_query_pkey(priv->ca, priv->port, 0, &priv->pkey);
        if (result) {
                pr_warn("%s: ib_query_pkey port %d failed (ret = %d)\n",
                        priv->ca->name, priv->port, result);
                return result;
        }

        result = rdma_query_gid(priv->ca, priv->port, 0, &priv->local_gid);
        if (result) {
                pr_warn("%s: rdma_query_gid port %d failed (ret = %d)\n",
                        priv->ca->name, priv->port, result);
                return result;
        }
        dev_addr_mod(priv->dev, 4, priv->local_gid.raw, sizeof(union ib_gid));

        SET_NETDEV_DEV(priv->dev, priv->ca->dev.parent);
        priv->dev->dev_port = priv->port - 1;
        /* Let's set this one too for backwards compatibility. */
        priv->dev->dev_id = priv->port - 1;

        return 0;
}

static void ipoib_child_init(struct net_device *ndev)
{
        struct ipoib_dev_priv *priv = ipoib_priv(ndev);
        struct ipoib_dev_priv *ppriv = ipoib_priv(priv->parent);

        priv->max_ib_mtu = ppriv->max_ib_mtu;
        set_bit(IPOIB_FLAG_SUBINTERFACE, &priv->flags);
        if (memchr_inv(priv->dev->dev_addr, 0, INFINIBAND_ALEN))
                memcpy(&priv->local_gid, priv->dev->dev_addr + 4,
                       sizeof(priv->local_gid));
        else {
                __dev_addr_set(priv->dev, ppriv->dev->dev_addr,
                               INFINIBAND_ALEN);
                memcpy(&priv->local_gid, &ppriv->local_gid,
                       sizeof(priv->local_gid));
        }
}

static int ipoib_ndo_init(struct net_device *ndev)
{
        struct ipoib_dev_priv *priv = ipoib_priv(ndev);
        int rc;
        struct rdma_netdev *rn = netdev_priv(ndev);

        if (priv->parent) {
                ipoib_child_init(ndev);
        } else {
                rc = ipoib_parent_init(ndev);
                if (rc)
                        return rc;
        }

        /* MTU will be reset when mcast join happens */
        ndev->mtu = IPOIB_UD_MTU(priv->max_ib_mtu);
        priv->mcast_mtu = priv->admin_mtu = ndev->mtu;
        rn->mtu = priv->mcast_mtu;
        ndev->max_mtu = IPOIB_CM_MTU;

        ndev->neigh_priv_len = sizeof(struct ipoib_neigh);

        /*
         * Set the full membership bit, so that we join the right
         * broadcast group, etc.
         */
        priv->pkey |= 0x8000;

        ndev->broadcast[8] = priv->pkey >> 8;
        ndev->broadcast[9] = priv->pkey & 0xff;
        set_bit(IPOIB_FLAG_DEV_ADDR_SET, &priv->flags);

        ipoib_set_dev_features(priv);

        rc = ipoib_dev_init(ndev);
        if (rc) {
                pr_warn("%s: failed to initialize device: %s port %d (ret = %d)\n",
                        priv->ca->name, priv->dev->name, priv->port, rc);
                return rc;
        }

        if (priv->parent) {
                struct ipoib_dev_priv *ppriv = ipoib_priv(priv->parent);

                dev_hold(priv->parent);

                down_write(&ppriv->vlan_rwsem);
                list_add_tail(&priv->list, &ppriv->child_intfs);
                up_write(&ppriv->vlan_rwsem);
        }

        return 0;
}

static void ipoib_ndo_uninit(struct net_device *dev)
{
        struct ipoib_dev_priv *priv = ipoib_priv(dev);

        ASSERT_RTNL();

        /*
         * ipoib_remove_one guarantees the children are removed before the
         * parent, and that is the only place where a parent can be removed.
         */
        WARN_ON(!list_empty(&priv->child_intfs));

        if (priv->parent) {
                struct ipoib_dev_priv *ppriv = ipoib_priv(priv->parent);

                down_write(&ppriv->vlan_rwsem);
                list_del(&priv->list);
                up_write(&ppriv->vlan_rwsem);
        }

        ipoib_neigh_hash_uninit(dev);

        ipoib_ib_dev_cleanup(dev);

        /* no more works over the priv->wq */
        if (priv->wq) {
                /* See ipoib_mcast_carrier_on_task() */
                WARN_ON(test_bit(IPOIB_FLAG_OPER_UP, &priv->flags));
                destroy_workqueue(priv->wq);
                priv->wq = NULL;
        }

        dev_put(priv->parent);
}

static int ipoib_set_vf_link_state(struct net_device *dev, int vf, int link_state)
{
        struct ipoib_dev_priv *priv = ipoib_priv(dev);

        return ib_set_vf_link_state(priv->ca, vf, priv->port, link_state);
}

static int ipoib_get_vf_config(struct net_device *dev, int vf,
                               struct ifla_vf_info *ivf)
{
        struct ipoib_dev_priv *priv = ipoib_priv(dev);
        int err;

        err = ib_get_vf_config(priv->ca, vf, priv->port, ivf);
        if (err)
                return err;

        ivf->vf = vf;
        memcpy(ivf->mac, dev->dev_addr, dev->addr_len);

        return 0;
}

static int ipoib_set_vf_guid(struct net_device *dev, int vf, u64 guid, int type)
{
        struct ipoib_dev_priv *priv = ipoib_priv(dev);

        if (type != IFLA_VF_IB_NODE_GUID && type != IFLA_VF_IB_PORT_GUID)
                return -EINVAL;

        return ib_set_vf_guid(priv->ca, vf, priv->port, guid, type);
}

static int ipoib_get_vf_guid(struct net_device *dev, int vf,
                             struct ifla_vf_guid *node_guid,
                             struct ifla_vf_guid *port_guid)
{
        struct ipoib_dev_priv *priv = ipoib_priv(dev);

        return ib_get_vf_guid(priv->ca, vf, priv->port, node_guid, port_guid);
}

static int ipoib_get_vf_stats(struct net_device *dev, int vf,
                              struct ifla_vf_stats *vf_stats)
{
        struct ipoib_dev_priv *priv = ipoib_priv(dev);

        return ib_get_vf_stats(priv->ca, vf, priv->port, vf_stats);
}

static const struct header_ops ipoib_header_ops = {
        .create        = ipoib_hard_header,
};

static const struct net_device_ops ipoib_netdev_ops_pf = {
        .ndo_init                 = ipoib_ndo_init,
        .ndo_uninit                 = ipoib_ndo_uninit,
        .ndo_open                 = ipoib_open,
        .ndo_stop                 = ipoib_stop,
        .ndo_change_mtu                 = ipoib_change_mtu,
        .ndo_fix_features         = ipoib_fix_features,
        .ndo_start_xmit                 = ipoib_start_xmit,
        .ndo_tx_timeout                 = ipoib_timeout,
        .ndo_set_rx_mode         = ipoib_set_mcast_list,
        .ndo_get_iflink                 = ipoib_get_iflink,
        .ndo_set_vf_link_state         = ipoib_set_vf_link_state,
        .ndo_get_vf_config         = ipoib_get_vf_config,
        .ndo_get_vf_stats         = ipoib_get_vf_stats,
        .ndo_get_vf_guid         = ipoib_get_vf_guid,
        .ndo_set_vf_guid         = ipoib_set_vf_guid,
        .ndo_set_mac_address         = ipoib_set_mac,
        .ndo_get_stats64         = ipoib_get_stats,
        .ndo_eth_ioctl                 = ipoib_ioctl,
};

static const struct net_device_ops ipoib_netdev_ops_vf = {
        .ndo_init                 = ipoib_ndo_init,
        .ndo_uninit                 = ipoib_ndo_uninit,
        .ndo_open                 = ipoib_open,
        .ndo_stop                 = ipoib_stop,
        .ndo_change_mtu                 = ipoib_change_mtu,
        .ndo_fix_features         = ipoib_fix_features,
        .ndo_start_xmit                  = ipoib_start_xmit,
        .ndo_tx_timeout                 = ipoib_timeout,
        .ndo_set_rx_mode         = ipoib_set_mcast_list,
        .ndo_get_iflink                 = ipoib_get_iflink,
        .ndo_get_stats64         = ipoib_get_stats,
        .ndo_eth_ioctl                 = ipoib_ioctl,
};

static const struct net_device_ops ipoib_netdev_default_pf = {
        .ndo_init                 = ipoib_dev_init_default,
        .ndo_uninit                 = ipoib_dev_uninit_default,
        .ndo_open                 = ipoib_ib_dev_open_default,
        .ndo_stop                 = ipoib_ib_dev_stop_default,
};

void ipoib_setup_common(struct net_device *dev)
{
        dev->header_ops                 = &ipoib_header_ops;
        dev->netdev_ops          = &ipoib_netdev_default_pf;

        ipoib_set_ethtool_ops(dev);

        dev->watchdog_timeo         = 10 * HZ;

        dev->flags                |= IFF_BROADCAST | IFF_MULTICAST;

        dev->hard_header_len         = IPOIB_HARD_LEN;
        dev->addr_len                 = INFINIBAND_ALEN;
        dev->type                 = ARPHRD_INFINIBAND;
        dev->tx_queue_len         = ipoib_sendq_size * 2;
        dev->features                 = (NETIF_F_VLAN_CHALLENGED        |
                                    NETIF_F_HIGHDMA);
        netif_keep_dst(dev);

        memcpy(dev->broadcast, ipv4_bcast_addr, INFINIBAND_ALEN);

        /*
         * unregister_netdev always frees the netdev, we use this mode
         * consistently to unify all the various unregister paths, including
         * those connected to rtnl_link_ops which require it.
         */
        dev->needs_free_netdev = true;
}

static void ipoib_build_priv(struct net_device *dev)
{
        struct ipoib_dev_priv *priv = ipoib_priv(dev);

        priv->dev = dev;
        spin_lock_init(&priv->lock);
        init_rwsem(&priv->vlan_rwsem);
        mutex_init(&priv->mcast_mutex);

        INIT_LIST_HEAD(&priv->path_list);
        INIT_LIST_HEAD(&priv->child_intfs);
        INIT_LIST_HEAD(&priv->dead_ahs);
        INIT_LIST_HEAD(&priv->multicast_list);

        INIT_DELAYED_WORK(&priv->mcast_task,   ipoib_mcast_join_task);
        INIT_WORK(&priv->carrier_on_task, ipoib_mcast_carrier_on_task);
        INIT_WORK(&priv->reschedule_napi_work, ipoib_napi_schedule_work);
        INIT_WORK(&priv->flush_light,   ipoib_ib_dev_flush_light);
        INIT_WORK(&priv->flush_normal,   ipoib_ib_dev_flush_normal);
        INIT_WORK(&priv->flush_heavy,   ipoib_ib_dev_flush_heavy);
        INIT_WORK(&priv->restart_task, ipoib_mcast_restart_task);
        INIT_WORK(&priv->tx_timeout_work, ipoib_ib_tx_timeout_work);
        INIT_DELAYED_WORK(&priv->ah_reap_task, ipoib_reap_ah);
        INIT_DELAYED_WORK(&priv->neigh_reap_task, ipoib_reap_neigh);
}

static struct net_device *ipoib_alloc_netdev(struct ib_device *hca, u32 port,
                                             const char *name)
{
        struct net_device *dev;

        dev = rdma_alloc_netdev(hca, port, RDMA_NETDEV_IPOIB, name,
                                NET_NAME_UNKNOWN, ipoib_setup_common);
        if (!IS_ERR(dev) || PTR_ERR(dev) != -EOPNOTSUPP)
                return dev;

        dev = alloc_netdev(sizeof(struct rdma_netdev), name, NET_NAME_UNKNOWN,
                           ipoib_setup_common);
        if (!dev)
                return ERR_PTR(-ENOMEM);
        return dev;
}

int ipoib_intf_init(struct ib_device *hca, u32 port, const char *name,
                    struct net_device *dev)
{
        struct rdma_netdev *rn = netdev_priv(dev);
        struct ipoib_dev_priv *priv;
        int rc;

        priv = kzalloc(sizeof(*priv), GFP_KERNEL);
        if (!priv)
                return -ENOMEM;

        priv->ca = hca;
        priv->port = port;

        rc = rdma_init_netdev(hca, port, RDMA_NETDEV_IPOIB, name,
                              NET_NAME_UNKNOWN, ipoib_setup_common, dev);
        if (rc) {
                if (rc != -EOPNOTSUPP)
                        goto out;

                rn->send = ipoib_send;
                rn->attach_mcast = ipoib_mcast_attach;
                rn->detach_mcast = ipoib_mcast_detach;
                rn->hca = hca;

                rc = netif_set_real_num_tx_queues(dev, 1);
                if (rc)
                        goto out;

                rc = netif_set_real_num_rx_queues(dev, 1);
                if (rc)
                        goto out;
        }

        priv->rn_ops = dev->netdev_ops;

        if (hca->attrs.kernel_cap_flags & IBK_VIRTUAL_FUNCTION)
                dev->netdev_ops        = &ipoib_netdev_ops_vf;
        else
                dev->netdev_ops        = &ipoib_netdev_ops_pf;

        rn->clnt_priv = priv;
        /*
         * Only the child register_netdev flows can handle priv_destructor
         * being set, so we force it to NULL here and handle manually until it
         * is safe to turn on.
         */
        priv->next_priv_destructor = dev->priv_destructor;
        dev->priv_destructor = NULL;

        ipoib_build_priv(dev);

        return 0;

out:
        kfree(priv);
        return rc;
}

struct net_device *ipoib_intf_alloc(struct ib_device *hca, u32 port,
                                    const char *name)
{
        struct net_device *dev;
        int rc;

        dev = ipoib_alloc_netdev(hca, port, name);
        if (IS_ERR(dev))
                return dev;

        rc = ipoib_intf_init(hca, port, name, dev);
        if (rc) {
                free_netdev(dev);
                return ERR_PTR(rc);
        }

        /*
         * Upon success the caller must ensure ipoib_intf_free is called or
         * register_netdevice succeed'd and priv_destructor is set to
         * ipoib_intf_free.
         */
        return dev;
}

void ipoib_intf_free(struct net_device *dev)
{
        struct ipoib_dev_priv *priv = ipoib_priv(dev);
        struct rdma_netdev *rn = netdev_priv(dev);

        dev->priv_destructor = priv->next_priv_destructor;
        if (dev->priv_destructor)
                dev->priv_destructor(dev);

        /*
         * There are some error flows around register_netdev failing that may
         * attempt to call priv_destructor twice, prevent that from happening.
         */
        dev->priv_destructor = NULL;

        /* unregister/destroy is very complicated. Make bugs more obvious. */
        rn->clnt_priv = NULL;

        kfree(priv);
}

static ssize_t pkey_show(struct device *dev, struct device_attribute *attr,
                         char *buf)
{
        struct net_device *ndev = to_net_dev(dev);
        struct ipoib_dev_priv *priv = ipoib_priv(ndev);

        return sysfs_emit(buf, "0x%04x\n", priv->pkey);
}
static DEVICE_ATTR_RO(pkey);

static ssize_t umcast_show(struct device *dev, struct device_attribute *attr,
                           char *buf)
{
        struct net_device *ndev = to_net_dev(dev);
        struct ipoib_dev_priv *priv = ipoib_priv(ndev);

        return sysfs_emit(buf, "%d\n",
                          test_bit(IPOIB_FLAG_UMCAST, &priv->flags));
}

void ipoib_set_umcast(struct net_device *ndev, int umcast_val)
{
        struct ipoib_dev_priv *priv = ipoib_priv(ndev);

        if (umcast_val > 0) {
                set_bit(IPOIB_FLAG_UMCAST, &priv->flags);
                ipoib_warn(priv, "ignoring multicast groups joined directly "
                                "by userspace\n");
        } else
                clear_bit(IPOIB_FLAG_UMCAST, &priv->flags);
}

static ssize_t umcast_store(struct device *dev, struct device_attribute *attr,
                            const char *buf, size_t count)
{
        unsigned long umcast_val = simple_strtoul(buf, NULL, 0);

        ipoib_set_umcast(to_net_dev(dev), umcast_val);

        return count;
}
static DEVICE_ATTR_RW(umcast);

int ipoib_add_umcast_attr(struct net_device *dev)
{
        return device_create_file(&dev->dev, &dev_attr_umcast);
}

static void set_base_guid(struct ipoib_dev_priv *priv, union ib_gid *gid)
{
        struct ipoib_dev_priv *child_priv;
        struct net_device *netdev = priv->dev;

        netif_addr_lock_bh(netdev);

        memcpy(&priv->local_gid.global.interface_id,
               &gid->global.interface_id,
               sizeof(gid->global.interface_id));
        dev_addr_mod(netdev, 4, (u8 *)&priv->local_gid, sizeof(priv->local_gid));
        clear_bit(IPOIB_FLAG_DEV_ADDR_SET, &priv->flags);

        netif_addr_unlock_bh(netdev);

        if (!test_bit(IPOIB_FLAG_SUBINTERFACE, &priv->flags)) {
                down_read(&priv->vlan_rwsem);
                list_for_each_entry(child_priv, &priv->child_intfs, list)
                        set_base_guid(child_priv, gid);
                up_read(&priv->vlan_rwsem);
        }
}

static int ipoib_check_lladdr(struct net_device *dev,
                              struct sockaddr_storage *ss)
{
        union ib_gid *gid = (union ib_gid *)(ss->__data + 4);
        int ret = 0;

        netif_addr_lock_bh(dev);

        /* Make sure the QPN, reserved and subnet prefix match the current
         * lladdr, it also makes sure the lladdr is unicast.
         */
        if (memcmp(dev->dev_addr, ss->__data,
                   4 + sizeof(gid->global.subnet_prefix)) ||
            gid->global.interface_id == 0)
                ret = -EINVAL;

        netif_addr_unlock_bh(dev);

        return ret;
}

static int ipoib_set_mac(struct net_device *dev, void *addr)
{
        struct ipoib_dev_priv *priv = ipoib_priv(dev);
        struct sockaddr_storage *ss = addr;
        int ret;

        if (!(dev->priv_flags & IFF_LIVE_ADDR_CHANGE) && netif_running(dev))
                return -EBUSY;

        ret = ipoib_check_lladdr(dev, ss);
        if (ret)
                return ret;

        set_base_guid(priv, (union ib_gid *)(ss->__data + 4));

        queue_work(ipoib_workqueue, &priv->flush_light);

        return 0;
}

static ssize_t create_child_store(struct device *dev,
                                  struct device_attribute *attr,
                                  const char *buf, size_t count)
{
        int pkey;
        int ret;

        if (sscanf(buf, "%i", &pkey) != 1)
                return -EINVAL;

        if (pkey <= 0 || pkey > 0xffff || pkey == 0x8000)
                return -EINVAL;

        ret = ipoib_vlan_add(to_net_dev(dev), pkey);

        return ret ? ret : count;
}
static DEVICE_ATTR_WO(create_child);

static ssize_t delete_child_store(struct device *dev,
                                  struct device_attribute *attr,
                                  const char *buf, size_t count)
{
        int pkey;
        int ret;

        if (sscanf(buf, "%i", &pkey) != 1)
                return -EINVAL;

        if (pkey < 0 || pkey > 0xffff)
                return -EINVAL;

        ret = ipoib_vlan_delete(to_net_dev(dev), pkey);

        return ret ? ret : count;

}
static DEVICE_ATTR_WO(delete_child);

int ipoib_add_pkey_attr(struct net_device *dev)
{
        return device_create_file(&dev->dev, &dev_attr_pkey);
}

/*
 * We erroneously exposed the iface's port number in the dev_id
 * sysfs field long after dev_port was introduced for that purpose[1],
 * and we need to stop everyone from relying on that.
 * Let's overload the shower routine for the dev_id file here
 * to gently bring the issue up.
 *
 * [1] https://www.spinics.net/lists/netdev/msg272123.html
 */
static ssize_t dev_id_show(struct device *dev,
                           struct device_attribute *attr, char *buf)
{
        struct net_device *ndev = to_net_dev(dev);

        /*
         * ndev->dev_port will be equal to 0 in old kernel prior to commit
         * 9b8b2a323008 ("IB/ipoib: Use dev_port to expose network interface
         * port numbers") Zero was chosen as special case for user space
         * applications to fallback and query dev_id to check if it has
         * different value or not.
         *
         * Don't print warning in such scenario.
         *
         * https://github.com/systemd/systemd/blob/master/src/udev/udev-builtin-net_id.c#L358
         */
        if (ndev->dev_port && ndev->dev_id == ndev->dev_port)
                netdev_info_once(ndev,
                        "\"%s\" wants to know my dev_id. Should it look at dev_port instead? See Documentation/ABI/testing/sysfs-class-net for more info.\n",
                        current->comm);

        return sysfs_emit(buf, "%#x\n", ndev->dev_id);
}
static DEVICE_ATTR_RO(dev_id);

static int ipoib_intercept_dev_id_attr(struct net_device *dev)
{
        device_remove_file(&dev->dev, &dev_attr_dev_id);
        return device_create_file(&dev->dev, &dev_attr_dev_id);
}

static struct net_device *ipoib_add_port(const char *format,
                                         struct ib_device *hca, u32 port)
{
        struct rtnl_link_ops *ops = ipoib_get_link_ops();
        struct rdma_netdev_alloc_params params;
        struct ipoib_dev_priv *priv;
        struct net_device *ndev;
        int result;

        ndev = ipoib_intf_alloc(hca, port, format);
        if (IS_ERR(ndev)) {
                pr_warn("%s, %d: ipoib_intf_alloc failed %ld\n", hca->name, port,
                        PTR_ERR(ndev));
                return ndev;
        }
        priv = ipoib_priv(ndev);

        INIT_IB_EVENT_HANDLER(&priv->event_handler,
                              priv->ca, ipoib_event);
        ib_register_event_handler(&priv->event_handler);

        /* call event handler to ensure pkey in sync */
        queue_work(ipoib_workqueue, &priv->flush_heavy);

        ndev->rtnl_link_ops = ipoib_get_link_ops();

        result = register_netdev(ndev);
        if (result) {
                pr_warn("%s: couldn't register ipoib port %d; error %d\n",
                        hca->name, port, result);

                ipoib_parent_unregister_pre(ndev);
                ipoib_intf_free(ndev);
                free_netdev(ndev);

                return ERR_PTR(result);
        }

        if (hca->ops.rdma_netdev_get_params) {
                int rc = hca->ops.rdma_netdev_get_params(hca, port,
                                                     RDMA_NETDEV_IPOIB,
                                                     &params);

                if (!rc && ops->priv_size < params.sizeof_priv)
                        ops->priv_size = params.sizeof_priv;
        }
        /*
         * We cannot set priv_destructor before register_netdev because we
         * need priv to be always valid during the error flow to execute
         * ipoib_parent_unregister_pre(). Instead handle it manually and only
         * enter priv_destructor mode once we are completely registered.
         */
        ndev->priv_destructor = ipoib_intf_free;

        if (ipoib_intercept_dev_id_attr(ndev))
                goto sysfs_failed;
        if (ipoib_cm_add_mode_attr(ndev))
                goto sysfs_failed;
        if (ipoib_add_pkey_attr(ndev))
                goto sysfs_failed;
        if (ipoib_add_umcast_attr(ndev))
                goto sysfs_failed;
        if (device_create_file(&ndev->dev, &dev_attr_create_child))
                goto sysfs_failed;
        if (device_create_file(&ndev->dev, &dev_attr_delete_child))
                goto sysfs_failed;

        return ndev;

sysfs_failed:
        ipoib_parent_unregister_pre(ndev);
        unregister_netdev(ndev);
        return ERR_PTR(-ENOMEM);
}

static int ipoib_add_one(struct ib_device *device)
{
        struct list_head *dev_list;
        struct net_device *dev;
        struct ipoib_dev_priv *priv;
        unsigned int p;
        int count = 0;

        dev_list = kmalloc(sizeof(*dev_list), GFP_KERNEL);
        if (!dev_list)
                return -ENOMEM;

        INIT_LIST_HEAD(dev_list);

        rdma_for_each_port (device, p) {
                if (!rdma_protocol_ib(device, p))
                        continue;
                dev = ipoib_add_port("ib%d", device, p);
                if (!IS_ERR(dev)) {
                        priv = ipoib_priv(dev);
                        list_add_tail(&priv->list, dev_list);
                        count++;
                }
        }

        if (!count) {
                kfree(dev_list);
                return -EOPNOTSUPP;
        }

        ib_set_client_data(device, &ipoib_client, dev_list);
        return 0;
}

static void ipoib_remove_one(struct ib_device *device, void *client_data)
{
        struct ipoib_dev_priv *priv, *tmp, *cpriv, *tcpriv;
        struct list_head *dev_list = client_data;

        list_for_each_entry_safe(priv, tmp, dev_list, list) {
                LIST_HEAD(head);
                ipoib_parent_unregister_pre(priv->dev);

                rtnl_lock();

                list_for_each_entry_safe(cpriv, tcpriv, &priv->child_intfs,
                                         list)
                        unregister_netdevice_queue(cpriv->dev, &head);
                unregister_netdevice_queue(priv->dev, &head);
                unregister_netdevice_many(&head);

                rtnl_unlock();
        }

        kfree(dev_list);
}

#ifdef CONFIG_INFINIBAND_IPOIB_DEBUG
static struct notifier_block ipoib_netdev_notifier = {
        .notifier_call = ipoib_netdev_event,
};
#endif

static int __init ipoib_init_module(void)
{
        int ret;

        ipoib_recvq_size = roundup_pow_of_two(ipoib_recvq_size);
        ipoib_recvq_size = min(ipoib_recvq_size, IPOIB_MAX_QUEUE_SIZE);
        ipoib_recvq_size = max(ipoib_recvq_size, IPOIB_MIN_QUEUE_SIZE);

        ipoib_sendq_size = roundup_pow_of_two(ipoib_sendq_size);
        ipoib_sendq_size = min(ipoib_sendq_size, IPOIB_MAX_QUEUE_SIZE);
        ipoib_sendq_size = max3(ipoib_sendq_size, 2 * MAX_SEND_CQE, IPOIB_MIN_QUEUE_SIZE);
#ifdef CONFIG_INFINIBAND_IPOIB_CM
        ipoib_max_conn_qp = min(ipoib_max_conn_qp, IPOIB_CM_MAX_CONN_QP);
        ipoib_max_conn_qp = max(ipoib_max_conn_qp, 0);
#endif

        /*
         * When copying small received packets, we only copy from the
         * linear data part of the SKB, so we rely on this condition.
         */
        BUILD_BUG_ON(IPOIB_CM_COPYBREAK > IPOIB_CM_HEAD_SIZE);

        ipoib_register_debugfs();

        /*
         * We create a global workqueue here that is used for all flush
         * operations.  However, if you attempt to flush a workqueue
         * from a task on that same workqueue, it deadlocks the system.
         * We want to be able to flush the tasks associated with a
         * specific net device, so we also create a workqueue for each
         * netdevice.  We queue up the tasks for that device only on
         * its private workqueue, and we only queue up flush events
         * on our global flush workqueue.  This avoids the deadlocks.
         */
        ipoib_workqueue = alloc_ordered_workqueue("ipoib_flush", 0);
        if (!ipoib_workqueue) {
                ret = -ENOMEM;
                goto err_fs;
        }

        ib_sa_register_client(&ipoib_sa_client);

        ret = ib_register_client(&ipoib_client);
        if (ret)
                goto err_sa;

        ret = ipoib_netlink_init();
        if (ret)
                goto err_client;

#ifdef CONFIG_INFINIBAND_IPOIB_DEBUG
        register_netdevice_notifier(&ipoib_netdev_notifier);
#endif
        return 0;

err_client:
        ib_unregister_client(&ipoib_client);

err_sa:
        ib_sa_unregister_client(&ipoib_sa_client);
        destroy_workqueue(ipoib_workqueue);

err_fs:
        ipoib_unregister_debugfs();

        return ret;
}

static void __exit ipoib_cleanup_module(void)
{
#ifdef CONFIG_INFINIBAND_IPOIB_DEBUG
        unregister_netdevice_notifier(&ipoib_netdev_notifier);
#endif
        ipoib_netlink_fini();
        ib_unregister_client(&ipoib_client);
        ib_sa_unregister_client(&ipoib_sa_client);
        ipoib_unregister_debugfs();
        destroy_workqueue(ipoib_workqueue);
}

module_init(ipoib_init_module);
module_exit(ipoib_cleanup_module);








































































































    2 
    2 















    6 




    2 







    1 

    2 



















    2 






    1 







    1 





















































































    1 







    1 







































    1 














    1 

    1 
    1 








































































































    2 


































    1 




    1 












   12 



   13 























    1 


































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * NetLabel CALIPSO/IPv6 Support
 *
 * This file defines the CALIPSO/IPv6 functions for the NetLabel system.  The
 * NetLabel system manages static and dynamic label mappings for network
 * protocols such as CIPSO and CALIPSO.
 *
 * Authors: Paul Moore <paul@paul-moore.com>
 *          Huw Davies <huw@codeweavers.com>
 */

/* (c) Copyright Hewlett-Packard Development Company, L.P., 2006
 * (c) Copyright Huw Davies <huw@codeweavers.com>, 2015
 */

#include <linux/types.h>
#include <linux/socket.h>
#include <linux/string.h>
#include <linux/skbuff.h>
#include <linux/audit.h>
#include <linux/slab.h>
#include <net/sock.h>
#include <net/netlink.h>
#include <net/genetlink.h>
#include <net/netlabel.h>
#include <net/calipso.h>
#include <linux/atomic.h>

#include "netlabel_user.h"
#include "netlabel_calipso.h"
#include "netlabel_mgmt.h"
#include "netlabel_domainhash.h"

/* Argument struct for calipso_doi_walk() */
struct netlbl_calipso_doiwalk_arg {
        struct netlink_callback *nl_cb;
        struct sk_buff *skb;
        u32 seq;
};

/* Argument struct for netlbl_domhsh_walk() */
struct netlbl_domhsh_walk_arg {
        struct netlbl_audit *audit_info;
        u32 doi;
};

/* NetLabel Generic NETLINK CALIPSO family */
static struct genl_family netlbl_calipso_gnl_family;

/* NetLabel Netlink attribute policy */
static const struct nla_policy calipso_genl_policy[NLBL_CALIPSO_A_MAX + 1] = {
        [NLBL_CALIPSO_A_DOI] = { .type = NLA_U32 },
        [NLBL_CALIPSO_A_MTYPE] = { .type = NLA_U32 },
};

static const struct netlbl_calipso_ops *calipso_ops;

/**
 * netlbl_calipso_ops_register - Register the CALIPSO operations
 * @ops: ops to register
 *
 * Description:
 * Register the CALIPSO packet engine operations.
 *
 */
const struct netlbl_calipso_ops *
netlbl_calipso_ops_register(const struct netlbl_calipso_ops *ops)
{
        return xchg(&calipso_ops, ops);
}
EXPORT_SYMBOL(netlbl_calipso_ops_register);

static const struct netlbl_calipso_ops *netlbl_calipso_ops_get(void)
{
        return READ_ONCE(calipso_ops);
}

/* NetLabel Command Handlers
 */
/**
 * netlbl_calipso_add_pass - Adds a CALIPSO pass DOI definition
 * @info: the Generic NETLINK info block
 * @audit_info: NetLabel audit information
 *
 * Description:
 * Create a new CALIPSO_MAP_PASS DOI definition based on the given ADD message
 * and add it to the CALIPSO engine.  Return zero on success and non-zero on
 * error.
 *
 */
static int netlbl_calipso_add_pass(struct genl_info *info,
                                   struct netlbl_audit *audit_info)
{
        int ret_val;
        struct calipso_doi *doi_def = NULL;

        doi_def = kmalloc(sizeof(*doi_def), GFP_KERNEL);
        if (!doi_def)
                return -ENOMEM;
        doi_def->type = CALIPSO_MAP_PASS;
        doi_def->doi = nla_get_u32(info->attrs[NLBL_CALIPSO_A_DOI]);
        ret_val = calipso_doi_add(doi_def, audit_info);
        if (ret_val != 0)
                calipso_doi_free(doi_def);

        return ret_val;
}

/**
 * netlbl_calipso_add - Handle an ADD message
 * @skb: the NETLINK buffer
 * @info: the Generic NETLINK info block
 *
 * Description:
 * Create a new DOI definition based on the given ADD message and add it to the
 * CALIPSO engine.  Returns zero on success, negative values on failure.
 *
 */
static int netlbl_calipso_add(struct sk_buff *skb, struct genl_info *info)
{
        int ret_val = -EINVAL;
        struct netlbl_audit audit_info;
        const struct netlbl_calipso_ops *ops = netlbl_calipso_ops_get();

        if (!info->attrs[NLBL_CALIPSO_A_DOI] ||
            !info->attrs[NLBL_CALIPSO_A_MTYPE])
                return -EINVAL;

        if (!ops)
                return -EOPNOTSUPP;

        netlbl_netlink_auditinfo(&audit_info);
        switch (nla_get_u32(info->attrs[NLBL_CALIPSO_A_MTYPE])) {
        case CALIPSO_MAP_PASS:
                ret_val = netlbl_calipso_add_pass(info, &audit_info);
                break;
        }
        if (ret_val == 0)
                atomic_inc(&netlabel_mgmt_protocount);

        return ret_val;
}

/**
 * netlbl_calipso_list - Handle a LIST message
 * @skb: the NETLINK buffer
 * @info: the Generic NETLINK info block
 *
 * Description:
 * Process a user generated LIST message and respond accordingly.
 * Returns zero on success and negative values on error.
 *
 */
static int netlbl_calipso_list(struct sk_buff *skb, struct genl_info *info)
{
        int ret_val;
        struct sk_buff *ans_skb = NULL;
        void *data;
        u32 doi;
        struct calipso_doi *doi_def;

        if (!info->attrs[NLBL_CALIPSO_A_DOI]) {
                ret_val = -EINVAL;
                goto list_failure;
        }

        doi = nla_get_u32(info->attrs[NLBL_CALIPSO_A_DOI]);

        doi_def = calipso_doi_getdef(doi);
        if (!doi_def) {
                ret_val = -EINVAL;
                goto list_failure;
        }

        ans_skb = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
        if (!ans_skb) {
                ret_val = -ENOMEM;
                goto list_failure_put;
        }
        data = genlmsg_put_reply(ans_skb, info, &netlbl_calipso_gnl_family,
                                 0, NLBL_CALIPSO_C_LIST);
        if (!data) {
                ret_val = -ENOMEM;
                goto list_failure_put;
        }

        ret_val = nla_put_u32(ans_skb, NLBL_CALIPSO_A_MTYPE, doi_def->type);
        if (ret_val != 0)
                goto list_failure_put;

        calipso_doi_putdef(doi_def);

        genlmsg_end(ans_skb, data);
        return genlmsg_reply(ans_skb, info);

list_failure_put:
        calipso_doi_putdef(doi_def);
list_failure:
        kfree_skb(ans_skb);
        return ret_val;
}

/**
 * netlbl_calipso_listall_cb - calipso_doi_walk() callback for LISTALL
 * @doi_def: the CALIPSO DOI definition
 * @arg: the netlbl_calipso_doiwalk_arg structure
 *
 * Description:
 * This function is designed to be used as a callback to the
 * calipso_doi_walk() function for use in generating a response for a LISTALL
 * message.  Returns the size of the message on success, negative values on
 * failure.
 *
 */
static int netlbl_calipso_listall_cb(struct calipso_doi *doi_def, void *arg)
{
        int ret_val = -ENOMEM;
        struct netlbl_calipso_doiwalk_arg *cb_arg = arg;
        void *data;

        data = genlmsg_put(cb_arg->skb, NETLINK_CB(cb_arg->nl_cb->skb).portid,
                           cb_arg->seq, &netlbl_calipso_gnl_family,
                           NLM_F_MULTI, NLBL_CALIPSO_C_LISTALL);
        if (!data)
                goto listall_cb_failure;

        ret_val = nla_put_u32(cb_arg->skb, NLBL_CALIPSO_A_DOI, doi_def->doi);
        if (ret_val != 0)
                goto listall_cb_failure;
        ret_val = nla_put_u32(cb_arg->skb,
                              NLBL_CALIPSO_A_MTYPE,
                              doi_def->type);
        if (ret_val != 0)
                goto listall_cb_failure;

        genlmsg_end(cb_arg->skb, data);
        return 0;

listall_cb_failure:
        genlmsg_cancel(cb_arg->skb, data);
        return ret_val;
}

/**
 * netlbl_calipso_listall - Handle a LISTALL message
 * @skb: the NETLINK buffer
 * @cb: the NETLINK callback
 *
 * Description:
 * Process a user generated LISTALL message and respond accordingly.  Returns
 * zero on success and negative values on error.
 *
 */
static int netlbl_calipso_listall(struct sk_buff *skb,
                                  struct netlink_callback *cb)
{
        struct netlbl_calipso_doiwalk_arg cb_arg;
        u32 doi_skip = cb->args[0];

        cb_arg.nl_cb = cb;
        cb_arg.skb = skb;
        cb_arg.seq = cb->nlh->nlmsg_seq;

        calipso_doi_walk(&doi_skip, netlbl_calipso_listall_cb, &cb_arg);

        cb->args[0] = doi_skip;
        return skb->len;
}

/**
 * netlbl_calipso_remove_cb - netlbl_calipso_remove() callback for REMOVE
 * @entry: LSM domain mapping entry
 * @arg: the netlbl_domhsh_walk_arg structure
 *
 * Description:
 * This function is intended for use by netlbl_calipso_remove() as the callback
 * for the netlbl_domhsh_walk() function; it removes LSM domain map entries
 * which are associated with the CALIPSO DOI specified in @arg.  Returns zero on
 * success, negative values on failure.
 *
 */
static int netlbl_calipso_remove_cb(struct netlbl_dom_map *entry, void *arg)
{
        struct netlbl_domhsh_walk_arg *cb_arg = arg;

        if (entry->def.type == NETLBL_NLTYPE_CALIPSO &&
            entry->def.calipso->doi == cb_arg->doi)
                return netlbl_domhsh_remove_entry(entry, cb_arg->audit_info);

        return 0;
}

/**
 * netlbl_calipso_remove - Handle a REMOVE message
 * @skb: the NETLINK buffer
 * @info: the Generic NETLINK info block
 *
 * Description:
 * Process a user generated REMOVE message and respond accordingly.  Returns
 * zero on success, negative values on failure.
 *
 */
static int netlbl_calipso_remove(struct sk_buff *skb, struct genl_info *info)
{
        int ret_val = -EINVAL;
        struct netlbl_domhsh_walk_arg cb_arg;
        struct netlbl_audit audit_info;
        u32 skip_bkt = 0;
        u32 skip_chain = 0;

        if (!info->attrs[NLBL_CALIPSO_A_DOI])
                return -EINVAL;

        netlbl_netlink_auditinfo(&audit_info);
        cb_arg.doi = nla_get_u32(info->attrs[NLBL_CALIPSO_A_DOI]);
        cb_arg.audit_info = &audit_info;
        ret_val = netlbl_domhsh_walk(&skip_bkt, &skip_chain,
                                     netlbl_calipso_remove_cb, &cb_arg);
        if (ret_val == 0 || ret_val == -ENOENT) {
                ret_val = calipso_doi_remove(cb_arg.doi, &audit_info);
                if (ret_val == 0)
                        atomic_dec(&netlabel_mgmt_protocount);
        }

        return ret_val;
}

/* NetLabel Generic NETLINK Command Definitions
 */

static const struct genl_small_ops netlbl_calipso_ops[] = {
        {
        .cmd = NLBL_CALIPSO_C_ADD,
        .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
        .flags = GENL_ADMIN_PERM,
        .doit = netlbl_calipso_add,
        .dumpit = NULL,
        },
        {
        .cmd = NLBL_CALIPSO_C_REMOVE,
        .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
        .flags = GENL_ADMIN_PERM,
        .doit = netlbl_calipso_remove,
        .dumpit = NULL,
        },
        {
        .cmd = NLBL_CALIPSO_C_LIST,
        .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
        .flags = 0,
        .doit = netlbl_calipso_list,
        .dumpit = NULL,
        },
        {
        .cmd = NLBL_CALIPSO_C_LISTALL,
        .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
        .flags = 0,
        .doit = NULL,
        .dumpit = netlbl_calipso_listall,
        },
};

static struct genl_family netlbl_calipso_gnl_family __ro_after_init = {
        .hdrsize = 0,
        .name = NETLBL_NLTYPE_CALIPSO_NAME,
        .version = NETLBL_PROTO_VERSION,
        .maxattr = NLBL_CALIPSO_A_MAX,
        .policy = calipso_genl_policy,
        .module = THIS_MODULE,
        .small_ops = netlbl_calipso_ops,
        .n_small_ops = ARRAY_SIZE(netlbl_calipso_ops),
        .resv_start_op = NLBL_CALIPSO_C_LISTALL + 1,
};

/* NetLabel Generic NETLINK Protocol Functions
 */

/**
 * netlbl_calipso_genl_init - Register the CALIPSO NetLabel component
 *
 * Description:
 * Register the CALIPSO packet NetLabel component with the Generic NETLINK
 * mechanism.  Returns zero on success, negative values on failure.
 *
 */
int __init netlbl_calipso_genl_init(void)
{
        return genl_register_family(&netlbl_calipso_gnl_family);
}

/**
 * calipso_doi_add - Add a new DOI to the CALIPSO protocol engine
 * @doi_def: the DOI structure
 * @audit_info: NetLabel audit information
 *
 * Description:
 * The caller defines a new DOI for use by the CALIPSO engine and calls this
 * function to add it to the list of acceptable domains.  The caller must
 * ensure that the mapping table specified in @doi_def->map meets all of the
 * requirements of the mapping type (see calipso.h for details).  Returns
 * zero on success and non-zero on failure.
 *
 */
int calipso_doi_add(struct calipso_doi *doi_def,
                    struct netlbl_audit *audit_info)
{
        int ret_val = -ENOMSG;
        const struct netlbl_calipso_ops *ops = netlbl_calipso_ops_get();

        if (ops)
                ret_val = ops->doi_add(doi_def, audit_info);
        return ret_val;
}

/**
 * calipso_doi_free - Frees a DOI definition
 * @doi_def: the DOI definition
 *
 * Description:
 * This function frees all of the memory associated with a DOI definition.
 *
 */
void calipso_doi_free(struct calipso_doi *doi_def)
{
        const struct netlbl_calipso_ops *ops = netlbl_calipso_ops_get();

        if (ops)
                ops->doi_free(doi_def);
}

/**
 * calipso_doi_remove - Remove an existing DOI from the CALIPSO protocol engine
 * @doi: the DOI value
 * @audit_info: NetLabel audit information
 *
 * Description:
 * Removes a DOI definition from the CALIPSO engine.  The NetLabel routines will
 * be called to release their own LSM domain mappings as well as our own
 * domain list.  Returns zero on success and negative values on failure.
 *
 */
int calipso_doi_remove(u32 doi, struct netlbl_audit *audit_info)
{
        int ret_val = -ENOMSG;
        const struct netlbl_calipso_ops *ops = netlbl_calipso_ops_get();

        if (ops)
                ret_val = ops->doi_remove(doi, audit_info);
        return ret_val;
}

/**
 * calipso_doi_getdef - Returns a reference to a valid DOI definition
 * @doi: the DOI value
 *
 * Description:
 * Searches for a valid DOI definition and if one is found it is returned to
 * the caller.  Otherwise NULL is returned.  The caller must ensure that
 * calipso_doi_putdef() is called when the caller is done.
 *
 */
struct calipso_doi *calipso_doi_getdef(u32 doi)
{
        struct calipso_doi *ret_val = NULL;
        const struct netlbl_calipso_ops *ops = netlbl_calipso_ops_get();

        if (ops)
                ret_val = ops->doi_getdef(doi);
        return ret_val;
}

/**
 * calipso_doi_putdef - Releases a reference for the given DOI definition
 * @doi_def: the DOI definition
 *
 * Description:
 * Releases a DOI definition reference obtained from calipso_doi_getdef().
 *
 */
void calipso_doi_putdef(struct calipso_doi *doi_def)
{
        const struct netlbl_calipso_ops *ops = netlbl_calipso_ops_get();

        if (ops)
                ops->doi_putdef(doi_def);
}

/**
 * calipso_doi_walk - Iterate through the DOI definitions
 * @skip_cnt: skip past this number of DOI definitions, updated
 * @callback: callback for each DOI definition
 * @cb_arg: argument for the callback function
 *
 * Description:
 * Iterate over the DOI definition list, skipping the first @skip_cnt entries.
 * For each entry call @callback, if @callback returns a negative value stop
 * 'walking' through the list and return.  Updates the value in @skip_cnt upon
 * return.  Returns zero on success, negative values on failure.
 *
 */
int calipso_doi_walk(u32 *skip_cnt,
                     int (*callback)(struct calipso_doi *doi_def, void *arg),
                     void *cb_arg)
{
        int ret_val = -ENOMSG;
        const struct netlbl_calipso_ops *ops = netlbl_calipso_ops_get();

        if (ops)
                ret_val = ops->doi_walk(skip_cnt, callback, cb_arg);
        return ret_val;
}

/**
 * calipso_sock_getattr - Get the security attributes from a sock
 * @sk: the sock
 * @secattr: the security attributes
 *
 * Description:
 * Query @sk to see if there is a CALIPSO option attached to the sock and if
 * there is return the CALIPSO security attributes in @secattr.  This function
 * requires that @sk be locked, or privately held, but it does not do any
 * locking itself.  Returns zero on success and negative values on failure.
 *
 */
int calipso_sock_getattr(struct sock *sk, struct netlbl_lsm_secattr *secattr)
{
        int ret_val = -ENOMSG;
        const struct netlbl_calipso_ops *ops = netlbl_calipso_ops_get();

        if (ops)
                ret_val = ops->sock_getattr(sk, secattr);
        return ret_val;
}

/**
 * calipso_sock_setattr - Add a CALIPSO option to a socket
 * @sk: the socket
 * @doi_def: the CALIPSO DOI to use
 * @secattr: the specific security attributes of the socket
 *
 * Description:
 * Set the CALIPSO option on the given socket using the DOI definition and
 * security attributes passed to the function.  This function requires
 * exclusive access to @sk, which means it either needs to be in the
 * process of being created or locked.  Returns zero on success and negative
 * values on failure.
 *
 */
int calipso_sock_setattr(struct sock *sk,
                         const struct calipso_doi *doi_def,
                         const struct netlbl_lsm_secattr *secattr)
{
        int ret_val = -ENOMSG;
        const struct netlbl_calipso_ops *ops = netlbl_calipso_ops_get();

        if (ops)
                ret_val = ops->sock_setattr(sk, doi_def, secattr);
        return ret_val;
}

/**
 * calipso_sock_delattr - Delete the CALIPSO option from a socket
 * @sk: the socket
 *
 * Description:
 * Removes the CALIPSO option from a socket, if present.
 *
 */
void calipso_sock_delattr(struct sock *sk)
{
        const struct netlbl_calipso_ops *ops = netlbl_calipso_ops_get();

        if (ops)
                ops->sock_delattr(sk);
}

/**
 * calipso_req_setattr - Add a CALIPSO option to a connection request socket
 * @req: the connection request socket
 * @doi_def: the CALIPSO DOI to use
 * @secattr: the specific security attributes of the socket
 *
 * Description:
 * Set the CALIPSO option on the given socket using the DOI definition and
 * security attributes passed to the function.  Returns zero on success and
 * negative values on failure.
 *
 */
int calipso_req_setattr(struct request_sock *req,
                        const struct calipso_doi *doi_def,
                        const struct netlbl_lsm_secattr *secattr)
{
        int ret_val = -ENOMSG;
        const struct netlbl_calipso_ops *ops = netlbl_calipso_ops_get();

        if (ops)
                ret_val = ops->req_setattr(req, doi_def, secattr);
        return ret_val;
}

/**
 * calipso_req_delattr - Delete the CALIPSO option from a request socket
 * @req: the request socket
 *
 * Description:
 * Removes the CALIPSO option from a request socket, if present.
 *
 */
void calipso_req_delattr(struct request_sock *req)
{
        const struct netlbl_calipso_ops *ops = netlbl_calipso_ops_get();

        if (ops)
                ops->req_delattr(req);
}

/**
 * calipso_optptr - Find the CALIPSO option in the packet
 * @skb: the packet
 *
 * Description:
 * Parse the packet's IP header looking for a CALIPSO option.  Returns a pointer
 * to the start of the CALIPSO option on success, NULL if one if not found.
 *
 */
unsigned char *calipso_optptr(const struct sk_buff *skb)
{
        unsigned char *ret_val = NULL;
        const struct netlbl_calipso_ops *ops = netlbl_calipso_ops_get();

        if (ops)
                ret_val = ops->skbuff_optptr(skb);
        return ret_val;
}

/**
 * calipso_getattr - Get the security attributes from a memory block.
 * @calipso: the CALIPSO option
 * @secattr: the security attributes
 *
 * Description:
 * Inspect @calipso and return the security attributes in @secattr.
 * Returns zero on success and negative values on failure.
 *
 */
int calipso_getattr(const unsigned char *calipso,
                    struct netlbl_lsm_secattr *secattr)
{
        int ret_val = -ENOMSG;
        const struct netlbl_calipso_ops *ops = netlbl_calipso_ops_get();

        if (ops)
                ret_val = ops->opt_getattr(calipso, secattr);
        return ret_val;
}

/**
 * calipso_skbuff_setattr - Set the CALIPSO option on a packet
 * @skb: the packet
 * @doi_def: the CALIPSO DOI to use
 * @secattr: the security attributes
 *
 * Description:
 * Set the CALIPSO option on the given packet based on the security attributes.
 * Returns a pointer to the IP header on success and NULL on failure.
 *
 */
int calipso_skbuff_setattr(struct sk_buff *skb,
                           const struct calipso_doi *doi_def,
                           const struct netlbl_lsm_secattr *secattr)
{
        int ret_val = -ENOMSG;
        const struct netlbl_calipso_ops *ops = netlbl_calipso_ops_get();

        if (ops)
                ret_val = ops->skbuff_setattr(skb, doi_def, secattr);
        return ret_val;
}

/**
 * calipso_skbuff_delattr - Delete any CALIPSO options from a packet
 * @skb: the packet
 *
 * Description:
 * Removes any and all CALIPSO options from the given packet.  Returns zero on
 * success, negative values on failure.
 *
 */
int calipso_skbuff_delattr(struct sk_buff *skb)
{
        int ret_val = -ENOMSG;
        const struct netlbl_calipso_ops *ops = netlbl_calipso_ops_get();

        if (ops)
                ret_val = ops->skbuff_delattr(skb);
        return ret_val;
}

/**
 * calipso_cache_invalidate - Invalidates the current CALIPSO cache
 *
 * Description:
 * Invalidates and frees any entries in the CALIPSO cache.  Returns zero on
 * success and negative values on failure.
 *
 */
void calipso_cache_invalidate(void)
{
        const struct netlbl_calipso_ops *ops = netlbl_calipso_ops_get();

        if (ops)
                ops->cache_invalidate();
}

/**
 * calipso_cache_add - Add an entry to the CALIPSO cache
 * @calipso_ptr: the CALIPSO option
 * @secattr: the packet's security attributes
 *
 * Description:
 * Add a new entry into the CALIPSO label mapping cache.
 * Returns zero on success, negative values on failure.
 *
 */
int calipso_cache_add(const unsigned char *calipso_ptr,
                      const struct netlbl_lsm_secattr *secattr)

{
        int ret_val = -ENOMSG;
        const struct netlbl_calipso_ops *ops = netlbl_calipso_ops_get();

        if (ops)
                ret_val = ops->cache_add(calipso_ptr, secattr);
        return ret_val;
}



















    6 


    6 

    6 








    6 











    3 


    3 







    3 









    5 


    5 















































    5 




    5 





    6 



    6 







    5 

    4 








    6 



   49 
   48 

   35 
   27 



    4 


    4 

    4 












    7 













    7 




    2 




    2 







    3 

    3 


    3 














    3 


    3 





























    3 































































    4 
















    4 










    4 
















    2 


















    2 









    6 











    3 




    5 



    6 







    6 



    5 

    1 
    4 



    2 



    1 
    1 



   10 

    1 
    9 



























    5 



    5 
    5 


    4 




    4 






   48 

















   48 




















    7 










1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
// SPDX-License-Identifier: GPL-2.0
/* Multipath TCP
 *
 * Copyright (c) 2019, Intel Corporation.
 */
#define pr_fmt(fmt) "MPTCP: " fmt

#include <linux/kernel.h>
#include <net/mptcp.h>
#include "protocol.h"

#include "mib.h"

/* path manager command handlers */

int mptcp_pm_announce_addr(struct mptcp_sock *msk,
                           const struct mptcp_addr_info *addr,
                           bool echo)
{
        u8 add_addr = READ_ONCE(msk->pm.addr_signal);

        pr_debug("msk=%p, local_id=%d, echo=%d\n", msk, addr->id, echo);

        lockdep_assert_held(&msk->pm.lock);

        if (add_addr &
            (echo ? BIT(MPTCP_ADD_ADDR_ECHO) : BIT(MPTCP_ADD_ADDR_SIGNAL))) {
                MPTCP_INC_STATS(sock_net((struct sock *)msk),
                                echo ? MPTCP_MIB_ECHOADDTXDROP : MPTCP_MIB_ADDADDRTXDROP);
                return -EINVAL;
        }

        if (echo) {
                msk->pm.remote = *addr;
                add_addr |= BIT(MPTCP_ADD_ADDR_ECHO);
        } else {
                msk->pm.local = *addr;
                add_addr |= BIT(MPTCP_ADD_ADDR_SIGNAL);
        }
        WRITE_ONCE(msk->pm.addr_signal, add_addr);
        return 0;
}

int mptcp_pm_remove_addr(struct mptcp_sock *msk, const struct mptcp_rm_list *rm_list)
{
        u8 rm_addr = READ_ONCE(msk->pm.addr_signal);

        pr_debug("msk=%p, rm_list_nr=%d\n", msk, rm_list->nr);

        if (rm_addr) {
                MPTCP_ADD_STATS(sock_net((struct sock *)msk),
                                MPTCP_MIB_RMADDRTXDROP, rm_list->nr);
                return -EINVAL;
        }

        msk->pm.rm_list_tx = *rm_list;
        rm_addr |= BIT(MPTCP_RM_ADDR_SIGNAL);
        WRITE_ONCE(msk->pm.addr_signal, rm_addr);
        mptcp_pm_nl_addr_send_ack(msk);
        return 0;
}

/* path manager event handlers */

void mptcp_pm_new_connection(struct mptcp_sock *msk, const struct sock *ssk, int server_side)
{
        struct mptcp_pm_data *pm = &msk->pm;

        pr_debug("msk=%p, token=%u side=%d\n", msk, READ_ONCE(msk->token), server_side);

        WRITE_ONCE(pm->server_side, server_side);
        mptcp_event(MPTCP_EVENT_CREATED, msk, ssk, GFP_ATOMIC);
}

bool mptcp_pm_allow_new_subflow(struct mptcp_sock *msk)
{
        struct mptcp_pm_data *pm = &msk->pm;
        unsigned int subflows_max;
        int ret = 0;

        if (mptcp_pm_is_userspace(msk)) {
                if (mptcp_userspace_pm_active(msk)) {
                        spin_lock_bh(&pm->lock);
                        pm->subflows++;
                        spin_unlock_bh(&pm->lock);
                        return true;
                }
                return false;
        }

        subflows_max = mptcp_pm_get_subflows_max(msk);

        pr_debug("msk=%p subflows=%d max=%d allow=%d\n", msk, pm->subflows,
                 subflows_max, READ_ONCE(pm->accept_subflow));

        /* try to avoid acquiring the lock below */
        if (!READ_ONCE(pm->accept_subflow))
                return false;

        spin_lock_bh(&pm->lock);
        if (READ_ONCE(pm->accept_subflow)) {
                ret = pm->subflows < subflows_max;
                if (ret && ++pm->subflows == subflows_max)
                        WRITE_ONCE(pm->accept_subflow, false);
        }
        spin_unlock_bh(&pm->lock);

        return ret;
}

/* return true if the new status bit is currently cleared, that is, this event
 * can be server, eventually by an already scheduled work
 */
static bool mptcp_pm_schedule_work(struct mptcp_sock *msk,
                                   enum mptcp_pm_status new_status)
{
        pr_debug("msk=%p status=%x new=%lx\n", msk, msk->pm.status,
                 BIT(new_status));
        if (msk->pm.status & BIT(new_status))
                return false;

        msk->pm.status |= BIT(new_status);
        mptcp_schedule_work((struct sock *)msk);
        return true;
}

void mptcp_pm_fully_established(struct mptcp_sock *msk, const struct sock *ssk)
{
        struct mptcp_pm_data *pm = &msk->pm;
        bool announce = false;

        pr_debug("msk=%p\n", msk);

        spin_lock_bh(&pm->lock);

        /* mptcp_pm_fully_established() can be invoked by multiple
         * racing paths - accept() and check_fully_established()
         * be sure to serve this event only once.
         */
        if (READ_ONCE(pm->work_pending) &&
            !(msk->pm.status & BIT(MPTCP_PM_ALREADY_ESTABLISHED)))
                mptcp_pm_schedule_work(msk, MPTCP_PM_ESTABLISHED);

        if ((msk->pm.status & BIT(MPTCP_PM_ALREADY_ESTABLISHED)) == 0)
                announce = true;

        msk->pm.status |= BIT(MPTCP_PM_ALREADY_ESTABLISHED);
        spin_unlock_bh(&pm->lock);

        if (announce)
                mptcp_event(MPTCP_EVENT_ESTABLISHED, msk, ssk, GFP_ATOMIC);
}

void mptcp_pm_connection_closed(struct mptcp_sock *msk)
{
        pr_debug("msk=%p\n", msk);

        if (msk->token)
                mptcp_event(MPTCP_EVENT_CLOSED, msk, NULL, GFP_KERNEL);
}

void mptcp_pm_subflow_established(struct mptcp_sock *msk)
{
        struct mptcp_pm_data *pm = &msk->pm;

        pr_debug("msk=%p\n", msk);

        if (!READ_ONCE(pm->work_pending))
                return;

        spin_lock_bh(&pm->lock);

        if (READ_ONCE(pm->work_pending))
                mptcp_pm_schedule_work(msk, MPTCP_PM_SUBFLOW_ESTABLISHED);

        spin_unlock_bh(&pm->lock);
}

void mptcp_pm_subflow_check_next(struct mptcp_sock *msk,
                                 const struct mptcp_subflow_context *subflow)
{
        struct mptcp_pm_data *pm = &msk->pm;
        bool update_subflows;

        update_subflows = subflow->request_join || subflow->mp_join;
        if (mptcp_pm_is_userspace(msk)) {
                if (update_subflows) {
                        spin_lock_bh(&pm->lock);
                        pm->subflows--;
                        spin_unlock_bh(&pm->lock);
                }
                return;
        }

        if (!READ_ONCE(pm->work_pending) && !update_subflows)
                return;

        spin_lock_bh(&pm->lock);
        if (update_subflows)
                __mptcp_pm_close_subflow(msk);

        /* Even if this subflow is not really established, tell the PM to try
         * to pick the next ones, if possible.
         */
        if (mptcp_pm_nl_check_work_pending(msk))
                mptcp_pm_schedule_work(msk, MPTCP_PM_SUBFLOW_ESTABLISHED);

        spin_unlock_bh(&pm->lock);
}

void mptcp_pm_add_addr_received(const struct sock *ssk,
                                const struct mptcp_addr_info *addr)
{
        struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);
        struct mptcp_sock *msk = mptcp_sk(subflow->conn);
        struct mptcp_pm_data *pm = &msk->pm;

        pr_debug("msk=%p remote_id=%d accept=%d\n", msk, addr->id,
                 READ_ONCE(pm->accept_addr));

        mptcp_event_addr_announced(ssk, addr);

        spin_lock_bh(&pm->lock);

        if (mptcp_pm_is_userspace(msk)) {
                if (mptcp_userspace_pm_active(msk)) {
                        mptcp_pm_announce_addr(msk, addr, true);
                        mptcp_pm_add_addr_send_ack(msk);
                } else {
                        __MPTCP_INC_STATS(sock_net((struct sock *)msk), MPTCP_MIB_ADDADDRDROP);
                }
        /* id0 should not have a different address */
        } else if ((addr->id == 0 && !mptcp_pm_nl_is_init_remote_addr(msk, addr)) ||
                   (addr->id > 0 && !READ_ONCE(pm->accept_addr))) {
                mptcp_pm_announce_addr(msk, addr, true);
                mptcp_pm_add_addr_send_ack(msk);
        } else if (mptcp_pm_schedule_work(msk, MPTCP_PM_ADD_ADDR_RECEIVED)) {
                pm->remote = *addr;
        } else {
                __MPTCP_INC_STATS(sock_net((struct sock *)msk), MPTCP_MIB_ADDADDRDROP);
        }

        spin_unlock_bh(&pm->lock);
}

void mptcp_pm_add_addr_echoed(struct mptcp_sock *msk,
                              const struct mptcp_addr_info *addr)
{
        struct mptcp_pm_data *pm = &msk->pm;

        pr_debug("msk=%p\n", msk);

        spin_lock_bh(&pm->lock);

        if (mptcp_lookup_anno_list_by_saddr(msk, addr) && READ_ONCE(pm->work_pending))
                mptcp_pm_schedule_work(msk, MPTCP_PM_SUBFLOW_ESTABLISHED);

        spin_unlock_bh(&pm->lock);
}

void mptcp_pm_add_addr_send_ack(struct mptcp_sock *msk)
{
        if (!mptcp_pm_should_add_signal(msk))
                return;

        mptcp_pm_schedule_work(msk, MPTCP_PM_ADD_ADDR_SEND_ACK);
}

void mptcp_pm_rm_addr_received(struct mptcp_sock *msk,
                               const struct mptcp_rm_list *rm_list)
{
        struct mptcp_pm_data *pm = &msk->pm;
        u8 i;

        pr_debug("msk=%p remote_ids_nr=%d\n", msk, rm_list->nr);

        for (i = 0; i < rm_list->nr; i++)
                mptcp_event_addr_removed(msk, rm_list->ids[i]);

        spin_lock_bh(&pm->lock);
        if (mptcp_pm_schedule_work(msk, MPTCP_PM_RM_ADDR_RECEIVED))
                pm->rm_list_rx = *rm_list;
        else
                __MPTCP_INC_STATS(sock_net((struct sock *)msk), MPTCP_MIB_RMADDRDROP);
        spin_unlock_bh(&pm->lock);
}

void mptcp_pm_mp_prio_received(struct sock *ssk, u8 bkup)
{
        struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);
        struct sock *sk = subflow->conn;
        struct mptcp_sock *msk;

        pr_debug("subflow->backup=%d, bkup=%d\n", subflow->backup, bkup);
        msk = mptcp_sk(sk);
        if (subflow->backup != bkup)
                subflow->backup = bkup;

        mptcp_event(MPTCP_EVENT_SUB_PRIORITY, msk, ssk, GFP_ATOMIC);
}

void mptcp_pm_mp_fail_received(struct sock *sk, u64 fail_seq)
{
        struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
        struct mptcp_sock *msk = mptcp_sk(subflow->conn);

        pr_debug("fail_seq=%llu\n", fail_seq);

        if (!READ_ONCE(msk->allow_infinite_fallback))
                return;

        if (!subflow->fail_tout) {
                pr_debug("send MP_FAIL response and infinite map\n");

                subflow->send_mp_fail = 1;
                subflow->send_infinite_map = 1;
                tcp_send_ack(sk);
        } else {
                pr_debug("MP_FAIL response received\n");
                WRITE_ONCE(subflow->fail_tout, 0);
        }
}

/* path manager helpers */

bool mptcp_pm_add_addr_signal(struct mptcp_sock *msk, const struct sk_buff *skb,
                              unsigned int opt_size, unsigned int remaining,
                              struct mptcp_addr_info *addr, bool *echo,
                              bool *drop_other_suboptions)
{
        int ret = false;
        u8 add_addr;
        u8 family;
        bool port;

        spin_lock_bh(&msk->pm.lock);

        /* double check after the lock is acquired */
        if (!mptcp_pm_should_add_signal(msk))
                goto out_unlock;

        /* always drop every other options for pure ack ADD_ADDR; this is a
         * plain dup-ack from TCP perspective. The other MPTCP-relevant info,
         * if any, will be carried by the 'original' TCP ack
         */
        if (skb && skb_is_tcp_pure_ack(skb)) {
                remaining += opt_size;
                *drop_other_suboptions = true;
        }

        *echo = mptcp_pm_should_add_signal_echo(msk);
        port = !!(*echo ? msk->pm.remote.port : msk->pm.local.port);

        family = *echo ? msk->pm.remote.family : msk->pm.local.family;
        if (remaining < mptcp_add_addr_len(family, *echo, port))
                goto out_unlock;

        if (*echo) {
                *addr = msk->pm.remote;
                add_addr = msk->pm.addr_signal & ~BIT(MPTCP_ADD_ADDR_ECHO);
        } else {
                *addr = msk->pm.local;
                add_addr = msk->pm.addr_signal & ~BIT(MPTCP_ADD_ADDR_SIGNAL);
        }
        WRITE_ONCE(msk->pm.addr_signal, add_addr);
        ret = true;

out_unlock:
        spin_unlock_bh(&msk->pm.lock);
        return ret;
}

bool mptcp_pm_rm_addr_signal(struct mptcp_sock *msk, unsigned int remaining,
                             struct mptcp_rm_list *rm_list)
{
        int ret = false, len;
        u8 rm_addr;

        spin_lock_bh(&msk->pm.lock);

        /* double check after the lock is acquired */
        if (!mptcp_pm_should_rm_signal(msk))
                goto out_unlock;

        rm_addr = msk->pm.addr_signal & ~BIT(MPTCP_RM_ADDR_SIGNAL);
        len = mptcp_rm_addr_len(&msk->pm.rm_list_tx);
        if (len < 0) {
                WRITE_ONCE(msk->pm.addr_signal, rm_addr);
                goto out_unlock;
        }
        if (remaining < len)
                goto out_unlock;

        *rm_list = msk->pm.rm_list_tx;
        WRITE_ONCE(msk->pm.addr_signal, rm_addr);
        ret = true;

out_unlock:
        spin_unlock_bh(&msk->pm.lock);
        return ret;
}

int mptcp_pm_get_local_id(struct mptcp_sock *msk, struct sock_common *skc)
{
        struct mptcp_addr_info skc_local;
        struct mptcp_addr_info msk_local;

        if (WARN_ON_ONCE(!msk))
                return -1;

        /* The 0 ID mapping is defined by the first subflow, copied into the msk
         * addr
         */
        mptcp_local_address((struct sock_common *)msk, &msk_local);
        mptcp_local_address((struct sock_common *)skc, &skc_local);
        if (mptcp_addresses_equal(&msk_local, &skc_local, false))
                return 0;

        if (mptcp_pm_is_userspace(msk))
                return mptcp_userspace_pm_get_local_id(msk, &skc_local);
        return mptcp_pm_nl_get_local_id(msk, &skc_local);
}

bool mptcp_pm_is_backup(struct mptcp_sock *msk, struct sock_common *skc)
{
        struct mptcp_addr_info skc_local;

        mptcp_local_address((struct sock_common *)skc, &skc_local);

        if (mptcp_pm_is_userspace(msk))
                return mptcp_userspace_pm_is_backup(msk, &skc_local);

        return mptcp_pm_nl_is_backup(msk, &skc_local);
}

int mptcp_pm_get_addr(struct sk_buff *skb, struct genl_info *info)
{
        if (info->attrs[MPTCP_PM_ATTR_TOKEN])
                return mptcp_userspace_pm_get_addr(skb, info);
        return mptcp_pm_nl_get_addr(skb, info);
}

int mptcp_pm_dump_addr(struct sk_buff *msg, struct netlink_callback *cb)
{
        const struct genl_info *info = genl_info_dump(cb);

        if (info->attrs[MPTCP_PM_ATTR_TOKEN])
                return mptcp_userspace_pm_dump_addr(msg, cb);
        return mptcp_pm_nl_dump_addr(msg, cb);
}

int mptcp_pm_set_flags(struct sk_buff *skb, struct genl_info *info)
{
        if (info->attrs[MPTCP_PM_ATTR_TOKEN])
                return mptcp_userspace_pm_set_flags(skb, info);
        return mptcp_pm_nl_set_flags(skb, info);
}

void mptcp_pm_subflow_chk_stale(const struct mptcp_sock *msk, struct sock *ssk)
{
        struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);
        u32 rcv_tstamp = READ_ONCE(tcp_sk(ssk)->rcv_tstamp);

        /* keep track of rtx periods with no progress */
        if (!subflow->stale_count) {
                subflow->stale_rcv_tstamp = rcv_tstamp;
                subflow->stale_count++;
        } else if (subflow->stale_rcv_tstamp == rcv_tstamp) {
                if (subflow->stale_count < U8_MAX)
                        subflow->stale_count++;
                mptcp_pm_nl_subflow_chk_stale(msk, ssk);
        } else {
                subflow->stale_count = 0;
                mptcp_subflow_set_active(subflow);
        }
}

/* if sk is ipv4 or ipv6_only allows only same-family local and remote addresses,
 * otherwise allow any matching local/remote pair
 */
bool mptcp_pm_addr_families_match(const struct sock *sk,
                                  const struct mptcp_addr_info *loc,
                                  const struct mptcp_addr_info *rem)
{
        bool mptcp_is_v4 = sk->sk_family == AF_INET;

#if IS_ENABLED(CONFIG_MPTCP_IPV6)
        bool loc_is_v4 = loc->family == AF_INET || ipv6_addr_v4mapped(&loc->addr6);
        bool rem_is_v4 = rem->family == AF_INET || ipv6_addr_v4mapped(&rem->addr6);

        if (mptcp_is_v4)
                return loc_is_v4 && rem_is_v4;

        if (ipv6_only_sock(sk))
                return !loc_is_v4 && !rem_is_v4;

        return loc_is_v4 == rem_is_v4;
#else
        return mptcp_is_v4 && loc->family == AF_INET && rem->family == AF_INET;
#endif
}

void mptcp_pm_data_reset(struct mptcp_sock *msk)
{
        u8 pm_type = mptcp_get_pm_type(sock_net((struct sock *)msk));
        struct mptcp_pm_data *pm = &msk->pm;

        pm->add_addr_signaled = 0;
        pm->add_addr_accepted = 0;
        pm->local_addr_used = 0;
        pm->subflows = 0;
        pm->rm_list_tx.nr = 0;
        pm->rm_list_rx.nr = 0;
        WRITE_ONCE(pm->pm_type, pm_type);

        if (pm_type == MPTCP_PM_TYPE_KERNEL) {
                bool subflows_allowed = !!mptcp_pm_get_subflows_max(msk);

                /* pm->work_pending must be only be set to 'true' when
                 * pm->pm_type is set to MPTCP_PM_TYPE_KERNEL
                 */
                WRITE_ONCE(pm->work_pending,
                           (!!mptcp_pm_get_local_addr_max(msk) &&
                            subflows_allowed) ||
                           !!mptcp_pm_get_add_addr_signal_max(msk));
                WRITE_ONCE(pm->accept_addr,
                           !!mptcp_pm_get_add_addr_accept_max(msk) &&
                           subflows_allowed);
                WRITE_ONCE(pm->accept_subflow, subflows_allowed);
        } else {
                WRITE_ONCE(pm->work_pending, 0);
                WRITE_ONCE(pm->accept_addr, 0);
                WRITE_ONCE(pm->accept_subflow, 0);
        }

        WRITE_ONCE(pm->addr_signal, 0);
        WRITE_ONCE(pm->remote_deny_join_id0, false);
        pm->status = 0;
        bitmap_fill(msk->pm.id_avail_bitmap, MPTCP_PM_MAX_ADDR_ID + 1);
}

void mptcp_pm_data_init(struct mptcp_sock *msk)
{
        spin_lock_init(&msk->pm.lock);
        INIT_LIST_HEAD(&msk->pm.anno_list);
        INIT_LIST_HEAD(&msk->pm.userspace_pm_local_addr_list);
        mptcp_pm_data_reset(msk);
}

void __init mptcp_pm_init(void)
{
        mptcp_pm_nl_init();
}








































































 8180 












   47 

 8373 
   11 









 8153 

 8180 











   54 


    3 


































    1 



    1 





    1 







    1 
    2 






  294 
    2 
    3 

  301 



   54 









   57 

   52 
    6 







  273 







   48 
   48 


    7 
    6 














1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __LINUX_NET_SCM_H
#define __LINUX_NET_SCM_H

#include <linux/limits.h>
#include <linux/net.h>
#include <linux/cred.h>
#include <linux/file.h>
#include <linux/security.h>
#include <linux/pid.h>
#include <linux/nsproxy.h>
#include <linux/sched/signal.h>
#include <net/compat.h>

/* Well, we should have at least one descriptor open
 * to accept passed FDs 8)
 */
#define SCM_MAX_FD        253

struct scm_creds {
        u32        pid;
        kuid_t        uid;
        kgid_t        gid;
};

#ifdef CONFIG_UNIX
struct unix_edge;
#endif

struct scm_fp_list {
        short                        count;
        short                        count_unix;
        short                        max;
#ifdef CONFIG_UNIX
        bool                        inflight;
        bool                        dead;
        struct list_head        vertices;
        struct unix_edge        *edges;
#endif
        struct user_struct        *user;
        struct file                *fp[SCM_MAX_FD];
};

struct scm_cookie {
        struct pid                *pid;                /* Skb credentials */
        struct scm_fp_list        *fp;                /* Passed files                */
        struct scm_creds        creds;                /* Skb credentials        */
#ifdef CONFIG_SECURITY_NETWORK
        u32                        secid;                /* Passed security ID         */
#endif
};

void scm_detach_fds(struct msghdr *msg, struct scm_cookie *scm);
void scm_detach_fds_compat(struct msghdr *msg, struct scm_cookie *scm);
int __scm_send(struct socket *sock, struct msghdr *msg, struct scm_cookie *scm);
void __scm_destroy(struct scm_cookie *scm);
struct scm_fp_list *scm_fp_dup(struct scm_fp_list *fpl);

#ifdef CONFIG_SECURITY_NETWORK
static __inline__ void unix_get_peersec_dgram(struct socket *sock, struct scm_cookie *scm)
{
        security_socket_getpeersec_dgram(sock, NULL, &scm->secid);
}
#else
static __inline__ void unix_get_peersec_dgram(struct socket *sock, struct scm_cookie *scm)
{ }
#endif /* CONFIG_SECURITY_NETWORK */

static __inline__ void scm_set_cred(struct scm_cookie *scm,
                                    struct pid *pid, kuid_t uid, kgid_t gid)
{
        scm->pid  = get_pid(pid);
        scm->creds.pid = pid_vnr(pid);
        scm->creds.uid = uid;
        scm->creds.gid = gid;
}

static __inline__ void scm_destroy_cred(struct scm_cookie *scm)
{
        put_pid(scm->pid);
        scm->pid  = NULL;
}

static __inline__ void scm_destroy(struct scm_cookie *scm)
{
        scm_destroy_cred(scm);
        if (scm->fp)
                __scm_destroy(scm);
}

static __inline__ int scm_send(struct socket *sock, struct msghdr *msg,
                               struct scm_cookie *scm, bool forcecreds)
{
        memset(scm, 0, sizeof(*scm));
        scm->creds.uid = INVALID_UID;
        scm->creds.gid = INVALID_GID;
        if (forcecreds)
                scm_set_cred(scm, task_tgid(current), current_uid(), current_gid());
        unix_get_peersec_dgram(sock, scm);
        if (msg->msg_controllen <= 0)
                return 0;
        return __scm_send(sock, msg, scm);
}

#ifdef CONFIG_SECURITY_NETWORK
static inline void scm_passec(struct socket *sock, struct msghdr *msg, struct scm_cookie *scm)
{
        char *secdata;
        u32 seclen;
        int err;

        if (test_bit(SOCK_PASSSEC, &sock->flags)) {
                err = security_secid_to_secctx(scm->secid, &secdata, &seclen);

                if (!err) {
                        put_cmsg(msg, SOL_SOCKET, SCM_SECURITY, seclen, secdata);
                        security_release_secctx(secdata, seclen);
                }
        }
}

static inline bool scm_has_secdata(struct socket *sock)
{
        return test_bit(SOCK_PASSSEC, &sock->flags);
}
#else
static inline void scm_passec(struct socket *sock, struct msghdr *msg, struct scm_cookie *scm)
{ }

static inline bool scm_has_secdata(struct socket *sock)
{
        return false;
}
#endif /* CONFIG_SECURITY_NETWORK */

static __inline__ void scm_pidfd_recv(struct msghdr *msg, struct scm_cookie *scm)
{
        struct file *pidfd_file = NULL;
        int len, pidfd;

        /* put_cmsg() doesn't return an error if CMSG is truncated,
         * that's why we need to opencode these checks here.
         */
        if (msg->msg_flags & MSG_CMSG_COMPAT)
                len = sizeof(struct compat_cmsghdr) + sizeof(int);
        else
                len = sizeof(struct cmsghdr) + sizeof(int);

        if (msg->msg_controllen < len) {
                msg->msg_flags |= MSG_CTRUNC;
                return;
        }

        if (!scm->pid)
                return;

        pidfd = pidfd_prepare(scm->pid, 0, &pidfd_file);

        if (put_cmsg(msg, SOL_SOCKET, SCM_PIDFD, sizeof(int), &pidfd)) {
                if (pidfd_file) {
                        put_unused_fd(pidfd);
                        fput(pidfd_file);
                }

                return;
        }

        if (pidfd_file)
                fd_install(pidfd, pidfd_file);
}

static inline bool __scm_recv_common(struct socket *sock, struct msghdr *msg,
                                     struct scm_cookie *scm, int flags)
{
        if (!msg->msg_control) {
                if (test_bit(SOCK_PASSCRED, &sock->flags) ||
                    test_bit(SOCK_PASSPIDFD, &sock->flags) ||
                    scm->fp || scm_has_secdata(sock))
                        msg->msg_flags |= MSG_CTRUNC;
                scm_destroy(scm);
                return false;
        }

        if (test_bit(SOCK_PASSCRED, &sock->flags)) {
                struct user_namespace *current_ns = current_user_ns();
                struct ucred ucreds = {
                        .pid = scm->creds.pid,
                        .uid = from_kuid_munged(current_ns, scm->creds.uid),
                        .gid = from_kgid_munged(current_ns, scm->creds.gid),
                };
                put_cmsg(msg, SOL_SOCKET, SCM_CREDENTIALS, sizeof(ucreds), &ucreds);
        }

        scm_passec(sock, msg, scm);

        if (scm->fp)
                scm_detach_fds(msg, scm);

        return true;
}

static inline void scm_recv(struct socket *sock, struct msghdr *msg,
                            struct scm_cookie *scm, int flags)
{
        if (!__scm_recv_common(sock, msg, scm, flags))
                return;

        scm_destroy_cred(scm);
}

static inline void scm_recv_unix(struct socket *sock, struct msghdr *msg,
                                 struct scm_cookie *scm, int flags)
{
        if (!__scm_recv_common(sock, msg, scm, flags))
                return;

        if (test_bit(SOCK_PASSPIDFD, &sock->flags))
                scm_pidfd_recv(msg, scm);

        scm_destroy_cred(scm);
}

static inline int scm_recv_one_fd(struct file *f, int __user *ufd,
                                  unsigned int flags)
{
        if (!ufd)
                return -EFAULT;
        return receive_fd(f, ufd, flags);
}

#endif /* __LINUX_NET_SCM_H */










































































































































    5 

    5 



    4 
    4 



    4 
































































   66 




   65 







  125 




















































   65 













   64 




   66 










    8 











    8 

























































































    4 










































  179 












  179 
























  183 



  182 















   67 




   66 















   29 









   12 












   17 



    8 
    8 














  123 


  185 












   67 
   67 





   27 
   26 
   19 



   16 
   17 






































































































































  195 

    4 

    1 




  192 



















































  111 
  120 


    4 
  116 





















   17 




   17 


   18 


   18 








































































































    4 
    4 






































    4 
    4 












    4 
    4 
    4 










    4 














    5 































    5 
    5 
    5 


























    5 



    5 






    5 























    5 



    5 

    5 








































    5 

    5 


    5 



























    6 










    6 



    6 




    6 






    6 


    6 





















































    5 











    5 

    5 












    5 



    5 



    5 




    5 

    5 





    5 





    5 












    4 









    4 
    4 

    4 











    4 















  796 



  794 











  734 











   63 












    4 

    4 













    4 










  820 
  130 


  794 







 1385 

 1391 

















































































































































































































    5 





    5 
    5 





    5 




    4 
    4 
    4 





















    4 














    4 











    4 



    4 



    4 


    4 





    4 
    4 



    4 


    4 



    4 
    4 



    4 





    4 






    4 



    4 
    4 


























































































































































































































































    5 









    4 





    2 

    2 















    4 





















    5 



    5 


    5 









    5 



    5 
    5 
    4 











    5 



    5 










    5 

































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































    4 

    4 

    4 












    5 





















    4 







    5 







   73 










   69 






















    5 













    5 



    5 
    4 


    4 
    4 



    5 
    5 



    5 





    1 
    3 
    1 
    4 
























































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































   70 
   71 













































































































































































    1 







    1 







    3 
















































    1 
































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965
2966
2967
2968
2969
2970
2971
2972
2973
2974
2975
2976
2977
2978
2979
2980
2981
2982
2983
2984
2985
2986
2987
2988
2989
2990
2991
2992
2993
2994
2995
2996
2997
2998
2999
3000
3001
3002
3003
3004
3005
3006
3007
3008
3009
3010
3011
3012
3013
3014
3015
3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029
3030
3031
3032
3033
3034
3035
3036
3037
3038
3039
3040
3041
3042
3043
3044
3045
3046
3047
3048
3049
3050
3051
3052
3053
3054
3055
3056
3057
3058
3059
3060
3061
3062
3063
3064
3065
3066
3067
3068
3069
3070
3071
3072
3073
3074
3075
3076
3077
3078
3079
3080
3081
3082
3083
3084
3085
3086
3087
3088
3089
3090
3091
3092
3093
3094
3095
3096
3097
3098
3099
3100
3101
3102
3103
3104
3105
3106
3107
3108
3109
3110
3111
3112
3113
3114
3115
3116
3117
3118
3119
3120
3121
3122
3123
3124
3125
3126
3127
3128
3129
3130
3131
3132
3133
3134
3135
3136
3137
3138
3139
3140
3141
3142
3143
3144
3145
3146
3147
3148
3149
3150
3151
3152
3153
3154
3155
3156
3157
3158
3159
3160
3161
3162
3163
3164
3165
3166
3167
3168
3169
3170
3171
3172
3173
3174
3175
3176
3177
3178
3179
3180
3181
3182
3183
3184
3185
3186
3187
3188
3189
3190
3191
3192
3193
3194
3195
3196
3197
3198
3199
3200
3201
3202
3203
3204
3205
3206
3207
3208
3209
3210
3211
3212
3213
3214
3215
3216
3217
3218
3219
3220
3221
3222
3223
3224
3225
3226
3227
3228
3229
3230
3231
3232
3233
3234
3235
3236
3237
3238
3239
3240
3241
3242
3243
3244
3245
3246
3247
3248
3249
3250
3251
3252
3253
3254
3255
3256
3257
3258
3259
3260
3261
3262
3263
3264
3265
3266
3267
3268
3269
3270
3271
3272
3273
3274
3275
3276
3277
3278
3279
3280
3281
3282
3283
3284
3285
3286
3287
3288
3289
3290
3291
3292
3293
3294
3295
3296
3297
3298
3299
3300
3301
3302
3303
3304
3305
3306
3307
3308
3309
3310
3311
3312
3313
3314
3315
3316
3317
3318
3319
3320
3321
3322
3323
3324
3325
3326
3327
3328
3329
3330
3331
3332
3333
3334
3335
3336
3337
3338
3339
3340
3341
3342
3343
3344
3345
3346
3347
3348
3349
3350
3351
3352
3353
3354
3355
3356
3357
3358
3359
3360
3361
3362
3363
3364
3365
3366
3367
3368
3369
3370
3371
3372
3373
3374
3375
3376
3377
3378
3379
3380
3381
3382
3383
3384
3385
3386
3387
3388
3389
3390
3391
3392
3393
3394
3395
3396
3397
3398
3399
3400
3401
3402
3403
3404
3405
3406
3407
3408
3409
3410
3411
3412
3413
3414
3415
3416
3417
3418
3419
3420
3421
3422
3423
3424
3425
3426
3427
3428
3429
3430
3431
3432
3433
3434
3435
3436
3437
3438
3439
3440
3441
3442
3443
3444
3445
3446
3447
3448
3449
3450
3451
3452
3453
3454
3455
3456
3457
3458
3459
3460
3461
3462
3463
3464
3465
3466
3467
3468
3469
3470
3471
3472
3473
3474
3475
3476
3477
3478
3479
3480
3481
3482
3483
3484
3485
3486
3487
3488
3489
3490
3491
3492
3493
3494
3495
3496
3497
3498
3499
3500
3501
3502
3503
3504
3505
3506
3507
3508
3509
3510
3511
3512
3513
3514
3515
3516
3517
3518
3519
3520
3521
3522
3523
3524
3525
3526
3527
3528
3529
3530
3531
3532
3533
3534
3535
3536
3537
3538
3539
3540
3541
3542
3543
3544
3545
3546
3547
3548
3549
3550
3551
3552
3553
3554
3555
3556
3557
3558
3559
3560
3561
3562
3563
3564
3565
3566
3567
3568
3569
3570
3571
3572
3573
3574
3575
3576
3577
3578
3579
3580
3581
3582
3583
3584
3585
3586
3587
3588
3589
3590
3591
3592
3593
3594
3595
3596
3597
3598
3599
3600
3601
3602
3603
3604
3605
3606
3607
3608
3609
3610
3611
3612
3613
3614
3615
3616
3617
3618
3619
3620
3621
3622
3623
3624
3625
3626
3627
3628
3629
3630
3631
3632
3633
3634
3635
3636
3637
3638
3639
3640
3641
3642
3643
3644
3645
3646
3647
3648
3649
3650
3651
3652
3653
3654
3655
3656
3657
3658
3659
3660
3661
3662
3663
3664
3665
3666
3667
3668
3669
3670
3671
3672
3673
3674
3675
3676
3677
3678
3679
3680
3681
3682
3683
3684
3685
3686
3687
3688
3689
3690
3691
3692
3693
3694
3695
3696
3697
3698
3699
3700
3701
3702
3703
3704
3705
3706
3707
3708
3709
3710
3711
3712
3713
3714
3715
3716
3717
3718
3719
3720
3721
3722
3723
3724
3725
3726
3727
3728
3729
3730
3731
3732
3733
3734
3735
3736
3737
3738
3739
3740
3741
3742
3743
3744
3745
3746
3747
3748
3749
3750
3751
3752
3753
3754
3755
3756
3757
3758
3759
3760
3761
3762
3763
3764
3765
3766
3767
3768
3769
3770
3771
3772
3773
3774
3775
3776
3777
3778
3779
3780
3781
3782
3783
3784
3785
3786
3787
3788
3789
3790
3791
3792
3793
3794
3795
3796
3797
3798
3799
3800
3801
3802
3803
3804
3805
3806
3807
3808
3809
3810
3811
3812
3813
3814
3815
3816
3817
3818
3819
3820
3821
3822
3823
3824
3825
3826
3827
3828
3829
3830
3831
3832
3833
3834
3835
3836
3837
3838
3839
3840
3841
3842
3843
3844
3845
3846
3847
3848
3849
3850
3851
3852
3853
3854
3855
3856
3857
3858
3859
3860
3861
3862
3863
3864
3865
3866
3867
3868
3869
3870
3871
3872
3873
3874
3875
3876
3877
3878
3879
3880
3881
3882
3883
3884
3885
3886
3887
3888
3889
3890
3891
3892
3893
3894
3895
3896
3897
3898
3899
3900
3901
3902
3903
3904
3905
3906
3907
3908
3909
3910
3911
3912
3913
3914
3915
3916
3917
3918
3919
3920
3921
3922
3923
3924
3925
3926
3927
3928
3929
3930
3931
3932
3933
3934
3935
3936
3937
3938
3939
3940
3941
3942
3943
3944
3945
3946
3947
3948
3949
3950
3951
3952
3953
3954
3955
3956
3957
3958
3959
3960
3961
3962
3963
3964
3965
3966
3967
3968
3969
3970
3971
3972
3973
3974
3975
3976
3977
3978
3979
3980
3981
3982
3983
3984
3985
3986
3987
3988
3989
3990
3991
3992
3993
3994
3995
3996
3997
3998
3999
4000
4001
4002
4003
4004
4005
4006
4007
4008
4009
4010
4011
4012
4013
4014
4015
4016
4017
4018
4019
4020
4021
4022
4023
4024
4025
4026
4027
4028
4029
4030
4031
4032
4033
4034
4035
4036
4037
4038
4039
4040
4041
4042
4043
4044
4045
4046
4047
4048
4049
4050
4051
4052
4053
4054
4055
4056
4057
4058
4059
4060
4061
4062
4063
4064
4065
4066
4067
4068
4069
4070
4071
4072
4073
4074
4075
4076
4077
4078
4079
4080
4081
4082
4083
4084
4085
4086
4087
4088
4089
4090
4091
4092
4093
4094
4095
4096
4097
4098
4099
4100
4101
4102
4103
4104
4105
4106
4107
4108
4109
4110
4111
4112
4113
4114
4115
4116
4117
4118
4119
4120
4121
4122
4123
4124
4125
4126
4127
4128
4129
4130
4131
4132
4133
4134
4135
4136
4137
4138
4139
4140
4141
4142
4143
4144
4145
4146
4147
4148
4149
4150
4151
4152
4153
4154
4155
4156
4157
4158
4159
4160
4161
4162
4163
4164
4165
4166
4167
4168
4169
4170
4171
4172
4173
4174
4175
4176
4177
4178
4179
4180
4181
4182
4183
4184
4185
4186
4187
4188
4189
4190
4191
4192
4193
4194
4195
4196
4197
4198
4199
4200
4201
4202
4203
4204
4205
4206
4207
4208
4209
4210
4211
4212
4213
4214
4215
4216
4217
4218
4219
4220
4221
4222
4223
4224
4225
4226
4227
4228
4229
4230
4231
4232
4233
4234
4235
4236
4237
4238
4239
4240
4241
4242
4243
4244
4245
4246
4247
4248
4249
4250
4251
4252
4253
4254
4255
4256
4257
4258
4259
4260
4261
4262
4263
4264
4265
4266
4267
4268
4269
4270
4271
4272
4273
4274
4275
4276
4277
4278
4279
4280
4281
4282
4283
4284
4285
4286
4287
4288
4289
4290
4291
4292
4293
4294
4295
4296
4297
4298
4299
4300
4301
4302
4303
4304
4305
4306
4307
4308
4309
4310
4311
4312
4313
4314
4315
4316
4317
4318
4319
4320
4321
4322
4323
4324
4325
4326
4327
4328
4329
4330
4331
4332
4333
4334
4335
4336
4337
4338
4339
4340
4341
4342
4343
4344
4345
4346
4347
4348
4349
4350
4351
4352
4353
4354
4355
4356
4357
4358
4359
4360
4361
4362
4363
4364
4365
4366
4367
4368
4369
4370
4371
4372
4373
4374
4375
4376
4377
4378
4379
4380
4381
4382
4383
4384
4385
4386
4387
4388
4389
4390
4391
4392
4393
4394
4395
4396
4397
4398
4399
4400
4401
4402
4403
4404
4405
4406
4407
4408
4409
4410
4411
4412
4413
4414
4415
4416
4417
4418
4419
4420
4421
4422
4423
4424
4425
4426
4427
4428
4429
4430
4431
4432
4433
4434
4435
4436
4437
4438
4439
4440
4441
4442
4443
4444
4445
4446
4447
4448
4449
4450
4451
4452
4453
4454
4455
4456
4457
4458
4459
4460
4461
4462
4463
4464
4465
4466
4467
4468
4469
4470
4471
4472
4473
4474
4475
4476
4477
4478
4479
4480
4481
4482
4483
4484
4485
4486
4487
4488
4489
4490
4491
4492
4493
4494
4495
4496
4497
4498
4499
4500
4501
4502
4503
4504
4505
4506
4507
4508
4509
4510
4511
4512
4513
4514
4515
4516
4517
4518
4519
4520
4521
4522
4523
4524
4525
4526
4527
4528
4529
4530
4531
4532
4533
4534
4535
4536
4537
4538
4539
4540
4541
4542
4543
4544
4545
4546
4547
4548
4549
4550
4551
4552
4553
4554
4555
4556
4557
4558
4559
4560
4561
4562
4563
4564
4565
4566
4567
4568
4569
4570
4571
4572
4573
4574
4575
4576
4577
4578
4579
4580
4581
4582
4583
4584
4585
4586
4587
4588
4589
4590
4591
4592
4593
4594
4595
4596
4597
4598
4599
4600
4601
4602
4603
4604
4605
4606
4607
4608
4609
4610
4611
4612
4613
4614
4615
4616
4617
4618
4619
4620
4621
4622
4623
4624
4625
4626
4627
4628
4629
4630
4631
4632
4633
4634
4635
4636
4637
4638
4639
4640
4641
4642
4643
4644
4645
4646
4647
4648
4649
4650
4651
4652
4653
4654
4655
4656
4657
4658
4659
4660
4661
4662
4663
4664
4665
4666
4667
4668
4669
4670
4671
4672
4673
4674
4675
4676
4677
4678
4679
4680
4681
4682
4683
4684
4685
4686
4687
4688
4689
4690
4691
4692
4693
4694
4695
4696
4697
4698
4699
4700
4701
4702
4703
4704
4705
4706
4707
4708
4709
4710
4711
4712
4713
4714
4715
4716
4717
4718
4719
4720
4721
4722
4723
4724
4725
4726
4727
4728
4729
4730
4731
4732
4733
4734
4735
4736
4737
4738
4739
4740
4741
4742
4743
4744
4745
4746
4747
4748
4749
4750
4751
4752
4753
4754
4755
4756
4757
4758
4759
4760
4761
4762
4763
4764
4765
4766
4767
4768
4769
4770
4771
4772
4773
4774
4775
4776
4777
4778
4779
4780
4781
4782
4783
4784
4785
4786
4787
4788
4789
4790
4791
4792
4793
4794
4795
4796
4797
4798
4799
4800
4801
4802
4803
4804
4805
4806
4807
4808
4809
4810
4811
4812
4813
4814
4815
4816
4817
4818
4819
4820
4821
4822
4823
4824
4825
4826
4827
4828
4829
4830
4831
4832
4833
4834
4835
4836
4837
4838
4839
4840
4841
4842
4843
4844
4845
4846
4847
4848
4849
4850
4851
4852
4853
4854
4855
4856
4857
4858
4859
4860
4861
4862
4863
4864
4865
4866
4867
4868
4869
4870
4871
4872
4873
4874
4875
4876
4877
4878
4879
4880
4881
4882
4883
4884
4885
4886
4887
4888
4889
4890
4891
4892
4893
4894
4895
4896
4897
4898
4899
4900
4901
4902
4903
4904
4905
4906
4907
4908
4909
4910
4911
4912
4913
4914
4915
4916
4917
4918
4919
4920
4921
4922
4923
4924
4925
4926
4927
4928
4929
4930
4931
4932
4933
4934
4935
4936
4937
4938
4939
4940
4941
4942
4943
4944
4945
4946
4947
4948
4949
4950
4951
4952
4953
4954
4955
4956
4957
4958
4959
4960
4961
4962
4963
4964
4965
4966
4967
4968
4969
4970
4971
4972
4973
4974
4975
4976
4977
4978
4979
4980
4981
4982
4983
4984
4985
4986
4987
4988
4989
4990
4991
4992
4993
4994
4995
4996
4997
4998
4999
5000
5001
5002
5003
5004
5005
5006
5007
5008
5009
5010
5011
5012
5013
5014
5015
5016
5017
5018
5019
5020
5021
5022
5023
5024
5025
5026
5027
5028
5029
5030
5031
5032
5033
5034
5035
5036
5037
5038
5039
5040
5041
5042
5043
5044
5045
5046
5047
5048
5049
5050
5051
5052
5053
5054
5055
5056
5057
5058
5059
5060
5061
5062
5063
5064
5065
5066
5067
5068
5069
5070
5071
5072
5073
5074
5075
5076
5077
5078
5079
5080
5081
5082
5083
5084
5085
5086
5087
5088
5089
5090
5091
5092
5093
5094
5095
5096
5097
5098
5099
5100
5101
5102
5103
5104
5105
5106
5107
5108
5109
5110
5111
5112
5113
5114
5115
5116
5117
5118
5119
5120
5121
5122
5123
5124
5125
5126
5127
5128
5129
5130
5131
5132
5133
5134
5135
5136
5137
5138
5139
5140
5141
5142
5143
5144
5145
5146
5147
5148
5149
5150
5151
5152
5153
5154
5155
5156
5157
5158
5159
5160
5161
5162
5163
5164
5165
5166
5167
5168
5169
5170
5171
5172
5173
5174
5175
5176
5177
5178
5179
5180
5181
5182
5183
5184
5185
5186
5187
5188
5189
5190
5191
5192
5193
5194
5195
5196
5197
5198
5199
5200
5201
5202
5203
5204
5205
5206
5207
5208
5209
5210
5211
5212
5213
5214
5215
5216
5217
5218
5219
5220
5221
5222
5223
5224
5225
5226
5227
5228
5229
5230
5231
5232
5233
5234
5235
5236
5237
5238
5239
5240
5241
5242
5243
5244
5245
5246
5247
5248
5249
5250
5251
5252
5253
5254
5255
5256
5257
5258
5259
5260
5261
5262
5263
5264
5265
5266
5267
5268
5269
5270
5271
5272
5273
5274
5275
5276
5277
5278
5279
5280
5281
5282
5283
5284
5285
5286
5287
5288
5289
5290
5291
5292
5293
5294
5295
5296
5297
5298
5299
5300
5301
5302
5303
5304
5305
5306
5307
5308
5309
5310
5311
5312
5313
5314
5315
5316
5317
5318
5319
5320
5321
5322
5323
5324
5325
5326
5327
5328
5329
5330
5331
5332
5333
5334
5335
5336
5337
5338
5339
5340
5341
5342
5343
5344
5345
5346
5347
5348
5349
5350
5351
5352
5353
5354
5355
5356
5357
5358
5359
5360
5361
5362
5363
5364
5365
5366
5367
5368
5369
5370
5371
5372
5373
5374
5375
5376
5377
5378
5379
5380
5381
5382
5383
5384
5385
5386
5387
5388
5389
5390
5391
5392
5393
5394
5395
5396
5397
5398
5399
5400
5401
5402
5403
5404
5405
5406
5407
5408
5409
5410
5411
5412
5413
5414
5415
5416
5417
5418
5419
5420
5421
5422
5423
5424
5425
5426
5427
5428
5429
5430
5431
5432
5433
5434
5435
5436
5437
5438
5439
5440
5441
5442
5443
5444
5445
5446
5447
5448
5449
5450
5451
5452
5453
5454
5455
5456
5457
5458
5459
5460
5461
5462
5463
5464
5465
5466
5467
5468
5469
5470
5471
5472
5473
5474
5475
5476
5477
5478
5479
5480
5481
5482
5483
5484
5485
5486
5487
5488
5489
5490
5491
5492
5493
5494
5495
5496
5497
5498
5499
5500
5501
5502
5503
5504
5505
5506
5507
5508
5509
5510
5511
5512
5513
5514
5515
5516
5517
5518
5519
5520
5521
5522
5523
5524
5525
5526
5527
5528
5529
5530
5531
5532
5533
5534
5535
5536
5537
5538
5539
5540
5541
5542
5543
5544
5545
5546
5547
5548
5549
5550
5551
5552
5553
5554
5555
5556
5557
5558
5559
5560
5561
5562
5563
5564
5565
5566
5567
5568
5569
5570
5571
5572
5573
5574
5575
5576
5577
5578
5579
5580
5581
5582
5583
5584
5585
5586
5587
5588
5589
5590
5591
5592
5593
5594
5595
5596
5597
5598
5599
5600
5601
5602
5603
5604
5605
5606
5607
5608
5609
5610
5611
5612
5613
5614
5615
5616
5617
5618
5619
5620
5621
5622
5623
5624
5625
5626
5627
5628
5629
5630
5631
5632
5633
5634
5635
5636
5637
5638
5639
5640
5641
5642
5643
5644
5645
5646
5647
5648
5649
5650
5651
5652
5653
5654
5655
5656
5657
5658
5659
5660
5661
5662
5663
5664
5665
5666
5667
5668
5669
5670
5671
5672
5673
5674
5675
5676
5677
5678
5679
5680
5681
5682
5683
5684
5685
5686
5687
5688
5689
5690
5691
5692
5693
5694
5695
5696
5697
5698
5699
5700
5701
5702
5703
5704
5705
5706
5707
5708
5709
5710
5711
5712
5713
5714
5715
5716
5717
5718
5719
5720
5721
5722
5723
5724
5725
5726
5727
5728
5729
5730
5731
5732
5733
5734
5735
5736
5737
5738
5739
5740
5741
5742
5743
5744
5745
5746
5747
5748
5749
5750
5751
5752
5753
5754
5755
5756
5757
5758
5759
5760
5761
5762
5763
5764
5765
5766
5767
5768
5769
5770
5771
5772
5773
5774
5775
5776
5777
5778
5779
5780
5781
5782
5783
5784
5785
5786
5787
5788
5789
5790
5791
5792
5793
5794
5795
5796
5797
5798
5799
5800
5801
5802
5803
5804
5805
5806
5807
5808
5809
5810
5811
5812
5813
5814
5815
5816
5817
5818
5819
5820
5821
5822
5823
5824
5825
5826
5827
5828
5829
5830
5831
5832
5833
5834
5835
5836
5837
5838
5839
5840
5841
5842
5843
5844
5845
5846
5847
5848
5849
5850
5851
5852
5853
5854
5855
5856
5857
5858
5859
5860
5861
5862
5863
5864
5865
5866
5867
5868
5869
5870
5871
5872
5873
5874
5875
5876
5877
5878
5879
5880
5881
5882
5883
5884
5885
5886
5887
5888
5889
5890
5891
5892
5893
5894
5895
5896
5897
5898
5899
5900
5901
5902
5903
5904
5905
5906
5907
5908
5909
5910
5911
5912
5913
5914
5915
5916
5917
5918
5919
5920
5921
5922
5923
5924
5925
5926
5927
5928
5929
5930
5931
5932
5933
5934
5935
5936
5937
5938
5939
5940
5941
5942
5943
5944
5945
5946
5947
5948
5949
5950
5951
5952
5953
5954
// SPDX-License-Identifier: GPL-2.0-only
/*
 *  linux/fs/namespace.c
 *
 * (C) Copyright Al Viro 2000, 2001
 *
 * Based on code from fs/super.c, copyright Linus Torvalds and others.
 * Heavily rewritten.
 */

#include <linux/syscalls.h>
#include <linux/export.h>
#include <linux/capability.h>
#include <linux/mnt_namespace.h>
#include <linux/user_namespace.h>
#include <linux/namei.h>
#include <linux/security.h>
#include <linux/cred.h>
#include <linux/idr.h>
#include <linux/init.h>                /* init_rootfs */
#include <linux/fs_struct.h>        /* get_fs_root et.al. */
#include <linux/fsnotify.h>        /* fsnotify_vfsmount_delete */
#include <linux/file.h>
#include <linux/uaccess.h>
#include <linux/proc_ns.h>
#include <linux/magic.h>
#include <linux/memblock.h>
#include <linux/proc_fs.h>
#include <linux/task_work.h>
#include <linux/sched/task.h>
#include <uapi/linux/mount.h>
#include <linux/fs_context.h>
#include <linux/shmem_fs.h>
#include <linux/mnt_idmapping.h>
#include <linux/nospec.h>

#include "pnode.h"
#include "internal.h"

/* Maximum number of mounts in a mount namespace */
static unsigned int sysctl_mount_max __read_mostly = 100000;

static unsigned int m_hash_mask __ro_after_init;
static unsigned int m_hash_shift __ro_after_init;
static unsigned int mp_hash_mask __ro_after_init;
static unsigned int mp_hash_shift __ro_after_init;

static __initdata unsigned long mhash_entries;
static int __init set_mhash_entries(char *str)
{
        if (!str)
                return 0;
        mhash_entries = simple_strtoul(str, &str, 0);
        return 1;
}
__setup("mhash_entries=", set_mhash_entries);

static __initdata unsigned long mphash_entries;
static int __init set_mphash_entries(char *str)
{
        if (!str)
                return 0;
        mphash_entries = simple_strtoul(str, &str, 0);
        return 1;
}
__setup("mphash_entries=", set_mphash_entries);

static u64 event;
static DEFINE_IDA(mnt_id_ida);
static DEFINE_IDA(mnt_group_ida);

/* Don't allow confusion with old 32bit mount ID */
#define MNT_UNIQUE_ID_OFFSET (1ULL << 31)
static atomic64_t mnt_id_ctr = ATOMIC64_INIT(MNT_UNIQUE_ID_OFFSET);

static struct hlist_head *mount_hashtable __ro_after_init;
static struct hlist_head *mountpoint_hashtable __ro_after_init;
static struct kmem_cache *mnt_cache __ro_after_init;
static DECLARE_RWSEM(namespace_sem);
static HLIST_HEAD(unmounted);        /* protected by namespace_sem */
static LIST_HEAD(ex_mountpoints); /* protected by namespace_sem */
static DEFINE_RWLOCK(mnt_ns_tree_lock);
static struct rb_root mnt_ns_tree = RB_ROOT; /* protected by mnt_ns_tree_lock */

struct mount_kattr {
        unsigned int attr_set;
        unsigned int attr_clr;
        unsigned int propagation;
        unsigned int lookup_flags;
        bool recurse;
        struct user_namespace *mnt_userns;
        struct mnt_idmap *mnt_idmap;
};

/* /sys/fs */
struct kobject *fs_kobj __ro_after_init;
EXPORT_SYMBOL_GPL(fs_kobj);

/*
 * vfsmount lock may be taken for read to prevent changes to the
 * vfsmount hash, ie. during mountpoint lookups or walking back
 * up the tree.
 *
 * It should be taken for write in all cases where the vfsmount
 * tree or hash is modified or when a vfsmount structure is modified.
 */
__cacheline_aligned_in_smp DEFINE_SEQLOCK(mount_lock);

static int mnt_ns_cmp(u64 seq, const struct mnt_namespace *ns)
{
        u64 seq_b = ns->seq;

        if (seq < seq_b)
                return -1;
        if (seq > seq_b)
                return 1;
        return 0;
}

static inline struct mnt_namespace *node_to_mnt_ns(const struct rb_node *node)
{
        if (!node)
                return NULL;
        return rb_entry(node, struct mnt_namespace, mnt_ns_tree_node);
}

static bool mnt_ns_less(struct rb_node *a, const struct rb_node *b)
{
        struct mnt_namespace *ns_a = node_to_mnt_ns(a);
        struct mnt_namespace *ns_b = node_to_mnt_ns(b);
        u64 seq_a = ns_a->seq;

        return mnt_ns_cmp(seq_a, ns_b) < 0;
}

static void mnt_ns_tree_add(struct mnt_namespace *ns)
{
        guard(write_lock)(&mnt_ns_tree_lock);
        rb_add(&ns->mnt_ns_tree_node, &mnt_ns_tree, mnt_ns_less);
}

static void mnt_ns_release(struct mnt_namespace *ns)
{
        lockdep_assert_not_held(&mnt_ns_tree_lock);

        /* keep alive for {list,stat}mount() */
        if (refcount_dec_and_test(&ns->passive)) {
                put_user_ns(ns->user_ns);
                kfree(ns);
        }
}
DEFINE_FREE(mnt_ns_release, struct mnt_namespace *, if (_T) mnt_ns_release(_T))

static void mnt_ns_tree_remove(struct mnt_namespace *ns)
{
        /* remove from global mount namespace list */
        if (!is_anon_ns(ns)) {
                guard(write_lock)(&mnt_ns_tree_lock);
                rb_erase(&ns->mnt_ns_tree_node, &mnt_ns_tree);
        }

        mnt_ns_release(ns);
}

/*
 * Returns the mount namespace which either has the specified id, or has the
 * next smallest id afer the specified one.
 */
static struct mnt_namespace *mnt_ns_find_id_at(u64 mnt_ns_id)
{
        struct rb_node *node = mnt_ns_tree.rb_node;
        struct mnt_namespace *ret = NULL;

        lockdep_assert_held(&mnt_ns_tree_lock);

        while (node) {
                struct mnt_namespace *n = node_to_mnt_ns(node);

                if (mnt_ns_id <= n->seq) {
                        ret = node_to_mnt_ns(node);
                        if (mnt_ns_id == n->seq)
                                break;
                        node = node->rb_left;
                } else {
                        node = node->rb_right;
                }
        }
        return ret;
}

/*
 * Lookup a mount namespace by id and take a passive reference count. Taking a
 * passive reference means the mount namespace can be emptied if e.g., the last
 * task holding an active reference exits. To access the mounts of the
 * namespace the @namespace_sem must first be acquired. If the namespace has
 * already shut down before acquiring @namespace_sem, {list,stat}mount() will
 * see that the mount rbtree of the namespace is empty.
 */
static struct mnt_namespace *lookup_mnt_ns(u64 mnt_ns_id)
{
       struct mnt_namespace *ns;

       guard(read_lock)(&mnt_ns_tree_lock);
       ns = mnt_ns_find_id_at(mnt_ns_id);
       if (!ns || ns->seq != mnt_ns_id)
               return NULL;

       refcount_inc(&ns->passive);
       return ns;
}

static inline void lock_mount_hash(void)
{
        write_seqlock(&mount_lock);
}

static inline void unlock_mount_hash(void)
{
        write_sequnlock(&mount_lock);
}

static inline struct hlist_head *m_hash(struct vfsmount *mnt, struct dentry *dentry)
{
        unsigned long tmp = ((unsigned long)mnt / L1_CACHE_BYTES);
        tmp += ((unsigned long)dentry / L1_CACHE_BYTES);
        tmp = tmp + (tmp >> m_hash_shift);
        return &mount_hashtable[tmp & m_hash_mask];
}

static inline struct hlist_head *mp_hash(struct dentry *dentry)
{
        unsigned long tmp = ((unsigned long)dentry / L1_CACHE_BYTES);
        tmp = tmp + (tmp >> mp_hash_shift);
        return &mountpoint_hashtable[tmp & mp_hash_mask];
}

static int mnt_alloc_id(struct mount *mnt)
{
        int res = ida_alloc(&mnt_id_ida, GFP_KERNEL);

        if (res < 0)
                return res;
        mnt->mnt_id = res;
        mnt->mnt_id_unique = atomic64_inc_return(&mnt_id_ctr);
        return 0;
}

static void mnt_free_id(struct mount *mnt)
{
        ida_free(&mnt_id_ida, mnt->mnt_id);
}

/*
 * Allocate a new peer group ID
 */
static int mnt_alloc_group_id(struct mount *mnt)
{
        int res = ida_alloc_min(&mnt_group_ida, 1, GFP_KERNEL);

        if (res < 0)
                return res;
        mnt->mnt_group_id = res;
        return 0;
}

/*
 * Release a peer group ID
 */
void mnt_release_group_id(struct mount *mnt)
{
        ida_free(&mnt_group_ida, mnt->mnt_group_id);
        mnt->mnt_group_id = 0;
}

/*
 * vfsmount lock must be held for read
 */
static inline void mnt_add_count(struct mount *mnt, int n)
{
#ifdef CONFIG_SMP
        this_cpu_add(mnt->mnt_pcp->mnt_count, n);
#else
        preempt_disable();
        mnt->mnt_count += n;
        preempt_enable();
#endif
}

/*
 * vfsmount lock must be held for write
 */
int mnt_get_count(struct mount *mnt)
{
#ifdef CONFIG_SMP
        int count = 0;
        int cpu;

        for_each_possible_cpu(cpu) {
                count += per_cpu_ptr(mnt->mnt_pcp, cpu)->mnt_count;
        }

        return count;
#else
        return mnt->mnt_count;
#endif
}

static struct mount *alloc_vfsmnt(const char *name)
{
        struct mount *mnt = kmem_cache_zalloc(mnt_cache, GFP_KERNEL);
        if (mnt) {
                int err;

                err = mnt_alloc_id(mnt);
                if (err)
                        goto out_free_cache;

                if (name) {
                        mnt->mnt_devname = kstrdup_const(name,
                                                         GFP_KERNEL_ACCOUNT);
                        if (!mnt->mnt_devname)
                                goto out_free_id;
                }

#ifdef CONFIG_SMP
                mnt->mnt_pcp = alloc_percpu(struct mnt_pcp);
                if (!mnt->mnt_pcp)
                        goto out_free_devname;

                this_cpu_add(mnt->mnt_pcp->mnt_count, 1);
#else
                mnt->mnt_count = 1;
                mnt->mnt_writers = 0;
#endif

                INIT_HLIST_NODE(&mnt->mnt_hash);
                INIT_LIST_HEAD(&mnt->mnt_child);
                INIT_LIST_HEAD(&mnt->mnt_mounts);
                INIT_LIST_HEAD(&mnt->mnt_list);
                INIT_LIST_HEAD(&mnt->mnt_expire);
                INIT_LIST_HEAD(&mnt->mnt_share);
                INIT_LIST_HEAD(&mnt->mnt_slave_list);
                INIT_LIST_HEAD(&mnt->mnt_slave);
                INIT_HLIST_NODE(&mnt->mnt_mp_list);
                INIT_LIST_HEAD(&mnt->mnt_umounting);
                INIT_HLIST_HEAD(&mnt->mnt_stuck_children);
                mnt->mnt.mnt_idmap = &nop_mnt_idmap;
        }
        return mnt;

#ifdef CONFIG_SMP
out_free_devname:
        kfree_const(mnt->mnt_devname);
#endif
out_free_id:
        mnt_free_id(mnt);
out_free_cache:
        kmem_cache_free(mnt_cache, mnt);
        return NULL;
}

/*
 * Most r/o checks on a fs are for operations that take
 * discrete amounts of time, like a write() or unlink().
 * We must keep track of when those operations start
 * (for permission checks) and when they end, so that
 * we can determine when writes are able to occur to
 * a filesystem.
 */
/*
 * __mnt_is_readonly: check whether a mount is read-only
 * @mnt: the mount to check for its write status
 *
 * This shouldn't be used directly ouside of the VFS.
 * It does not guarantee that the filesystem will stay
 * r/w, just that it is right *now*.  This can not and
 * should not be used in place of IS_RDONLY(inode).
 * mnt_want/drop_write() will _keep_ the filesystem
 * r/w.
 */
bool __mnt_is_readonly(struct vfsmount *mnt)
{
        return (mnt->mnt_flags & MNT_READONLY) || sb_rdonly(mnt->mnt_sb);
}
EXPORT_SYMBOL_GPL(__mnt_is_readonly);

static inline void mnt_inc_writers(struct mount *mnt)
{
#ifdef CONFIG_SMP
        this_cpu_inc(mnt->mnt_pcp->mnt_writers);
#else
        mnt->mnt_writers++;
#endif
}

static inline void mnt_dec_writers(struct mount *mnt)
{
#ifdef CONFIG_SMP
        this_cpu_dec(mnt->mnt_pcp->mnt_writers);
#else
        mnt->mnt_writers--;
#endif
}

static unsigned int mnt_get_writers(struct mount *mnt)
{
#ifdef CONFIG_SMP
        unsigned int count = 0;
        int cpu;

        for_each_possible_cpu(cpu) {
                count += per_cpu_ptr(mnt->mnt_pcp, cpu)->mnt_writers;
        }

        return count;
#else
        return mnt->mnt_writers;
#endif
}

static int mnt_is_readonly(struct vfsmount *mnt)
{
        if (READ_ONCE(mnt->mnt_sb->s_readonly_remount))
                return 1;
        /*
         * The barrier pairs with the barrier in sb_start_ro_state_change()
         * making sure if we don't see s_readonly_remount set yet, we also will
         * not see any superblock / mount flag changes done by remount.
         * It also pairs with the barrier in sb_end_ro_state_change()
         * assuring that if we see s_readonly_remount already cleared, we will
         * see the values of superblock / mount flags updated by remount.
         */
        smp_rmb();
        return __mnt_is_readonly(mnt);
}

/*
 * Most r/o & frozen checks on a fs are for operations that take discrete
 * amounts of time, like a write() or unlink().  We must keep track of when
 * those operations start (for permission checks) and when they end, so that we
 * can determine when writes are able to occur to a filesystem.
 */
/**
 * mnt_get_write_access - get write access to a mount without freeze protection
 * @m: the mount on which to take a write
 *
 * This tells the low-level filesystem that a write is about to be performed to
 * it, and makes sure that writes are allowed (mnt it read-write) before
 * returning success. This operation does not protect against filesystem being
 * frozen. When the write operation is finished, mnt_put_write_access() must be
 * called. This is effectively a refcount.
 */
int mnt_get_write_access(struct vfsmount *m)
{
        struct mount *mnt = real_mount(m);
        int ret = 0;

        preempt_disable();
        mnt_inc_writers(mnt);
        /*
         * The store to mnt_inc_writers must be visible before we pass
         * MNT_WRITE_HOLD loop below, so that the slowpath can see our
         * incremented count after it has set MNT_WRITE_HOLD.
         */
        smp_mb();
        might_lock(&mount_lock.lock);
        while (READ_ONCE(mnt->mnt.mnt_flags) & MNT_WRITE_HOLD) {
                if (!IS_ENABLED(CONFIG_PREEMPT_RT)) {
                        cpu_relax();
                } else {
                        /*
                         * This prevents priority inversion, if the task
                         * setting MNT_WRITE_HOLD got preempted on a remote
                         * CPU, and it prevents life lock if the task setting
                         * MNT_WRITE_HOLD has a lower priority and is bound to
                         * the same CPU as the task that is spinning here.
                         */
                        preempt_enable();
                        lock_mount_hash();
                        unlock_mount_hash();
                        preempt_disable();
                }
        }
        /*
         * The barrier pairs with the barrier sb_start_ro_state_change() making
         * sure that if we see MNT_WRITE_HOLD cleared, we will also see
         * s_readonly_remount set (or even SB_RDONLY / MNT_READONLY flags) in
         * mnt_is_readonly() and bail in case we are racing with remount
         * read-only.
         */
        smp_rmb();
        if (mnt_is_readonly(m)) {
                mnt_dec_writers(mnt);
                ret = -EROFS;
        }
        preempt_enable();

        return ret;
}
EXPORT_SYMBOL_GPL(mnt_get_write_access);

/**
 * mnt_want_write - get write access to a mount
 * @m: the mount on which to take a write
 *
 * This tells the low-level filesystem that a write is about to be performed to
 * it, and makes sure that writes are allowed (mount is read-write, filesystem
 * is not frozen) before returning success.  When the write operation is
 * finished, mnt_drop_write() must be called.  This is effectively a refcount.
 */
int mnt_want_write(struct vfsmount *m)
{
        int ret;

        sb_start_write(m->mnt_sb);
        ret = mnt_get_write_access(m);
        if (ret)
                sb_end_write(m->mnt_sb);
        return ret;
}
EXPORT_SYMBOL_GPL(mnt_want_write);

/**
 * mnt_get_write_access_file - get write access to a file's mount
 * @file: the file who's mount on which to take a write
 *
 * This is like mnt_get_write_access, but if @file is already open for write it
 * skips incrementing mnt_writers (since the open file already has a reference)
 * and instead only does the check for emergency r/o remounts.  This must be
 * paired with mnt_put_write_access_file.
 */
int mnt_get_write_access_file(struct file *file)
{
        if (file->f_mode & FMODE_WRITER) {
                /*
                 * Superblock may have become readonly while there are still
                 * writable fd's, e.g. due to a fs error with errors=remount-ro
                 */
                if (__mnt_is_readonly(file->f_path.mnt))
                        return -EROFS;
                return 0;
        }
        return mnt_get_write_access(file->f_path.mnt);
}

/**
 * mnt_want_write_file - get write access to a file's mount
 * @file: the file who's mount on which to take a write
 *
 * This is like mnt_want_write, but if the file is already open for writing it
 * skips incrementing mnt_writers (since the open file already has a reference)
 * and instead only does the freeze protection and the check for emergency r/o
 * remounts.  This must be paired with mnt_drop_write_file.
 */
int mnt_want_write_file(struct file *file)
{
        int ret;

        sb_start_write(file_inode(file)->i_sb);
        ret = mnt_get_write_access_file(file);
        if (ret)
                sb_end_write(file_inode(file)->i_sb);
        return ret;
}
EXPORT_SYMBOL_GPL(mnt_want_write_file);

/**
 * mnt_put_write_access - give up write access to a mount
 * @mnt: the mount on which to give up write access
 *
 * Tells the low-level filesystem that we are done
 * performing writes to it.  Must be matched with
 * mnt_get_write_access() call above.
 */
void mnt_put_write_access(struct vfsmount *mnt)
{
        preempt_disable();
        mnt_dec_writers(real_mount(mnt));
        preempt_enable();
}
EXPORT_SYMBOL_GPL(mnt_put_write_access);

/**
 * mnt_drop_write - give up write access to a mount
 * @mnt: the mount on which to give up write access
 *
 * Tells the low-level filesystem that we are done performing writes to it and
 * also allows filesystem to be frozen again.  Must be matched with
 * mnt_want_write() call above.
 */
void mnt_drop_write(struct vfsmount *mnt)
{
        mnt_put_write_access(mnt);
        sb_end_write(mnt->mnt_sb);
}
EXPORT_SYMBOL_GPL(mnt_drop_write);

void mnt_put_write_access_file(struct file *file)
{
        if (!(file->f_mode & FMODE_WRITER))
                mnt_put_write_access(file->f_path.mnt);
}

void mnt_drop_write_file(struct file *file)
{
        mnt_put_write_access_file(file);
        sb_end_write(file_inode(file)->i_sb);
}
EXPORT_SYMBOL(mnt_drop_write_file);

/**
 * mnt_hold_writers - prevent write access to the given mount
 * @mnt: mnt to prevent write access to
 *
 * Prevents write access to @mnt if there are no active writers for @mnt.
 * This function needs to be called and return successfully before changing
 * properties of @mnt that need to remain stable for callers with write access
 * to @mnt.
 *
 * After this functions has been called successfully callers must pair it with
 * a call to mnt_unhold_writers() in order to stop preventing write access to
 * @mnt.
 *
 * Context: This function expects lock_mount_hash() to be held serializing
 *          setting MNT_WRITE_HOLD.
 * Return: On success 0 is returned.
 *           On error, -EBUSY is returned.
 */
static inline int mnt_hold_writers(struct mount *mnt)
{
        mnt->mnt.mnt_flags |= MNT_WRITE_HOLD;
        /*
         * After storing MNT_WRITE_HOLD, we'll read the counters. This store
         * should be visible before we do.
         */
        smp_mb();

        /*
         * With writers on hold, if this value is zero, then there are
         * definitely no active writers (although held writers may subsequently
         * increment the count, they'll have to wait, and decrement it after
         * seeing MNT_READONLY).
         *
         * It is OK to have counter incremented on one CPU and decremented on
         * another: the sum will add up correctly. The danger would be when we
         * sum up each counter, if we read a counter before it is incremented,
         * but then read another CPU's count which it has been subsequently
         * decremented from -- we would see more decrements than we should.
         * MNT_WRITE_HOLD protects against this scenario, because
         * mnt_want_write first increments count, then smp_mb, then spins on
         * MNT_WRITE_HOLD, so it can't be decremented by another CPU while
         * we're counting up here.
         */
        if (mnt_get_writers(mnt) > 0)
                return -EBUSY;

        return 0;
}

/**
 * mnt_unhold_writers - stop preventing write access to the given mount
 * @mnt: mnt to stop preventing write access to
 *
 * Stop preventing write access to @mnt allowing callers to gain write access
 * to @mnt again.
 *
 * This function can only be called after a successful call to
 * mnt_hold_writers().
 *
 * Context: This function expects lock_mount_hash() to be held.
 */
static inline void mnt_unhold_writers(struct mount *mnt)
{
        /*
         * MNT_READONLY must become visible before ~MNT_WRITE_HOLD, so writers
         * that become unheld will see MNT_READONLY.
         */
        smp_wmb();
        mnt->mnt.mnt_flags &= ~MNT_WRITE_HOLD;
}

static int mnt_make_readonly(struct mount *mnt)
{
        int ret;

        ret = mnt_hold_writers(mnt);
        if (!ret)
                mnt->mnt.mnt_flags |= MNT_READONLY;
        mnt_unhold_writers(mnt);
        return ret;
}

int sb_prepare_remount_readonly(struct super_block *sb)
{
        struct mount *mnt;
        int err = 0;

        /* Racy optimization.  Recheck the counter under MNT_WRITE_HOLD */
        if (atomic_long_read(&sb->s_remove_count))
                return -EBUSY;

        lock_mount_hash();
        list_for_each_entry(mnt, &sb->s_mounts, mnt_instance) {
                if (!(mnt->mnt.mnt_flags & MNT_READONLY)) {
                        err = mnt_hold_writers(mnt);
                        if (err)
                                break;
                }
        }
        if (!err && atomic_long_read(&sb->s_remove_count))
                err = -EBUSY;

        if (!err)
                sb_start_ro_state_change(sb);
        list_for_each_entry(mnt, &sb->s_mounts, mnt_instance) {
                if (mnt->mnt.mnt_flags & MNT_WRITE_HOLD)
                        mnt->mnt.mnt_flags &= ~MNT_WRITE_HOLD;
        }
        unlock_mount_hash();

        return err;
}

static void free_vfsmnt(struct mount *mnt)
{
        mnt_idmap_put(mnt_idmap(&mnt->mnt));
        kfree_const(mnt->mnt_devname);
#ifdef CONFIG_SMP
        free_percpu(mnt->mnt_pcp);
#endif
        kmem_cache_free(mnt_cache, mnt);
}

static void delayed_free_vfsmnt(struct rcu_head *head)
{
        free_vfsmnt(container_of(head, struct mount, mnt_rcu));
}

/* call under rcu_read_lock */
int __legitimize_mnt(struct vfsmount *bastard, unsigned seq)
{
        struct mount *mnt;
        if (read_seqretry(&mount_lock, seq))
                return 1;
        if (bastard == NULL)
                return 0;
        mnt = real_mount(bastard);
        mnt_add_count(mnt, 1);
        smp_mb();                        // see mntput_no_expire()
        if (likely(!read_seqretry(&mount_lock, seq)))
                return 0;
        if (bastard->mnt_flags & MNT_SYNC_UMOUNT) {
                mnt_add_count(mnt, -1);
                return 1;
        }
        lock_mount_hash();
        if (unlikely(bastard->mnt_flags & MNT_DOOMED)) {
                mnt_add_count(mnt, -1);
                unlock_mount_hash();
                return 1;
        }
        unlock_mount_hash();
        /* caller will mntput() */
        return -1;
}

/* call under rcu_read_lock */
static bool legitimize_mnt(struct vfsmount *bastard, unsigned seq)
{
        int res = __legitimize_mnt(bastard, seq);
        if (likely(!res))
                return true;
        if (unlikely(res < 0)) {
                rcu_read_unlock();
                mntput(bastard);
                rcu_read_lock();
        }
        return false;
}

/**
 * __lookup_mnt - find first child mount
 * @mnt:        parent mount
 * @dentry:        mountpoint
 *
 * If @mnt has a child mount @c mounted @dentry find and return it.
 *
 * Note that the child mount @c need not be unique. There are cases
 * where shadow mounts are created. For example, during mount
 * propagation when a source mount @mnt whose root got overmounted by a
 * mount @o after path lookup but before @namespace_sem could be
 * acquired gets copied and propagated. So @mnt gets copied including
 * @o. When @mnt is propagated to a destination mount @d that already
 * has another mount @n mounted at the same mountpoint then the source
 * mount @mnt will be tucked beneath @n, i.e., @n will be mounted on
 * @mnt and @mnt mounted on @d. Now both @n and @o are mounted at @mnt
 * on @dentry.
 *
 * Return: The first child of @mnt mounted @dentry or NULL.
 */
struct mount *__lookup_mnt(struct vfsmount *mnt, struct dentry *dentry)
{
        struct hlist_head *head = m_hash(mnt, dentry);
        struct mount *p;

        hlist_for_each_entry_rcu(p, head, mnt_hash)
                if (&p->mnt_parent->mnt == mnt && p->mnt_mountpoint == dentry)
                        return p;
        return NULL;
}

/*
 * lookup_mnt - Return the first child mount mounted at path
 *
 * "First" means first mounted chronologically.  If you create the
 * following mounts:
 *
 * mount /dev/sda1 /mnt
 * mount /dev/sda2 /mnt
 * mount /dev/sda3 /mnt
 *
 * Then lookup_mnt() on the base /mnt dentry in the root mount will
 * return successively the root dentry and vfsmount of /dev/sda1, then
 * /dev/sda2, then /dev/sda3, then NULL.
 *
 * lookup_mnt takes a reference to the found vfsmount.
 */
struct vfsmount *lookup_mnt(const struct path *path)
{
        struct mount *child_mnt;
        struct vfsmount *m;
        unsigned seq;

        rcu_read_lock();
        do {
                seq = read_seqbegin(&mount_lock);
                child_mnt = __lookup_mnt(path->mnt, path->dentry);
                m = child_mnt ? &child_mnt->mnt : NULL;
        } while (!legitimize_mnt(m, seq));
        rcu_read_unlock();
        return m;
}

/*
 * __is_local_mountpoint - Test to see if dentry is a mountpoint in the
 *                         current mount namespace.
 *
 * The common case is dentries are not mountpoints at all and that
 * test is handled inline.  For the slow case when we are actually
 * dealing with a mountpoint of some kind, walk through all of the
 * mounts in the current mount namespace and test to see if the dentry
 * is a mountpoint.
 *
 * The mount_hashtable is not usable in the context because we
 * need to identify all mounts that may be in the current mount
 * namespace not just a mount that happens to have some specified
 * parent mount.
 */
bool __is_local_mountpoint(struct dentry *dentry)
{
        struct mnt_namespace *ns = current->nsproxy->mnt_ns;
        struct mount *mnt, *n;
        bool is_covered = false;

        down_read(&namespace_sem);
        rbtree_postorder_for_each_entry_safe(mnt, n, &ns->mounts, mnt_node) {
                is_covered = (mnt->mnt_mountpoint == dentry);
                if (is_covered)
                        break;
        }
        up_read(&namespace_sem);

        return is_covered;
}

static struct mountpoint *lookup_mountpoint(struct dentry *dentry)
{
        struct hlist_head *chain = mp_hash(dentry);
        struct mountpoint *mp;

        hlist_for_each_entry(mp, chain, m_hash) {
                if (mp->m_dentry == dentry) {
                        mp->m_count++;
                        return mp;
                }
        }
        return NULL;
}

static struct mountpoint *get_mountpoint(struct dentry *dentry)
{
        struct mountpoint *mp, *new = NULL;
        int ret;

        if (d_mountpoint(dentry)) {
                /* might be worth a WARN_ON() */
                if (d_unlinked(dentry))
                        return ERR_PTR(-ENOENT);
mountpoint:
                read_seqlock_excl(&mount_lock);
                mp = lookup_mountpoint(dentry);
                read_sequnlock_excl(&mount_lock);
                if (mp)
                        goto done;
        }

        if (!new)
                new = kmalloc(sizeof(struct mountpoint), GFP_KERNEL);
        if (!new)
                return ERR_PTR(-ENOMEM);


        /* Exactly one processes may set d_mounted */
        ret = d_set_mounted(dentry);

        /* Someone else set d_mounted? */
        if (ret == -EBUSY)
                goto mountpoint;

        /* The dentry is not available as a mountpoint? */
        mp = ERR_PTR(ret);
        if (ret)
                goto done;

        /* Add the new mountpoint to the hash table */
        read_seqlock_excl(&mount_lock);
        new->m_dentry = dget(dentry);
        new->m_count = 1;
        hlist_add_head(&new->m_hash, mp_hash(dentry));
        INIT_HLIST_HEAD(&new->m_list);
        read_sequnlock_excl(&mount_lock);

        mp = new;
        new = NULL;
done:
        kfree(new);
        return mp;
}

/*
 * vfsmount lock must be held.  Additionally, the caller is responsible
 * for serializing calls for given disposal list.
 */
static void __put_mountpoint(struct mountpoint *mp, struct list_head *list)
{
        if (!--mp->m_count) {
                struct dentry *dentry = mp->m_dentry;
                BUG_ON(!hlist_empty(&mp->m_list));
                spin_lock(&dentry->d_lock);
                dentry->d_flags &= ~DCACHE_MOUNTED;
                spin_unlock(&dentry->d_lock);
                dput_to_list(dentry, list);
                hlist_del(&mp->m_hash);
                kfree(mp);
        }
}

/* called with namespace_lock and vfsmount lock */
static void put_mountpoint(struct mountpoint *mp)
{
        __put_mountpoint(mp, &ex_mountpoints);
}

static inline int check_mnt(struct mount *mnt)
{
        return mnt->mnt_ns == current->nsproxy->mnt_ns;
}

/*
 * vfsmount lock must be held for write
 */
static void touch_mnt_namespace(struct mnt_namespace *ns)
{
        if (ns) {
                ns->event = ++event;
                wake_up_interruptible(&ns->poll);
        }
}

/*
 * vfsmount lock must be held for write
 */
static void __touch_mnt_namespace(struct mnt_namespace *ns)
{
        if (ns && ns->event != event) {
                ns->event = event;
                wake_up_interruptible(&ns->poll);
        }
}

/*
 * vfsmount lock must be held for write
 */
static struct mountpoint *unhash_mnt(struct mount *mnt)
{
        struct mountpoint *mp;
        mnt->mnt_parent = mnt;
        mnt->mnt_mountpoint = mnt->mnt.mnt_root;
        list_del_init(&mnt->mnt_child);
        hlist_del_init_rcu(&mnt->mnt_hash);
        hlist_del_init(&mnt->mnt_mp_list);
        mp = mnt->mnt_mp;
        mnt->mnt_mp = NULL;
        return mp;
}

/*
 * vfsmount lock must be held for write
 */
static void umount_mnt(struct mount *mnt)
{
        put_mountpoint(unhash_mnt(mnt));
}

/*
 * vfsmount lock must be held for write
 */
void mnt_set_mountpoint(struct mount *mnt,
                        struct mountpoint *mp,
                        struct mount *child_mnt)
{
        mp->m_count++;
        mnt_add_count(mnt, 1);        /* essentially, that's mntget */
        child_mnt->mnt_mountpoint = mp->m_dentry;
        child_mnt->mnt_parent = mnt;
        child_mnt->mnt_mp = mp;
        hlist_add_head(&child_mnt->mnt_mp_list, &mp->m_list);
}

/**
 * mnt_set_mountpoint_beneath - mount a mount beneath another one
 *
 * @new_parent: the source mount
 * @top_mnt:    the mount beneath which @new_parent is mounted
 * @new_mp:     the new mountpoint of @top_mnt on @new_parent
 *
 * Remove @top_mnt from its current mountpoint @top_mnt->mnt_mp and
 * parent @top_mnt->mnt_parent and mount it on top of @new_parent at
 * @new_mp. And mount @new_parent on the old parent and old
 * mountpoint of @top_mnt.
 *
 * Context: This function expects namespace_lock() and lock_mount_hash()
 *          to have been acquired in that order.
 */
static void mnt_set_mountpoint_beneath(struct mount *new_parent,
                                       struct mount *top_mnt,
                                       struct mountpoint *new_mp)
{
        struct mount *old_top_parent = top_mnt->mnt_parent;
        struct mountpoint *old_top_mp = top_mnt->mnt_mp;

        mnt_set_mountpoint(old_top_parent, old_top_mp, new_parent);
        mnt_change_mountpoint(new_parent, new_mp, top_mnt);
}


static void __attach_mnt(struct mount *mnt, struct mount *parent)
{
        hlist_add_head_rcu(&mnt->mnt_hash,
                           m_hash(&parent->mnt, mnt->mnt_mountpoint));
        list_add_tail(&mnt->mnt_child, &parent->mnt_mounts);
}

/**
 * attach_mnt - mount a mount, attach to @mount_hashtable and parent's
 *              list of child mounts
 * @parent:  the parent
 * @mnt:     the new mount
 * @mp:      the new mountpoint
 * @beneath: whether to mount @mnt beneath or on top of @parent
 *
 * If @beneath is false, mount @mnt at @mp on @parent. Then attach @mnt
 * to @parent's child mount list and to @mount_hashtable.
 *
 * If @beneath is true, remove @mnt from its current parent and
 * mountpoint and mount it on @mp on @parent, and mount @parent on the
 * old parent and old mountpoint of @mnt. Finally, attach @parent to
 * @mnt_hashtable and @parent->mnt_parent->mnt_mounts.
 *
 * Note, when __attach_mnt() is called @mnt->mnt_parent already points
 * to the correct parent.
 *
 * Context: This function expects namespace_lock() and lock_mount_hash()
 *          to have been acquired in that order.
 */
static void attach_mnt(struct mount *mnt, struct mount *parent,
                       struct mountpoint *mp, bool beneath)
{
        if (beneath)
                mnt_set_mountpoint_beneath(mnt, parent, mp);
        else
                mnt_set_mountpoint(parent, mp, mnt);
        /*
         * Note, @mnt->mnt_parent has to be used. If @mnt was mounted
         * beneath @parent then @mnt will need to be attached to
         * @parent's old parent, not @parent. IOW, @mnt->mnt_parent
         * isn't the same mount as @parent.
         */
        __attach_mnt(mnt, mnt->mnt_parent);
}

void mnt_change_mountpoint(struct mount *parent, struct mountpoint *mp, struct mount *mnt)
{
        struct mountpoint *old_mp = mnt->mnt_mp;
        struct mount *old_parent = mnt->mnt_parent;

        list_del_init(&mnt->mnt_child);
        hlist_del_init(&mnt->mnt_mp_list);
        hlist_del_init_rcu(&mnt->mnt_hash);

        attach_mnt(mnt, parent, mp, false);

        put_mountpoint(old_mp);
        mnt_add_count(old_parent, -1);
}

static inline struct mount *node_to_mount(struct rb_node *node)
{
        return node ? rb_entry(node, struct mount, mnt_node) : NULL;
}

static void mnt_add_to_ns(struct mnt_namespace *ns, struct mount *mnt)
{
        struct rb_node **link = &ns->mounts.rb_node;
        struct rb_node *parent = NULL;

        WARN_ON(mnt->mnt.mnt_flags & MNT_ONRB);
        mnt->mnt_ns = ns;
        while (*link) {
                parent = *link;
                if (mnt->mnt_id_unique < node_to_mount(parent)->mnt_id_unique)
                        link = &parent->rb_left;
                else
                        link = &parent->rb_right;
        }
        rb_link_node(&mnt->mnt_node, parent, link);
        rb_insert_color(&mnt->mnt_node, &ns->mounts);
        mnt->mnt.mnt_flags |= MNT_ONRB;
}

/*
 * vfsmount lock must be held for write
 */
static void commit_tree(struct mount *mnt)
{
        struct mount *parent = mnt->mnt_parent;
        struct mount *m;
        LIST_HEAD(head);
        struct mnt_namespace *n = parent->mnt_ns;

        BUG_ON(parent == mnt);

        list_add_tail(&head, &mnt->mnt_list);
        while (!list_empty(&head)) {
                m = list_first_entry(&head, typeof(*m), mnt_list);
                list_del(&m->mnt_list);

                mnt_add_to_ns(n, m);
        }
        n->nr_mounts += n->pending_mounts;
        n->pending_mounts = 0;

        __attach_mnt(mnt, parent);
        touch_mnt_namespace(n);
}

static struct mount *next_mnt(struct mount *p, struct mount *root)
{
        struct list_head *next = p->mnt_mounts.next;
        if (next == &p->mnt_mounts) {
                while (1) {
                        if (p == root)
                                return NULL;
                        next = p->mnt_child.next;
                        if (next != &p->mnt_parent->mnt_mounts)
                                break;
                        p = p->mnt_parent;
                }
        }
        return list_entry(next, struct mount, mnt_child);
}

static struct mount *skip_mnt_tree(struct mount *p)
{
        struct list_head *prev = p->mnt_mounts.prev;
        while (prev != &p->mnt_mounts) {
                p = list_entry(prev, struct mount, mnt_child);
                prev = p->mnt_mounts.prev;
        }
        return p;
}

/**
 * vfs_create_mount - Create a mount for a configured superblock
 * @fc: The configuration context with the superblock attached
 *
 * Create a mount to an already configured superblock.  If necessary, the
 * caller should invoke vfs_get_tree() before calling this.
 *
 * Note that this does not attach the mount to anything.
 */
struct vfsmount *vfs_create_mount(struct fs_context *fc)
{
        struct mount *mnt;

        if (!fc->root)
                return ERR_PTR(-EINVAL);

        mnt = alloc_vfsmnt(fc->source ?: "none");
        if (!mnt)
                return ERR_PTR(-ENOMEM);

        if (fc->sb_flags & SB_KERNMOUNT)
                mnt->mnt.mnt_flags = MNT_INTERNAL;

        atomic_inc(&fc->root->d_sb->s_active);
        mnt->mnt.mnt_sb                = fc->root->d_sb;
        mnt->mnt.mnt_root        = dget(fc->root);
        mnt->mnt_mountpoint        = mnt->mnt.mnt_root;
        mnt->mnt_parent                = mnt;

        lock_mount_hash();
        list_add_tail(&mnt->mnt_instance, &mnt->mnt.mnt_sb->s_mounts);
        unlock_mount_hash();
        return &mnt->mnt;
}
EXPORT_SYMBOL(vfs_create_mount);

struct vfsmount *fc_mount(struct fs_context *fc)
{
        int err = vfs_get_tree(fc);
        if (!err) {
                up_write(&fc->root->d_sb->s_umount);
                return vfs_create_mount(fc);
        }
        return ERR_PTR(err);
}
EXPORT_SYMBOL(fc_mount);

struct vfsmount *vfs_kern_mount(struct file_system_type *type,
                                int flags, const char *name,
                                void *data)
{
        struct fs_context *fc;
        struct vfsmount *mnt;
        int ret = 0;

        if (!type)
                return ERR_PTR(-EINVAL);

        fc = fs_context_for_mount(type, flags);
        if (IS_ERR(fc))
                return ERR_CAST(fc);

        if (name)
                ret = vfs_parse_fs_string(fc, "source",
                                          name, strlen(name));
        if (!ret)
                ret = parse_monolithic_mount_data(fc, data);
        if (!ret)
                mnt = fc_mount(fc);
        else
                mnt = ERR_PTR(ret);

        put_fs_context(fc);
        return mnt;
}
EXPORT_SYMBOL_GPL(vfs_kern_mount);

struct vfsmount *
vfs_submount(const struct dentry *mountpoint, struct file_system_type *type,
             const char *name, void *data)
{
        /* Until it is worked out how to pass the user namespace
         * through from the parent mount to the submount don't support
         * unprivileged mounts with submounts.
         */
        if (mountpoint->d_sb->s_user_ns != &init_user_ns)
                return ERR_PTR(-EPERM);

        return vfs_kern_mount(type, SB_SUBMOUNT, name, data);
}
EXPORT_SYMBOL_GPL(vfs_submount);

static struct mount *clone_mnt(struct mount *old, struct dentry *root,
                                        int flag)
{
        struct super_block *sb = old->mnt.mnt_sb;
        struct mount *mnt;
        int err;

        mnt = alloc_vfsmnt(old->mnt_devname);
        if (!mnt)
                return ERR_PTR(-ENOMEM);

        if (flag & (CL_SLAVE | CL_PRIVATE | CL_SHARED_TO_SLAVE))
                mnt->mnt_group_id = 0; /* not a peer of original */
        else
                mnt->mnt_group_id = old->mnt_group_id;

        if ((flag & CL_MAKE_SHARED) && !mnt->mnt_group_id) {
                err = mnt_alloc_group_id(mnt);
                if (err)
                        goto out_free;
        }

        mnt->mnt.mnt_flags = old->mnt.mnt_flags;
        mnt->mnt.mnt_flags &= ~(MNT_WRITE_HOLD|MNT_MARKED|MNT_INTERNAL|MNT_ONRB);

        atomic_inc(&sb->s_active);
        mnt->mnt.mnt_idmap = mnt_idmap_get(mnt_idmap(&old->mnt));

        mnt->mnt.mnt_sb = sb;
        mnt->mnt.mnt_root = dget(root);
        mnt->mnt_mountpoint = mnt->mnt.mnt_root;
        mnt->mnt_parent = mnt;
        lock_mount_hash();
        list_add_tail(&mnt->mnt_instance, &sb->s_mounts);
        unlock_mount_hash();

        if ((flag & CL_SLAVE) ||
            ((flag & CL_SHARED_TO_SLAVE) && IS_MNT_SHARED(old))) {
                list_add(&mnt->mnt_slave, &old->mnt_slave_list);
                mnt->mnt_master = old;
                CLEAR_MNT_SHARED(mnt);
        } else if (!(flag & CL_PRIVATE)) {
                if ((flag & CL_MAKE_SHARED) || IS_MNT_SHARED(old))
                        list_add(&mnt->mnt_share, &old->mnt_share);
                if (IS_MNT_SLAVE(old))
                        list_add(&mnt->mnt_slave, &old->mnt_slave);
                mnt->mnt_master = old->mnt_master;
        } else {
                CLEAR_MNT_SHARED(mnt);
        }
        if (flag & CL_MAKE_SHARED)
                set_mnt_shared(mnt);

        /* stick the duplicate mount on the same expiry list
         * as the original if that was on one */
        if (flag & CL_EXPIRE) {
                if (!list_empty(&old->mnt_expire))
                        list_add(&mnt->mnt_expire, &old->mnt_expire);
        }

        return mnt;

 out_free:
        mnt_free_id(mnt);
        free_vfsmnt(mnt);
        return ERR_PTR(err);
}

static void cleanup_mnt(struct mount *mnt)
{
        struct hlist_node *p;
        struct mount *m;
        /*
         * The warning here probably indicates that somebody messed
         * up a mnt_want/drop_write() pair.  If this happens, the
         * filesystem was probably unable to make r/w->r/o transitions.
         * The locking used to deal with mnt_count decrement provides barriers,
         * so mnt_get_writers() below is safe.
         */
        WARN_ON(mnt_get_writers(mnt));
        if (unlikely(mnt->mnt_pins.first))
                mnt_pin_kill(mnt);
        hlist_for_each_entry_safe(m, p, &mnt->mnt_stuck_children, mnt_umount) {
                hlist_del(&m->mnt_umount);
                mntput(&m->mnt);
        }
        fsnotify_vfsmount_delete(&mnt->mnt);
        dput(mnt->mnt.mnt_root);
        deactivate_super(mnt->mnt.mnt_sb);
        mnt_free_id(mnt);
        call_rcu(&mnt->mnt_rcu, delayed_free_vfsmnt);
}

static void __cleanup_mnt(struct rcu_head *head)
{
        cleanup_mnt(container_of(head, struct mount, mnt_rcu));
}

static LLIST_HEAD(delayed_mntput_list);
static void delayed_mntput(struct work_struct *unused)
{
        struct llist_node *node = llist_del_all(&delayed_mntput_list);
        struct mount *m, *t;

        llist_for_each_entry_safe(m, t, node, mnt_llist)
                cleanup_mnt(m);
}
static DECLARE_DELAYED_WORK(delayed_mntput_work, delayed_mntput);

static void mntput_no_expire(struct mount *mnt)
{
        LIST_HEAD(list);
        int count;

        rcu_read_lock();
        if (likely(READ_ONCE(mnt->mnt_ns))) {
                /*
                 * Since we don't do lock_mount_hash() here,
                 * ->mnt_ns can change under us.  However, if it's
                 * non-NULL, then there's a reference that won't
                 * be dropped until after an RCU delay done after
                 * turning ->mnt_ns NULL.  So if we observe it
                 * non-NULL under rcu_read_lock(), the reference
                 * we are dropping is not the final one.
                 */
                mnt_add_count(mnt, -1);
                rcu_read_unlock();
                return;
        }
        lock_mount_hash();
        /*
         * make sure that if __legitimize_mnt() has not seen us grab
         * mount_lock, we'll see their refcount increment here.
         */
        smp_mb();
        mnt_add_count(mnt, -1);
        count = mnt_get_count(mnt);
        if (count != 0) {
                WARN_ON(count < 0);
                rcu_read_unlock();
                unlock_mount_hash();
                return;
        }
        if (unlikely(mnt->mnt.mnt_flags & MNT_DOOMED)) {
                rcu_read_unlock();
                unlock_mount_hash();
                return;
        }
        mnt->mnt.mnt_flags |= MNT_DOOMED;
        rcu_read_unlock();

        list_del(&mnt->mnt_instance);

        if (unlikely(!list_empty(&mnt->mnt_mounts))) {
                struct mount *p, *tmp;
                list_for_each_entry_safe(p, tmp, &mnt->mnt_mounts,  mnt_child) {
                        __put_mountpoint(unhash_mnt(p), &list);
                        hlist_add_head(&p->mnt_umount, &mnt->mnt_stuck_children);
                }
        }
        unlock_mount_hash();
        shrink_dentry_list(&list);

        if (likely(!(mnt->mnt.mnt_flags & MNT_INTERNAL))) {
                struct task_struct *task = current;
                if (likely(!(task->flags & PF_KTHREAD))) {
                        init_task_work(&mnt->mnt_rcu, __cleanup_mnt);
                        if (!task_work_add(task, &mnt->mnt_rcu, TWA_RESUME))
                                return;
                }
                if (llist_add(&mnt->mnt_llist, &delayed_mntput_list))
                        schedule_delayed_work(&delayed_mntput_work, 1);
                return;
        }
        cleanup_mnt(mnt);
}

void mntput(struct vfsmount *mnt)
{
        if (mnt) {
                struct mount *m = real_mount(mnt);
                /* avoid cacheline pingpong */
                if (unlikely(m->mnt_expiry_mark))
                        WRITE_ONCE(m->mnt_expiry_mark, 0);
                mntput_no_expire(m);
        }
}
EXPORT_SYMBOL(mntput);

struct vfsmount *mntget(struct vfsmount *mnt)
{
        if (mnt)
                mnt_add_count(real_mount(mnt), 1);
        return mnt;
}
EXPORT_SYMBOL(mntget);

/*
 * Make a mount point inaccessible to new lookups.
 * Because there may still be current users, the caller MUST WAIT
 * for an RCU grace period before destroying the mount point.
 */
void mnt_make_shortterm(struct vfsmount *mnt)
{
        if (mnt)
                real_mount(mnt)->mnt_ns = NULL;
}

/**
 * path_is_mountpoint() - Check if path is a mount in the current namespace.
 * @path: path to check
 *
 *  d_mountpoint() can only be used reliably to establish if a dentry is
 *  not mounted in any namespace and that common case is handled inline.
 *  d_mountpoint() isn't aware of the possibility there may be multiple
 *  mounts using a given dentry in a different namespace. This function
 *  checks if the passed in path is a mountpoint rather than the dentry
 *  alone.
 */
bool path_is_mountpoint(const struct path *path)
{
        unsigned seq;
        bool res;

        if (!d_mountpoint(path->dentry))
                return false;

        rcu_read_lock();
        do {
                seq = read_seqbegin(&mount_lock);
                res = __path_is_mountpoint(path);
        } while (read_seqretry(&mount_lock, seq));
        rcu_read_unlock();

        return res;
}
EXPORT_SYMBOL(path_is_mountpoint);

struct vfsmount *mnt_clone_internal(const struct path *path)
{
        struct mount *p;
        p = clone_mnt(real_mount(path->mnt), path->dentry, CL_PRIVATE);
        if (IS_ERR(p))
                return ERR_CAST(p);
        p->mnt.mnt_flags |= MNT_INTERNAL;
        return &p->mnt;
}

/*
 * Returns the mount which either has the specified mnt_id, or has the next
 * smallest id afer the specified one.
 */
static struct mount *mnt_find_id_at(struct mnt_namespace *ns, u64 mnt_id)
{
        struct rb_node *node = ns->mounts.rb_node;
        struct mount *ret = NULL;

        while (node) {
                struct mount *m = node_to_mount(node);

                if (mnt_id <= m->mnt_id_unique) {
                        ret = node_to_mount(node);
                        if (mnt_id == m->mnt_id_unique)
                                break;
                        node = node->rb_left;
                } else {
                        node = node->rb_right;
                }
        }
        return ret;
}

/*
 * Returns the mount which either has the specified mnt_id, or has the next
 * greater id before the specified one.
 */
static struct mount *mnt_find_id_at_reverse(struct mnt_namespace *ns, u64 mnt_id)
{
        struct rb_node *node = ns->mounts.rb_node;
        struct mount *ret = NULL;

        while (node) {
                struct mount *m = node_to_mount(node);

                if (mnt_id >= m->mnt_id_unique) {
                        ret = node_to_mount(node);
                        if (mnt_id == m->mnt_id_unique)
                                break;
                        node = node->rb_right;
                } else {
                        node = node->rb_left;
                }
        }
        return ret;
}

#ifdef CONFIG_PROC_FS

/* iterator; we want it to have access to namespace_sem, thus here... */
static void *m_start(struct seq_file *m, loff_t *pos)
{
        struct proc_mounts *p = m->private;

        down_read(&namespace_sem);

        return mnt_find_id_at(p->ns, *pos);
}

static void *m_next(struct seq_file *m, void *v, loff_t *pos)
{
        struct mount *next = NULL, *mnt = v;
        struct rb_node *node = rb_next(&mnt->mnt_node);

        ++*pos;
        if (node) {
                next = node_to_mount(node);
                *pos = next->mnt_id_unique;
        }
        return next;
}

static void m_stop(struct seq_file *m, void *v)
{
        up_read(&namespace_sem);
}

static int m_show(struct seq_file *m, void *v)
{
        struct proc_mounts *p = m->private;
        struct mount *r = v;
        return p->show(m, &r->mnt);
}

const struct seq_operations mounts_op = {
        .start        = m_start,
        .next        = m_next,
        .stop        = m_stop,
        .show        = m_show,
};

#endif  /* CONFIG_PROC_FS */

/**
 * may_umount_tree - check if a mount tree is busy
 * @m: root of mount tree
 *
 * This is called to check if a tree of mounts has any
 * open files, pwds, chroots or sub mounts that are
 * busy.
 */
int may_umount_tree(struct vfsmount *m)
{
        struct mount *mnt = real_mount(m);
        int actual_refs = 0;
        int minimum_refs = 0;
        struct mount *p;
        BUG_ON(!m);

        /* write lock needed for mnt_get_count */
        lock_mount_hash();
        for (p = mnt; p; p = next_mnt(p, mnt)) {
                actual_refs += mnt_get_count(p);
                minimum_refs += 2;
        }
        unlock_mount_hash();

        if (actual_refs > minimum_refs)
                return 0;

        return 1;
}

EXPORT_SYMBOL(may_umount_tree);

/**
 * may_umount - check if a mount point is busy
 * @mnt: root of mount
 *
 * This is called to check if a mount point has any
 * open files, pwds, chroots or sub mounts. If the
 * mount has sub mounts this will return busy
 * regardless of whether the sub mounts are busy.
 *
 * Doesn't take quota and stuff into account. IOW, in some cases it will
 * give false negatives. The main reason why it's here is that we need
 * a non-destructive way to look for easily umountable filesystems.
 */
int may_umount(struct vfsmount *mnt)
{
        int ret = 1;
        down_read(&namespace_sem);
        lock_mount_hash();
        if (propagate_mount_busy(real_mount(mnt), 2))
                ret = 0;
        unlock_mount_hash();
        up_read(&namespace_sem);
        return ret;
}

EXPORT_SYMBOL(may_umount);

static void namespace_unlock(void)
{
        struct hlist_head head;
        struct hlist_node *p;
        struct mount *m;
        LIST_HEAD(list);

        hlist_move_list(&unmounted, &head);
        list_splice_init(&ex_mountpoints, &list);

        up_write(&namespace_sem);

        shrink_dentry_list(&list);

        if (likely(hlist_empty(&head)))
                return;

        synchronize_rcu_expedited();

        hlist_for_each_entry_safe(m, p, &head, mnt_umount) {
                hlist_del(&m->mnt_umount);
                mntput(&m->mnt);
        }
}

static inline void namespace_lock(void)
{
        down_write(&namespace_sem);
}

enum umount_tree_flags {
        UMOUNT_SYNC = 1,
        UMOUNT_PROPAGATE = 2,
        UMOUNT_CONNECTED = 4,
};

static bool disconnect_mount(struct mount *mnt, enum umount_tree_flags how)
{
        /* Leaving mounts connected is only valid for lazy umounts */
        if (how & UMOUNT_SYNC)
                return true;

        /* A mount without a parent has nothing to be connected to */
        if (!mnt_has_parent(mnt))
                return true;

        /* Because the reference counting rules change when mounts are
         * unmounted and connected, umounted mounts may not be
         * connected to mounted mounts.
         */
        if (!(mnt->mnt_parent->mnt.mnt_flags & MNT_UMOUNT))
                return true;

        /* Has it been requested that the mount remain connected? */
        if (how & UMOUNT_CONNECTED)
                return false;

        /* Is the mount locked such that it needs to remain connected? */
        if (IS_MNT_LOCKED(mnt))
                return false;

        /* By default disconnect the mount */
        return true;
}

/*
 * mount_lock must be held
 * namespace_sem must be held for write
 */
static void umount_tree(struct mount *mnt, enum umount_tree_flags how)
{
        LIST_HEAD(tmp_list);
        struct mount *p;

        if (how & UMOUNT_PROPAGATE)
                propagate_mount_unlock(mnt);

        /* Gather the mounts to umount */
        for (p = mnt; p; p = next_mnt(p, mnt)) {
                p->mnt.mnt_flags |= MNT_UMOUNT;
                if (p->mnt.mnt_flags & MNT_ONRB)
                        move_from_ns(p, &tmp_list);
                else
                        list_move(&p->mnt_list, &tmp_list);
        }

        /* Hide the mounts from mnt_mounts */
        list_for_each_entry(p, &tmp_list, mnt_list) {
                list_del_init(&p->mnt_child);
        }

        /* Add propagated mounts to the tmp_list */
        if (how & UMOUNT_PROPAGATE)
                propagate_umount(&tmp_list);

        while (!list_empty(&tmp_list)) {
                struct mnt_namespace *ns;
                bool disconnect;
                p = list_first_entry(&tmp_list, struct mount, mnt_list);
                list_del_init(&p->mnt_expire);
                list_del_init(&p->mnt_list);
                ns = p->mnt_ns;
                if (ns) {
                        ns->nr_mounts--;
                        __touch_mnt_namespace(ns);
                }
                p->mnt_ns = NULL;
                if (how & UMOUNT_SYNC)
                        p->mnt.mnt_flags |= MNT_SYNC_UMOUNT;

                disconnect = disconnect_mount(p, how);
                if (mnt_has_parent(p)) {
                        mnt_add_count(p->mnt_parent, -1);
                        if (!disconnect) {
                                /* Don't forget about p */
                                list_add_tail(&p->mnt_child, &p->mnt_parent->mnt_mounts);
                        } else {
                                umount_mnt(p);
                        }
                }
                change_mnt_propagation(p, MS_PRIVATE);
                if (disconnect)
                        hlist_add_head(&p->mnt_umount, &unmounted);
        }
}

static void shrink_submounts(struct mount *mnt);

static int do_umount_root(struct super_block *sb)
{
        int ret = 0;

        down_write(&sb->s_umount);
        if (!sb_rdonly(sb)) {
                struct fs_context *fc;

                fc = fs_context_for_reconfigure(sb->s_root, SB_RDONLY,
                                                SB_RDONLY);
                if (IS_ERR(fc)) {
                        ret = PTR_ERR(fc);
                } else {
                        ret = parse_monolithic_mount_data(fc, NULL);
                        if (!ret)
                                ret = reconfigure_super(fc);
                        put_fs_context(fc);
                }
        }
        up_write(&sb->s_umount);
        return ret;
}

static int do_umount(struct mount *mnt, int flags)
{
        struct super_block *sb = mnt->mnt.mnt_sb;
        int retval;

        retval = security_sb_umount(&mnt->mnt, flags);
        if (retval)
                return retval;

        /*
         * Allow userspace to request a mountpoint be expired rather than
         * unmounting unconditionally. Unmount only happens if:
         *  (1) the mark is already set (the mark is cleared by mntput())
         *  (2) the usage count == 1 [parent vfsmount] + 1 [sys_umount]
         */
        if (flags & MNT_EXPIRE) {
                if (&mnt->mnt == current->fs->root.mnt ||
                    flags & (MNT_FORCE | MNT_DETACH))
                        return -EINVAL;

                /*
                 * probably don't strictly need the lock here if we examined
                 * all race cases, but it's a slowpath.
                 */
                lock_mount_hash();
                if (mnt_get_count(mnt) != 2) {
                        unlock_mount_hash();
                        return -EBUSY;
                }
                unlock_mount_hash();

                if (!xchg(&mnt->mnt_expiry_mark, 1))
                        return -EAGAIN;
        }

        /*
         * If we may have to abort operations to get out of this
         * mount, and they will themselves hold resources we must
         * allow the fs to do things. In the Unix tradition of
         * 'Gee thats tricky lets do it in userspace' the umount_begin
         * might fail to complete on the first run through as other tasks
         * must return, and the like. Thats for the mount program to worry
         * about for the moment.
         */

        if (flags & MNT_FORCE && sb->s_op->umount_begin) {
                sb->s_op->umount_begin(sb);
        }

        /*
         * No sense to grab the lock for this test, but test itself looks
         * somewhat bogus. Suggestions for better replacement?
         * Ho-hum... In principle, we might treat that as umount + switch
         * to rootfs. GC would eventually take care of the old vfsmount.
         * Actually it makes sense, especially if rootfs would contain a
         * /reboot - static binary that would close all descriptors and
         * call reboot(9). Then init(8) could umount root and exec /reboot.
         */
        if (&mnt->mnt == current->fs->root.mnt && !(flags & MNT_DETACH)) {
                /*
                 * Special case for "unmounting" root ...
                 * we just try to remount it readonly.
                 */
                if (!ns_capable(sb->s_user_ns, CAP_SYS_ADMIN))
                        return -EPERM;
                return do_umount_root(sb);
        }

        namespace_lock();
        lock_mount_hash();

        /* Recheck MNT_LOCKED with the locks held */
        retval = -EINVAL;
        if (mnt->mnt.mnt_flags & MNT_LOCKED)
                goto out;

        event++;
        if (flags & MNT_DETACH) {
                if (mnt->mnt.mnt_flags & MNT_ONRB ||
                    !list_empty(&mnt->mnt_list))
                        umount_tree(mnt, UMOUNT_PROPAGATE);
                retval = 0;
        } else {
                shrink_submounts(mnt);
                retval = -EBUSY;
                if (!propagate_mount_busy(mnt, 2)) {
                        if (mnt->mnt.mnt_flags & MNT_ONRB ||
                            !list_empty(&mnt->mnt_list))
                                umount_tree(mnt, UMOUNT_PROPAGATE|UMOUNT_SYNC);
                        retval = 0;
                }
        }
out:
        unlock_mount_hash();
        namespace_unlock();
        return retval;
}

/*
 * __detach_mounts - lazily unmount all mounts on the specified dentry
 *
 * During unlink, rmdir, and d_drop it is possible to loose the path
 * to an existing mountpoint, and wind up leaking the mount.
 * detach_mounts allows lazily unmounting those mounts instead of
 * leaking them.
 *
 * The caller may hold dentry->d_inode->i_mutex.
 */
void __detach_mounts(struct dentry *dentry)
{
        struct mountpoint *mp;
        struct mount *mnt;

        namespace_lock();
        lock_mount_hash();
        mp = lookup_mountpoint(dentry);
        if (!mp)
                goto out_unlock;

        event++;
        while (!hlist_empty(&mp->m_list)) {
                mnt = hlist_entry(mp->m_list.first, struct mount, mnt_mp_list);
                if (mnt->mnt.mnt_flags & MNT_UMOUNT) {
                        umount_mnt(mnt);
                        hlist_add_head(&mnt->mnt_umount, &unmounted);
                }
                else umount_tree(mnt, UMOUNT_CONNECTED);
        }
        put_mountpoint(mp);
out_unlock:
        unlock_mount_hash();
        namespace_unlock();
}

/*
 * Is the caller allowed to modify his namespace?
 */
bool may_mount(void)
{
        return ns_capable(current->nsproxy->mnt_ns->user_ns, CAP_SYS_ADMIN);
}

static void warn_mandlock(void)
{
        pr_warn_once("=======================================================\n"
                     "WARNING: The mand mount option has been deprecated and\n"
                     "         and is ignored by this kernel. Remove the mand\n"
                     "         option from the mount to silence this warning.\n"
                     "=======================================================\n");
}

static int can_umount(const struct path *path, int flags)
{
        struct mount *mnt = real_mount(path->mnt);

        if (!may_mount())
                return -EPERM;
        if (!path_mounted(path))
                return -EINVAL;
        if (!check_mnt(mnt))
                return -EINVAL;
        if (mnt->mnt.mnt_flags & MNT_LOCKED) /* Check optimistically */
                return -EINVAL;
        if (flags & MNT_FORCE && !capable(CAP_SYS_ADMIN))
                return -EPERM;
        return 0;
}

// caller is responsible for flags being sane
int path_umount(struct path *path, int flags)
{
        struct mount *mnt = real_mount(path->mnt);
        int ret;

        ret = can_umount(path, flags);
        if (!ret)
                ret = do_umount(mnt, flags);

        /* we mustn't call path_put() as that would clear mnt_expiry_mark */
        dput(path->dentry);
        mntput_no_expire(mnt);
        return ret;
}

static int ksys_umount(char __user *name, int flags)
{
        int lookup_flags = LOOKUP_MOUNTPOINT;
        struct path path;
        int ret;

        // basic validity checks done first
        if (flags & ~(MNT_FORCE | MNT_DETACH | MNT_EXPIRE | UMOUNT_NOFOLLOW))
                return -EINVAL;

        if (!(flags & UMOUNT_NOFOLLOW))
                lookup_flags |= LOOKUP_FOLLOW;
        ret = user_path_at(AT_FDCWD, name, lookup_flags, &path);
        if (ret)
                return ret;
        return path_umount(&path, flags);
}

SYSCALL_DEFINE2(umount, char __user *, name, int, flags)
{
        return ksys_umount(name, flags);
}

#ifdef __ARCH_WANT_SYS_OLDUMOUNT

/*
 *        The 2.0 compatible umount. No flags.
 */
SYSCALL_DEFINE1(oldumount, char __user *, name)
{
        return ksys_umount(name, 0);
}

#endif

static bool is_mnt_ns_file(struct dentry *dentry)
{
        /* Is this a proxy for a mount namespace? */
        return dentry->d_op == &ns_dentry_operations &&
               dentry->d_fsdata == &mntns_operations;
}

struct ns_common *from_mnt_ns(struct mnt_namespace *mnt)
{
        return &mnt->ns;
}

struct mnt_namespace *__lookup_next_mnt_ns(struct mnt_namespace *mntns, bool previous)
{
        guard(read_lock)(&mnt_ns_tree_lock);
        for (;;) {
                struct rb_node *node;

                if (previous)
                        node = rb_prev(&mntns->mnt_ns_tree_node);
                else
                        node = rb_next(&mntns->mnt_ns_tree_node);
                if (!node)
                        return ERR_PTR(-ENOENT);

                mntns = node_to_mnt_ns(node);
                node = &mntns->mnt_ns_tree_node;

                if (!ns_capable_noaudit(mntns->user_ns, CAP_SYS_ADMIN))
                        continue;

                /*
                 * Holding mnt_ns_tree_lock prevents the mount namespace from
                 * being freed but it may well be on it's deathbed. We want an
                 * active reference, not just a passive one here as we're
                 * persisting the mount namespace.
                 */
                if (!refcount_inc_not_zero(&mntns->ns.count))
                        continue;

                return mntns;
        }
}

static bool mnt_ns_loop(struct dentry *dentry)
{
        /* Could bind mounting the mount namespace inode cause a
         * mount namespace loop?
         */
        struct mnt_namespace *mnt_ns;
        if (!is_mnt_ns_file(dentry))
                return false;

        mnt_ns = to_mnt_ns(get_proc_ns(dentry->d_inode));
        return current->nsproxy->mnt_ns->seq >= mnt_ns->seq;
}

struct mount *copy_tree(struct mount *src_root, struct dentry *dentry,
                                        int flag)
{
        struct mount *res, *src_parent, *src_root_child, *src_mnt,
                *dst_parent, *dst_mnt;

        if (!(flag & CL_COPY_UNBINDABLE) && IS_MNT_UNBINDABLE(src_root))
                return ERR_PTR(-EINVAL);

        if (!(flag & CL_COPY_MNT_NS_FILE) && is_mnt_ns_file(dentry))
                return ERR_PTR(-EINVAL);

        res = dst_mnt = clone_mnt(src_root, dentry, flag);
        if (IS_ERR(dst_mnt))
                return dst_mnt;

        src_parent = src_root;
        dst_mnt->mnt_mountpoint = src_root->mnt_mountpoint;

        list_for_each_entry(src_root_child, &src_root->mnt_mounts, mnt_child) {
                if (!is_subdir(src_root_child->mnt_mountpoint, dentry))
                        continue;

                for (src_mnt = src_root_child; src_mnt;
                    src_mnt = next_mnt(src_mnt, src_root_child)) {
                        if (!(flag & CL_COPY_UNBINDABLE) &&
                            IS_MNT_UNBINDABLE(src_mnt)) {
                                if (src_mnt->mnt.mnt_flags & MNT_LOCKED) {
                                        /* Both unbindable and locked. */
                                        dst_mnt = ERR_PTR(-EPERM);
                                        goto out;
                                } else {
                                        src_mnt = skip_mnt_tree(src_mnt);
                                        continue;
                                }
                        }
                        if (!(flag & CL_COPY_MNT_NS_FILE) &&
                            is_mnt_ns_file(src_mnt->mnt.mnt_root)) {
                                src_mnt = skip_mnt_tree(src_mnt);
                                continue;
                        }
                        while (src_parent != src_mnt->mnt_parent) {
                                src_parent = src_parent->mnt_parent;
                                dst_mnt = dst_mnt->mnt_parent;
                        }

                        src_parent = src_mnt;
                        dst_parent = dst_mnt;
                        dst_mnt = clone_mnt(src_mnt, src_mnt->mnt.mnt_root, flag);
                        if (IS_ERR(dst_mnt))
                                goto out;
                        lock_mount_hash();
                        list_add_tail(&dst_mnt->mnt_list, &res->mnt_list);
                        attach_mnt(dst_mnt, dst_parent, src_parent->mnt_mp, false);
                        unlock_mount_hash();
                }
        }
        return res;

out:
        if (res) {
                lock_mount_hash();
                umount_tree(res, UMOUNT_SYNC);
                unlock_mount_hash();
        }
        return dst_mnt;
}

/* Caller should check returned pointer for errors */

struct vfsmount *collect_mounts(const struct path *path)
{
        struct mount *tree;
        namespace_lock();
        if (!check_mnt(real_mount(path->mnt)))
                tree = ERR_PTR(-EINVAL);
        else
                tree = copy_tree(real_mount(path->mnt), path->dentry,
                                 CL_COPY_ALL | CL_PRIVATE);
        namespace_unlock();
        if (IS_ERR(tree))
                return ERR_CAST(tree);
        return &tree->mnt;
}

static void free_mnt_ns(struct mnt_namespace *);
static struct mnt_namespace *alloc_mnt_ns(struct user_namespace *, bool);

void dissolve_on_fput(struct vfsmount *mnt)
{
        struct mnt_namespace *ns;
        namespace_lock();
        lock_mount_hash();
        ns = real_mount(mnt)->mnt_ns;
        if (ns) {
                if (is_anon_ns(ns))
                        umount_tree(real_mount(mnt), UMOUNT_CONNECTED);
                else
                        ns = NULL;
        }
        unlock_mount_hash();
        namespace_unlock();
        if (ns)
                free_mnt_ns(ns);
}

void drop_collected_mounts(struct vfsmount *mnt)
{
        namespace_lock();
        lock_mount_hash();
        umount_tree(real_mount(mnt), 0);
        unlock_mount_hash();
        namespace_unlock();
}

bool has_locked_children(struct mount *mnt, struct dentry *dentry)
{
        struct mount *child;

        list_for_each_entry(child, &mnt->mnt_mounts, mnt_child) {
                if (!is_subdir(child->mnt_mountpoint, dentry))
                        continue;

                if (child->mnt.mnt_flags & MNT_LOCKED)
                        return true;
        }
        return false;
}

/**
 * clone_private_mount - create a private clone of a path
 * @path: path to clone
 *
 * This creates a new vfsmount, which will be the clone of @path.  The new mount
 * will not be attached anywhere in the namespace and will be private (i.e.
 * changes to the originating mount won't be propagated into this).
 *
 * Release with mntput().
 */
struct vfsmount *clone_private_mount(const struct path *path)
{
        struct mount *old_mnt = real_mount(path->mnt);
        struct mount *new_mnt;

        down_read(&namespace_sem);
        if (IS_MNT_UNBINDABLE(old_mnt))
                goto invalid;

        if (!check_mnt(old_mnt))
                goto invalid;

        if (has_locked_children(old_mnt, path->dentry))
                goto invalid;

        new_mnt = clone_mnt(old_mnt, path->dentry, CL_PRIVATE);
        up_read(&namespace_sem);

        if (IS_ERR(new_mnt))
                return ERR_CAST(new_mnt);

        /* Longterm mount to be removed by kern_unmount*() */
        new_mnt->mnt_ns = MNT_NS_INTERNAL;

        return &new_mnt->mnt;

invalid:
        up_read(&namespace_sem);
        return ERR_PTR(-EINVAL);
}
EXPORT_SYMBOL_GPL(clone_private_mount);

int iterate_mounts(int (*f)(struct vfsmount *, void *), void *arg,
                   struct vfsmount *root)
{
        struct mount *mnt;
        int res = f(root, arg);
        if (res)
                return res;
        list_for_each_entry(mnt, &real_mount(root)->mnt_list, mnt_list) {
                res = f(&mnt->mnt, arg);
                if (res)
                        return res;
        }
        return 0;
}

static void lock_mnt_tree(struct mount *mnt)
{
        struct mount *p;

        for (p = mnt; p; p = next_mnt(p, mnt)) {
                int flags = p->mnt.mnt_flags;
                /* Don't allow unprivileged users to change mount flags */
                flags |= MNT_LOCK_ATIME;

                if (flags & MNT_READONLY)
                        flags |= MNT_LOCK_READONLY;

                if (flags & MNT_NODEV)
                        flags |= MNT_LOCK_NODEV;

                if (flags & MNT_NOSUID)
                        flags |= MNT_LOCK_NOSUID;

                if (flags & MNT_NOEXEC)
                        flags |= MNT_LOCK_NOEXEC;
                /* Don't allow unprivileged users to reveal what is under a mount */
                if (list_empty(&p->mnt_expire))
                        flags |= MNT_LOCKED;
                p->mnt.mnt_flags = flags;
        }
}

static void cleanup_group_ids(struct mount *mnt, struct mount *end)
{
        struct mount *p;

        for (p = mnt; p != end; p = next_mnt(p, mnt)) {
                if (p->mnt_group_id && !IS_MNT_SHARED(p))
                        mnt_release_group_id(p);
        }
}

static int invent_group_ids(struct mount *mnt, bool recurse)
{
        struct mount *p;

        for (p = mnt; p; p = recurse ? next_mnt(p, mnt) : NULL) {
                if (!p->mnt_group_id && !IS_MNT_SHARED(p)) {
                        int err = mnt_alloc_group_id(p);
                        if (err) {
                                cleanup_group_ids(mnt, p);
                                return err;
                        }
                }
        }

        return 0;
}

int count_mounts(struct mnt_namespace *ns, struct mount *mnt)
{
        unsigned int max = READ_ONCE(sysctl_mount_max);
        unsigned int mounts = 0;
        struct mount *p;

        if (ns->nr_mounts >= max)
                return -ENOSPC;
        max -= ns->nr_mounts;
        if (ns->pending_mounts >= max)
                return -ENOSPC;
        max -= ns->pending_mounts;

        for (p = mnt; p; p = next_mnt(p, mnt))
                mounts++;

        if (mounts > max)
                return -ENOSPC;

        ns->pending_mounts += mounts;
        return 0;
}

enum mnt_tree_flags_t {
        MNT_TREE_MOVE = BIT(0),
        MNT_TREE_BENEATH = BIT(1),
};

/**
 * attach_recursive_mnt - attach a source mount tree
 * @source_mnt: mount tree to be attached
 * @top_mnt:    mount that @source_mnt will be mounted on or mounted beneath
 * @dest_mp:    the mountpoint @source_mnt will be mounted at
 * @flags:      modify how @source_mnt is supposed to be attached
 *
 *  NOTE: in the table below explains the semantics when a source mount
 *  of a given type is attached to a destination mount of a given type.
 * ---------------------------------------------------------------------------
 * |         BIND MOUNT OPERATION                                            |
 * |**************************************************************************
 * | source-->| shared        |       private  |       slave    | unbindable |
 * | dest     |               |                |                |            |
 * |   |      |               |                |                |            |
 * |   v      |               |                |                |            |
 * |**************************************************************************
 * |  shared  | shared (++)   |     shared (+) |     shared(+++)|  invalid   |
 * |          |               |                |                |            |
 * |non-shared| shared (+)    |      private   |      slave (*) |  invalid   |
 * ***************************************************************************
 * A bind operation clones the source mount and mounts the clone on the
 * destination mount.
 *
 * (++)  the cloned mount is propagated to all the mounts in the propagation
 *          tree of the destination mount and the cloned mount is added to
 *          the peer group of the source mount.
 * (+)   the cloned mount is created under the destination mount and is marked
 *       as shared. The cloned mount is added to the peer group of the source
 *       mount.
 * (+++) the mount is propagated to all the mounts in the propagation tree
 *       of the destination mount and the cloned mount is made slave
 *       of the same master as that of the source mount. The cloned mount
 *       is marked as 'shared and slave'.
 * (*)   the cloned mount is made a slave of the same master as that of the
 *          source mount.
 *
 * ---------------------------------------------------------------------------
 * |                         MOVE MOUNT OPERATION                                 |
 * |**************************************************************************
 * | source-->| shared        |       private  |       slave    | unbindable |
 * | dest     |               |                |                |            |
 * |   |      |               |                |                |            |
 * |   v      |               |                |                |            |
 * |**************************************************************************
 * |  shared  | shared (+)    |     shared (+) |    shared(+++) |  invalid   |
 * |          |               |                |                |            |
 * |non-shared| shared (+*)   |      private   |    slave (*)   | unbindable |
 * ***************************************************************************
 *
 * (+)  the mount is moved to the destination. And is then propagated to
 *         all the mounts in the propagation tree of the destination mount.
 * (+*)  the mount is moved to the destination.
 * (+++)  the mount is moved to the destination and is then propagated to
 *         all the mounts belonging to the destination mount's propagation tree.
 *         the mount is marked as 'shared and slave'.
 * (*)        the mount continues to be a slave at the new location.
 *
 * if the source mount is a tree, the operations explained above is
 * applied to each mount in the tree.
 * Must be called without spinlocks held, since this function can sleep
 * in allocations.
 *
 * Context: The function expects namespace_lock() to be held.
 * Return: If @source_mnt was successfully attached 0 is returned.
 *         Otherwise a negative error code is returned.
 */
static int attach_recursive_mnt(struct mount *source_mnt,
                                struct mount *top_mnt,
                                struct mountpoint *dest_mp,
                                enum mnt_tree_flags_t flags)
{
        struct user_namespace *user_ns = current->nsproxy->mnt_ns->user_ns;
        HLIST_HEAD(tree_list);
        struct mnt_namespace *ns = top_mnt->mnt_ns;
        struct mountpoint *smp;
        struct mount *child, *dest_mnt, *p;
        struct hlist_node *n;
        int err = 0;
        bool moving = flags & MNT_TREE_MOVE, beneath = flags & MNT_TREE_BENEATH;

        /*
         * Preallocate a mountpoint in case the new mounts need to be
         * mounted beneath mounts on the same mountpoint.
         */
        smp = get_mountpoint(source_mnt->mnt.mnt_root);
        if (IS_ERR(smp))
                return PTR_ERR(smp);

        /* Is there space to add these mounts to the mount namespace? */
        if (!moving) {
                err = count_mounts(ns, source_mnt);
                if (err)
                        goto out;
        }

        if (beneath)
                dest_mnt = top_mnt->mnt_parent;
        else
                dest_mnt = top_mnt;

        if (IS_MNT_SHARED(dest_mnt)) {
                err = invent_group_ids(source_mnt, true);
                if (err)
                        goto out;
                err = propagate_mnt(dest_mnt, dest_mp, source_mnt, &tree_list);
        }
        lock_mount_hash();
        if (err)
                goto out_cleanup_ids;

        if (IS_MNT_SHARED(dest_mnt)) {
                for (p = source_mnt; p; p = next_mnt(p, source_mnt))
                        set_mnt_shared(p);
        }

        if (moving) {
                if (beneath)
                        dest_mp = smp;
                unhash_mnt(source_mnt);
                attach_mnt(source_mnt, top_mnt, dest_mp, beneath);
                touch_mnt_namespace(source_mnt->mnt_ns);
        } else {
                if (source_mnt->mnt_ns) {
                        LIST_HEAD(head);

                        /* move from anon - the caller will destroy */
                        for (p = source_mnt; p; p = next_mnt(p, source_mnt))
                                move_from_ns(p, &head);
                        list_del_init(&head);
                }
                if (beneath)
                        mnt_set_mountpoint_beneath(source_mnt, top_mnt, smp);
                else
                        mnt_set_mountpoint(dest_mnt, dest_mp, source_mnt);
                commit_tree(source_mnt);
        }

        hlist_for_each_entry_safe(child, n, &tree_list, mnt_hash) {
                struct mount *q;
                hlist_del_init(&child->mnt_hash);
                q = __lookup_mnt(&child->mnt_parent->mnt,
                                 child->mnt_mountpoint);
                if (q)
                        mnt_change_mountpoint(child, smp, q);
                /* Notice when we are propagating across user namespaces */
                if (child->mnt_parent->mnt_ns->user_ns != user_ns)
                        lock_mnt_tree(child);
                child->mnt.mnt_flags &= ~MNT_LOCKED;
                commit_tree(child);
        }
        put_mountpoint(smp);
        unlock_mount_hash();

        return 0;

 out_cleanup_ids:
        while (!hlist_empty(&tree_list)) {
                child = hlist_entry(tree_list.first, struct mount, mnt_hash);
                child->mnt_parent->mnt_ns->pending_mounts = 0;
                umount_tree(child, UMOUNT_SYNC);
        }
        unlock_mount_hash();
        cleanup_group_ids(source_mnt, NULL);
 out:
        ns->pending_mounts = 0;

        read_seqlock_excl(&mount_lock);
        put_mountpoint(smp);
        read_sequnlock_excl(&mount_lock);

        return err;
}

/**
 * do_lock_mount - lock mount and mountpoint
 * @path:    target path
 * @beneath: whether the intention is to mount beneath @path
 *
 * Follow the mount stack on @path until the top mount @mnt is found. If
 * the initial @path->{mnt,dentry} is a mountpoint lookup the first
 * mount stacked on top of it. Then simply follow @{mnt,mnt->mnt_root}
 * until nothing is stacked on top of it anymore.
 *
 * Acquire the inode_lock() on the top mount's ->mnt_root to protect
 * against concurrent removal of the new mountpoint from another mount
 * namespace.
 *
 * If @beneath is requested, acquire inode_lock() on @mnt's mountpoint
 * @mp on @mnt->mnt_parent must be acquired. This protects against a
 * concurrent unlink of @mp->mnt_dentry from another mount namespace
 * where @mnt doesn't have a child mount mounted @mp. A concurrent
 * removal of @mnt->mnt_root doesn't matter as nothing will be mounted
 * on top of it for @beneath.
 *
 * In addition, @beneath needs to make sure that @mnt hasn't been
 * unmounted or moved from its current mountpoint in between dropping
 * @mount_lock and acquiring @namespace_sem. For the !@beneath case @mnt
 * being unmounted would be detected later by e.g., calling
 * check_mnt(mnt) in the function it's called from. For the @beneath
 * case however, it's useful to detect it directly in do_lock_mount().
 * If @mnt hasn't been unmounted then @mnt->mnt_mountpoint still points
 * to @mnt->mnt_mp->m_dentry. But if @mnt has been unmounted it will
 * point to @mnt->mnt_root and @mnt->mnt_mp will be NULL.
 *
 * Return: Either the target mountpoint on the top mount or the top
 *         mount's mountpoint.
 */
static struct mountpoint *do_lock_mount(struct path *path, bool beneath)
{
        struct vfsmount *mnt = path->mnt;
        struct dentry *dentry;
        struct mountpoint *mp = ERR_PTR(-ENOENT);

        for (;;) {
                struct mount *m;

                if (beneath) {
                        m = real_mount(mnt);
                        read_seqlock_excl(&mount_lock);
                        dentry = dget(m->mnt_mountpoint);
                        read_sequnlock_excl(&mount_lock);
                } else {
                        dentry = path->dentry;
                }

                inode_lock(dentry->d_inode);
                if (unlikely(cant_mount(dentry))) {
                        inode_unlock(dentry->d_inode);
                        goto out;
                }

                namespace_lock();

                if (beneath && (!is_mounted(mnt) || m->mnt_mountpoint != dentry)) {
                        namespace_unlock();
                        inode_unlock(dentry->d_inode);
                        goto out;
                }

                mnt = lookup_mnt(path);
                if (likely(!mnt))
                        break;

                namespace_unlock();
                inode_unlock(dentry->d_inode);
                if (beneath)
                        dput(dentry);
                path_put(path);
                path->mnt = mnt;
                path->dentry = dget(mnt->mnt_root);
        }

        mp = get_mountpoint(dentry);
        if (IS_ERR(mp)) {
                namespace_unlock();
                inode_unlock(dentry->d_inode);
        }

out:
        if (beneath)
                dput(dentry);

        return mp;
}

static inline struct mountpoint *lock_mount(struct path *path)
{
        return do_lock_mount(path, false);
}

static void unlock_mount(struct mountpoint *where)
{
        struct dentry *dentry = where->m_dentry;

        read_seqlock_excl(&mount_lock);
        put_mountpoint(where);
        read_sequnlock_excl(&mount_lock);

        namespace_unlock();
        inode_unlock(dentry->d_inode);
}

static int graft_tree(struct mount *mnt, struct mount *p, struct mountpoint *mp)
{
        if (mnt->mnt.mnt_sb->s_flags & SB_NOUSER)
                return -EINVAL;

        if (d_is_dir(mp->m_dentry) !=
              d_is_dir(mnt->mnt.mnt_root))
                return -ENOTDIR;

        return attach_recursive_mnt(mnt, p, mp, 0);
}

/*
 * Sanity check the flags to change_mnt_propagation.
 */

static int flags_to_propagation_type(int ms_flags)
{
        int type = ms_flags & ~(MS_REC | MS_SILENT);

        /* Fail if any non-propagation flags are set */
        if (type & ~(MS_SHARED | MS_PRIVATE | MS_SLAVE | MS_UNBINDABLE))
                return 0;
        /* Only one propagation flag should be set */
        if (!is_power_of_2(type))
                return 0;
        return type;
}

/*
 * recursively change the type of the mountpoint.
 */
static int do_change_type(struct path *path, int ms_flags)
{
        struct mount *m;
        struct mount *mnt = real_mount(path->mnt);
        int recurse = ms_flags & MS_REC;
        int type;
        int err = 0;

        if (!path_mounted(path))
                return -EINVAL;

        type = flags_to_propagation_type(ms_flags);
        if (!type)
                return -EINVAL;

        namespace_lock();
        if (type == MS_SHARED) {
                err = invent_group_ids(mnt, recurse);
                if (err)
                        goto out_unlock;
        }

        lock_mount_hash();
        for (m = mnt; m; m = (recurse ? next_mnt(m, mnt) : NULL))
                change_mnt_propagation(m, type);
        unlock_mount_hash();

 out_unlock:
        namespace_unlock();
        return err;
}

static struct mount *__do_loopback(struct path *old_path, int recurse)
{
        struct mount *mnt = ERR_PTR(-EINVAL), *old = real_mount(old_path->mnt);

        if (IS_MNT_UNBINDABLE(old))
                return mnt;

        if (!check_mnt(old) && old_path->dentry->d_op != &ns_dentry_operations)
                return mnt;

        if (!recurse && has_locked_children(old, old_path->dentry))
                return mnt;

        if (recurse)
                mnt = copy_tree(old, old_path->dentry, CL_COPY_MNT_NS_FILE);
        else
                mnt = clone_mnt(old, old_path->dentry, 0);

        if (!IS_ERR(mnt))
                mnt->mnt.mnt_flags &= ~MNT_LOCKED;

        return mnt;
}

/*
 * do loopback mount.
 */
static int do_loopback(struct path *path, const char *old_name,
                                int recurse)
{
        struct path old_path;
        struct mount *mnt = NULL, *parent;
        struct mountpoint *mp;
        int err;
        if (!old_name || !*old_name)
                return -EINVAL;
        err = kern_path(old_name, LOOKUP_FOLLOW|LOOKUP_AUTOMOUNT, &old_path);
        if (err)
                return err;

        err = -EINVAL;
        if (mnt_ns_loop(old_path.dentry))
                goto out;

        mp = lock_mount(path);
        if (IS_ERR(mp)) {
                err = PTR_ERR(mp);
                goto out;
        }

        parent = real_mount(path->mnt);
        if (!check_mnt(parent))
                goto out2;

        mnt = __do_loopback(&old_path, recurse);
        if (IS_ERR(mnt)) {
                err = PTR_ERR(mnt);
                goto out2;
        }

        err = graft_tree(mnt, parent, mp);
        if (err) {
                lock_mount_hash();
                umount_tree(mnt, UMOUNT_SYNC);
                unlock_mount_hash();
        }
out2:
        unlock_mount(mp);
out:
        path_put(&old_path);
        return err;
}

static struct file *open_detached_copy(struct path *path, bool recursive)
{
        struct user_namespace *user_ns = current->nsproxy->mnt_ns->user_ns;
        struct mnt_namespace *ns = alloc_mnt_ns(user_ns, true);
        struct mount *mnt, *p;
        struct file *file;

        if (IS_ERR(ns))
                return ERR_CAST(ns);

        namespace_lock();
        mnt = __do_loopback(path, recursive);
        if (IS_ERR(mnt)) {
                namespace_unlock();
                free_mnt_ns(ns);
                return ERR_CAST(mnt);
        }

        lock_mount_hash();
        for (p = mnt; p; p = next_mnt(p, mnt)) {
                mnt_add_to_ns(ns, p);
                ns->nr_mounts++;
        }
        ns->root = mnt;
        mntget(&mnt->mnt);
        unlock_mount_hash();
        namespace_unlock();

        mntput(path->mnt);
        path->mnt = &mnt->mnt;
        file = dentry_open(path, O_PATH, current_cred());
        if (IS_ERR(file))
                dissolve_on_fput(path->mnt);
        else
                file->f_mode |= FMODE_NEED_UNMOUNT;
        return file;
}

SYSCALL_DEFINE3(open_tree, int, dfd, const char __user *, filename, unsigned, flags)
{
        struct file *file;
        struct path path;
        int lookup_flags = LOOKUP_AUTOMOUNT | LOOKUP_FOLLOW;
        bool detached = flags & OPEN_TREE_CLONE;
        int error;
        int fd;

        BUILD_BUG_ON(OPEN_TREE_CLOEXEC != O_CLOEXEC);

        if (flags & ~(AT_EMPTY_PATH | AT_NO_AUTOMOUNT | AT_RECURSIVE |
                      AT_SYMLINK_NOFOLLOW | OPEN_TREE_CLONE |
                      OPEN_TREE_CLOEXEC))
                return -EINVAL;

        if ((flags & (AT_RECURSIVE | OPEN_TREE_CLONE)) == AT_RECURSIVE)
                return -EINVAL;

        if (flags & AT_NO_AUTOMOUNT)
                lookup_flags &= ~LOOKUP_AUTOMOUNT;
        if (flags & AT_SYMLINK_NOFOLLOW)
                lookup_flags &= ~LOOKUP_FOLLOW;
        if (flags & AT_EMPTY_PATH)
                lookup_flags |= LOOKUP_EMPTY;

        if (detached && !may_mount())
                return -EPERM;

        fd = get_unused_fd_flags(flags & O_CLOEXEC);
        if (fd < 0)
                return fd;

        error = user_path_at(dfd, filename, lookup_flags, &path);
        if (unlikely(error)) {
                file = ERR_PTR(error);
        } else {
                if (detached)
                        file = open_detached_copy(&path, flags & AT_RECURSIVE);
                else
                        file = dentry_open(&path, O_PATH, current_cred());
                path_put(&path);
        }
        if (IS_ERR(file)) {
                put_unused_fd(fd);
                return PTR_ERR(file);
        }
        fd_install(fd, file);
        return fd;
}

/*
 * Don't allow locked mount flags to be cleared.
 *
 * No locks need to be held here while testing the various MNT_LOCK
 * flags because those flags can never be cleared once they are set.
 */
static bool can_change_locked_flags(struct mount *mnt, unsigned int mnt_flags)
{
        unsigned int fl = mnt->mnt.mnt_flags;

        if ((fl & MNT_LOCK_READONLY) &&
            !(mnt_flags & MNT_READONLY))
                return false;

        if ((fl & MNT_LOCK_NODEV) &&
            !(mnt_flags & MNT_NODEV))
                return false;

        if ((fl & MNT_LOCK_NOSUID) &&
            !(mnt_flags & MNT_NOSUID))
                return false;

        if ((fl & MNT_LOCK_NOEXEC) &&
            !(mnt_flags & MNT_NOEXEC))
                return false;

        if ((fl & MNT_LOCK_ATIME) &&
            ((fl & MNT_ATIME_MASK) != (mnt_flags & MNT_ATIME_MASK)))
                return false;

        return true;
}

static int change_mount_ro_state(struct mount *mnt, unsigned int mnt_flags)
{
        bool readonly_request = (mnt_flags & MNT_READONLY);

        if (readonly_request == __mnt_is_readonly(&mnt->mnt))
                return 0;

        if (readonly_request)
                return mnt_make_readonly(mnt);

        mnt->mnt.mnt_flags &= ~MNT_READONLY;
        return 0;
}

static void set_mount_attributes(struct mount *mnt, unsigned int mnt_flags)
{
        mnt_flags |= mnt->mnt.mnt_flags & ~MNT_USER_SETTABLE_MASK;
        mnt->mnt.mnt_flags = mnt_flags;
        touch_mnt_namespace(mnt->mnt_ns);
}

static void mnt_warn_timestamp_expiry(struct path *mountpoint, struct vfsmount *mnt)
{
        struct super_block *sb = mnt->mnt_sb;

        if (!__mnt_is_readonly(mnt) &&
           (!(sb->s_iflags & SB_I_TS_EXPIRY_WARNED)) &&
           (ktime_get_real_seconds() + TIME_UPTIME_SEC_MAX > sb->s_time_max)) {
                char *buf, *mntpath;

                buf = (char *)__get_free_page(GFP_KERNEL);
                if (buf)
                        mntpath = d_path(mountpoint, buf, PAGE_SIZE);
                else
                        mntpath = ERR_PTR(-ENOMEM);
                if (IS_ERR(mntpath))
                        mntpath = "(unknown)";

                pr_warn("%s filesystem being %s at %s supports timestamps until %ptTd (0x%llx)\n",
                        sb->s_type->name,
                        is_mounted(mnt) ? "remounted" : "mounted",
                        mntpath, &sb->s_time_max,
                        (unsigned long long)sb->s_time_max);

                sb->s_iflags |= SB_I_TS_EXPIRY_WARNED;
                if (buf)
                        free_page((unsigned long)buf);
        }
}

/*
 * Handle reconfiguration of the mountpoint only without alteration of the
 * superblock it refers to.  This is triggered by specifying MS_REMOUNT|MS_BIND
 * to mount(2).
 */
static int do_reconfigure_mnt(struct path *path, unsigned int mnt_flags)
{
        struct super_block *sb = path->mnt->mnt_sb;
        struct mount *mnt = real_mount(path->mnt);
        int ret;

        if (!check_mnt(mnt))
                return -EINVAL;

        if (!path_mounted(path))
                return -EINVAL;

        if (!can_change_locked_flags(mnt, mnt_flags))
                return -EPERM;

        /*
         * We're only checking whether the superblock is read-only not
         * changing it, so only take down_read(&sb->s_umount).
         */
        down_read(&sb->s_umount);
        lock_mount_hash();
        ret = change_mount_ro_state(mnt, mnt_flags);
        if (ret == 0)
                set_mount_attributes(mnt, mnt_flags);
        unlock_mount_hash();
        up_read(&sb->s_umount);

        mnt_warn_timestamp_expiry(path, &mnt->mnt);

        return ret;
}

/*
 * change filesystem flags. dir should be a physical root of filesystem.
 * If you've mounted a non-root directory somewhere and want to do remount
 * on it - tough luck.
 */
static int do_remount(struct path *path, int ms_flags, int sb_flags,
                      int mnt_flags, void *data)
{
        int err;
        struct super_block *sb = path->mnt->mnt_sb;
        struct mount *mnt = real_mount(path->mnt);
        struct fs_context *fc;

        if (!check_mnt(mnt))
                return -EINVAL;

        if (!path_mounted(path))
                return -EINVAL;

        if (!can_change_locked_flags(mnt, mnt_flags))
                return -EPERM;

        fc = fs_context_for_reconfigure(path->dentry, sb_flags, MS_RMT_MASK);
        if (IS_ERR(fc))
                return PTR_ERR(fc);

        /*
         * Indicate to the filesystem that the remount request is coming
         * from the legacy mount system call.
         */
        fc->oldapi = true;

        err = parse_monolithic_mount_data(fc, data);
        if (!err) {
                down_write(&sb->s_umount);
                err = -EPERM;
                if (ns_capable(sb->s_user_ns, CAP_SYS_ADMIN)) {
                        err = reconfigure_super(fc);
                        if (!err) {
                                lock_mount_hash();
                                set_mount_attributes(mnt, mnt_flags);
                                unlock_mount_hash();
                        }
                }
                up_write(&sb->s_umount);
        }

        mnt_warn_timestamp_expiry(path, &mnt->mnt);

        put_fs_context(fc);
        return err;
}

static inline int tree_contains_unbindable(struct mount *mnt)
{
        struct mount *p;
        for (p = mnt; p; p = next_mnt(p, mnt)) {
                if (IS_MNT_UNBINDABLE(p))
                        return 1;
        }
        return 0;
}

/*
 * Check that there aren't references to earlier/same mount namespaces in the
 * specified subtree.  Such references can act as pins for mount namespaces
 * that aren't checked by the mount-cycle checking code, thereby allowing
 * cycles to be made.
 */
static bool check_for_nsfs_mounts(struct mount *subtree)
{
        struct mount *p;
        bool ret = false;

        lock_mount_hash();
        for (p = subtree; p; p = next_mnt(p, subtree))
                if (mnt_ns_loop(p->mnt.mnt_root))
                        goto out;

        ret = true;
out:
        unlock_mount_hash();
        return ret;
}

static int do_set_group(struct path *from_path, struct path *to_path)
{
        struct mount *from, *to;
        int err;

        from = real_mount(from_path->mnt);
        to = real_mount(to_path->mnt);

        namespace_lock();

        err = -EINVAL;
        /* To and From must be mounted */
        if (!is_mounted(&from->mnt))
                goto out;
        if (!is_mounted(&to->mnt))
                goto out;

        err = -EPERM;
        /* We should be allowed to modify mount namespaces of both mounts */
        if (!ns_capable(from->mnt_ns->user_ns, CAP_SYS_ADMIN))
                goto out;
        if (!ns_capable(to->mnt_ns->user_ns, CAP_SYS_ADMIN))
                goto out;

        err = -EINVAL;
        /* To and From paths should be mount roots */
        if (!path_mounted(from_path))
                goto out;
        if (!path_mounted(to_path))
                goto out;

        /* Setting sharing groups is only allowed across same superblock */
        if (from->mnt.mnt_sb != to->mnt.mnt_sb)
                goto out;

        /* From mount root should be wider than To mount root */
        if (!is_subdir(to->mnt.mnt_root, from->mnt.mnt_root))
                goto out;

        /* From mount should not have locked children in place of To's root */
        if (has_locked_children(from, to->mnt.mnt_root))
                goto out;

        /* Setting sharing groups is only allowed on private mounts */
        if (IS_MNT_SHARED(to) || IS_MNT_SLAVE(to))
                goto out;

        /* From should not be private */
        if (!IS_MNT_SHARED(from) && !IS_MNT_SLAVE(from))
                goto out;

        if (IS_MNT_SLAVE(from)) {
                struct mount *m = from->mnt_master;

                list_add(&to->mnt_slave, &m->mnt_slave_list);
                to->mnt_master = m;
        }

        if (IS_MNT_SHARED(from)) {
                to->mnt_group_id = from->mnt_group_id;
                list_add(&to->mnt_share, &from->mnt_share);
                lock_mount_hash();
                set_mnt_shared(to);
                unlock_mount_hash();
        }

        err = 0;
out:
        namespace_unlock();
        return err;
}

/**
 * path_overmounted - check if path is overmounted
 * @path: path to check
 *
 * Check if path is overmounted, i.e., if there's a mount on top of
 * @path->mnt with @path->dentry as mountpoint.
 *
 * Context: This function expects namespace_lock() to be held.
 * Return: If path is overmounted true is returned, false if not.
 */
static inline bool path_overmounted(const struct path *path)
{
        rcu_read_lock();
        if (unlikely(__lookup_mnt(path->mnt, path->dentry))) {
                rcu_read_unlock();
                return true;
        }
        rcu_read_unlock();
        return false;
}

/**
 * can_move_mount_beneath - check that we can mount beneath the top mount
 * @from: mount to mount beneath
 * @to:   mount under which to mount
 * @mp:   mountpoint of @to
 *
 * - Make sure that @to->dentry is actually the root of a mount under
 *   which we can mount another mount.
 * - Make sure that nothing can be mounted beneath the caller's current
 *   root or the rootfs of the namespace.
 * - Make sure that the caller can unmount the topmost mount ensuring
 *   that the caller could reveal the underlying mountpoint.
 * - Ensure that nothing has been mounted on top of @from before we
 *   grabbed @namespace_sem to avoid creating pointless shadow mounts.
 * - Prevent mounting beneath a mount if the propagation relationship
 *   between the source mount, parent mount, and top mount would lead to
 *   nonsensical mount trees.
 *
 * Context: This function expects namespace_lock() to be held.
 * Return: On success 0, and on error a negative error code is returned.
 */
static int can_move_mount_beneath(const struct path *from,
                                  const struct path *to,
                                  const struct mountpoint *mp)
{
        struct mount *mnt_from = real_mount(from->mnt),
                     *mnt_to = real_mount(to->mnt),
                     *parent_mnt_to = mnt_to->mnt_parent;

        if (!mnt_has_parent(mnt_to))
                return -EINVAL;

        if (!path_mounted(to))
                return -EINVAL;

        if (IS_MNT_LOCKED(mnt_to))
                return -EINVAL;

        /* Avoid creating shadow mounts during mount propagation. */
        if (path_overmounted(from))
                return -EINVAL;

        /*
         * Mounting beneath the rootfs only makes sense when the
         * semantics of pivot_root(".", ".") are used.
         */
        if (&mnt_to->mnt == current->fs->root.mnt)
                return -EINVAL;
        if (parent_mnt_to == current->nsproxy->mnt_ns->root)
                return -EINVAL;

        for (struct mount *p = mnt_from; mnt_has_parent(p); p = p->mnt_parent)
                if (p == mnt_to)
                        return -EINVAL;

        /*
         * If the parent mount propagates to the child mount this would
         * mean mounting @mnt_from on @mnt_to->mnt_parent and then
         * propagating a copy @c of @mnt_from on top of @mnt_to. This
         * defeats the whole purpose of mounting beneath another mount.
         */
        if (propagation_would_overmount(parent_mnt_to, mnt_to, mp))
                return -EINVAL;

        /*
         * If @mnt_to->mnt_parent propagates to @mnt_from this would
         * mean propagating a copy @c of @mnt_from on top of @mnt_from.
         * Afterwards @mnt_from would be mounted on top of
         * @mnt_to->mnt_parent and @mnt_to would be unmounted from
         * @mnt->mnt_parent and remounted on @mnt_from. But since @c is
         * already mounted on @mnt_from, @mnt_to would ultimately be
         * remounted on top of @c. Afterwards, @mnt_from would be
         * covered by a copy @c of @mnt_from and @c would be covered by
         * @mnt_from itself. This defeats the whole purpose of mounting
         * @mnt_from beneath @mnt_to.
         */
        if (propagation_would_overmount(parent_mnt_to, mnt_from, mp))
                return -EINVAL;

        return 0;
}

static int do_move_mount(struct path *old_path, struct path *new_path,
                         bool beneath)
{
        struct mnt_namespace *ns;
        struct mount *p;
        struct mount *old;
        struct mount *parent;
        struct mountpoint *mp, *old_mp;
        int err;
        bool attached;
        enum mnt_tree_flags_t flags = 0;

        mp = do_lock_mount(new_path, beneath);
        if (IS_ERR(mp))
                return PTR_ERR(mp);

        old = real_mount(old_path->mnt);
        p = real_mount(new_path->mnt);
        parent = old->mnt_parent;
        attached = mnt_has_parent(old);
        if (attached)
                flags |= MNT_TREE_MOVE;
        old_mp = old->mnt_mp;
        ns = old->mnt_ns;

        err = -EINVAL;
        /* The mountpoint must be in our namespace. */
        if (!check_mnt(p))
                goto out;

        /* The thing moved must be mounted... */
        if (!is_mounted(&old->mnt))
                goto out;

        /* ... and either ours or the root of anon namespace */
        if (!(attached ? check_mnt(old) : is_anon_ns(ns)))
                goto out;

        if (old->mnt.mnt_flags & MNT_LOCKED)
                goto out;

        if (!path_mounted(old_path))
                goto out;

        if (d_is_dir(new_path->dentry) !=
            d_is_dir(old_path->dentry))
                goto out;
        /*
         * Don't move a mount residing in a shared parent.
         */
        if (attached && IS_MNT_SHARED(parent))
                goto out;

        if (beneath) {
                err = can_move_mount_beneath(old_path, new_path, mp);
                if (err)
                        goto out;

                err = -EINVAL;
                p = p->mnt_parent;
                flags |= MNT_TREE_BENEATH;
        }

        /*
         * Don't move a mount tree containing unbindable mounts to a destination
         * mount which is shared.
         */
        if (IS_MNT_SHARED(p) && tree_contains_unbindable(old))
                goto out;
        err = -ELOOP;
        if (!check_for_nsfs_mounts(old))
                goto out;
        for (; mnt_has_parent(p); p = p->mnt_parent)
                if (p == old)
                        goto out;

        err = attach_recursive_mnt(old, real_mount(new_path->mnt), mp, flags);
        if (err)
                goto out;

        /* if the mount is moved, it should no longer be expire
         * automatically */
        list_del_init(&old->mnt_expire);
        if (attached)
                put_mountpoint(old_mp);
out:
        unlock_mount(mp);
        if (!err) {
                if (attached)
                        mntput_no_expire(parent);
                else
                        free_mnt_ns(ns);
        }
        return err;
}

static int do_move_mount_old(struct path *path, const char *old_name)
{
        struct path old_path;
        int err;

        if (!old_name || !*old_name)
                return -EINVAL;

        err = kern_path(old_name, LOOKUP_FOLLOW, &old_path);
        if (err)
                return err;

        err = do_move_mount(&old_path, path, false);
        path_put(&old_path);
        return err;
}

/*
 * add a mount into a namespace's mount tree
 */
static int do_add_mount(struct mount *newmnt, struct mountpoint *mp,
                        const struct path *path, int mnt_flags)
{
        struct mount *parent = real_mount(path->mnt);

        mnt_flags &= ~MNT_INTERNAL_FLAGS;

        if (unlikely(!check_mnt(parent))) {
                /* that's acceptable only for automounts done in private ns */
                if (!(mnt_flags & MNT_SHRINKABLE))
                        return -EINVAL;
                /* ... and for those we'd better have mountpoint still alive */
                if (!parent->mnt_ns)
                        return -EINVAL;
        }

        /* Refuse the same filesystem on the same mount point */
        if (path->mnt->mnt_sb == newmnt->mnt.mnt_sb && path_mounted(path))
                return -EBUSY;

        if (d_is_symlink(newmnt->mnt.mnt_root))
                return -EINVAL;

        newmnt->mnt.mnt_flags = mnt_flags;
        return graft_tree(newmnt, parent, mp);
}

static bool mount_too_revealing(const struct super_block *sb, int *new_mnt_flags);

/*
 * Create a new mount using a superblock configuration and request it
 * be added to the namespace tree.
 */
static int do_new_mount_fc(struct fs_context *fc, struct path *mountpoint,
                           unsigned int mnt_flags)
{
        struct vfsmount *mnt;
        struct mountpoint *mp;
        struct super_block *sb = fc->root->d_sb;
        int error;

        error = security_sb_kern_mount(sb);
        if (!error && mount_too_revealing(sb, &mnt_flags))
                error = -EPERM;

        if (unlikely(error)) {
                fc_drop_locked(fc);
                return error;
        }

        up_write(&sb->s_umount);

        mnt = vfs_create_mount(fc);
        if (IS_ERR(mnt))
                return PTR_ERR(mnt);

        mnt_warn_timestamp_expiry(mountpoint, mnt);

        mp = lock_mount(mountpoint);
        if (IS_ERR(mp)) {
                mntput(mnt);
                return PTR_ERR(mp);
        }
        error = do_add_mount(real_mount(mnt), mp, mountpoint, mnt_flags);
        unlock_mount(mp);
        if (error < 0)
                mntput(mnt);
        return error;
}

/*
 * create a new mount for userspace and request it to be added into the
 * namespace's tree
 */
static int do_new_mount(struct path *path, const char *fstype, int sb_flags,
                        int mnt_flags, const char *name, void *data)
{
        struct file_system_type *type;
        struct fs_context *fc;
        const char *subtype = NULL;
        int err = 0;

        if (!fstype)
                return -EINVAL;

        type = get_fs_type(fstype);
        if (!type)
                return -ENODEV;

        if (type->fs_flags & FS_HAS_SUBTYPE) {
                subtype = strchr(fstype, '.');
                if (subtype) {
                        subtype++;
                        if (!*subtype) {
                                put_filesystem(type);
                                return -EINVAL;
                        }
                }
        }

        fc = fs_context_for_mount(type, sb_flags);
        put_filesystem(type);
        if (IS_ERR(fc))
                return PTR_ERR(fc);

        /*
         * Indicate to the filesystem that the mount request is coming
         * from the legacy mount system call.
         */
        fc->oldapi = true;

        if (subtype)
                err = vfs_parse_fs_string(fc, "subtype",
                                          subtype, strlen(subtype));
        if (!err && name)
                err = vfs_parse_fs_string(fc, "source", name, strlen(name));
        if (!err)
                err = parse_monolithic_mount_data(fc, data);
        if (!err && !mount_capable(fc))
                err = -EPERM;
        if (!err)
                err = vfs_get_tree(fc);
        if (!err)
                err = do_new_mount_fc(fc, path, mnt_flags);

        put_fs_context(fc);
        return err;
}

int finish_automount(struct vfsmount *m, const struct path *path)
{
        struct dentry *dentry = path->dentry;
        struct mountpoint *mp;
        struct mount *mnt;
        int err;

        if (!m)
                return 0;
        if (IS_ERR(m))
                return PTR_ERR(m);

        mnt = real_mount(m);
        /* The new mount record should have at least 2 refs to prevent it being
         * expired before we get a chance to add it
         */
        BUG_ON(mnt_get_count(mnt) < 2);

        if (m->mnt_sb == path->mnt->mnt_sb &&
            m->mnt_root == dentry) {
                err = -ELOOP;
                goto discard;
        }

        /*
         * we don't want to use lock_mount() - in this case finding something
         * that overmounts our mountpoint to be means "quitely drop what we've
         * got", not "try to mount it on top".
         */
        inode_lock(dentry->d_inode);
        namespace_lock();
        if (unlikely(cant_mount(dentry))) {
                err = -ENOENT;
                goto discard_locked;
        }
        if (path_overmounted(path)) {
                err = 0;
                goto discard_locked;
        }
        mp = get_mountpoint(dentry);
        if (IS_ERR(mp)) {
                err = PTR_ERR(mp);
                goto discard_locked;
        }

        err = do_add_mount(mnt, mp, path, path->mnt->mnt_flags | MNT_SHRINKABLE);
        unlock_mount(mp);
        if (unlikely(err))
                goto discard;
        mntput(m);
        return 0;

discard_locked:
        namespace_unlock();
        inode_unlock(dentry->d_inode);
discard:
        /* remove m from any expiration list it may be on */
        if (!list_empty(&mnt->mnt_expire)) {
                namespace_lock();
                list_del_init(&mnt->mnt_expire);
                namespace_unlock();
        }
        mntput(m);
        mntput(m);
        return err;
}

/**
 * mnt_set_expiry - Put a mount on an expiration list
 * @mnt: The mount to list.
 * @expiry_list: The list to add the mount to.
 */
void mnt_set_expiry(struct vfsmount *mnt, struct list_head *expiry_list)
{
        namespace_lock();

        list_add_tail(&real_mount(mnt)->mnt_expire, expiry_list);

        namespace_unlock();
}
EXPORT_SYMBOL(mnt_set_expiry);

/*
 * process a list of expirable mountpoints with the intent of discarding any
 * mountpoints that aren't in use and haven't been touched since last we came
 * here
 */
void mark_mounts_for_expiry(struct list_head *mounts)
{
        struct mount *mnt, *next;
        LIST_HEAD(graveyard);

        if (list_empty(mounts))
                return;

        namespace_lock();
        lock_mount_hash();

        /* extract from the expiration list every vfsmount that matches the
         * following criteria:
         * - only referenced by its parent vfsmount
         * - still marked for expiry (marked on the last call here; marks are
         *   cleared by mntput())
         */
        list_for_each_entry_safe(mnt, next, mounts, mnt_expire) {
                if (!xchg(&mnt->mnt_expiry_mark, 1) ||
                        propagate_mount_busy(mnt, 1))
                        continue;
                list_move(&mnt->mnt_expire, &graveyard);
        }
        while (!list_empty(&graveyard)) {
                mnt = list_first_entry(&graveyard, struct mount, mnt_expire);
                touch_mnt_namespace(mnt->mnt_ns);
                umount_tree(mnt, UMOUNT_PROPAGATE|UMOUNT_SYNC);
        }
        unlock_mount_hash();
        namespace_unlock();
}

EXPORT_SYMBOL_GPL(mark_mounts_for_expiry);

/*
 * Ripoff of 'select_parent()'
 *
 * search the list of submounts for a given mountpoint, and move any
 * shrinkable submounts to the 'graveyard' list.
 */
static int select_submounts(struct mount *parent, struct list_head *graveyard)
{
        struct mount *this_parent = parent;
        struct list_head *next;
        int found = 0;

repeat:
        next = this_parent->mnt_mounts.next;
resume:
        while (next != &this_parent->mnt_mounts) {
                struct list_head *tmp = next;
                struct mount *mnt = list_entry(tmp, struct mount, mnt_child);

                next = tmp->next;
                if (!(mnt->mnt.mnt_flags & MNT_SHRINKABLE))
                        continue;
                /*
                 * Descend a level if the d_mounts list is non-empty.
                 */
                if (!list_empty(&mnt->mnt_mounts)) {
                        this_parent = mnt;
                        goto repeat;
                }

                if (!propagate_mount_busy(mnt, 1)) {
                        list_move_tail(&mnt->mnt_expire, graveyard);
                        found++;
                }
        }
        /*
         * All done at this level ... ascend and resume the search
         */
        if (this_parent != parent) {
                next = this_parent->mnt_child.next;
                this_parent = this_parent->mnt_parent;
                goto resume;
        }
        return found;
}

/*
 * process a list of expirable mountpoints with the intent of discarding any
 * submounts of a specific parent mountpoint
 *
 * mount_lock must be held for write
 */
static void shrink_submounts(struct mount *mnt)
{
        LIST_HEAD(graveyard);
        struct mount *m;

        /* extract submounts of 'mountpoint' from the expiration list */
        while (select_submounts(mnt, &graveyard)) {
                while (!list_empty(&graveyard)) {
                        m = list_first_entry(&graveyard, struct mount,
                                                mnt_expire);
                        touch_mnt_namespace(m->mnt_ns);
                        umount_tree(m, UMOUNT_PROPAGATE|UMOUNT_SYNC);
                }
        }
}

static void *copy_mount_options(const void __user * data)
{
        char *copy;
        unsigned left, offset;

        if (!data)
                return NULL;

        copy = kmalloc(PAGE_SIZE, GFP_KERNEL);
        if (!copy)
                return ERR_PTR(-ENOMEM);

        left = copy_from_user(copy, data, PAGE_SIZE);

        /*
         * Not all architectures have an exact copy_from_user(). Resort to
         * byte at a time.
         */
        offset = PAGE_SIZE - left;
        while (left) {
                char c;
                if (get_user(c, (const char __user *)data + offset))
                        break;
                copy[offset] = c;
                left--;
                offset++;
        }

        if (left == PAGE_SIZE) {
                kfree(copy);
                return ERR_PTR(-EFAULT);
        }

        return copy;
}

static char *copy_mount_string(const void __user *data)
{
        return data ? strndup_user(data, PATH_MAX) : NULL;
}

/*
 * Flags is a 32-bit value that allows up to 31 non-fs dependent flags to
 * be given to the mount() call (ie: read-only, no-dev, no-suid etc).
 *
 * data is a (void *) that can point to any structure up to
 * PAGE_SIZE-1 bytes, which can contain arbitrary fs-dependent
 * information (or be NULL).
 *
 * Pre-0.97 versions of mount() didn't have a flags word.
 * When the flags word was introduced its top half was required
 * to have the magic value 0xC0ED, and this remained so until 2.4.0-test9.
 * Therefore, if this magic number is present, it carries no information
 * and must be discarded.
 */
int path_mount(const char *dev_name, struct path *path,
                const char *type_page, unsigned long flags, void *data_page)
{
        unsigned int mnt_flags = 0, sb_flags;
        int ret;

        /* Discard magic */
        if ((flags & MS_MGC_MSK) == MS_MGC_VAL)
                flags &= ~MS_MGC_MSK;

        /* Basic sanity checks */
        if (data_page)
                ((char *)data_page)[PAGE_SIZE - 1] = 0;

        if (flags & MS_NOUSER)
                return -EINVAL;

        ret = security_sb_mount(dev_name, path, type_page, flags, data_page);
        if (ret)
                return ret;
        if (!may_mount())
                return -EPERM;
        if (flags & SB_MANDLOCK)
                warn_mandlock();

        /* Default to relatime unless overriden */
        if (!(flags & MS_NOATIME))
                mnt_flags |= MNT_RELATIME;

        /* Separate the per-mountpoint flags */
        if (flags & MS_NOSUID)
                mnt_flags |= MNT_NOSUID;
        if (flags & MS_NODEV)
                mnt_flags |= MNT_NODEV;
        if (flags & MS_NOEXEC)
                mnt_flags |= MNT_NOEXEC;
        if (flags & MS_NOATIME)
                mnt_flags |= MNT_NOATIME;
        if (flags & MS_NODIRATIME)
                mnt_flags |= MNT_NODIRATIME;
        if (flags & MS_STRICTATIME)
                mnt_flags &= ~(MNT_RELATIME | MNT_NOATIME);
        if (flags & MS_RDONLY)
                mnt_flags |= MNT_READONLY;
        if (flags & MS_NOSYMFOLLOW)
                mnt_flags |= MNT_NOSYMFOLLOW;

        /* The default atime for remount is preservation */
        if ((flags & MS_REMOUNT) &&
            ((flags & (MS_NOATIME | MS_NODIRATIME | MS_RELATIME |
                       MS_STRICTATIME)) == 0)) {
                mnt_flags &= ~MNT_ATIME_MASK;
                mnt_flags |= path->mnt->mnt_flags & MNT_ATIME_MASK;
        }

        sb_flags = flags & (SB_RDONLY |
                            SB_SYNCHRONOUS |
                            SB_MANDLOCK |
                            SB_DIRSYNC |
                            SB_SILENT |
                            SB_POSIXACL |
                            SB_LAZYTIME |
                            SB_I_VERSION);

        if ((flags & (MS_REMOUNT | MS_BIND)) == (MS_REMOUNT | MS_BIND))
                return do_reconfigure_mnt(path, mnt_flags);
        if (flags & MS_REMOUNT)
                return do_remount(path, flags, sb_flags, mnt_flags, data_page);
        if (flags & MS_BIND)
                return do_loopback(path, dev_name, flags & MS_REC);
        if (flags & (MS_SHARED | MS_PRIVATE | MS_SLAVE | MS_UNBINDABLE))
                return do_change_type(path, flags);
        if (flags & MS_MOVE)
                return do_move_mount_old(path, dev_name);

        return do_new_mount(path, type_page, sb_flags, mnt_flags, dev_name,
                            data_page);
}

long do_mount(const char *dev_name, const char __user *dir_name,
                const char *type_page, unsigned long flags, void *data_page)
{
        struct path path;
        int ret;

        ret = user_path_at(AT_FDCWD, dir_name, LOOKUP_FOLLOW, &path);
        if (ret)
                return ret;
        ret = path_mount(dev_name, &path, type_page, flags, data_page);
        path_put(&path);
        return ret;
}

static struct ucounts *inc_mnt_namespaces(struct user_namespace *ns)
{
        return inc_ucount(ns, current_euid(), UCOUNT_MNT_NAMESPACES);
}

static void dec_mnt_namespaces(struct ucounts *ucounts)
{
        dec_ucount(ucounts, UCOUNT_MNT_NAMESPACES);
}

static void free_mnt_ns(struct mnt_namespace *ns)
{
        if (!is_anon_ns(ns))
                ns_free_inum(&ns->ns);
        dec_mnt_namespaces(ns->ucounts);
        mnt_ns_tree_remove(ns);
}

/*
 * Assign a sequence number so we can detect when we attempt to bind
 * mount a reference to an older mount namespace into the current
 * mount namespace, preventing reference counting loops.  A 64bit
 * number incrementing at 10Ghz will take 12,427 years to wrap which
 * is effectively never, so we can ignore the possibility.
 */
static atomic64_t mnt_ns_seq = ATOMIC64_INIT(1);

static struct mnt_namespace *alloc_mnt_ns(struct user_namespace *user_ns, bool anon)
{
        struct mnt_namespace *new_ns;
        struct ucounts *ucounts;
        int ret;

        ucounts = inc_mnt_namespaces(user_ns);
        if (!ucounts)
                return ERR_PTR(-ENOSPC);

        new_ns = kzalloc(sizeof(struct mnt_namespace), GFP_KERNEL_ACCOUNT);
        if (!new_ns) {
                dec_mnt_namespaces(ucounts);
                return ERR_PTR(-ENOMEM);
        }
        if (!anon) {
                ret = ns_alloc_inum(&new_ns->ns);
                if (ret) {
                        kfree(new_ns);
                        dec_mnt_namespaces(ucounts);
                        return ERR_PTR(ret);
                }
        }
        new_ns->ns.ops = &mntns_operations;
        if (!anon)
                new_ns->seq = atomic64_inc_return(&mnt_ns_seq);
        refcount_set(&new_ns->ns.count, 1);
        refcount_set(&new_ns->passive, 1);
        new_ns->mounts = RB_ROOT;
        RB_CLEAR_NODE(&new_ns->mnt_ns_tree_node);
        init_waitqueue_head(&new_ns->poll);
        new_ns->user_ns = get_user_ns(user_ns);
        new_ns->ucounts = ucounts;
        return new_ns;
}

__latent_entropy
struct mnt_namespace *copy_mnt_ns(unsigned long flags, struct mnt_namespace *ns,
                struct user_namespace *user_ns, struct fs_struct *new_fs)
{
        struct mnt_namespace *new_ns;
        struct vfsmount *rootmnt = NULL, *pwdmnt = NULL;
        struct mount *p, *q;
        struct mount *old;
        struct mount *new;
        int copy_flags;

        BUG_ON(!ns);

        if (likely(!(flags & CLONE_NEWNS))) {
                get_mnt_ns(ns);
                return ns;
        }

        old = ns->root;

        new_ns = alloc_mnt_ns(user_ns, false);
        if (IS_ERR(new_ns))
                return new_ns;

        namespace_lock();
        /* First pass: copy the tree topology */
        copy_flags = CL_COPY_UNBINDABLE | CL_EXPIRE;
        if (user_ns != ns->user_ns)
                copy_flags |= CL_SHARED_TO_SLAVE;
        new = copy_tree(old, old->mnt.mnt_root, copy_flags);
        if (IS_ERR(new)) {
                namespace_unlock();
                ns_free_inum(&new_ns->ns);
                dec_mnt_namespaces(new_ns->ucounts);
                mnt_ns_release(new_ns);
                return ERR_CAST(new);
        }
        if (user_ns != ns->user_ns) {
                lock_mount_hash();
                lock_mnt_tree(new);
                unlock_mount_hash();
        }
        new_ns->root = new;

        /*
         * Second pass: switch the tsk->fs->* elements and mark new vfsmounts
         * as belonging to new namespace.  We have already acquired a private
         * fs_struct, so tsk->fs->lock is not needed.
         */
        p = old;
        q = new;
        while (p) {
                mnt_add_to_ns(new_ns, q);
                new_ns->nr_mounts++;
                if (new_fs) {
                        if (&p->mnt == new_fs->root.mnt) {
                                new_fs->root.mnt = mntget(&q->mnt);
                                rootmnt = &p->mnt;
                        }
                        if (&p->mnt == new_fs->pwd.mnt) {
                                new_fs->pwd.mnt = mntget(&q->mnt);
                                pwdmnt = &p->mnt;
                        }
                }
                p = next_mnt(p, old);
                q = next_mnt(q, new);
                if (!q)
                        break;
                // an mntns binding we'd skipped?
                while (p->mnt.mnt_root != q->mnt.mnt_root)
                        p = next_mnt(skip_mnt_tree(p), old);
        }
        mnt_ns_tree_add(new_ns);
        namespace_unlock();

        if (rootmnt)
                mntput(rootmnt);
        if (pwdmnt)
                mntput(pwdmnt);

        return new_ns;
}

struct dentry *mount_subtree(struct vfsmount *m, const char *name)
{
        struct mount *mnt = real_mount(m);
        struct mnt_namespace *ns;
        struct super_block *s;
        struct path path;
        int err;

        ns = alloc_mnt_ns(&init_user_ns, true);
        if (IS_ERR(ns)) {
                mntput(m);
                return ERR_CAST(ns);
        }
        ns->root = mnt;
        ns->nr_mounts++;
        mnt_add_to_ns(ns, mnt);

        err = vfs_path_lookup(m->mnt_root, m,
                        name, LOOKUP_FOLLOW|LOOKUP_AUTOMOUNT, &path);

        put_mnt_ns(ns);

        if (err)
                return ERR_PTR(err);

        /* trade a vfsmount reference for active sb one */
        s = path.mnt->mnt_sb;
        atomic_inc(&s->s_active);
        mntput(path.mnt);
        /* lock the sucker */
        down_write(&s->s_umount);
        /* ... and return the root of (sub)tree on it */
        return path.dentry;
}
EXPORT_SYMBOL(mount_subtree);

SYSCALL_DEFINE5(mount, char __user *, dev_name, char __user *, dir_name,
                char __user *, type, unsigned long, flags, void __user *, data)
{
        int ret;
        char *kernel_type;
        char *kernel_dev;
        void *options;

        kernel_type = copy_mount_string(type);
        ret = PTR_ERR(kernel_type);
        if (IS_ERR(kernel_type))
                goto out_type;

        kernel_dev = copy_mount_string(dev_name);
        ret = PTR_ERR(kernel_dev);
        if (IS_ERR(kernel_dev))
                goto out_dev;

        options = copy_mount_options(data);
        ret = PTR_ERR(options);
        if (IS_ERR(options))
                goto out_data;

        ret = do_mount(kernel_dev, dir_name, kernel_type, flags, options);

        kfree(options);
out_data:
        kfree(kernel_dev);
out_dev:
        kfree(kernel_type);
out_type:
        return ret;
}

#define FSMOUNT_VALID_FLAGS                                                    \
        (MOUNT_ATTR_RDONLY | MOUNT_ATTR_NOSUID | MOUNT_ATTR_NODEV |            \
         MOUNT_ATTR_NOEXEC | MOUNT_ATTR__ATIME | MOUNT_ATTR_NODIRATIME |       \
         MOUNT_ATTR_NOSYMFOLLOW)

#define MOUNT_SETATTR_VALID_FLAGS (FSMOUNT_VALID_FLAGS | MOUNT_ATTR_IDMAP)

#define MOUNT_SETATTR_PROPAGATION_FLAGS \
        (MS_UNBINDABLE | MS_PRIVATE | MS_SLAVE | MS_SHARED)

static unsigned int attr_flags_to_mnt_flags(u64 attr_flags)
{
        unsigned int mnt_flags = 0;

        if (attr_flags & MOUNT_ATTR_RDONLY)
                mnt_flags |= MNT_READONLY;
        if (attr_flags & MOUNT_ATTR_NOSUID)
                mnt_flags |= MNT_NOSUID;
        if (attr_flags & MOUNT_ATTR_NODEV)
                mnt_flags |= MNT_NODEV;
        if (attr_flags & MOUNT_ATTR_NOEXEC)
                mnt_flags |= MNT_NOEXEC;
        if (attr_flags & MOUNT_ATTR_NODIRATIME)
                mnt_flags |= MNT_NODIRATIME;
        if (attr_flags & MOUNT_ATTR_NOSYMFOLLOW)
                mnt_flags |= MNT_NOSYMFOLLOW;

        return mnt_flags;
}

/*
 * Create a kernel mount representation for a new, prepared superblock
 * (specified by fs_fd) and attach to an open_tree-like file descriptor.
 */
SYSCALL_DEFINE3(fsmount, int, fs_fd, unsigned int, flags,
                unsigned int, attr_flags)
{
        struct mnt_namespace *ns;
        struct fs_context *fc;
        struct file *file;
        struct path newmount;
        struct mount *mnt;
        unsigned int mnt_flags = 0;
        long ret;

        if (!may_mount())
                return -EPERM;

        if ((flags & ~(FSMOUNT_CLOEXEC)) != 0)
                return -EINVAL;

        if (attr_flags & ~FSMOUNT_VALID_FLAGS)
                return -EINVAL;

        mnt_flags = attr_flags_to_mnt_flags(attr_flags);

        switch (attr_flags & MOUNT_ATTR__ATIME) {
        case MOUNT_ATTR_STRICTATIME:
                break;
        case MOUNT_ATTR_NOATIME:
                mnt_flags |= MNT_NOATIME;
                break;
        case MOUNT_ATTR_RELATIME:
                mnt_flags |= MNT_RELATIME;
                break;
        default:
                return -EINVAL;
        }

        CLASS(fd, f)(fs_fd);
        if (fd_empty(f))
                return -EBADF;

        if (fd_file(f)->f_op != &fscontext_fops)
                return -EINVAL;

        fc = fd_file(f)->private_data;

        ret = mutex_lock_interruptible(&fc->uapi_mutex);
        if (ret < 0)
                return ret;

        /* There must be a valid superblock or we can't mount it */
        ret = -EINVAL;
        if (!fc->root)
                goto err_unlock;

        ret = -EPERM;
        if (mount_too_revealing(fc->root->d_sb, &mnt_flags)) {
                pr_warn("VFS: Mount too revealing\n");
                goto err_unlock;
        }

        ret = -EBUSY;
        if (fc->phase != FS_CONTEXT_AWAITING_MOUNT)
                goto err_unlock;

        if (fc->sb_flags & SB_MANDLOCK)
                warn_mandlock();

        newmount.mnt = vfs_create_mount(fc);
        if (IS_ERR(newmount.mnt)) {
                ret = PTR_ERR(newmount.mnt);
                goto err_unlock;
        }
        newmount.dentry = dget(fc->root);
        newmount.mnt->mnt_flags = mnt_flags;

        /* We've done the mount bit - now move the file context into more or
         * less the same state as if we'd done an fspick().  We don't want to
         * do any memory allocation or anything like that at this point as we
         * don't want to have to handle any errors incurred.
         */
        vfs_clean_context(fc);

        ns = alloc_mnt_ns(current->nsproxy->mnt_ns->user_ns, true);
        if (IS_ERR(ns)) {
                ret = PTR_ERR(ns);
                goto err_path;
        }
        mnt = real_mount(newmount.mnt);
        ns->root = mnt;
        ns->nr_mounts = 1;
        mnt_add_to_ns(ns, mnt);
        mntget(newmount.mnt);

        /* Attach to an apparent O_PATH fd with a note that we need to unmount
         * it, not just simply put it.
         */
        file = dentry_open(&newmount, O_PATH, fc->cred);
        if (IS_ERR(file)) {
                dissolve_on_fput(newmount.mnt);
                ret = PTR_ERR(file);
                goto err_path;
        }
        file->f_mode |= FMODE_NEED_UNMOUNT;

        ret = get_unused_fd_flags((flags & FSMOUNT_CLOEXEC) ? O_CLOEXEC : 0);
        if (ret >= 0)
                fd_install(ret, file);
        else
                fput(file);

err_path:
        path_put(&newmount);
err_unlock:
        mutex_unlock(&fc->uapi_mutex);
        return ret;
}

/*
 * Move a mount from one place to another.  In combination with
 * fsopen()/fsmount() this is used to install a new mount and in combination
 * with open_tree(OPEN_TREE_CLONE [| AT_RECURSIVE]) it can be used to copy
 * a mount subtree.
 *
 * Note the flags value is a combination of MOVE_MOUNT_* flags.
 */
SYSCALL_DEFINE5(move_mount,
                int, from_dfd, const char __user *, from_pathname,
                int, to_dfd, const char __user *, to_pathname,
                unsigned int, flags)
{
        struct path from_path, to_path;
        unsigned int lflags;
        int ret = 0;

        if (!may_mount())
                return -EPERM;

        if (flags & ~MOVE_MOUNT__MASK)
                return -EINVAL;

        if ((flags & (MOVE_MOUNT_BENEATH | MOVE_MOUNT_SET_GROUP)) ==
            (MOVE_MOUNT_BENEATH | MOVE_MOUNT_SET_GROUP))
                return -EINVAL;

        /* If someone gives a pathname, they aren't permitted to move
         * from an fd that requires unmount as we can't get at the flag
         * to clear it afterwards.
         */
        lflags = 0;
        if (flags & MOVE_MOUNT_F_SYMLINKS)        lflags |= LOOKUP_FOLLOW;
        if (flags & MOVE_MOUNT_F_AUTOMOUNTS)        lflags |= LOOKUP_AUTOMOUNT;
        if (flags & MOVE_MOUNT_F_EMPTY_PATH)        lflags |= LOOKUP_EMPTY;

        ret = user_path_at(from_dfd, from_pathname, lflags, &from_path);
        if (ret < 0)
                return ret;

        lflags = 0;
        if (flags & MOVE_MOUNT_T_SYMLINKS)        lflags |= LOOKUP_FOLLOW;
        if (flags & MOVE_MOUNT_T_AUTOMOUNTS)        lflags |= LOOKUP_AUTOMOUNT;
        if (flags & MOVE_MOUNT_T_EMPTY_PATH)        lflags |= LOOKUP_EMPTY;

        ret = user_path_at(to_dfd, to_pathname, lflags, &to_path);
        if (ret < 0)
                goto out_from;

        ret = security_move_mount(&from_path, &to_path);
        if (ret < 0)
                goto out_to;

        if (flags & MOVE_MOUNT_SET_GROUP)
                ret = do_set_group(&from_path, &to_path);
        else
                ret = do_move_mount(&from_path, &to_path,
                                    (flags & MOVE_MOUNT_BENEATH));

out_to:
        path_put(&to_path);
out_from:
        path_put(&from_path);
        return ret;
}

/*
 * Return true if path is reachable from root
 *
 * namespace_sem or mount_lock is held
 */
bool is_path_reachable(struct mount *mnt, struct dentry *dentry,
                         const struct path *root)
{
        while (&mnt->mnt != root->mnt && mnt_has_parent(mnt)) {
                dentry = mnt->mnt_mountpoint;
                mnt = mnt->mnt_parent;
        }
        return &mnt->mnt == root->mnt && is_subdir(dentry, root->dentry);
}

bool path_is_under(const struct path *path1, const struct path *path2)
{
        bool res;
        read_seqlock_excl(&mount_lock);
        res = is_path_reachable(real_mount(path1->mnt), path1->dentry, path2);
        read_sequnlock_excl(&mount_lock);
        return res;
}
EXPORT_SYMBOL(path_is_under);

/*
 * pivot_root Semantics:
 * Moves the root file system of the current process to the directory put_old,
 * makes new_root as the new root file system of the current process, and sets
 * root/cwd of all processes which had them on the current root to new_root.
 *
 * Restrictions:
 * The new_root and put_old must be directories, and  must not be on the
 * same file  system as the current process root. The put_old  must  be
 * underneath new_root,  i.e. adding a non-zero number of /.. to the string
 * pointed to by put_old must yield the same directory as new_root. No other
 * file system may be mounted on put_old. After all, new_root is a mountpoint.
 *
 * Also, the current root cannot be on the 'rootfs' (initial ramfs) filesystem.
 * See Documentation/filesystems/ramfs-rootfs-initramfs.rst for alternatives
 * in this situation.
 *
 * Notes:
 *  - we don't move root/cwd if they are not at the root (reason: if something
 *    cared enough to change them, it's probably wrong to force them elsewhere)
 *  - it's okay to pick a root that isn't the root of a file system, e.g.
 *    /nfs/my_root where /nfs is the mount point. It must be a mountpoint,
 *    though, so you may need to say mount --bind /nfs/my_root /nfs/my_root
 *    first.
 */
SYSCALL_DEFINE2(pivot_root, const char __user *, new_root,
                const char __user *, put_old)
{
        struct path new, old, root;
        struct mount *new_mnt, *root_mnt, *old_mnt, *root_parent, *ex_parent;
        struct mountpoint *old_mp, *root_mp;
        int error;

        if (!may_mount())
                return -EPERM;

        error = user_path_at(AT_FDCWD, new_root,
                             LOOKUP_FOLLOW | LOOKUP_DIRECTORY, &new);
        if (error)
                goto out0;

        error = user_path_at(AT_FDCWD, put_old,
                             LOOKUP_FOLLOW | LOOKUP_DIRECTORY, &old);
        if (error)
                goto out1;

        error = security_sb_pivotroot(&old, &new);
        if (error)
                goto out2;

        get_fs_root(current->fs, &root);
        old_mp = lock_mount(&old);
        error = PTR_ERR(old_mp);
        if (IS_ERR(old_mp))
                goto out3;

        error = -EINVAL;
        new_mnt = real_mount(new.mnt);
        root_mnt = real_mount(root.mnt);
        old_mnt = real_mount(old.mnt);
        ex_parent = new_mnt->mnt_parent;
        root_parent = root_mnt->mnt_parent;
        if (IS_MNT_SHARED(old_mnt) ||
                IS_MNT_SHARED(ex_parent) ||
                IS_MNT_SHARED(root_parent))
                goto out4;
        if (!check_mnt(root_mnt) || !check_mnt(new_mnt))
                goto out4;
        if (new_mnt->mnt.mnt_flags & MNT_LOCKED)
                goto out4;
        error = -ENOENT;
        if (d_unlinked(new.dentry))
                goto out4;
        error = -EBUSY;
        if (new_mnt == root_mnt || old_mnt == root_mnt)
                goto out4; /* loop, on the same file system  */
        error = -EINVAL;
        if (!path_mounted(&root))
                goto out4; /* not a mountpoint */
        if (!mnt_has_parent(root_mnt))
                goto out4; /* not attached */
        if (!path_mounted(&new))
                goto out4; /* not a mountpoint */
        if (!mnt_has_parent(new_mnt))
                goto out4; /* not attached */
        /* make sure we can reach put_old from new_root */
        if (!is_path_reachable(old_mnt, old.dentry, &new))
                goto out4;
        /* make certain new is below the root */
        if (!is_path_reachable(new_mnt, new.dentry, &root))
                goto out4;
        lock_mount_hash();
        umount_mnt(new_mnt);
        root_mp = unhash_mnt(root_mnt);  /* we'll need its mountpoint */
        if (root_mnt->mnt.mnt_flags & MNT_LOCKED) {
                new_mnt->mnt.mnt_flags |= MNT_LOCKED;
                root_mnt->mnt.mnt_flags &= ~MNT_LOCKED;
        }
        /* mount old root on put_old */
        attach_mnt(root_mnt, old_mnt, old_mp, false);
        /* mount new_root on / */
        attach_mnt(new_mnt, root_parent, root_mp, false);
        mnt_add_count(root_parent, -1);
        touch_mnt_namespace(current->nsproxy->mnt_ns);
        /* A moved mount should not expire automatically */
        list_del_init(&new_mnt->mnt_expire);
        put_mountpoint(root_mp);
        unlock_mount_hash();
        chroot_fs_refs(&root, &new);
        error = 0;
out4:
        unlock_mount(old_mp);
        if (!error)
                mntput_no_expire(ex_parent);
out3:
        path_put(&root);
out2:
        path_put(&old);
out1:
        path_put(&new);
out0:
        return error;
}

static unsigned int recalc_flags(struct mount_kattr *kattr, struct mount *mnt)
{
        unsigned int flags = mnt->mnt.mnt_flags;

        /*  flags to clear */
        flags &= ~kattr->attr_clr;
        /* flags to raise */
        flags |= kattr->attr_set;

        return flags;
}

static int can_idmap_mount(const struct mount_kattr *kattr, struct mount *mnt)
{
        struct vfsmount *m = &mnt->mnt;
        struct user_namespace *fs_userns = m->mnt_sb->s_user_ns;

        if (!kattr->mnt_idmap)
                return 0;

        /*
         * Creating an idmapped mount with the filesystem wide idmapping
         * doesn't make sense so block that. We don't allow mushy semantics.
         */
        if (kattr->mnt_userns == m->mnt_sb->s_user_ns)
                return -EINVAL;

        /*
         * Once a mount has been idmapped we don't allow it to change its
         * mapping. It makes things simpler and callers can just create
         * another bind-mount they can idmap if they want to.
         */
        if (is_idmapped_mnt(m))
                return -EPERM;

        /* The underlying filesystem doesn't support idmapped mounts yet. */
        if (!(m->mnt_sb->s_type->fs_flags & FS_ALLOW_IDMAP))
                return -EINVAL;

        /* The filesystem has turned off idmapped mounts. */
        if (m->mnt_sb->s_iflags & SB_I_NOIDMAP)
                return -EINVAL;

        /* We're not controlling the superblock. */
        if (!ns_capable(fs_userns, CAP_SYS_ADMIN))
                return -EPERM;

        /* Mount has already been visible in the filesystem hierarchy. */
        if (!is_anon_ns(mnt->mnt_ns))
                return -EINVAL;

        return 0;
}

/**
 * mnt_allow_writers() - check whether the attribute change allows writers
 * @kattr: the new mount attributes
 * @mnt: the mount to which @kattr will be applied
 *
 * Check whether thew new mount attributes in @kattr allow concurrent writers.
 *
 * Return: true if writers need to be held, false if not
 */
static inline bool mnt_allow_writers(const struct mount_kattr *kattr,
                                     const struct mount *mnt)
{
        return (!(kattr->attr_set & MNT_READONLY) ||
                (mnt->mnt.mnt_flags & MNT_READONLY)) &&
               !kattr->mnt_idmap;
}

static int mount_setattr_prepare(struct mount_kattr *kattr, struct mount *mnt)
{
        struct mount *m;
        int err;

        for (m = mnt; m; m = next_mnt(m, mnt)) {
                if (!can_change_locked_flags(m, recalc_flags(kattr, m))) {
                        err = -EPERM;
                        break;
                }

                err = can_idmap_mount(kattr, m);
                if (err)
                        break;

                if (!mnt_allow_writers(kattr, m)) {
                        err = mnt_hold_writers(m);
                        if (err)
                                break;
                }

                if (!kattr->recurse)
                        return 0;
        }

        if (err) {
                struct mount *p;

                /*
                 * If we had to call mnt_hold_writers() MNT_WRITE_HOLD will
                 * be set in @mnt_flags. The loop unsets MNT_WRITE_HOLD for all
                 * mounts and needs to take care to include the first mount.
                 */
                for (p = mnt; p; p = next_mnt(p, mnt)) {
                        /* If we had to hold writers unblock them. */
                        if (p->mnt.mnt_flags & MNT_WRITE_HOLD)
                                mnt_unhold_writers(p);

                        /*
                         * We're done once the first mount we changed got
                         * MNT_WRITE_HOLD unset.
                         */
                        if (p == m)
                                break;
                }
        }
        return err;
}

static void do_idmap_mount(const struct mount_kattr *kattr, struct mount *mnt)
{
        if (!kattr->mnt_idmap)
                return;

        /*
         * Pairs with smp_load_acquire() in mnt_idmap().
         *
         * Since we only allow a mount to change the idmapping once and
         * verified this in can_idmap_mount() we know that the mount has
         * @nop_mnt_idmap attached to it. So there's no need to drop any
         * references.
         */
        smp_store_release(&mnt->mnt.mnt_idmap, mnt_idmap_get(kattr->mnt_idmap));
}

static void mount_setattr_commit(struct mount_kattr *kattr, struct mount *mnt)
{
        struct mount *m;

        for (m = mnt; m; m = next_mnt(m, mnt)) {
                unsigned int flags;

                do_idmap_mount(kattr, m);
                flags = recalc_flags(kattr, m);
                WRITE_ONCE(m->mnt.mnt_flags, flags);

                /* If we had to hold writers unblock them. */
                if (m->mnt.mnt_flags & MNT_WRITE_HOLD)
                        mnt_unhold_writers(m);

                if (kattr->propagation)
                        change_mnt_propagation(m, kattr->propagation);
                if (!kattr->recurse)
                        break;
        }
        touch_mnt_namespace(mnt->mnt_ns);
}

static int do_mount_setattr(struct path *path, struct mount_kattr *kattr)
{
        struct mount *mnt = real_mount(path->mnt);
        int err = 0;

        if (!path_mounted(path))
                return -EINVAL;

        if (kattr->mnt_userns) {
                struct mnt_idmap *mnt_idmap;

                mnt_idmap = alloc_mnt_idmap(kattr->mnt_userns);
                if (IS_ERR(mnt_idmap))
                        return PTR_ERR(mnt_idmap);
                kattr->mnt_idmap = mnt_idmap;
        }

        if (kattr->propagation) {
                /*
                 * Only take namespace_lock() if we're actually changing
                 * propagation.
                 */
                namespace_lock();
                if (kattr->propagation == MS_SHARED) {
                        err = invent_group_ids(mnt, kattr->recurse);
                        if (err) {
                                namespace_unlock();
                                return err;
                        }
                }
        }

        err = -EINVAL;
        lock_mount_hash();

        /* Ensure that this isn't anything purely vfs internal. */
        if (!is_mounted(&mnt->mnt))
                goto out;

        /*
         * If this is an attached mount make sure it's located in the callers
         * mount namespace. If it's not don't let the caller interact with it.
         *
         * If this mount doesn't have a parent it's most often simply a
         * detached mount with an anonymous mount namespace. IOW, something
         * that's simply not attached yet. But there are apparently also users
         * that do change mount properties on the rootfs itself. That obviously
         * neither has a parent nor is it a detached mount so we cannot
         * unconditionally check for detached mounts.
         */
        if ((mnt_has_parent(mnt) || !is_anon_ns(mnt->mnt_ns)) && !check_mnt(mnt))
                goto out;

        /*
         * First, we get the mount tree in a shape where we can change mount
         * properties without failure. If we succeeded to do so we commit all
         * changes and if we failed we clean up.
         */
        err = mount_setattr_prepare(kattr, mnt);
        if (!err)
                mount_setattr_commit(kattr, mnt);

out:
        unlock_mount_hash();

        if (kattr->propagation) {
                if (err)
                        cleanup_group_ids(mnt, NULL);
                namespace_unlock();
        }

        return err;
}

static int build_mount_idmapped(const struct mount_attr *attr, size_t usize,
                                struct mount_kattr *kattr, unsigned int flags)
{
        struct ns_common *ns;
        struct user_namespace *mnt_userns;

        if (!((attr->attr_set | attr->attr_clr) & MOUNT_ATTR_IDMAP))
                return 0;

        /*
         * We currently do not support clearing an idmapped mount. If this ever
         * is a use-case we can revisit this but for now let's keep it simple
         * and not allow it.
         */
        if (attr->attr_clr & MOUNT_ATTR_IDMAP)
                return -EINVAL;

        if (attr->userns_fd > INT_MAX)
                return -EINVAL;

        CLASS(fd, f)(attr->userns_fd);
        if (fd_empty(f))
                return -EBADF;

        if (!proc_ns_file(fd_file(f)))
                return -EINVAL;

        ns = get_proc_ns(file_inode(fd_file(f)));
        if (ns->ops->type != CLONE_NEWUSER)
                return -EINVAL;

        /*
         * The initial idmapping cannot be used to create an idmapped
         * mount. We use the initial idmapping as an indicator of a mount
         * that is not idmapped. It can simply be passed into helpers that
         * are aware of idmapped mounts as a convenient shortcut. A user
         * can just create a dedicated identity mapping to achieve the same
         * result.
         */
        mnt_userns = container_of(ns, struct user_namespace, ns);
        if (mnt_userns == &init_user_ns)
                return -EPERM;

        /* We're not controlling the target namespace. */
        if (!ns_capable(mnt_userns, CAP_SYS_ADMIN))
                return -EPERM;

        kattr->mnt_userns = get_user_ns(mnt_userns);
        return 0;
}

static int build_mount_kattr(const struct mount_attr *attr, size_t usize,
                             struct mount_kattr *kattr, unsigned int flags)
{
        unsigned int lookup_flags = LOOKUP_AUTOMOUNT | LOOKUP_FOLLOW;

        if (flags & AT_NO_AUTOMOUNT)
                lookup_flags &= ~LOOKUP_AUTOMOUNT;
        if (flags & AT_SYMLINK_NOFOLLOW)
                lookup_flags &= ~LOOKUP_FOLLOW;
        if (flags & AT_EMPTY_PATH)
                lookup_flags |= LOOKUP_EMPTY;

        *kattr = (struct mount_kattr) {
                .lookup_flags        = lookup_flags,
                .recurse        = !!(flags & AT_RECURSIVE),
        };

        if (attr->propagation & ~MOUNT_SETATTR_PROPAGATION_FLAGS)
                return -EINVAL;
        if (hweight32(attr->propagation & MOUNT_SETATTR_PROPAGATION_FLAGS) > 1)
                return -EINVAL;
        kattr->propagation = attr->propagation;

        if ((attr->attr_set | attr->attr_clr) & ~MOUNT_SETATTR_VALID_FLAGS)
                return -EINVAL;

        kattr->attr_set = attr_flags_to_mnt_flags(attr->attr_set);
        kattr->attr_clr = attr_flags_to_mnt_flags(attr->attr_clr);

        /*
         * Since the MOUNT_ATTR_<atime> values are an enum, not a bitmap,
         * users wanting to transition to a different atime setting cannot
         * simply specify the atime setting in @attr_set, but must also
         * specify MOUNT_ATTR__ATIME in the @attr_clr field.
         * So ensure that MOUNT_ATTR__ATIME can't be partially set in
         * @attr_clr and that @attr_set can't have any atime bits set if
         * MOUNT_ATTR__ATIME isn't set in @attr_clr.
         */
        if (attr->attr_clr & MOUNT_ATTR__ATIME) {
                if ((attr->attr_clr & MOUNT_ATTR__ATIME) != MOUNT_ATTR__ATIME)
                        return -EINVAL;

                /*
                 * Clear all previous time settings as they are mutually
                 * exclusive.
                 */
                kattr->attr_clr |= MNT_RELATIME | MNT_NOATIME;
                switch (attr->attr_set & MOUNT_ATTR__ATIME) {
                case MOUNT_ATTR_RELATIME:
                        kattr->attr_set |= MNT_RELATIME;
                        break;
                case MOUNT_ATTR_NOATIME:
                        kattr->attr_set |= MNT_NOATIME;
                        break;
                case MOUNT_ATTR_STRICTATIME:
                        break;
                default:
                        return -EINVAL;
                }
        } else {
                if (attr->attr_set & MOUNT_ATTR__ATIME)
                        return -EINVAL;
        }

        return build_mount_idmapped(attr, usize, kattr, flags);
}

static void finish_mount_kattr(struct mount_kattr *kattr)
{
        put_user_ns(kattr->mnt_userns);
        kattr->mnt_userns = NULL;

        if (kattr->mnt_idmap)
                mnt_idmap_put(kattr->mnt_idmap);
}

SYSCALL_DEFINE5(mount_setattr, int, dfd, const char __user *, path,
                unsigned int, flags, struct mount_attr __user *, uattr,
                size_t, usize)
{
        int err;
        struct path target;
        struct mount_attr attr;
        struct mount_kattr kattr;

        BUILD_BUG_ON(sizeof(struct mount_attr) != MOUNT_ATTR_SIZE_VER0);

        if (flags & ~(AT_EMPTY_PATH |
                      AT_RECURSIVE |
                      AT_SYMLINK_NOFOLLOW |
                      AT_NO_AUTOMOUNT))
                return -EINVAL;

        if (unlikely(usize > PAGE_SIZE))
                return -E2BIG;
        if (unlikely(usize < MOUNT_ATTR_SIZE_VER0))
                return -EINVAL;

        if (!may_mount())
                return -EPERM;

        err = copy_struct_from_user(&attr, sizeof(attr), uattr, usize);
        if (err)
                return err;

        /* Don't bother walking through the mounts if this is a nop. */
        if (attr.attr_set == 0 &&
            attr.attr_clr == 0 &&
            attr.propagation == 0)
                return 0;

        err = build_mount_kattr(&attr, usize, &kattr, flags);
        if (err)
                return err;

        err = user_path_at(dfd, path, kattr.lookup_flags, &target);
        if (!err) {
                err = do_mount_setattr(&target, &kattr);
                path_put(&target);
        }
        finish_mount_kattr(&kattr);
        return err;
}

int show_path(struct seq_file *m, struct dentry *root)
{
        if (root->d_sb->s_op->show_path)
                return root->d_sb->s_op->show_path(m, root);

        seq_dentry(m, root, " \t\n\\");
        return 0;
}

static struct vfsmount *lookup_mnt_in_ns(u64 id, struct mnt_namespace *ns)
{
        struct mount *mnt = mnt_find_id_at(ns, id);

        if (!mnt || mnt->mnt_id_unique != id)
                return NULL;

        return &mnt->mnt;
}

struct kstatmount {
        struct statmount __user *buf;
        size_t bufsize;
        struct vfsmount *mnt;
        u64 mask;
        struct path root;
        struct statmount sm;
        struct seq_file seq;
};

static u64 mnt_to_attr_flags(struct vfsmount *mnt)
{
        unsigned int mnt_flags = READ_ONCE(mnt->mnt_flags);
        u64 attr_flags = 0;

        if (mnt_flags & MNT_READONLY)
                attr_flags |= MOUNT_ATTR_RDONLY;
        if (mnt_flags & MNT_NOSUID)
                attr_flags |= MOUNT_ATTR_NOSUID;
        if (mnt_flags & MNT_NODEV)
                attr_flags |= MOUNT_ATTR_NODEV;
        if (mnt_flags & MNT_NOEXEC)
                attr_flags |= MOUNT_ATTR_NOEXEC;
        if (mnt_flags & MNT_NODIRATIME)
                attr_flags |= MOUNT_ATTR_NODIRATIME;
        if (mnt_flags & MNT_NOSYMFOLLOW)
                attr_flags |= MOUNT_ATTR_NOSYMFOLLOW;

        if (mnt_flags & MNT_NOATIME)
                attr_flags |= MOUNT_ATTR_NOATIME;
        else if (mnt_flags & MNT_RELATIME)
                attr_flags |= MOUNT_ATTR_RELATIME;
        else
                attr_flags |= MOUNT_ATTR_STRICTATIME;

        if (is_idmapped_mnt(mnt))
                attr_flags |= MOUNT_ATTR_IDMAP;

        return attr_flags;
}

static u64 mnt_to_propagation_flags(struct mount *m)
{
        u64 propagation = 0;

        if (IS_MNT_SHARED(m))
                propagation |= MS_SHARED;
        if (IS_MNT_SLAVE(m))
                propagation |= MS_SLAVE;
        if (IS_MNT_UNBINDABLE(m))
                propagation |= MS_UNBINDABLE;
        if (!propagation)
                propagation |= MS_PRIVATE;

        return propagation;
}

static void statmount_sb_basic(struct kstatmount *s)
{
        struct super_block *sb = s->mnt->mnt_sb;

        s->sm.mask |= STATMOUNT_SB_BASIC;
        s->sm.sb_dev_major = MAJOR(sb->s_dev);
        s->sm.sb_dev_minor = MINOR(sb->s_dev);
        s->sm.sb_magic = sb->s_magic;
        s->sm.sb_flags = sb->s_flags & (SB_RDONLY|SB_SYNCHRONOUS|SB_DIRSYNC|SB_LAZYTIME);
}

static void statmount_mnt_basic(struct kstatmount *s)
{
        struct mount *m = real_mount(s->mnt);

        s->sm.mask |= STATMOUNT_MNT_BASIC;
        s->sm.mnt_id = m->mnt_id_unique;
        s->sm.mnt_parent_id = m->mnt_parent->mnt_id_unique;
        s->sm.mnt_id_old = m->mnt_id;
        s->sm.mnt_parent_id_old = m->mnt_parent->mnt_id;
        s->sm.mnt_attr = mnt_to_attr_flags(&m->mnt);
        s->sm.mnt_propagation = mnt_to_propagation_flags(m);
        s->sm.mnt_peer_group = IS_MNT_SHARED(m) ? m->mnt_group_id : 0;
        s->sm.mnt_master = IS_MNT_SLAVE(m) ? m->mnt_master->mnt_group_id : 0;
}

static void statmount_propagate_from(struct kstatmount *s)
{
        struct mount *m = real_mount(s->mnt);

        s->sm.mask |= STATMOUNT_PROPAGATE_FROM;
        if (IS_MNT_SLAVE(m))
                s->sm.propagate_from = get_dominating_id(m, &current->fs->root);
}

static int statmount_mnt_root(struct kstatmount *s, struct seq_file *seq)
{
        int ret;
        size_t start = seq->count;

        ret = show_path(seq, s->mnt->mnt_root);
        if (ret)
                return ret;

        if (unlikely(seq_has_overflowed(seq)))
                return -EAGAIN;

        /*
         * Unescape the result. It would be better if supplied string was not
         * escaped in the first place, but that's a pretty invasive change.
         */
        seq->buf[seq->count] = '\0';
        seq->count = start;
        seq_commit(seq, string_unescape_inplace(seq->buf + start, UNESCAPE_OCTAL));
        return 0;
}

static int statmount_mnt_point(struct kstatmount *s, struct seq_file *seq)
{
        struct vfsmount *mnt = s->mnt;
        struct path mnt_path = { .dentry = mnt->mnt_root, .mnt = mnt };
        int err;

        err = seq_path_root(seq, &mnt_path, &s->root, "");
        return err == SEQ_SKIP ? 0 : err;
}

static int statmount_fs_type(struct kstatmount *s, struct seq_file *seq)
{
        struct super_block *sb = s->mnt->mnt_sb;

        seq_puts(seq, sb->s_type->name);
        return 0;
}

static void statmount_fs_subtype(struct kstatmount *s, struct seq_file *seq)
{
        struct super_block *sb = s->mnt->mnt_sb;

        if (sb->s_subtype)
                seq_puts(seq, sb->s_subtype);
}

static int statmount_sb_source(struct kstatmount *s, struct seq_file *seq)
{
        struct super_block *sb = s->mnt->mnt_sb;
        struct mount *r = real_mount(s->mnt);

        if (sb->s_op->show_devname) {
                size_t start = seq->count;
                int ret;

                ret = sb->s_op->show_devname(seq, s->mnt->mnt_root);
                if (ret)
                        return ret;

                if (unlikely(seq_has_overflowed(seq)))
                        return -EAGAIN;

                /* Unescape the result */
                seq->buf[seq->count] = '\0';
                seq->count = start;
                seq_commit(seq, string_unescape_inplace(seq->buf + start, UNESCAPE_OCTAL));
        } else if (r->mnt_devname) {
                seq_puts(seq, r->mnt_devname);
        }
        return 0;
}

static void statmount_mnt_ns_id(struct kstatmount *s, struct mnt_namespace *ns)
{
        s->sm.mask |= STATMOUNT_MNT_NS_ID;
        s->sm.mnt_ns_id = ns->seq;
}

static int statmount_mnt_opts(struct kstatmount *s, struct seq_file *seq)
{
        struct vfsmount *mnt = s->mnt;
        struct super_block *sb = mnt->mnt_sb;
        int err;

        if (sb->s_op->show_options) {
                size_t start = seq->count;

                err = sb->s_op->show_options(seq, mnt->mnt_root);
                if (err)
                        return err;

                if (unlikely(seq_has_overflowed(seq)))
                        return -EAGAIN;

                if (seq->count == start)
                        return 0;

                /* skip leading comma */
                memmove(seq->buf + start, seq->buf + start + 1,
                        seq->count - start - 1);
                seq->count--;
        }

        return 0;
}

static inline int statmount_opt_unescape(struct seq_file *seq, char *buf_start)
{
        char *buf_end, *opt_start, *opt_end;
        int count = 0;

        buf_end = seq->buf + seq->count;
        *buf_end = '\0';
        for (opt_start = buf_start + 1; opt_start < buf_end; opt_start = opt_end + 1) {
                opt_end = strchrnul(opt_start, ',');
                *opt_end = '\0';
                buf_start += string_unescape(opt_start, buf_start, 0, UNESCAPE_OCTAL) + 1;
                if (WARN_ON_ONCE(++count == INT_MAX))
                        return -EOVERFLOW;
        }
        seq->count = buf_start - 1 - seq->buf;
        return count;
}

static int statmount_opt_array(struct kstatmount *s, struct seq_file *seq)
{
        struct vfsmount *mnt = s->mnt;
        struct super_block *sb = mnt->mnt_sb;
        size_t start = seq->count;
        char *buf_start;
        int err;

        if (!sb->s_op->show_options)
                return 0;

        buf_start = seq->buf + start;
        err = sb->s_op->show_options(seq, mnt->mnt_root);
        if (err)
                return err;

        if (unlikely(seq_has_overflowed(seq)))
                return -EAGAIN;

        if (seq->count == start)
                return 0;

        err = statmount_opt_unescape(seq, buf_start);
        if (err < 0)
                return err;

        s->sm.opt_num = err;
        return 0;
}

static int statmount_opt_sec_array(struct kstatmount *s, struct seq_file *seq)
{
        struct vfsmount *mnt = s->mnt;
        struct super_block *sb = mnt->mnt_sb;
        size_t start = seq->count;
        char *buf_start;
        int err;

        buf_start = seq->buf + start;

        err = security_sb_show_options(seq, sb);
        if (!err)
                return err;

        if (unlikely(seq_has_overflowed(seq)))
                return -EAGAIN;

        if (seq->count == start)
                return 0;

        err = statmount_opt_unescape(seq, buf_start);
        if (err < 0)
                return err;

        s->sm.opt_sec_num = err;
        return 0;
}

static int statmount_string(struct kstatmount *s, u64 flag)
{
        int ret = 0;
        size_t kbufsize;
        struct seq_file *seq = &s->seq;
        struct statmount *sm = &s->sm;
        u32 start = seq->count;

        switch (flag) {
        case STATMOUNT_FS_TYPE:
                sm->fs_type = start;
                ret = statmount_fs_type(s, seq);
                break;
        case STATMOUNT_MNT_ROOT:
                sm->mnt_root = start;
                ret = statmount_mnt_root(s, seq);
                break;
        case STATMOUNT_MNT_POINT:
                sm->mnt_point = start;
                ret = statmount_mnt_point(s, seq);
                break;
        case STATMOUNT_MNT_OPTS:
                sm->mnt_opts = start;
                ret = statmount_mnt_opts(s, seq);
                break;
        case STATMOUNT_OPT_ARRAY:
                sm->opt_array = start;
                ret = statmount_opt_array(s, seq);
                break;
        case STATMOUNT_OPT_SEC_ARRAY:
                sm->opt_sec_array = start;
                ret = statmount_opt_sec_array(s, seq);
                break;
        case STATMOUNT_FS_SUBTYPE:
                sm->fs_subtype = start;
                statmount_fs_subtype(s, seq);
                break;
        case STATMOUNT_SB_SOURCE:
                sm->sb_source = start;
                ret = statmount_sb_source(s, seq);
                break;
        default:
                WARN_ON_ONCE(true);
                return -EINVAL;
        }

        /*
         * If nothing was emitted, return to avoid setting the flag
         * and terminating the buffer.
         */
        if (seq->count == start)
                return ret;
        if (unlikely(check_add_overflow(sizeof(*sm), seq->count, &kbufsize)))
                return -EOVERFLOW;
        if (kbufsize >= s->bufsize)
                return -EOVERFLOW;

        /* signal a retry */
        if (unlikely(seq_has_overflowed(seq)))
                return -EAGAIN;

        if (ret)
                return ret;

        seq->buf[seq->count++] = '\0';
        sm->mask |= flag;
        return 0;
}

static int copy_statmount_to_user(struct kstatmount *s)
{
        struct statmount *sm = &s->sm;
        struct seq_file *seq = &s->seq;
        char __user *str = ((char __user *)s->buf) + sizeof(*sm);
        size_t copysize = min_t(size_t, s->bufsize, sizeof(*sm));

        if (seq->count && copy_to_user(str, seq->buf, seq->count))
                return -EFAULT;

        /* Return the number of bytes copied to the buffer */
        sm->size = copysize + seq->count;
        if (copy_to_user(s->buf, sm, copysize))
                return -EFAULT;

        return 0;
}

static struct mount *listmnt_next(struct mount *curr, bool reverse)
{
        struct rb_node *node;

        if (reverse)
                node = rb_prev(&curr->mnt_node);
        else
                node = rb_next(&curr->mnt_node);

        return node_to_mount(node);
}

static int grab_requested_root(struct mnt_namespace *ns, struct path *root)
{
        struct mount *first, *child;

        rwsem_assert_held(&namespace_sem);

        /* We're looking at our own ns, just use get_fs_root. */
        if (ns == current->nsproxy->mnt_ns) {
                get_fs_root(current->fs, root);
                return 0;
        }

        /*
         * We have to find the first mount in our ns and use that, however it
         * may not exist, so handle that properly.
         */
        if (RB_EMPTY_ROOT(&ns->mounts))
                return -ENOENT;

        first = child = ns->root;
        for (;;) {
                child = listmnt_next(child, false);
                if (!child)
                        return -ENOENT;
                if (child->mnt_parent == first)
                        break;
        }

        root->mnt = mntget(&child->mnt);
        root->dentry = dget(root->mnt->mnt_root);
        return 0;
}

static int do_statmount(struct kstatmount *s, u64 mnt_id, u64 mnt_ns_id,
                        struct mnt_namespace *ns)
{
        struct path root __free(path_put) = {};
        struct mount *m;
        int err;

        /* Has the namespace already been emptied? */
        if (mnt_ns_id && RB_EMPTY_ROOT(&ns->mounts))
                return -ENOENT;

        s->mnt = lookup_mnt_in_ns(mnt_id, ns);
        if (!s->mnt)
                return -ENOENT;

        err = grab_requested_root(ns, &root);
        if (err)
                return err;

        /*
         * Don't trigger audit denials. We just want to determine what
         * mounts to show users.
         */
        m = real_mount(s->mnt);
        if (!is_path_reachable(m, m->mnt.mnt_root, &root) &&
            !ns_capable_noaudit(ns->user_ns, CAP_SYS_ADMIN))
                return -EPERM;

        err = security_sb_statfs(s->mnt->mnt_root);
        if (err)
                return err;

        s->root = root;
        if (s->mask & STATMOUNT_SB_BASIC)
                statmount_sb_basic(s);

        if (s->mask & STATMOUNT_MNT_BASIC)
                statmount_mnt_basic(s);

        if (s->mask & STATMOUNT_PROPAGATE_FROM)
                statmount_propagate_from(s);

        if (s->mask & STATMOUNT_FS_TYPE)
                err = statmount_string(s, STATMOUNT_FS_TYPE);

        if (!err && s->mask & STATMOUNT_MNT_ROOT)
                err = statmount_string(s, STATMOUNT_MNT_ROOT);

        if (!err && s->mask & STATMOUNT_MNT_POINT)
                err = statmount_string(s, STATMOUNT_MNT_POINT);

        if (!err && s->mask & STATMOUNT_MNT_OPTS)
                err = statmount_string(s, STATMOUNT_MNT_OPTS);

        if (!err && s->mask & STATMOUNT_OPT_ARRAY)
                err = statmount_string(s, STATMOUNT_OPT_ARRAY);

        if (!err && s->mask & STATMOUNT_OPT_SEC_ARRAY)
                err = statmount_string(s, STATMOUNT_OPT_SEC_ARRAY);

        if (!err && s->mask & STATMOUNT_FS_SUBTYPE)
                err = statmount_string(s, STATMOUNT_FS_SUBTYPE);

        if (!err && s->mask & STATMOUNT_SB_SOURCE)
                err = statmount_string(s, STATMOUNT_SB_SOURCE);

        if (!err && s->mask & STATMOUNT_MNT_NS_ID)
                statmount_mnt_ns_id(s, ns);

        if (err)
                return err;

        return 0;
}

static inline bool retry_statmount(const long ret, size_t *seq_size)
{
        if (likely(ret != -EAGAIN))
                return false;
        if (unlikely(check_mul_overflow(*seq_size, 2, seq_size)))
                return false;
        if (unlikely(*seq_size > MAX_RW_COUNT))
                return false;
        return true;
}

#define STATMOUNT_STRING_REQ (STATMOUNT_MNT_ROOT | STATMOUNT_MNT_POINT | \
                              STATMOUNT_FS_TYPE | STATMOUNT_MNT_OPTS | \
                              STATMOUNT_FS_SUBTYPE | STATMOUNT_SB_SOURCE | \
                              STATMOUNT_OPT_ARRAY | STATMOUNT_OPT_SEC_ARRAY)

static int prepare_kstatmount(struct kstatmount *ks, struct mnt_id_req *kreq,
                              struct statmount __user *buf, size_t bufsize,
                              size_t seq_size)
{
        if (!access_ok(buf, bufsize))
                return -EFAULT;

        memset(ks, 0, sizeof(*ks));
        ks->mask = kreq->param;
        ks->buf = buf;
        ks->bufsize = bufsize;

        if (ks->mask & STATMOUNT_STRING_REQ) {
                if (bufsize == sizeof(ks->sm))
                        return -EOVERFLOW;

                ks->seq.buf = kvmalloc(seq_size, GFP_KERNEL_ACCOUNT);
                if (!ks->seq.buf)
                        return -ENOMEM;

                ks->seq.size = seq_size;
        }

        return 0;
}

static int copy_mnt_id_req(const struct mnt_id_req __user *req,
                           struct mnt_id_req *kreq)
{
        int ret;
        size_t usize;

        BUILD_BUG_ON(sizeof(struct mnt_id_req) != MNT_ID_REQ_SIZE_VER1);

        ret = get_user(usize, &req->size);
        if (ret)
                return -EFAULT;
        if (unlikely(usize > PAGE_SIZE))
                return -E2BIG;
        if (unlikely(usize < MNT_ID_REQ_SIZE_VER0))
                return -EINVAL;
        memset(kreq, 0, sizeof(*kreq));
        ret = copy_struct_from_user(kreq, sizeof(*kreq), req, usize);
        if (ret)
                return ret;
        if (kreq->spare != 0)
                return -EINVAL;
        /* The first valid unique mount id is MNT_UNIQUE_ID_OFFSET + 1. */
        if (kreq->mnt_id <= MNT_UNIQUE_ID_OFFSET)
                return -EINVAL;
        return 0;
}

/*
 * If the user requested a specific mount namespace id, look that up and return
 * that, or if not simply grab a passive reference on our mount namespace and
 * return that.
 */
static struct mnt_namespace *grab_requested_mnt_ns(const struct mnt_id_req *kreq)
{
        struct mnt_namespace *mnt_ns;

        if (kreq->mnt_ns_id && kreq->spare)
                return ERR_PTR(-EINVAL);

        if (kreq->mnt_ns_id)
                return lookup_mnt_ns(kreq->mnt_ns_id);

        if (kreq->spare) {
                struct ns_common *ns;

                CLASS(fd, f)(kreq->spare);
                if (fd_empty(f))
                        return ERR_PTR(-EBADF);

                if (!proc_ns_file(fd_file(f)))
                        return ERR_PTR(-EINVAL);

                ns = get_proc_ns(file_inode(fd_file(f)));
                if (ns->ops->type != CLONE_NEWNS)
                        return ERR_PTR(-EINVAL);

                mnt_ns = to_mnt_ns(ns);
        } else {
                mnt_ns = current->nsproxy->mnt_ns;
        }

        refcount_inc(&mnt_ns->passive);
        return mnt_ns;
}

SYSCALL_DEFINE4(statmount, const struct mnt_id_req __user *, req,
                struct statmount __user *, buf, size_t, bufsize,
                unsigned int, flags)
{
        struct mnt_namespace *ns __free(mnt_ns_release) = NULL;
        struct kstatmount *ks __free(kfree) = NULL;
        struct mnt_id_req kreq;
        /* We currently support retrieval of 3 strings. */
        size_t seq_size = 3 * PATH_MAX;
        int ret;

        if (flags)
                return -EINVAL;

        ret = copy_mnt_id_req(req, &kreq);
        if (ret)
                return ret;

        ns = grab_requested_mnt_ns(&kreq);
        if (!ns)
                return -ENOENT;

        if (kreq.mnt_ns_id && (ns != current->nsproxy->mnt_ns) &&
            !ns_capable_noaudit(ns->user_ns, CAP_SYS_ADMIN))
                return -ENOENT;

        ks = kmalloc(sizeof(*ks), GFP_KERNEL_ACCOUNT);
        if (!ks)
                return -ENOMEM;

retry:
        ret = prepare_kstatmount(ks, &kreq, buf, bufsize, seq_size);
        if (ret)
                return ret;

        scoped_guard(rwsem_read, &namespace_sem)
                ret = do_statmount(ks, kreq.mnt_id, kreq.mnt_ns_id, ns);

        if (!ret)
                ret = copy_statmount_to_user(ks);
        kvfree(ks->seq.buf);
        if (retry_statmount(ret, &seq_size))
                goto retry;
        return ret;
}

static ssize_t do_listmount(struct mnt_namespace *ns, u64 mnt_parent_id,
                            u64 last_mnt_id, u64 *mnt_ids, size_t nr_mnt_ids,
                            bool reverse)
{
        struct path root __free(path_put) = {};
        struct path orig;
        struct mount *r, *first;
        ssize_t ret;

        rwsem_assert_held(&namespace_sem);

        ret = grab_requested_root(ns, &root);
        if (ret)
                return ret;

        if (mnt_parent_id == LSMT_ROOT) {
                orig = root;
        } else {
                orig.mnt = lookup_mnt_in_ns(mnt_parent_id, ns);
                if (!orig.mnt)
                        return -ENOENT;
                orig.dentry = orig.mnt->mnt_root;
        }

        /*
         * Don't trigger audit denials. We just want to determine what
         * mounts to show users.
         */
        if (!is_path_reachable(real_mount(orig.mnt), orig.dentry, &root) &&
            !ns_capable_noaudit(ns->user_ns, CAP_SYS_ADMIN))
                return -EPERM;

        ret = security_sb_statfs(orig.dentry);
        if (ret)
                return ret;

        if (!last_mnt_id) {
                if (reverse)
                        first = node_to_mount(rb_last(&ns->mounts));
                else
                        first = node_to_mount(rb_first(&ns->mounts));
        } else {
                if (reverse)
                        first = mnt_find_id_at_reverse(ns, last_mnt_id - 1);
                else
                        first = mnt_find_id_at(ns, last_mnt_id + 1);
        }

        for (ret = 0, r = first; r && nr_mnt_ids; r = listmnt_next(r, reverse)) {
                if (r->mnt_id_unique == mnt_parent_id)
                        continue;
                if (!is_path_reachable(r, r->mnt.mnt_root, &orig))
                        continue;
                *mnt_ids = r->mnt_id_unique;
                mnt_ids++;
                nr_mnt_ids--;
                ret++;
        }
        return ret;
}

SYSCALL_DEFINE4(listmount, const struct mnt_id_req __user *, req,
                u64 __user *, mnt_ids, size_t, nr_mnt_ids, unsigned int, flags)
{
        u64 *kmnt_ids __free(kvfree) = NULL;
        const size_t maxcount = 1000000;
        struct mnt_namespace *ns __free(mnt_ns_release) = NULL;
        struct mnt_id_req kreq;
        u64 last_mnt_id;
        ssize_t ret;

        if (flags & ~LISTMOUNT_REVERSE)
                return -EINVAL;

        /*
         * If the mount namespace really has more than 1 million mounts the
         * caller must iterate over the mount namespace (and reconsider their
         * system design...).
         */
        if (unlikely(nr_mnt_ids > maxcount))
                return -EOVERFLOW;

        if (!access_ok(mnt_ids, nr_mnt_ids * sizeof(*mnt_ids)))
                return -EFAULT;

        ret = copy_mnt_id_req(req, &kreq);
        if (ret)
                return ret;

        last_mnt_id = kreq.param;
        /* The first valid unique mount id is MNT_UNIQUE_ID_OFFSET + 1. */
        if (last_mnt_id != 0 && last_mnt_id <= MNT_UNIQUE_ID_OFFSET)
                return -EINVAL;

        kmnt_ids = kvmalloc_array(nr_mnt_ids, sizeof(*kmnt_ids),
                                  GFP_KERNEL_ACCOUNT);
        if (!kmnt_ids)
                return -ENOMEM;

        ns = grab_requested_mnt_ns(&kreq);
        if (!ns)
                return -ENOENT;

        if (kreq.mnt_ns_id && (ns != current->nsproxy->mnt_ns) &&
            !ns_capable_noaudit(ns->user_ns, CAP_SYS_ADMIN))
                return -ENOENT;

        scoped_guard(rwsem_read, &namespace_sem)
                ret = do_listmount(ns, kreq.mnt_id, last_mnt_id, kmnt_ids,
                                   nr_mnt_ids, (flags & LISTMOUNT_REVERSE));
        if (ret <= 0)
                return ret;

        if (copy_to_user(mnt_ids, kmnt_ids, ret * sizeof(*mnt_ids)))
                return -EFAULT;

        return ret;
}

static void __init init_mount_tree(void)
{
        struct vfsmount *mnt;
        struct mount *m;
        struct mnt_namespace *ns;
        struct path root;

        mnt = vfs_kern_mount(&rootfs_fs_type, 0, "rootfs", NULL);
        if (IS_ERR(mnt))
                panic("Can't create rootfs");

        ns = alloc_mnt_ns(&init_user_ns, false);
        if (IS_ERR(ns))
                panic("Can't allocate initial namespace");
        m = real_mount(mnt);
        ns->root = m;
        ns->nr_mounts = 1;
        mnt_add_to_ns(ns, m);
        init_task.nsproxy->mnt_ns = ns;
        get_mnt_ns(ns);

        root.mnt = mnt;
        root.dentry = mnt->mnt_root;
        mnt->mnt_flags |= MNT_LOCKED;

        set_fs_pwd(current->fs, &root);
        set_fs_root(current->fs, &root);

        mnt_ns_tree_add(ns);
}

void __init mnt_init(void)
{
        int err;

        mnt_cache = kmem_cache_create("mnt_cache", sizeof(struct mount),
                        0, SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_ACCOUNT, NULL);

        mount_hashtable = alloc_large_system_hash("Mount-cache",
                                sizeof(struct hlist_head),
                                mhash_entries, 19,
                                HASH_ZERO,
                                &m_hash_shift, &m_hash_mask, 0, 0);
        mountpoint_hashtable = alloc_large_system_hash("Mountpoint-cache",
                                sizeof(struct hlist_head),
                                mphash_entries, 19,
                                HASH_ZERO,
                                &mp_hash_shift, &mp_hash_mask, 0, 0);

        if (!mount_hashtable || !mountpoint_hashtable)
                panic("Failed to allocate mount hash table\n");

        kernfs_init();

        err = sysfs_init();
        if (err)
                printk(KERN_WARNING "%s: sysfs_init error: %d\n",
                        __func__, err);
        fs_kobj = kobject_create_and_add("fs", NULL);
        if (!fs_kobj)
                printk(KERN_WARNING "%s: kobj create error\n", __func__);
        shmem_init();
        init_rootfs();
        init_mount_tree();
}

void put_mnt_ns(struct mnt_namespace *ns)
{
        if (!refcount_dec_and_test(&ns->ns.count))
                return;
        drop_collected_mounts(&ns->root->mnt);
        free_mnt_ns(ns);
}

struct vfsmount *kern_mount(struct file_system_type *type)
{
        struct vfsmount *mnt;
        mnt = vfs_kern_mount(type, SB_KERNMOUNT, type->name, NULL);
        if (!IS_ERR(mnt)) {
                /*
                 * it is a longterm mount, don't release mnt until
                 * we unmount before file sys is unregistered
                */
                real_mount(mnt)->mnt_ns = MNT_NS_INTERNAL;
        }
        return mnt;
}
EXPORT_SYMBOL_GPL(kern_mount);

void kern_unmount(struct vfsmount *mnt)
{
        /* release long term mount so mount point can be released */
        if (!IS_ERR(mnt)) {
                mnt_make_shortterm(mnt);
                synchronize_rcu();        /* yecchhh... */
                mntput(mnt);
        }
}
EXPORT_SYMBOL(kern_unmount);

void kern_unmount_array(struct vfsmount *mnt[], unsigned int num)
{
        unsigned int i;

        for (i = 0; i < num; i++)
                mnt_make_shortterm(mnt[i]);
        synchronize_rcu_expedited();
        for (i = 0; i < num; i++)
                mntput(mnt[i]);
}
EXPORT_SYMBOL(kern_unmount_array);

bool our_mnt(struct vfsmount *mnt)
{
        return check_mnt(real_mount(mnt));
}

bool current_chrooted(void)
{
        /* Does the current process have a non-standard root */
        struct path ns_root;
        struct path fs_root;
        bool chrooted;

        /* Find the namespace root */
        ns_root.mnt = &current->nsproxy->mnt_ns->root->mnt;
        ns_root.dentry = ns_root.mnt->mnt_root;
        path_get(&ns_root);
        while (d_mountpoint(ns_root.dentry) && follow_down_one(&ns_root))
                ;

        get_fs_root(current->fs, &fs_root);

        chrooted = !path_equal(&fs_root, &ns_root);

        path_put(&fs_root);
        path_put(&ns_root);

        return chrooted;
}

static bool mnt_already_visible(struct mnt_namespace *ns,
                                const struct super_block *sb,
                                int *new_mnt_flags)
{
        int new_flags = *new_mnt_flags;
        struct mount *mnt, *n;
        bool visible = false;

        down_read(&namespace_sem);
        rbtree_postorder_for_each_entry_safe(mnt, n, &ns->mounts, mnt_node) {
                struct mount *child;
                int mnt_flags;

                if (mnt->mnt.mnt_sb->s_type != sb->s_type)
                        continue;

                /* This mount is not fully visible if it's root directory
                 * is not the root directory of the filesystem.
                 */
                if (mnt->mnt.mnt_root != mnt->mnt.mnt_sb->s_root)
                        continue;

                /* A local view of the mount flags */
                mnt_flags = mnt->mnt.mnt_flags;

                /* Don't miss readonly hidden in the superblock flags */
                if (sb_rdonly(mnt->mnt.mnt_sb))
                        mnt_flags |= MNT_LOCK_READONLY;

                /* Verify the mount flags are equal to or more permissive
                 * than the proposed new mount.
                 */
                if ((mnt_flags & MNT_LOCK_READONLY) &&
                    !(new_flags & MNT_READONLY))
                        continue;
                if ((mnt_flags & MNT_LOCK_ATIME) &&
                    ((mnt_flags & MNT_ATIME_MASK) != (new_flags & MNT_ATIME_MASK)))
                        continue;

                /* This mount is not fully visible if there are any
                 * locked child mounts that cover anything except for
                 * empty directories.
                 */
                list_for_each_entry(child, &mnt->mnt_mounts, mnt_child) {
                        struct inode *inode = child->mnt_mountpoint->d_inode;
                        /* Only worry about locked mounts */
                        if (!(child->mnt.mnt_flags & MNT_LOCKED))
                                continue;
                        /* Is the directory permanently empty? */
                        if (!is_empty_dir_inode(inode))
                                goto next;
                }
                /* Preserve the locked attributes */
                *new_mnt_flags |= mnt_flags & (MNT_LOCK_READONLY | \
                                               MNT_LOCK_ATIME);
                visible = true;
                goto found;
        next:        ;
        }
found:
        up_read(&namespace_sem);
        return visible;
}

static bool mount_too_revealing(const struct super_block *sb, int *new_mnt_flags)
{
        const unsigned long required_iflags = SB_I_NOEXEC | SB_I_NODEV;
        struct mnt_namespace *ns = current->nsproxy->mnt_ns;
        unsigned long s_iflags;

        if (ns->user_ns == &init_user_ns)
                return false;

        /* Can this filesystem be too revealing? */
        s_iflags = sb->s_iflags;
        if (!(s_iflags & SB_I_USERNS_VISIBLE))
                return false;

        if ((s_iflags & required_iflags) != required_iflags) {
                WARN_ONCE(1, "Expected s_iflags to contain 0x%lx\n",
                          required_iflags);
                return true;
        }

        return !mnt_already_visible(ns, sb, new_mnt_flags);
}

bool mnt_may_suid(struct vfsmount *mnt)
{
        /*
         * Foreign mounts (accessed via fchdir or through /proc
         * symlinks) are always treated as if they are nosuid.  This
         * prevents namespaces from trusting potentially unsafe
         * suid/sgid bits, file caps, or security labels that originate
         * in other namespaces.
         */
        return !(mnt->mnt_flags & MNT_NOSUID) && check_mnt(real_mount(mnt)) &&
               current_in_userns(mnt->mnt_sb->s_user_ns);
}

static struct ns_common *mntns_get(struct task_struct *task)
{
        struct ns_common *ns = NULL;
        struct nsproxy *nsproxy;

        task_lock(task);
        nsproxy = task->nsproxy;
        if (nsproxy) {
                ns = &nsproxy->mnt_ns->ns;
                get_mnt_ns(to_mnt_ns(ns));
        }
        task_unlock(task);

        return ns;
}

static void mntns_put(struct ns_common *ns)
{
        put_mnt_ns(to_mnt_ns(ns));
}

static int mntns_install(struct nsset *nsset, struct ns_common *ns)
{
        struct nsproxy *nsproxy = nsset->nsproxy;
        struct fs_struct *fs = nsset->fs;
        struct mnt_namespace *mnt_ns = to_mnt_ns(ns), *old_mnt_ns;
        struct user_namespace *user_ns = nsset->cred->user_ns;
        struct path root;
        int err;

        if (!ns_capable(mnt_ns->user_ns, CAP_SYS_ADMIN) ||
            !ns_capable(user_ns, CAP_SYS_CHROOT) ||
            !ns_capable(user_ns, CAP_SYS_ADMIN))
                return -EPERM;

        if (is_anon_ns(mnt_ns))
                return -EINVAL;

        if (fs->users != 1)
                return -EINVAL;

        get_mnt_ns(mnt_ns);
        old_mnt_ns = nsproxy->mnt_ns;
        nsproxy->mnt_ns = mnt_ns;

        /* Find the root */
        err = vfs_path_lookup(mnt_ns->root->mnt.mnt_root, &mnt_ns->root->mnt,
                                "/", LOOKUP_DOWN, &root);
        if (err) {
                /* revert to old namespace */
                nsproxy->mnt_ns = old_mnt_ns;
                put_mnt_ns(mnt_ns);
                return err;
        }

        put_mnt_ns(old_mnt_ns);

        /* Update the pwd and root */
        set_fs_pwd(fs, &root);
        set_fs_root(fs, &root);

        path_put(&root);
        return 0;
}

static struct user_namespace *mntns_owner(struct ns_common *ns)
{
        return to_mnt_ns(ns)->user_ns;
}

const struct proc_ns_operations mntns_operations = {
        .name                = "mnt",
        .type                = CLONE_NEWNS,
        .get                = mntns_get,
        .put                = mntns_put,
        .install        = mntns_install,
        .owner                = mntns_owner,
};

#ifdef CONFIG_SYSCTL
static struct ctl_table fs_namespace_sysctls[] = {
        {
                .procname        = "mount-max",
                .data                = &sysctl_mount_max,
                .maxlen                = sizeof(unsigned int),
                .mode                = 0644,
                .proc_handler        = proc_dointvec_minmax,
                .extra1                = SYSCTL_ONE,
        },
};

static int __init init_fs_namespace_sysctls(void)
{
        register_sysctl_init("fs", fs_namespace_sysctls);
        return 0;
}
fs_initcall(init_fs_namespace_sysctls);

#endif /* CONFIG_SYSCTL */














   11 










   11 
















































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
// SPDX-License-Identifier: GPL-2.0
#include <linux/cache.h>
#include <linux/sched.h>
#include <linux/slab.h>
#include <linux/pid_namespace.h>
#include "internal.h"

/*
 * /proc/thread_self:
 */
static const char *proc_thread_self_get_link(struct dentry *dentry,
                                             struct inode *inode,
                                             struct delayed_call *done)
{
        struct pid_namespace *ns = proc_pid_ns(inode->i_sb);
        pid_t tgid = task_tgid_nr_ns(current, ns);
        pid_t pid = task_pid_nr_ns(current, ns);
        char *name;

        if (!pid)
                return ERR_PTR(-ENOENT);
        name = kmalloc(10 + 6 + 10 + 1, dentry ? GFP_KERNEL : GFP_ATOMIC);
        if (unlikely(!name))
                return dentry ? ERR_PTR(-ENOMEM) : ERR_PTR(-ECHILD);
        sprintf(name, "%u/task/%u", tgid, pid);
        set_delayed_call(done, kfree_link, name);
        return name;
}

static const struct inode_operations proc_thread_self_inode_operations = {
        .get_link        = proc_thread_self_get_link,
};

static unsigned thread_self_inum __ro_after_init;

int proc_setup_thread_self(struct super_block *s)
{
        struct inode *root_inode = d_inode(s->s_root);
        struct proc_fs_info *fs_info = proc_sb_info(s);
        struct dentry *thread_self;
        int ret = -ENOMEM;

        inode_lock(root_inode);
        thread_self = d_alloc_name(s->s_root, "thread-self");
        if (thread_self) {
                struct inode *inode = new_inode(s);
                if (inode) {
                        inode->i_ino = thread_self_inum;
                        simple_inode_init_ts(inode);
                        inode->i_mode = S_IFLNK | S_IRWXUGO;
                        inode->i_uid = GLOBAL_ROOT_UID;
                        inode->i_gid = GLOBAL_ROOT_GID;
                        inode->i_op = &proc_thread_self_inode_operations;
                        d_add(thread_self, inode);
                        ret = 0;
                } else {
                        dput(thread_self);
                }
        }
        inode_unlock(root_inode);

        if (ret)
                pr_err("proc_fill_super: can't allocate /proc/thread-self\n");
        else
                fs_info->proc_thread_self = thread_self;

        return ret;
}

void __init proc_thread_self_init(void)
{
        proc_alloc_inum(&thread_self_inum);
}














































   49 






   37 
   10 

   46 
    2 
   48 
    1 
   44 


















   67 






   29 





   31 




    3 
    2 

    1 





   19 




    3 
    9 
    2 

    3 


    3 
    3 

    3 





    1 



    1 
    1 










   17 









    5 

    9 








    1 
    8 









   23 
    6 





   31 













   19 





   19 
    3 



    1 






    1 
    4 















    2 
   13 











   93 








   71 


   24 


   56 


   46 
   33 
    6 









    1 



    8 
    1 



   22 






    1 




    1 



   15 





    4 






    1 







    2 



    5 




    2 
    7 









    1 













    1 











    1 



    1 
    8 








    1 






    4 








    2 





    2 






    2 






    2 










    4 
    4 


   11 





   11 






    3 














    2 
   28 






   57 




   34 
   25 









    6 
    5 





    5 





    3 


    1 


    1 
    1 

    2 
    1 







   27 






   25 
    1 



    6 

   17 
   20 
    4 
















































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
// SPDX-License-Identifier: GPL-2.0
/*
 * INET                An implementation of the TCP/IP protocol suite for the LINUX
 *                operating system.  INET is implemented using the  BSD Socket
 *                interface as the means of communication with the user level.
 *
 *                The options processing module for ip.c
 *
 * Authors:        A.N.Kuznetsov
 *
 */

#define pr_fmt(fmt) "IPv4: " fmt

#include <linux/capability.h>
#include <linux/module.h>
#include <linux/slab.h>
#include <linux/types.h>
#include <linux/uaccess.h>
#include <linux/unaligned.h>
#include <linux/skbuff.h>
#include <linux/ip.h>
#include <linux/icmp.h>
#include <linux/netdevice.h>
#include <linux/rtnetlink.h>
#include <net/sock.h>
#include <net/ip.h>
#include <net/icmp.h>
#include <net/route.h>
#include <net/cipso_ipv4.h>
#include <net/ip_fib.h>

/*
 * Write options to IP header, record destination address to
 * source route option, address of outgoing interface
 * (we should already know it, so that this  function is allowed be
 * called only after routing decision) and timestamp,
 * if we originate this datagram.
 *
 * daddr is real destination address, next hop is recorded in IP header.
 * saddr is address of outgoing interface.
 */

void ip_options_build(struct sk_buff *skb, struct ip_options *opt,
                      __be32 daddr, struct rtable *rt)
{
        unsigned char *iph = skb_network_header(skb);

        memcpy(&(IPCB(skb)->opt), opt, sizeof(struct ip_options));
        memcpy(iph + sizeof(struct iphdr), opt->__data, opt->optlen);
        opt = &(IPCB(skb)->opt);

        if (opt->srr)
                memcpy(iph + opt->srr + iph[opt->srr + 1] - 4, &daddr, 4);

        if (opt->rr_needaddr)
                ip_rt_get_source(iph + opt->rr + iph[opt->rr + 2] - 5, skb, rt);
        if (opt->ts_needaddr)
                ip_rt_get_source(iph + opt->ts + iph[opt->ts + 2] - 9, skb, rt);
        if (opt->ts_needtime) {
                __be32 midtime;

                midtime = inet_current_timestamp();
                memcpy(iph + opt->ts + iph[opt->ts + 2] - 5, &midtime, 4);
        }
}

/*
 * Provided (sopt, skb) points to received options,
 * build in dopt compiled option set appropriate for answering.
 * i.e. invert SRR option, copy anothers,
 * and grab room in RR/TS options.
 *
 * NOTE: dopt cannot point to skb.
 */

int __ip_options_echo(struct net *net, struct ip_options *dopt,
                      struct sk_buff *skb, const struct ip_options *sopt)
{
        unsigned char *sptr, *dptr;
        int soffset, doffset;
        int        optlen;

        memset(dopt, 0, sizeof(struct ip_options));

        if (sopt->optlen == 0)
                return 0;

        sptr = skb_network_header(skb);
        dptr = dopt->__data;

        if (sopt->rr) {
                optlen  = sptr[sopt->rr+1];
                soffset = sptr[sopt->rr+2];
                dopt->rr = dopt->optlen + sizeof(struct iphdr);
                memcpy(dptr, sptr+sopt->rr, optlen);
                if (sopt->rr_needaddr && soffset <= optlen) {
                        if (soffset + 3 > optlen)
                                return -EINVAL;
                        dptr[2] = soffset + 4;
                        dopt->rr_needaddr = 1;
                }
                dptr += optlen;
                dopt->optlen += optlen;
        }
        if (sopt->ts) {
                optlen = sptr[sopt->ts+1];
                soffset = sptr[sopt->ts+2];
                dopt->ts = dopt->optlen + sizeof(struct iphdr);
                memcpy(dptr, sptr+sopt->ts, optlen);
                if (soffset <= optlen) {
                        if (sopt->ts_needaddr) {
                                if (soffset + 3 > optlen)
                                        return -EINVAL;
                                dopt->ts_needaddr = 1;
                                soffset += 4;
                        }
                        if (sopt->ts_needtime) {
                                if (soffset + 3 > optlen)
                                        return -EINVAL;
                                if ((dptr[3]&0xF) != IPOPT_TS_PRESPEC) {
                                        dopt->ts_needtime = 1;
                                        soffset += 4;
                                } else {
                                        dopt->ts_needtime = 0;

                                        if (soffset + 7 <= optlen) {
                                                __be32 addr;

                                                memcpy(&addr, dptr+soffset-1, 4);
                                                if (inet_addr_type(net, addr) != RTN_UNICAST) {
                                                        dopt->ts_needtime = 1;
                                                        soffset += 8;
                                                }
                                        }
                                }
                        }
                        dptr[2] = soffset;
                }
                dptr += optlen;
                dopt->optlen += optlen;
        }
        if (sopt->srr) {
                unsigned char *start = sptr+sopt->srr;
                __be32 faddr;

                optlen  = start[1];
                soffset = start[2];
                doffset = 0;
                if (soffset > optlen)
                        soffset = optlen + 1;
                soffset -= 4;
                if (soffset > 3) {
                        memcpy(&faddr, &start[soffset-1], 4);
                        for (soffset -= 4, doffset = 4; soffset > 3; soffset -= 4, doffset += 4)
                                memcpy(&dptr[doffset-1], &start[soffset-1], 4);
                        /*
                         * RFC1812 requires to fix illegal source routes.
                         */
                        if (memcmp(&ip_hdr(skb)->saddr,
                                   &start[soffset + 3], 4) == 0)
                                doffset -= 4;
                }
                if (doffset > 3) {
                        dopt->faddr = faddr;
                        dptr[0] = start[0];
                        dptr[1] = doffset+3;
                        dptr[2] = 4;
                        dptr += doffset+3;
                        dopt->srr = dopt->optlen + sizeof(struct iphdr);
                        dopt->optlen += doffset+3;
                        dopt->is_strictroute = sopt->is_strictroute;
                }
        }
        if (sopt->cipso) {
                optlen  = sptr[sopt->cipso+1];
                dopt->cipso = dopt->optlen+sizeof(struct iphdr);
                memcpy(dptr, sptr+sopt->cipso, optlen);
                dptr += optlen;
                dopt->optlen += optlen;
        }
        while (dopt->optlen & 3) {
                *dptr++ = IPOPT_END;
                dopt->optlen++;
        }
        return 0;
}

/*
 *        Options "fragmenting", just fill options not
 *        allowed in fragments with NOOPs.
 *        Simple and stupid 8), but the most efficient way.
 */

void ip_options_fragment(struct sk_buff *skb)
{
        unsigned char *optptr = skb_network_header(skb) + sizeof(struct iphdr);
        struct ip_options *opt = &(IPCB(skb)->opt);
        int  l = opt->optlen;
        int  optlen;

        while (l > 0) {
                switch (*optptr) {
                case IPOPT_END:
                        return;
                case IPOPT_NOOP:
                        l--;
                        optptr++;
                        continue;
                }
                optlen = optptr[1];
                if (optlen < 2 || optlen > l)
                  return;
                if (!IPOPT_COPIED(*optptr))
                        memset(optptr, IPOPT_NOOP, optlen);
                l -= optlen;
                optptr += optlen;
        }
        opt->ts = 0;
        opt->rr = 0;
        opt->rr_needaddr = 0;
        opt->ts_needaddr = 0;
        opt->ts_needtime = 0;
}

/* helper used by ip_options_compile() to call fib_compute_spec_dst()
 * at most one time.
 */
static void spec_dst_fill(__be32 *spec_dst, struct sk_buff *skb)
{
        if (*spec_dst == htonl(INADDR_ANY))
                *spec_dst = fib_compute_spec_dst(skb);
}

/*
 * Verify options and fill pointers in struct options.
 * Caller should clear *opt, and set opt->data.
 * If opt == NULL, then skb->data should point to IP header.
 */

int __ip_options_compile(struct net *net,
                         struct ip_options *opt, struct sk_buff *skb,
                         __be32 *info)
{
        __be32 spec_dst = htonl(INADDR_ANY);
        unsigned char *pp_ptr = NULL;
        struct rtable *rt = NULL;
        unsigned char *optptr;
        unsigned char *iph;
        int optlen, l;

        if (skb) {
                rt = skb_rtable(skb);
                optptr = (unsigned char *)&(ip_hdr(skb)[1]);
        } else
                optptr = opt->__data;
        iph = optptr - sizeof(struct iphdr);

        for (l = opt->optlen; l > 0; ) {
                switch (*optptr) {
                case IPOPT_END:
                        for (optptr++, l--; l > 0; optptr++, l--) {
                                if (*optptr != IPOPT_END) {
                                        *optptr = IPOPT_END;
                                        opt->is_changed = 1;
                                }
                        }
                        goto eol;
                case IPOPT_NOOP:
                        l--;
                        optptr++;
                        continue;
                }
                if (unlikely(l < 2)) {
                        pp_ptr = optptr;
                        goto error;
                }
                optlen = optptr[1];
                if (optlen < 2 || optlen > l) {
                        pp_ptr = optptr;
                        goto error;
                }
                switch (*optptr) {
                case IPOPT_SSRR:
                case IPOPT_LSRR:
                        if (optlen < 3) {
                                pp_ptr = optptr + 1;
                                goto error;
                        }
                        if (optptr[2] < 4) {
                                pp_ptr = optptr + 2;
                                goto error;
                        }
                        /* NB: cf RFC-1812 5.2.4.1 */
                        if (opt->srr) {
                                pp_ptr = optptr;
                                goto error;
                        }
                        if (!skb) {
                                if (optptr[2] != 4 || optlen < 7 || ((optlen-3) & 3)) {
                                        pp_ptr = optptr + 1;
                                        goto error;
                                }
                                memcpy(&opt->faddr, &optptr[3], 4);
                                if (optlen > 7)
                                        memmove(&optptr[3], &optptr[7], optlen-7);
                        }
                        opt->is_strictroute = (optptr[0] == IPOPT_SSRR);
                        opt->srr = optptr - iph;
                        break;
                case IPOPT_RR:
                        if (opt->rr) {
                                pp_ptr = optptr;
                                goto error;
                        }
                        if (optlen < 3) {
                                pp_ptr = optptr + 1;
                                goto error;
                        }
                        if (optptr[2] < 4) {
                                pp_ptr = optptr + 2;
                                goto error;
                        }
                        if (optptr[2] <= optlen) {
                                if (optptr[2]+3 > optlen) {
                                        pp_ptr = optptr + 2;
                                        goto error;
                                }
                                if (rt) {
                                        spec_dst_fill(&spec_dst, skb);
                                        memcpy(&optptr[optptr[2]-1], &spec_dst, 4);
                                        opt->is_changed = 1;
                                }
                                optptr[2] += 4;
                                opt->rr_needaddr = 1;
                        }
                        opt->rr = optptr - iph;
                        break;
                case IPOPT_TIMESTAMP:
                        if (opt->ts) {
                                pp_ptr = optptr;
                                goto error;
                        }
                        if (optlen < 4) {
                                pp_ptr = optptr + 1;
                                goto error;
                        }
                        if (optptr[2] < 5) {
                                pp_ptr = optptr + 2;
                                goto error;
                        }
                        if (optptr[2] <= optlen) {
                                unsigned char *timeptr = NULL;
                                if (optptr[2]+3 > optlen) {
                                        pp_ptr = optptr + 2;
                                        goto error;
                                }
                                switch (optptr[3]&0xF) {
                                case IPOPT_TS_TSONLY:
                                        if (skb)
                                                timeptr = &optptr[optptr[2]-1];
                                        opt->ts_needtime = 1;
                                        optptr[2] += 4;
                                        break;
                                case IPOPT_TS_TSANDADDR:
                                        if (optptr[2]+7 > optlen) {
                                                pp_ptr = optptr + 2;
                                                goto error;
                                        }
                                        if (rt)  {
                                                spec_dst_fill(&spec_dst, skb);
                                                memcpy(&optptr[optptr[2]-1], &spec_dst, 4);
                                                timeptr = &optptr[optptr[2]+3];
                                        }
                                        opt->ts_needaddr = 1;
                                        opt->ts_needtime = 1;
                                        optptr[2] += 8;
                                        break;
                                case IPOPT_TS_PRESPEC:
                                        if (optptr[2]+7 > optlen) {
                                                pp_ptr = optptr + 2;
                                                goto error;
                                        }
                                        {
                                                __be32 addr;
                                                memcpy(&addr, &optptr[optptr[2]-1], 4);
                                                if (inet_addr_type(net, addr) == RTN_UNICAST)
                                                        break;
                                                if (skb)
                                                        timeptr = &optptr[optptr[2]+3];
                                        }
                                        opt->ts_needtime = 1;
                                        optptr[2] += 8;
                                        break;
                                default:
                                        if (!skb && !ns_capable(net->user_ns, CAP_NET_RAW)) {
                                                pp_ptr = optptr + 3;
                                                goto error;
                                        }
                                        break;
                                }
                                if (timeptr) {
                                        __be32 midtime;

                                        midtime = inet_current_timestamp();
                                        memcpy(timeptr, &midtime, 4);
                                        opt->is_changed = 1;
                                }
                        } else if ((optptr[3]&0xF) != IPOPT_TS_PRESPEC) {
                                unsigned int overflow = optptr[3]>>4;
                                if (overflow == 15) {
                                        pp_ptr = optptr + 3;
                                        goto error;
                                }
                                if (skb) {
                                        optptr[3] = (optptr[3]&0xF)|((overflow+1)<<4);
                                        opt->is_changed = 1;
                                }
                        }
                        opt->ts = optptr - iph;
                        break;
                case IPOPT_RA:
                        if (optlen < 4) {
                                pp_ptr = optptr + 1;
                                goto error;
                        }
                        if (optptr[2] == 0 && optptr[3] == 0)
                                opt->router_alert = optptr - iph;
                        break;
                case IPOPT_CIPSO:
                        if ((!skb && !ns_capable(net->user_ns, CAP_NET_RAW)) || opt->cipso) {
                                pp_ptr = optptr;
                                goto error;
                        }
                        opt->cipso = optptr - iph;
                        if (cipso_v4_validate(skb, &optptr)) {
                                pp_ptr = optptr;
                                goto error;
                        }
                        break;
                case IPOPT_SEC:
                case IPOPT_SID:
                default:
                        if (!skb && !ns_capable(net->user_ns, CAP_NET_RAW)) {
                                pp_ptr = optptr;
                                goto error;
                        }
                        break;
                }
                l -= optlen;
                optptr += optlen;
        }

eol:
        if (!pp_ptr)
                return 0;

error:
        if (info)
                *info = htonl((pp_ptr-iph)<<24);
        return -EINVAL;
}
EXPORT_SYMBOL(__ip_options_compile);

int ip_options_compile(struct net *net,
                       struct ip_options *opt, struct sk_buff *skb)
{
        int ret;
        __be32 info;

        ret = __ip_options_compile(net, opt, skb, &info);
        if (ret != 0 && skb)
                icmp_send(skb, ICMP_PARAMETERPROB, 0, info);
        return ret;
}
EXPORT_SYMBOL(ip_options_compile);

/*
 *        Undo all the changes done by ip_options_compile().
 */

void ip_options_undo(struct ip_options *opt)
{
        if (opt->srr) {
                unsigned char *optptr = opt->__data + opt->srr - sizeof(struct iphdr);

                memmove(optptr + 7, optptr + 3, optptr[1] - 7);
                memcpy(optptr + 3, &opt->faddr, 4);
        }
        if (opt->rr_needaddr) {
                unsigned char *optptr = opt->__data + opt->rr - sizeof(struct iphdr);

                optptr[2] -= 4;
                memset(&optptr[optptr[2] - 1], 0, 4);
        }
        if (opt->ts) {
                unsigned char *optptr = opt->__data + opt->ts - sizeof(struct iphdr);

                if (opt->ts_needtime) {
                        optptr[2] -= 4;
                        memset(&optptr[optptr[2] - 1], 0, 4);
                        if ((optptr[3] & 0xF) == IPOPT_TS_PRESPEC)
                                optptr[2] -= 4;
                }
                if (opt->ts_needaddr) {
                        optptr[2] -= 4;
                        memset(&optptr[optptr[2] - 1], 0, 4);
                }
        }
}

int ip_options_get(struct net *net, struct ip_options_rcu **optp,
                   sockptr_t data, int optlen)
{
        struct ip_options_rcu *opt;

        opt = kzalloc(sizeof(struct ip_options_rcu) + ((optlen + 3) & ~3),
                       GFP_KERNEL);
        if (!opt)
                return -ENOMEM;
        if (optlen && copy_from_sockptr(opt->opt.__data, data, optlen)) {
                kfree(opt);
                return -EFAULT;
        }

        while (optlen & 3)
                opt->opt.__data[optlen++] = IPOPT_END;
        opt->opt.optlen = optlen;
        if (optlen && ip_options_compile(net, &opt->opt, NULL)) {
                kfree(opt);
                return -EINVAL;
        }
        kfree(*optp);
        *optp = opt;
        return 0;
}

void ip_forward_options(struct sk_buff *skb)
{
        struct   ip_options *opt        = &(IPCB(skb)->opt);
        unsigned char *optptr;
        struct rtable *rt = skb_rtable(skb);
        unsigned char *raw = skb_network_header(skb);

        if (opt->rr_needaddr) {
                optptr = (unsigned char *)raw + opt->rr;
                ip_rt_get_source(&optptr[optptr[2]-5], skb, rt);
                opt->is_changed = 1;
        }
        if (opt->srr_is_hit) {
                int srrptr, srrspace;

                optptr = raw + opt->srr;

                for ( srrptr = optptr[2], srrspace = optptr[1];
                     srrptr <= srrspace;
                     srrptr += 4
                     ) {
                        if (srrptr + 3 > srrspace)
                                break;
                        if (memcmp(&opt->nexthop, &optptr[srrptr-1], 4) == 0)
                                break;
                }
                if (srrptr + 3 <= srrspace) {
                        opt->is_changed = 1;
                        ip_hdr(skb)->daddr = opt->nexthop;
                        ip_rt_get_source(&optptr[srrptr-1], skb, rt);
                        optptr[2] = srrptr+4;
                } else {
                        net_crit_ratelimited("%s(): Argh! Destination lost!\n",
                                             __func__);
                }
                if (opt->ts_needaddr) {
                        optptr = raw + opt->ts;
                        ip_rt_get_source(&optptr[optptr[2]-9], skb, rt);
                        opt->is_changed = 1;
                }
        }
        if (opt->is_changed) {
                opt->is_changed = 0;
                ip_send_check(ip_hdr(skb));
        }
}

int ip_options_rcv_srr(struct sk_buff *skb, struct net_device *dev)
{
        struct ip_options *opt = &(IPCB(skb)->opt);
        int srrspace, srrptr;
        __be32 nexthop;
        struct iphdr *iph = ip_hdr(skb);
        unsigned char *optptr = skb_network_header(skb) + opt->srr;
        struct rtable *rt = skb_rtable(skb);
        struct rtable *rt2;
        unsigned long orefdst;
        int err;

        if (!rt)
                return 0;

        if (skb->pkt_type != PACKET_HOST)
                return -EINVAL;
        if (rt->rt_type == RTN_UNICAST) {
                if (!opt->is_strictroute)
                        return 0;
                icmp_send(skb, ICMP_PARAMETERPROB, 0, htonl(16<<24));
                return -EINVAL;
        }
        if (rt->rt_type != RTN_LOCAL)
                return -EINVAL;

        for (srrptr = optptr[2], srrspace = optptr[1]; srrptr <= srrspace; srrptr += 4) {
                if (srrptr + 3 > srrspace) {
                        icmp_send(skb, ICMP_PARAMETERPROB, 0, htonl((opt->srr+2)<<24));
                        return -EINVAL;
                }
                memcpy(&nexthop, &optptr[srrptr-1], 4);

                orefdst = skb->_skb_refdst;
                skb_dst_set(skb, NULL);
                err = ip_route_input(skb, nexthop, iph->saddr, ip4h_dscp(iph),
                                     dev) ? -EINVAL : 0;
                rt2 = skb_rtable(skb);
                if (err || (rt2->rt_type != RTN_UNICAST && rt2->rt_type != RTN_LOCAL)) {
                        skb_dst_drop(skb);
                        skb->_skb_refdst = orefdst;
                        return -EINVAL;
                }
                refdst_drop(orefdst);
                if (rt2->rt_type != RTN_LOCAL)
                        break;
                /* Superfast 8) loopback forward */
                iph->daddr = nexthop;
                opt->is_changed = 1;
        }
        if (srrptr <= srrspace) {
                opt->srr_is_hit = 1;
                opt->nexthop = nexthop;
                opt->is_changed = 1;
        }
        return 0;
}
EXPORT_SYMBOL(ip_options_rcv_srr);















   23 














    1 


   25 







   37 












    5 
    5 
    1 





   27 
    1 



   26 


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
// SPDX-License-Identifier: GPL-2.0-only
/*
 * VMware vSockets Driver
 *
 * Copyright (C) 2007-2012 VMware, Inc. All rights reserved.
 */

#include <linux/types.h>
#include <linux/socket.h>
#include <linux/stddef.h>
#include <net/sock.h>
#include <net/vsock_addr.h>

void vsock_addr_init(struct sockaddr_vm *addr, u32 cid, u32 port)
{
        memset(addr, 0, sizeof(*addr));
        addr->svm_family = AF_VSOCK;
        addr->svm_cid = cid;
        addr->svm_port = port;
}
EXPORT_SYMBOL_GPL(vsock_addr_init);

int vsock_addr_validate(const struct sockaddr_vm *addr)
{
        __u8 svm_valid_flags = VMADDR_FLAG_TO_HOST;

        if (!addr)
                return -EFAULT;

        if (addr->svm_family != AF_VSOCK)
                return -EAFNOSUPPORT;

        if (addr->svm_flags & ~svm_valid_flags)
                return -EINVAL;

        return 0;
}
EXPORT_SYMBOL_GPL(vsock_addr_validate);

bool vsock_addr_bound(const struct sockaddr_vm *addr)
{
        return addr->svm_port != VMADDR_PORT_ANY;
}
EXPORT_SYMBOL_GPL(vsock_addr_bound);

void vsock_addr_unbind(struct sockaddr_vm *addr)
{
        vsock_addr_init(addr, VMADDR_CID_ANY, VMADDR_PORT_ANY);
}
EXPORT_SYMBOL_GPL(vsock_addr_unbind);

bool vsock_addr_equals_addr(const struct sockaddr_vm *addr,
                            const struct sockaddr_vm *other)
{
        return addr->svm_cid == other->svm_cid &&
                addr->svm_port == other->svm_port;
}
EXPORT_SYMBOL_GPL(vsock_addr_equals_addr);

int vsock_addr_cast(const struct sockaddr *addr,
                    size_t len, struct sockaddr_vm **out_addr)
{
        if (len < sizeof(**out_addr))
                return -EFAULT;

        *out_addr = (struct sockaddr_vm *)addr;
        return vsock_addr_validate(*out_addr);
}
EXPORT_SYMBOL_GPL(vsock_addr_cast);

























































































































































































































































































































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
/* SPDX-License-Identifier: GPL-2.0-only */
/*
 * pm_runtime.h - Device run-time power management helper functions.
 *
 * Copyright (C) 2009 Rafael J. Wysocki <rjw@sisk.pl>
 */

#ifndef _LINUX_PM_RUNTIME_H
#define _LINUX_PM_RUNTIME_H

#include <linux/device.h>
#include <linux/notifier.h>
#include <linux/pm.h>

#include <linux/jiffies.h>

/* Runtime PM flag argument bits */
#define RPM_ASYNC                0x01        /* Request is asynchronous */
#define RPM_NOWAIT                0x02        /* Don't wait for concurrent
                                            state change */
#define RPM_GET_PUT                0x04        /* Increment/decrement the
                                            usage_count */
#define RPM_AUTO                0x08        /* Use autosuspend_delay */

/*
 * Use this for defining a set of PM operations to be used in all situations
 * (system suspend, hibernation or runtime PM).
 *
 * Note that the behaviour differs from the deprecated UNIVERSAL_DEV_PM_OPS()
 * macro, which uses the provided callbacks for both runtime PM and system
 * sleep, while DEFINE_RUNTIME_DEV_PM_OPS() uses pm_runtime_force_suspend()
 * and pm_runtime_force_resume() for its system sleep callbacks.
 *
 * If the underlying dev_pm_ops struct symbol has to be exported, use
 * EXPORT_RUNTIME_DEV_PM_OPS() or EXPORT_GPL_RUNTIME_DEV_PM_OPS() instead.
 */
#define DEFINE_RUNTIME_DEV_PM_OPS(name, suspend_fn, resume_fn, idle_fn) \
        _DEFINE_DEV_PM_OPS(name, pm_runtime_force_suspend, \
                           pm_runtime_force_resume, suspend_fn, \
                           resume_fn, idle_fn)

#define EXPORT_RUNTIME_DEV_PM_OPS(name, suspend_fn, resume_fn, idle_fn) \
        EXPORT_DEV_PM_OPS(name) = { \
                RUNTIME_PM_OPS(suspend_fn, resume_fn, idle_fn) \
        }
#define EXPORT_GPL_RUNTIME_DEV_PM_OPS(name, suspend_fn, resume_fn, idle_fn) \
        EXPORT_GPL_DEV_PM_OPS(name) = { \
                RUNTIME_PM_OPS(suspend_fn, resume_fn, idle_fn) \
        }
#define EXPORT_NS_RUNTIME_DEV_PM_OPS(name, suspend_fn, resume_fn, idle_fn, ns) \
        EXPORT_NS_DEV_PM_OPS(name, ns) = { \
                RUNTIME_PM_OPS(suspend_fn, resume_fn, idle_fn) \
        }
#define EXPORT_NS_GPL_RUNTIME_DEV_PM_OPS(name, suspend_fn, resume_fn, idle_fn, ns) \
        EXPORT_NS_GPL_DEV_PM_OPS(name, ns) = { \
                RUNTIME_PM_OPS(suspend_fn, resume_fn, idle_fn) \
        }

#ifdef CONFIG_PM
extern struct workqueue_struct *pm_wq;

static inline bool queue_pm_work(struct work_struct *work)
{
        return queue_work(pm_wq, work);
}

extern int pm_generic_runtime_suspend(struct device *dev);
extern int pm_generic_runtime_resume(struct device *dev);
extern int pm_runtime_force_suspend(struct device *dev);
extern int pm_runtime_force_resume(struct device *dev);

extern int __pm_runtime_idle(struct device *dev, int rpmflags);
extern int __pm_runtime_suspend(struct device *dev, int rpmflags);
extern int __pm_runtime_resume(struct device *dev, int rpmflags);
extern int pm_runtime_get_if_active(struct device *dev);
extern int pm_runtime_get_if_in_use(struct device *dev);
extern int pm_schedule_suspend(struct device *dev, unsigned int delay);
extern int __pm_runtime_set_status(struct device *dev, unsigned int status);
extern int pm_runtime_barrier(struct device *dev);
extern void pm_runtime_enable(struct device *dev);
extern void __pm_runtime_disable(struct device *dev, bool check_resume);
extern void pm_runtime_allow(struct device *dev);
extern void pm_runtime_forbid(struct device *dev);
extern void pm_runtime_no_callbacks(struct device *dev);
extern void pm_runtime_irq_safe(struct device *dev);
extern void __pm_runtime_use_autosuspend(struct device *dev, bool use);
extern void pm_runtime_set_autosuspend_delay(struct device *dev, int delay);
extern u64 pm_runtime_autosuspend_expiration(struct device *dev);
extern void pm_runtime_set_memalloc_noio(struct device *dev, bool enable);
extern void pm_runtime_get_suppliers(struct device *dev);
extern void pm_runtime_put_suppliers(struct device *dev);
extern void pm_runtime_new_link(struct device *dev);
extern void pm_runtime_drop_link(struct device_link *link);
extern void pm_runtime_release_supplier(struct device_link *link);

extern int devm_pm_runtime_enable(struct device *dev);

/**
 * pm_suspend_ignore_children - Set runtime PM behavior regarding children.
 * @dev: Target device.
 * @enable: Whether or not to ignore possible dependencies on children.
 *
 * The dependencies of @dev on its children will not be taken into account by
 * the runtime PM framework going forward if @enable is %true, or they will
 * be taken into account otherwise.
 */
static inline void pm_suspend_ignore_children(struct device *dev, bool enable)
{
        dev->power.ignore_children = enable;
}

/**
 * pm_runtime_get_noresume - Bump up runtime PM usage counter of a device.
 * @dev: Target device.
 */
static inline void pm_runtime_get_noresume(struct device *dev)
{
        atomic_inc(&dev->power.usage_count);
}

/**
 * pm_runtime_put_noidle - Drop runtime PM usage counter of a device.
 * @dev: Target device.
 *
 * Decrement the runtime PM usage counter of @dev unless it is 0 already.
 */
static inline void pm_runtime_put_noidle(struct device *dev)
{
        atomic_add_unless(&dev->power.usage_count, -1, 0);
}

/**
 * pm_runtime_suspended - Check whether or not a device is runtime-suspended.
 * @dev: Target device.
 *
 * Return %true if runtime PM is enabled for @dev and its runtime PM status is
 * %RPM_SUSPENDED, or %false otherwise.
 *
 * Note that the return value of this function can only be trusted if it is
 * called under the runtime PM lock of @dev or under conditions in which
 * runtime PM cannot be either disabled or enabled for @dev and its runtime PM
 * status cannot change.
 */
static inline bool pm_runtime_suspended(struct device *dev)
{
        return dev->power.runtime_status == RPM_SUSPENDED
                && !dev->power.disable_depth;
}

/**
 * pm_runtime_active - Check whether or not a device is runtime-active.
 * @dev: Target device.
 *
 * Return %true if runtime PM is disabled for @dev or its runtime PM status is
 * %RPM_ACTIVE, or %false otherwise.
 *
 * Note that the return value of this function can only be trusted if it is
 * called under the runtime PM lock of @dev or under conditions in which
 * runtime PM cannot be either disabled or enabled for @dev and its runtime PM
 * status cannot change.
 */
static inline bool pm_runtime_active(struct device *dev)
{
        return dev->power.runtime_status == RPM_ACTIVE
                || dev->power.disable_depth;
}

/**
 * pm_runtime_status_suspended - Check if runtime PM status is "suspended".
 * @dev: Target device.
 *
 * Return %true if the runtime PM status of @dev is %RPM_SUSPENDED, or %false
 * otherwise, regardless of whether or not runtime PM has been enabled for @dev.
 *
 * Note that the return value of this function can only be trusted if it is
 * called under the runtime PM lock of @dev or under conditions in which the
 * runtime PM status of @dev cannot change.
 */
static inline bool pm_runtime_status_suspended(struct device *dev)
{
        return dev->power.runtime_status == RPM_SUSPENDED;
}

/**
 * pm_runtime_enabled - Check if runtime PM is enabled.
 * @dev: Target device.
 *
 * Return %true if runtime PM is enabled for @dev or %false otherwise.
 *
 * Note that the return value of this function can only be trusted if it is
 * called under the runtime PM lock of @dev or under conditions in which
 * runtime PM cannot be either disabled or enabled for @dev.
 */
static inline bool pm_runtime_enabled(struct device *dev)
{
        return !dev->power.disable_depth;
}

/**
 * pm_runtime_has_no_callbacks - Check if runtime PM callbacks may be present.
 * @dev: Target device.
 *
 * Return %true if @dev is a special device without runtime PM callbacks or
 * %false otherwise.
 */
static inline bool pm_runtime_has_no_callbacks(struct device *dev)
{
        return dev->power.no_callbacks;
}

/**
 * pm_runtime_mark_last_busy - Update the last access time of a device.
 * @dev: Target device.
 *
 * Update the last access time of @dev used by the runtime PM autosuspend
 * mechanism to the current time as returned by ktime_get_mono_fast_ns().
 */
static inline void pm_runtime_mark_last_busy(struct device *dev)
{
        WRITE_ONCE(dev->power.last_busy, ktime_get_mono_fast_ns());
}

/**
 * pm_runtime_is_irq_safe - Check if runtime PM can work in interrupt context.
 * @dev: Target device.
 *
 * Return %true if @dev has been marked as an "IRQ-safe" device (with respect
 * to runtime PM), in which case its runtime PM callabcks can be expected to
 * work correctly when invoked from interrupt handlers.
 */
static inline bool pm_runtime_is_irq_safe(struct device *dev)
{
        return dev->power.irq_safe;
}

extern u64 pm_runtime_suspended_time(struct device *dev);

#else /* !CONFIG_PM */

static inline bool queue_pm_work(struct work_struct *work) { return false; }

static inline int pm_generic_runtime_suspend(struct device *dev) { return 0; }
static inline int pm_generic_runtime_resume(struct device *dev) { return 0; }
static inline int pm_runtime_force_suspend(struct device *dev) { return 0; }
static inline int pm_runtime_force_resume(struct device *dev) { return 0; }

static inline int __pm_runtime_idle(struct device *dev, int rpmflags)
{
        return -ENOSYS;
}
static inline int __pm_runtime_suspend(struct device *dev, int rpmflags)
{
        return -ENOSYS;
}
static inline int __pm_runtime_resume(struct device *dev, int rpmflags)
{
        return 1;
}
static inline int pm_schedule_suspend(struct device *dev, unsigned int delay)
{
        return -ENOSYS;
}
static inline int pm_runtime_get_if_in_use(struct device *dev)
{
        return -EINVAL;
}
static inline int pm_runtime_get_if_active(struct device *dev)
{
        return -EINVAL;
}
static inline int __pm_runtime_set_status(struct device *dev,
                                            unsigned int status) { return 0; }
static inline int pm_runtime_barrier(struct device *dev) { return 0; }
static inline void pm_runtime_enable(struct device *dev) {}
static inline void __pm_runtime_disable(struct device *dev, bool c) {}
static inline void pm_runtime_allow(struct device *dev) {}
static inline void pm_runtime_forbid(struct device *dev) {}

static inline int devm_pm_runtime_enable(struct device *dev) { return 0; }

static inline void pm_suspend_ignore_children(struct device *dev, bool enable) {}
static inline void pm_runtime_get_noresume(struct device *dev) {}
static inline void pm_runtime_put_noidle(struct device *dev) {}
static inline bool pm_runtime_suspended(struct device *dev) { return false; }
static inline bool pm_runtime_active(struct device *dev) { return true; }
static inline bool pm_runtime_status_suspended(struct device *dev) { return false; }
static inline bool pm_runtime_enabled(struct device *dev) { return false; }

static inline void pm_runtime_no_callbacks(struct device *dev) {}
static inline void pm_runtime_irq_safe(struct device *dev) {}
static inline bool pm_runtime_is_irq_safe(struct device *dev) { return false; }

static inline bool pm_runtime_has_no_callbacks(struct device *dev) { return false; }
static inline void pm_runtime_mark_last_busy(struct device *dev) {}
static inline void __pm_runtime_use_autosuspend(struct device *dev,
                                                bool use) {}
static inline void pm_runtime_set_autosuspend_delay(struct device *dev,
                                                int delay) {}
static inline u64 pm_runtime_autosuspend_expiration(
                                struct device *dev) { return 0; }
static inline void pm_runtime_set_memalloc_noio(struct device *dev,
                                                bool enable){}
static inline void pm_runtime_get_suppliers(struct device *dev) {}
static inline void pm_runtime_put_suppliers(struct device *dev) {}
static inline void pm_runtime_new_link(struct device *dev) {}
static inline void pm_runtime_drop_link(struct device_link *link) {}
static inline void pm_runtime_release_supplier(struct device_link *link) {}

#endif /* !CONFIG_PM */

/**
 * pm_runtime_idle - Conditionally set up autosuspend of a device or suspend it.
 * @dev: Target device.
 *
 * Invoke the "idle check" callback of @dev and, depending on its return value,
 * set up autosuspend of @dev or suspend it (depending on whether or not
 * autosuspend has been enabled for it).
 */
static inline int pm_runtime_idle(struct device *dev)
{
        return __pm_runtime_idle(dev, 0);
}

/**
 * pm_runtime_suspend - Suspend a device synchronously.
 * @dev: Target device.
 */
static inline int pm_runtime_suspend(struct device *dev)
{
        return __pm_runtime_suspend(dev, 0);
}

/**
 * pm_runtime_autosuspend - Set up autosuspend of a device or suspend it.
 * @dev: Target device.
 *
 * Set up autosuspend of @dev or suspend it (depending on whether or not
 * autosuspend is enabled for it) without engaging its "idle check" callback.
 */
static inline int pm_runtime_autosuspend(struct device *dev)
{
        return __pm_runtime_suspend(dev, RPM_AUTO);
}

/**
 * pm_runtime_resume - Resume a device synchronously.
 * @dev: Target device.
 */
static inline int pm_runtime_resume(struct device *dev)
{
        return __pm_runtime_resume(dev, 0);
}

/**
 * pm_request_idle - Queue up "idle check" execution for a device.
 * @dev: Target device.
 *
 * Queue up a work item to run an equivalent of pm_runtime_idle() for @dev
 * asynchronously.
 */
static inline int pm_request_idle(struct device *dev)
{
        return __pm_runtime_idle(dev, RPM_ASYNC);
}

/**
 * pm_request_resume - Queue up runtime-resume of a device.
 * @dev: Target device.
 */
static inline int pm_request_resume(struct device *dev)
{
        return __pm_runtime_resume(dev, RPM_ASYNC);
}

/**
 * pm_request_autosuspend - Queue up autosuspend of a device.
 * @dev: Target device.
 *
 * Queue up a work item to run an equivalent pm_runtime_autosuspend() for @dev
 * asynchronously.
 */
static inline int pm_request_autosuspend(struct device *dev)
{
        return __pm_runtime_suspend(dev, RPM_ASYNC | RPM_AUTO);
}

/**
 * pm_runtime_get - Bump up usage counter and queue up resume of a device.
 * @dev: Target device.
 *
 * Bump up the runtime PM usage counter of @dev and queue up a work item to
 * carry out runtime-resume of it.
 */
static inline int pm_runtime_get(struct device *dev)
{
        return __pm_runtime_resume(dev, RPM_GET_PUT | RPM_ASYNC);
}

/**
 * pm_runtime_get_sync - Bump up usage counter of a device and resume it.
 * @dev: Target device.
 *
 * Bump up the runtime PM usage counter of @dev and carry out runtime-resume of
 * it synchronously.
 *
 * The possible return values of this function are the same as for
 * pm_runtime_resume() and the runtime PM usage counter of @dev remains
 * incremented in all cases, even if it returns an error code.
 * Consider using pm_runtime_resume_and_get() instead of it, especially
 * if its return value is checked by the caller, as this is likely to result
 * in cleaner code.
 */
static inline int pm_runtime_get_sync(struct device *dev)
{
        return __pm_runtime_resume(dev, RPM_GET_PUT);
}

/**
 * pm_runtime_resume_and_get - Bump up usage counter of a device and resume it.
 * @dev: Target device.
 *
 * Resume @dev synchronously and if that is successful, increment its runtime
 * PM usage counter. Return 0 if the runtime PM usage counter of @dev has been
 * incremented or a negative error code otherwise.
 */
static inline int pm_runtime_resume_and_get(struct device *dev)
{
        int ret;

        ret = __pm_runtime_resume(dev, RPM_GET_PUT);
        if (ret < 0) {
                pm_runtime_put_noidle(dev);
                return ret;
        }

        return 0;
}

/**
 * pm_runtime_put - Drop device usage counter and queue up "idle check" if 0.
 * @dev: Target device.
 *
 * Decrement the runtime PM usage counter of @dev and if it turns out to be
 * equal to 0, queue up a work item for @dev like in pm_request_idle().
 */
static inline int pm_runtime_put(struct device *dev)
{
        return __pm_runtime_idle(dev, RPM_GET_PUT | RPM_ASYNC);
}

/**
 * __pm_runtime_put_autosuspend - Drop device usage counter and queue autosuspend if 0.
 * @dev: Target device.
 *
 * Decrement the runtime PM usage counter of @dev and if it turns out to be
 * equal to 0, queue up a work item for @dev like in pm_request_autosuspend().
 */
static inline int __pm_runtime_put_autosuspend(struct device *dev)
{
        return __pm_runtime_suspend(dev, RPM_GET_PUT | RPM_ASYNC | RPM_AUTO);
}

/**
 * pm_runtime_put_autosuspend - Drop device usage counter and queue autosuspend if 0.
 * @dev: Target device.
 *
 * Decrement the runtime PM usage counter of @dev and if it turns out to be
 * equal to 0, queue up a work item for @dev like in pm_request_autosuspend().
 */
static inline int pm_runtime_put_autosuspend(struct device *dev)
{
        return __pm_runtime_suspend(dev,
            RPM_GET_PUT | RPM_ASYNC | RPM_AUTO);
}

/**
 * pm_runtime_put_sync - Drop device usage counter and run "idle check" if 0.
 * @dev: Target device.
 *
 * Decrement the runtime PM usage counter of @dev and if it turns out to be
 * equal to 0, invoke the "idle check" callback of @dev and, depending on its
 * return value, set up autosuspend of @dev or suspend it (depending on whether
 * or not autosuspend has been enabled for it).
 *
 * The possible return values of this function are the same as for
 * pm_runtime_idle() and the runtime PM usage counter of @dev remains
 * decremented in all cases, even if it returns an error code.
 */
static inline int pm_runtime_put_sync(struct device *dev)
{
        return __pm_runtime_idle(dev, RPM_GET_PUT);
}

/**
 * pm_runtime_put_sync_suspend - Drop device usage counter and suspend if 0.
 * @dev: Target device.
 *
 * Decrement the runtime PM usage counter of @dev and if it turns out to be
 * equal to 0, carry out runtime-suspend of @dev synchronously.
 *
 * The possible return values of this function are the same as for
 * pm_runtime_suspend() and the runtime PM usage counter of @dev remains
 * decremented in all cases, even if it returns an error code.
 */
static inline int pm_runtime_put_sync_suspend(struct device *dev)
{
        return __pm_runtime_suspend(dev, RPM_GET_PUT);
}

/**
 * pm_runtime_put_sync_autosuspend - Drop device usage counter and autosuspend if 0.
 * @dev: Target device.
 *
 * Decrement the runtime PM usage counter of @dev and if it turns out to be
 * equal to 0, set up autosuspend of @dev or suspend it synchronously (depending
 * on whether or not autosuspend has been enabled for it).
 *
 * The possible return values of this function are the same as for
 * pm_runtime_autosuspend() and the runtime PM usage counter of @dev remains
 * decremented in all cases, even if it returns an error code.
 */
static inline int pm_runtime_put_sync_autosuspend(struct device *dev)
{
        return __pm_runtime_suspend(dev, RPM_GET_PUT | RPM_AUTO);
}

/**
 * pm_runtime_set_active - Set runtime PM status to "active".
 * @dev: Target device.
 *
 * Set the runtime PM status of @dev to %RPM_ACTIVE and ensure that dependencies
 * of it will be taken into account.
 *
 * It is not valid to call this function for devices with runtime PM enabled.
 */
static inline int pm_runtime_set_active(struct device *dev)
{
        return __pm_runtime_set_status(dev, RPM_ACTIVE);
}

/**
 * pm_runtime_set_suspended - Set runtime PM status to "suspended".
 * @dev: Target device.
 *
 * Set the runtime PM status of @dev to %RPM_SUSPENDED and ensure that
 * dependencies of it will be taken into account.
 *
 * It is not valid to call this function for devices with runtime PM enabled.
 */
static inline int pm_runtime_set_suspended(struct device *dev)
{
        return __pm_runtime_set_status(dev, RPM_SUSPENDED);
}

/**
 * pm_runtime_disable - Disable runtime PM for a device.
 * @dev: Target device.
 *
 * Prevent the runtime PM framework from working with @dev (by incrementing its
 * "blocking" counter).
 *
 * For each invocation of this function for @dev there must be a matching
 * pm_runtime_enable() call in order for runtime PM to be enabled for it.
 */
static inline void pm_runtime_disable(struct device *dev)
{
        __pm_runtime_disable(dev, true);
}

/**
 * pm_runtime_use_autosuspend - Allow autosuspend to be used for a device.
 * @dev: Target device.
 *
 * Allow the runtime PM autosuspend mechanism to be used for @dev whenever
 * requested (or "autosuspend" will be handled as direct runtime-suspend for
 * it).
 *
 * NOTE: It's important to undo this with pm_runtime_dont_use_autosuspend()
 * at driver exit time unless your driver initially enabled pm_runtime
 * with devm_pm_runtime_enable() (which handles it for you).
 */
static inline void pm_runtime_use_autosuspend(struct device *dev)
{
        __pm_runtime_use_autosuspend(dev, true);
}

/**
 * pm_runtime_dont_use_autosuspend - Prevent autosuspend from being used.
 * @dev: Target device.
 *
 * Prevent the runtime PM autosuspend mechanism from being used for @dev which
 * means that "autosuspend" will be handled as direct runtime-suspend for it
 * going forward.
 */
static inline void pm_runtime_dont_use_autosuspend(struct device *dev)
{
        __pm_runtime_use_autosuspend(dev, false);
}

#endif




































































































































































































































































    1 

















    1 






    1 














    1 




    1 



























































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































    2 






    2 









































































































































































































































































































   27 






   27 





    2 



    1 



1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
// SPDX-License-Identifier: GPL-2.0-only

#include <linux/if_bridge.h>
#include <linux/in.h>
#include <linux/list.h>
#include <linux/netdevice.h>
#include <linux/netlink.h>
#include <linux/rhashtable.h>
#include <linux/rhashtable-types.h>
#include <linux/rtnetlink.h>
#include <linux/skbuff.h>
#include <linux/types.h>
#include <net/netlink.h>
#include <net/vxlan.h>

#include "vxlan_private.h"

struct vxlan_mdb_entry_key {
        union vxlan_addr src;
        union vxlan_addr dst;
        __be32 vni;
};

struct vxlan_mdb_entry {
        struct rhash_head rhnode;
        struct list_head remotes;
        struct vxlan_mdb_entry_key key;
        struct hlist_node mdb_node;
        struct rcu_head rcu;
};

#define VXLAN_MDB_REMOTE_F_BLOCKED        BIT(0)

struct vxlan_mdb_remote {
        struct list_head list;
        struct vxlan_rdst __rcu *rd;
        u8 flags;
        u8 filter_mode;
        u8 rt_protocol;
        struct hlist_head src_list;
        struct rcu_head rcu;
};

#define VXLAN_SGRP_F_DELETE        BIT(0)

struct vxlan_mdb_src_entry {
        struct hlist_node node;
        union vxlan_addr addr;
        u8 flags;
};

struct vxlan_mdb_dump_ctx {
        long reserved;
        long entry_idx;
        long remote_idx;
};

struct vxlan_mdb_config_src_entry {
        union vxlan_addr addr;
        struct list_head node;
};

struct vxlan_mdb_config {
        struct vxlan_dev *vxlan;
        struct vxlan_mdb_entry_key group;
        struct list_head src_list;
        union vxlan_addr remote_ip;
        u32 remote_ifindex;
        __be32 remote_vni;
        __be16 remote_port;
        u16 nlflags;
        u8 flags;
        u8 filter_mode;
        u8 rt_protocol;
};

struct vxlan_mdb_flush_desc {
        union vxlan_addr remote_ip;
        __be32 src_vni;
        __be32 remote_vni;
        __be16 remote_port;
        u8 rt_protocol;
};

static const struct rhashtable_params vxlan_mdb_rht_params = {
        .head_offset = offsetof(struct vxlan_mdb_entry, rhnode),
        .key_offset = offsetof(struct vxlan_mdb_entry, key),
        .key_len = sizeof(struct vxlan_mdb_entry_key),
        .automatic_shrinking = true,
};

static int __vxlan_mdb_add(const struct vxlan_mdb_config *cfg,
                           struct netlink_ext_ack *extack);
static int __vxlan_mdb_del(const struct vxlan_mdb_config *cfg,
                           struct netlink_ext_ack *extack);

static void vxlan_br_mdb_entry_fill(const struct vxlan_dev *vxlan,
                                    const struct vxlan_mdb_entry *mdb_entry,
                                    const struct vxlan_mdb_remote *remote,
                                    struct br_mdb_entry *e)
{
        const union vxlan_addr *dst = &mdb_entry->key.dst;

        memset(e, 0, sizeof(*e));
        e->ifindex = vxlan->dev->ifindex;
        e->state = MDB_PERMANENT;

        if (remote->flags & VXLAN_MDB_REMOTE_F_BLOCKED)
                e->flags |= MDB_FLAGS_BLOCKED;

        switch (dst->sa.sa_family) {
        case AF_INET:
                e->addr.u.ip4 = dst->sin.sin_addr.s_addr;
                e->addr.proto = htons(ETH_P_IP);
                break;
#if IS_ENABLED(CONFIG_IPV6)
        case AF_INET6:
                e->addr.u.ip6 = dst->sin6.sin6_addr;
                e->addr.proto = htons(ETH_P_IPV6);
                break;
#endif
        }
}

static int vxlan_mdb_entry_info_fill_srcs(struct sk_buff *skb,
                                          const struct vxlan_mdb_remote *remote)
{
        struct vxlan_mdb_src_entry *ent;
        struct nlattr *nest;

        if (hlist_empty(&remote->src_list))
                return 0;

        nest = nla_nest_start(skb, MDBA_MDB_EATTR_SRC_LIST);
        if (!nest)
                return -EMSGSIZE;

        hlist_for_each_entry(ent, &remote->src_list, node) {
                struct nlattr *nest_ent;

                nest_ent = nla_nest_start(skb, MDBA_MDB_SRCLIST_ENTRY);
                if (!nest_ent)
                        goto out_cancel_err;

                if (vxlan_nla_put_addr(skb, MDBA_MDB_SRCATTR_ADDRESS,
                                       &ent->addr) ||
                    nla_put_u32(skb, MDBA_MDB_SRCATTR_TIMER, 0))
                        goto out_cancel_err;

                nla_nest_end(skb, nest_ent);
        }

        nla_nest_end(skb, nest);

        return 0;

out_cancel_err:
        nla_nest_cancel(skb, nest);
        return -EMSGSIZE;
}

static int vxlan_mdb_entry_info_fill(const struct vxlan_dev *vxlan,
                                     struct sk_buff *skb,
                                     const struct vxlan_mdb_entry *mdb_entry,
                                     const struct vxlan_mdb_remote *remote)
{
        struct vxlan_rdst *rd = rtnl_dereference(remote->rd);
        struct br_mdb_entry e;
        struct nlattr *nest;

        nest = nla_nest_start_noflag(skb, MDBA_MDB_ENTRY_INFO);
        if (!nest)
                return -EMSGSIZE;

        vxlan_br_mdb_entry_fill(vxlan, mdb_entry, remote, &e);

        if (nla_put_nohdr(skb, sizeof(e), &e) ||
            nla_put_u32(skb, MDBA_MDB_EATTR_TIMER, 0))
                goto nest_err;

        if (!vxlan_addr_any(&mdb_entry->key.src) &&
            vxlan_nla_put_addr(skb, MDBA_MDB_EATTR_SOURCE, &mdb_entry->key.src))
                goto nest_err;

        if (nla_put_u8(skb, MDBA_MDB_EATTR_RTPROT, remote->rt_protocol) ||
            nla_put_u8(skb, MDBA_MDB_EATTR_GROUP_MODE, remote->filter_mode) ||
            vxlan_mdb_entry_info_fill_srcs(skb, remote) ||
            vxlan_nla_put_addr(skb, MDBA_MDB_EATTR_DST, &rd->remote_ip))
                goto nest_err;

        if (rd->remote_port && rd->remote_port != vxlan->cfg.dst_port &&
            nla_put_u16(skb, MDBA_MDB_EATTR_DST_PORT,
                        be16_to_cpu(rd->remote_port)))
                goto nest_err;

        if (rd->remote_vni != vxlan->default_dst.remote_vni &&
            nla_put_u32(skb, MDBA_MDB_EATTR_VNI, be32_to_cpu(rd->remote_vni)))
                goto nest_err;

        if (rd->remote_ifindex &&
            nla_put_u32(skb, MDBA_MDB_EATTR_IFINDEX, rd->remote_ifindex))
                goto nest_err;

        if ((vxlan->cfg.flags & VXLAN_F_COLLECT_METADATA) &&
            mdb_entry->key.vni && nla_put_u32(skb, MDBA_MDB_EATTR_SRC_VNI,
                                              be32_to_cpu(mdb_entry->key.vni)))
                goto nest_err;

        nla_nest_end(skb, nest);

        return 0;

nest_err:
        nla_nest_cancel(skb, nest);
        return -EMSGSIZE;
}

static int vxlan_mdb_entry_fill(const struct vxlan_dev *vxlan,
                                struct sk_buff *skb,
                                struct vxlan_mdb_dump_ctx *ctx,
                                const struct vxlan_mdb_entry *mdb_entry)
{
        int remote_idx = 0, s_remote_idx = ctx->remote_idx;
        struct vxlan_mdb_remote *remote;
        struct nlattr *nest;
        int err = 0;

        nest = nla_nest_start_noflag(skb, MDBA_MDB_ENTRY);
        if (!nest)
                return -EMSGSIZE;

        list_for_each_entry(remote, &mdb_entry->remotes, list) {
                if (remote_idx < s_remote_idx)
                        goto skip;

                err = vxlan_mdb_entry_info_fill(vxlan, skb, mdb_entry, remote);
                if (err)
                        break;
skip:
                remote_idx++;
        }

        ctx->remote_idx = err ? remote_idx : 0;
        nla_nest_end(skb, nest);
        return err;
}

static int vxlan_mdb_fill(const struct vxlan_dev *vxlan, struct sk_buff *skb,
                          struct vxlan_mdb_dump_ctx *ctx)
{
        int entry_idx = 0, s_entry_idx = ctx->entry_idx;
        struct vxlan_mdb_entry *mdb_entry;
        struct nlattr *nest;
        int err = 0;

        nest = nla_nest_start_noflag(skb, MDBA_MDB);
        if (!nest)
                return -EMSGSIZE;

        hlist_for_each_entry(mdb_entry, &vxlan->mdb_list, mdb_node) {
                if (entry_idx < s_entry_idx)
                        goto skip;

                err = vxlan_mdb_entry_fill(vxlan, skb, ctx, mdb_entry);
                if (err)
                        break;
skip:
                entry_idx++;
        }

        ctx->entry_idx = err ? entry_idx : 0;
        nla_nest_end(skb, nest);
        return err;
}

int vxlan_mdb_dump(struct net_device *dev, struct sk_buff *skb,
                   struct netlink_callback *cb)
{
        struct vxlan_mdb_dump_ctx *ctx = (void *)cb->ctx;
        struct vxlan_dev *vxlan = netdev_priv(dev);
        struct br_port_msg *bpm;
        struct nlmsghdr *nlh;
        int err;

        ASSERT_RTNL();

        NL_ASSERT_CTX_FITS(struct vxlan_mdb_dump_ctx);

        nlh = nlmsg_put(skb, NETLINK_CB(cb->skb).portid,
                        cb->nlh->nlmsg_seq, RTM_NEWMDB, sizeof(*bpm),
                        NLM_F_MULTI);
        if (!nlh)
                return -EMSGSIZE;

        bpm = nlmsg_data(nlh);
        memset(bpm, 0, sizeof(*bpm));
        bpm->family = AF_BRIDGE;
        bpm->ifindex = dev->ifindex;

        err = vxlan_mdb_fill(vxlan, skb, ctx);

        nlmsg_end(skb, nlh);

        cb->seq = vxlan->mdb_seq;
        nl_dump_check_consistent(cb, nlh);

        return err;
}

static const struct nla_policy
vxlan_mdbe_src_list_entry_pol[MDBE_SRCATTR_MAX + 1] = {
        [MDBE_SRCATTR_ADDRESS] = NLA_POLICY_RANGE(NLA_BINARY,
                                                  sizeof(struct in_addr),
                                                  sizeof(struct in6_addr)),
};

static const struct nla_policy
vxlan_mdbe_src_list_pol[MDBE_SRC_LIST_MAX + 1] = {
        [MDBE_SRC_LIST_ENTRY] = NLA_POLICY_NESTED(vxlan_mdbe_src_list_entry_pol),
};

static const struct netlink_range_validation vni_range = {
        .max = VXLAN_N_VID - 1,
};

static const struct nla_policy vxlan_mdbe_attrs_pol[MDBE_ATTR_MAX + 1] = {
        [MDBE_ATTR_SOURCE] = NLA_POLICY_RANGE(NLA_BINARY,
                                              sizeof(struct in_addr),
                                              sizeof(struct in6_addr)),
        [MDBE_ATTR_GROUP_MODE] = NLA_POLICY_RANGE(NLA_U8, MCAST_EXCLUDE,
                                                  MCAST_INCLUDE),
        [MDBE_ATTR_SRC_LIST] = NLA_POLICY_NESTED(vxlan_mdbe_src_list_pol),
        [MDBE_ATTR_RTPROT] = NLA_POLICY_MIN(NLA_U8, RTPROT_STATIC),
        [MDBE_ATTR_DST] = NLA_POLICY_RANGE(NLA_BINARY,
                                           sizeof(struct in_addr),
                                           sizeof(struct in6_addr)),
        [MDBE_ATTR_DST_PORT] = { .type = NLA_U16 },
        [MDBE_ATTR_VNI] = NLA_POLICY_FULL_RANGE(NLA_U32, &vni_range),
        [MDBE_ATTR_IFINDEX] = NLA_POLICY_MIN(NLA_S32, 1),
        [MDBE_ATTR_SRC_VNI] = NLA_POLICY_FULL_RANGE(NLA_U32, &vni_range),
};

static bool vxlan_mdb_is_valid_source(const struct nlattr *attr, __be16 proto,
                                      struct netlink_ext_ack *extack)
{
        switch (proto) {
        case htons(ETH_P_IP):
                if (nla_len(attr) != sizeof(struct in_addr)) {
                        NL_SET_ERR_MSG_MOD(extack, "IPv4 invalid source address length");
                        return false;
                }
                if (ipv4_is_multicast(nla_get_in_addr(attr))) {
                        NL_SET_ERR_MSG_MOD(extack, "IPv4 multicast source address is not allowed");
                        return false;
                }
                break;
#if IS_ENABLED(CONFIG_IPV6)
        case htons(ETH_P_IPV6): {
                struct in6_addr src;

                if (nla_len(attr) != sizeof(struct in6_addr)) {
                        NL_SET_ERR_MSG_MOD(extack, "IPv6 invalid source address length");
                        return false;
                }
                src = nla_get_in6_addr(attr);
                if (ipv6_addr_is_multicast(&src)) {
                        NL_SET_ERR_MSG_MOD(extack, "IPv6 multicast source address is not allowed");
                        return false;
                }
                break;
        }
#endif
        default:
                NL_SET_ERR_MSG_MOD(extack, "Invalid protocol used with source address");
                return false;
        }

        return true;
}

static void vxlan_mdb_group_set(struct vxlan_mdb_entry_key *group,
                                const struct br_mdb_entry *entry,
                                const struct nlattr *source_attr)
{
        switch (entry->addr.proto) {
        case htons(ETH_P_IP):
                group->dst.sa.sa_family = AF_INET;
                group->dst.sin.sin_addr.s_addr = entry->addr.u.ip4;
                break;
#if IS_ENABLED(CONFIG_IPV6)
        case htons(ETH_P_IPV6):
                group->dst.sa.sa_family = AF_INET6;
                group->dst.sin6.sin6_addr = entry->addr.u.ip6;
                break;
#endif
        }

        if (source_attr)
                vxlan_nla_get_addr(&group->src, source_attr);
}

static bool vxlan_mdb_is_star_g(const struct vxlan_mdb_entry_key *group)
{
        return !vxlan_addr_any(&group->dst) && vxlan_addr_any(&group->src);
}

static bool vxlan_mdb_is_sg(const struct vxlan_mdb_entry_key *group)
{
        return !vxlan_addr_any(&group->dst) && !vxlan_addr_any(&group->src);
}

static int vxlan_mdb_config_src_entry_init(struct vxlan_mdb_config *cfg,
                                           __be16 proto,
                                           const struct nlattr *src_entry,
                                           struct netlink_ext_ack *extack)
{
        struct nlattr *tb[MDBE_SRCATTR_MAX + 1];
        struct vxlan_mdb_config_src_entry *src;
        int err;

        err = nla_parse_nested(tb, MDBE_SRCATTR_MAX, src_entry,
                               vxlan_mdbe_src_list_entry_pol, extack);
        if (err)
                return err;

        if (NL_REQ_ATTR_CHECK(extack, src_entry, tb, MDBE_SRCATTR_ADDRESS))
                return -EINVAL;

        if (!vxlan_mdb_is_valid_source(tb[MDBE_SRCATTR_ADDRESS], proto,
                                       extack))
                return -EINVAL;

        src = kzalloc(sizeof(*src), GFP_KERNEL);
        if (!src)
                return -ENOMEM;

        err = vxlan_nla_get_addr(&src->addr, tb[MDBE_SRCATTR_ADDRESS]);
        if (err)
                goto err_free_src;

        list_add_tail(&src->node, &cfg->src_list);

        return 0;

err_free_src:
        kfree(src);
        return err;
}

static void
vxlan_mdb_config_src_entry_fini(struct vxlan_mdb_config_src_entry *src)
{
        list_del(&src->node);
        kfree(src);
}

static int vxlan_mdb_config_src_list_init(struct vxlan_mdb_config *cfg,
                                          __be16 proto,
                                          const struct nlattr *src_list,
                                          struct netlink_ext_ack *extack)
{
        struct vxlan_mdb_config_src_entry *src, *tmp;
        struct nlattr *src_entry;
        int rem, err;

        nla_for_each_nested(src_entry, src_list, rem) {
                err = vxlan_mdb_config_src_entry_init(cfg, proto, src_entry,
                                                      extack);
                if (err)
                        goto err_src_entry_init;
        }

        return 0;

err_src_entry_init:
        list_for_each_entry_safe_reverse(src, tmp, &cfg->src_list, node)
                vxlan_mdb_config_src_entry_fini(src);
        return err;
}

static void vxlan_mdb_config_src_list_fini(struct vxlan_mdb_config *cfg)
{
        struct vxlan_mdb_config_src_entry *src, *tmp;

        list_for_each_entry_safe_reverse(src, tmp, &cfg->src_list, node)
                vxlan_mdb_config_src_entry_fini(src);
}

static int vxlan_mdb_config_attrs_init(struct vxlan_mdb_config *cfg,
                                       const struct br_mdb_entry *entry,
                                       const struct nlattr *set_attrs,
                                       struct netlink_ext_ack *extack)
{
        struct nlattr *mdbe_attrs[MDBE_ATTR_MAX + 1];
        int err;

        err = nla_parse_nested(mdbe_attrs, MDBE_ATTR_MAX, set_attrs,
                               vxlan_mdbe_attrs_pol, extack);
        if (err)
                return err;

        if (NL_REQ_ATTR_CHECK(extack, set_attrs, mdbe_attrs, MDBE_ATTR_DST)) {
                NL_SET_ERR_MSG_MOD(extack, "Missing remote destination IP address");
                return -EINVAL;
        }

        if (mdbe_attrs[MDBE_ATTR_SOURCE] &&
            !vxlan_mdb_is_valid_source(mdbe_attrs[MDBE_ATTR_SOURCE],
                                       entry->addr.proto, extack))
                return -EINVAL;

        vxlan_mdb_group_set(&cfg->group, entry, mdbe_attrs[MDBE_ATTR_SOURCE]);

        /* rtnetlink code only validates that IPv4 group address is
         * multicast.
         */
        if (!vxlan_addr_is_multicast(&cfg->group.dst) &&
            !vxlan_addr_any(&cfg->group.dst)) {
                NL_SET_ERR_MSG_MOD(extack, "Group address is not multicast");
                return -EINVAL;
        }

        if (vxlan_addr_any(&cfg->group.dst) &&
            mdbe_attrs[MDBE_ATTR_SOURCE]) {
                NL_SET_ERR_MSG_MOD(extack, "Source cannot be specified for the all-zeros entry");
                return -EINVAL;
        }

        if (vxlan_mdb_is_sg(&cfg->group))
                cfg->filter_mode = MCAST_INCLUDE;

        if (mdbe_attrs[MDBE_ATTR_GROUP_MODE]) {
                if (!vxlan_mdb_is_star_g(&cfg->group)) {
                        NL_SET_ERR_MSG_MOD(extack, "Filter mode can only be set for (*, G) entries");
                        return -EINVAL;
                }
                cfg->filter_mode = nla_get_u8(mdbe_attrs[MDBE_ATTR_GROUP_MODE]);
        }

        if (mdbe_attrs[MDBE_ATTR_SRC_LIST]) {
                if (!vxlan_mdb_is_star_g(&cfg->group)) {
                        NL_SET_ERR_MSG_MOD(extack, "Source list can only be set for (*, G) entries");
                        return -EINVAL;
                }
                if (!mdbe_attrs[MDBE_ATTR_GROUP_MODE]) {
                        NL_SET_ERR_MSG_MOD(extack, "Source list cannot be set without filter mode");
                        return -EINVAL;
                }
                err = vxlan_mdb_config_src_list_init(cfg, entry->addr.proto,
                                                     mdbe_attrs[MDBE_ATTR_SRC_LIST],
                                                     extack);
                if (err)
                        return err;
        }

        if (vxlan_mdb_is_star_g(&cfg->group) && list_empty(&cfg->src_list) &&
            cfg->filter_mode == MCAST_INCLUDE) {
                NL_SET_ERR_MSG_MOD(extack, "Cannot add (*, G) INCLUDE with an empty source list");
                return -EINVAL;
        }

        if (mdbe_attrs[MDBE_ATTR_RTPROT])
                cfg->rt_protocol = nla_get_u8(mdbe_attrs[MDBE_ATTR_RTPROT]);

        err = vxlan_nla_get_addr(&cfg->remote_ip, mdbe_attrs[MDBE_ATTR_DST]);
        if (err) {
                NL_SET_ERR_MSG_MOD(extack, "Invalid remote destination address");
                goto err_src_list_fini;
        }

        if (mdbe_attrs[MDBE_ATTR_DST_PORT])
                cfg->remote_port =
                        cpu_to_be16(nla_get_u16(mdbe_attrs[MDBE_ATTR_DST_PORT]));

        if (mdbe_attrs[MDBE_ATTR_VNI])
                cfg->remote_vni =
                        cpu_to_be32(nla_get_u32(mdbe_attrs[MDBE_ATTR_VNI]));

        if (mdbe_attrs[MDBE_ATTR_IFINDEX]) {
                cfg->remote_ifindex =
                        nla_get_s32(mdbe_attrs[MDBE_ATTR_IFINDEX]);
                if (!__dev_get_by_index(cfg->vxlan->net, cfg->remote_ifindex)) {
                        NL_SET_ERR_MSG_MOD(extack, "Outgoing interface not found");
                        err = -EINVAL;
                        goto err_src_list_fini;
                }
        }

        if (mdbe_attrs[MDBE_ATTR_SRC_VNI])
                cfg->group.vni =
                        cpu_to_be32(nla_get_u32(mdbe_attrs[MDBE_ATTR_SRC_VNI]));

        return 0;

err_src_list_fini:
        vxlan_mdb_config_src_list_fini(cfg);
        return err;
}

static int vxlan_mdb_config_init(struct vxlan_mdb_config *cfg,
                                 struct net_device *dev, struct nlattr *tb[],
                                 u16 nlmsg_flags,
                                 struct netlink_ext_ack *extack)
{
        struct br_mdb_entry *entry = nla_data(tb[MDBA_SET_ENTRY]);
        struct vxlan_dev *vxlan = netdev_priv(dev);

        memset(cfg, 0, sizeof(*cfg));
        cfg->vxlan = vxlan;
        cfg->group.vni = vxlan->default_dst.remote_vni;
        INIT_LIST_HEAD(&cfg->src_list);
        cfg->nlflags = nlmsg_flags;
        cfg->filter_mode = MCAST_EXCLUDE;
        cfg->rt_protocol = RTPROT_STATIC;
        cfg->remote_vni = vxlan->default_dst.remote_vni;
        cfg->remote_port = vxlan->cfg.dst_port;

        if (entry->ifindex != dev->ifindex) {
                NL_SET_ERR_MSG_MOD(extack, "Port net device must be the VXLAN net device");
                return -EINVAL;
        }

        /* State is not part of the entry key and can be ignored on deletion
         * requests.
         */
        if ((nlmsg_flags & (NLM_F_CREATE | NLM_F_REPLACE)) &&
            entry->state != MDB_PERMANENT) {
                NL_SET_ERR_MSG_MOD(extack, "MDB entry must be permanent");
                return -EINVAL;
        }

        if (entry->flags) {
                NL_SET_ERR_MSG_MOD(extack, "Invalid MDB entry flags");
                return -EINVAL;
        }

        if (entry->vid) {
                NL_SET_ERR_MSG_MOD(extack, "VID must not be specified");
                return -EINVAL;
        }

        if (entry->addr.proto != htons(ETH_P_IP) &&
            entry->addr.proto != htons(ETH_P_IPV6)) {
                NL_SET_ERR_MSG_MOD(extack, "Group address must be an IPv4 / IPv6 address");
                return -EINVAL;
        }

        if (NL_REQ_ATTR_CHECK(extack, NULL, tb, MDBA_SET_ENTRY_ATTRS)) {
                NL_SET_ERR_MSG_MOD(extack, "Missing MDBA_SET_ENTRY_ATTRS attribute");
                return -EINVAL;
        }

        return vxlan_mdb_config_attrs_init(cfg, entry, tb[MDBA_SET_ENTRY_ATTRS],
                                           extack);
}

static void vxlan_mdb_config_fini(struct vxlan_mdb_config *cfg)
{
        vxlan_mdb_config_src_list_fini(cfg);
}

static struct vxlan_mdb_entry *
vxlan_mdb_entry_lookup(struct vxlan_dev *vxlan,
                       const struct vxlan_mdb_entry_key *group)
{
        return rhashtable_lookup_fast(&vxlan->mdb_tbl, group,
                                      vxlan_mdb_rht_params);
}

static struct vxlan_mdb_remote *
vxlan_mdb_remote_lookup(const struct vxlan_mdb_entry *mdb_entry,
                        const union vxlan_addr *addr)
{
        struct vxlan_mdb_remote *remote;

        list_for_each_entry(remote, &mdb_entry->remotes, list) {
                struct vxlan_rdst *rd = rtnl_dereference(remote->rd);

                if (vxlan_addr_equal(addr, &rd->remote_ip))
                        return remote;
        }

        return NULL;
}

static void vxlan_mdb_rdst_free(struct rcu_head *head)
{
        struct vxlan_rdst *rd = container_of(head, struct vxlan_rdst, rcu);

        dst_cache_destroy(&rd->dst_cache);
        kfree(rd);
}

static int vxlan_mdb_remote_rdst_init(const struct vxlan_mdb_config *cfg,
                                      struct vxlan_mdb_remote *remote)
{
        struct vxlan_rdst *rd;
        int err;

        rd = kzalloc(sizeof(*rd), GFP_KERNEL);
        if (!rd)
                return -ENOMEM;

        err = dst_cache_init(&rd->dst_cache, GFP_KERNEL);
        if (err)
                goto err_free_rdst;

        rd->remote_ip = cfg->remote_ip;
        rd->remote_port = cfg->remote_port;
        rd->remote_vni = cfg->remote_vni;
        rd->remote_ifindex = cfg->remote_ifindex;
        rcu_assign_pointer(remote->rd, rd);

        return 0;

err_free_rdst:
        kfree(rd);
        return err;
}

static void vxlan_mdb_remote_rdst_fini(struct vxlan_rdst *rd)
{
        call_rcu(&rd->rcu, vxlan_mdb_rdst_free);
}

static int vxlan_mdb_remote_init(const struct vxlan_mdb_config *cfg,
                                 struct vxlan_mdb_remote *remote)
{
        int err;

        err = vxlan_mdb_remote_rdst_init(cfg, remote);
        if (err)
                return err;

        remote->flags = cfg->flags;
        remote->filter_mode = cfg->filter_mode;
        remote->rt_protocol = cfg->rt_protocol;
        INIT_HLIST_HEAD(&remote->src_list);

        return 0;
}

static void vxlan_mdb_remote_fini(struct vxlan_dev *vxlan,
                                  struct vxlan_mdb_remote *remote)
{
        WARN_ON_ONCE(!hlist_empty(&remote->src_list));
        vxlan_mdb_remote_rdst_fini(rtnl_dereference(remote->rd));
}

static struct vxlan_mdb_src_entry *
vxlan_mdb_remote_src_entry_lookup(const struct vxlan_mdb_remote *remote,
                                  const union vxlan_addr *addr)
{
        struct vxlan_mdb_src_entry *ent;

        hlist_for_each_entry(ent, &remote->src_list, node) {
                if (vxlan_addr_equal(&ent->addr, addr))
                        return ent;
        }

        return NULL;
}

static struct vxlan_mdb_src_entry *
vxlan_mdb_remote_src_entry_add(struct vxlan_mdb_remote *remote,
                               const union vxlan_addr *addr)
{
        struct vxlan_mdb_src_entry *ent;

        ent = kzalloc(sizeof(*ent), GFP_KERNEL);
        if (!ent)
                return NULL;

        ent->addr = *addr;
        hlist_add_head(&ent->node, &remote->src_list);

        return ent;
}

static void
vxlan_mdb_remote_src_entry_del(struct vxlan_mdb_src_entry *ent)
{
        hlist_del(&ent->node);
        kfree(ent);
}

static int
vxlan_mdb_remote_src_fwd_add(const struct vxlan_mdb_config *cfg,
                             const union vxlan_addr *addr,
                             struct netlink_ext_ack *extack)
{
        struct vxlan_mdb_config sg_cfg;

        memset(&sg_cfg, 0, sizeof(sg_cfg));
        sg_cfg.vxlan = cfg->vxlan;
        sg_cfg.group.src = *addr;
        sg_cfg.group.dst = cfg->group.dst;
        sg_cfg.group.vni = cfg->group.vni;
        INIT_LIST_HEAD(&sg_cfg.src_list);
        sg_cfg.remote_ip = cfg->remote_ip;
        sg_cfg.remote_ifindex = cfg->remote_ifindex;
        sg_cfg.remote_vni = cfg->remote_vni;
        sg_cfg.remote_port = cfg->remote_port;
        sg_cfg.nlflags = cfg->nlflags;
        sg_cfg.filter_mode = MCAST_INCLUDE;
        if (cfg->filter_mode == MCAST_EXCLUDE)
                sg_cfg.flags = VXLAN_MDB_REMOTE_F_BLOCKED;
        sg_cfg.rt_protocol = cfg->rt_protocol;

        return __vxlan_mdb_add(&sg_cfg, extack);
}

static void
vxlan_mdb_remote_src_fwd_del(struct vxlan_dev *vxlan,
                             const struct vxlan_mdb_entry_key *group,
                             const struct vxlan_mdb_remote *remote,
                             const union vxlan_addr *addr)
{
        struct vxlan_rdst *rd = rtnl_dereference(remote->rd);
        struct vxlan_mdb_config sg_cfg;

        memset(&sg_cfg, 0, sizeof(sg_cfg));
        sg_cfg.vxlan = vxlan;
        sg_cfg.group.src = *addr;
        sg_cfg.group.dst = group->dst;
        sg_cfg.group.vni = group->vni;
        INIT_LIST_HEAD(&sg_cfg.src_list);
        sg_cfg.remote_ip = rd->remote_ip;

        __vxlan_mdb_del(&sg_cfg, NULL);
}

static int
vxlan_mdb_remote_src_add(const struct vxlan_mdb_config *cfg,
                         struct vxlan_mdb_remote *remote,
                         const struct vxlan_mdb_config_src_entry *src,
                         struct netlink_ext_ack *extack)
{
        struct vxlan_mdb_src_entry *ent;
        int err;

        ent = vxlan_mdb_remote_src_entry_lookup(remote, &src->addr);
        if (!ent) {
                ent = vxlan_mdb_remote_src_entry_add(remote, &src->addr);
                if (!ent)
                        return -ENOMEM;
        } else if (!(cfg->nlflags & NLM_F_REPLACE)) {
                NL_SET_ERR_MSG_MOD(extack, "Source entry already exists");
                return -EEXIST;
        }

        err = vxlan_mdb_remote_src_fwd_add(cfg, &ent->addr, extack);
        if (err)
                goto err_src_del;

        /* Clear flags in case source entry was marked for deletion as part of
         * replace flow.
         */
        ent->flags = 0;

        return 0;

err_src_del:
        vxlan_mdb_remote_src_entry_del(ent);
        return err;
}

static void vxlan_mdb_remote_src_del(struct vxlan_dev *vxlan,
                                     const struct vxlan_mdb_entry_key *group,
                                     const struct vxlan_mdb_remote *remote,
                                     struct vxlan_mdb_src_entry *ent)
{
        vxlan_mdb_remote_src_fwd_del(vxlan, group, remote, &ent->addr);
        vxlan_mdb_remote_src_entry_del(ent);
}

static int vxlan_mdb_remote_srcs_add(const struct vxlan_mdb_config *cfg,
                                     struct vxlan_mdb_remote *remote,
                                     struct netlink_ext_ack *extack)
{
        struct vxlan_mdb_config_src_entry *src;
        struct vxlan_mdb_src_entry *ent;
        struct hlist_node *tmp;
        int err;

        list_for_each_entry(src, &cfg->src_list, node) {
                err = vxlan_mdb_remote_src_add(cfg, remote, src, extack);
                if (err)
                        goto err_src_del;
        }

        return 0;

err_src_del:
        hlist_for_each_entry_safe(ent, tmp, &remote->src_list, node)
                vxlan_mdb_remote_src_del(cfg->vxlan, &cfg->group, remote, ent);
        return err;
}

static void vxlan_mdb_remote_srcs_del(struct vxlan_dev *vxlan,
                                      const struct vxlan_mdb_entry_key *group,
                                      struct vxlan_mdb_remote *remote)
{
        struct vxlan_mdb_src_entry *ent;
        struct hlist_node *tmp;

        hlist_for_each_entry_safe(ent, tmp, &remote->src_list, node)
                vxlan_mdb_remote_src_del(vxlan, group, remote, ent);
}

static size_t
vxlan_mdb_nlmsg_src_list_size(const struct vxlan_mdb_entry_key *group,
                              const struct vxlan_mdb_remote *remote)
{
        struct vxlan_mdb_src_entry *ent;
        size_t nlmsg_size;

        if (hlist_empty(&remote->src_list))
                return 0;

        /* MDBA_MDB_EATTR_SRC_LIST */
        nlmsg_size = nla_total_size(0);

        hlist_for_each_entry(ent, &remote->src_list, node) {
                              /* MDBA_MDB_SRCLIST_ENTRY */
                nlmsg_size += nla_total_size(0) +
                              /* MDBA_MDB_SRCATTR_ADDRESS */
                              nla_total_size(vxlan_addr_size(&group->dst)) +
                              /* MDBA_MDB_SRCATTR_TIMER */
                              nla_total_size(sizeof(u8));
        }

        return nlmsg_size;
}

static size_t
vxlan_mdb_nlmsg_remote_size(const struct vxlan_dev *vxlan,
                            const struct vxlan_mdb_entry *mdb_entry,
                            const struct vxlan_mdb_remote *remote)
{
        const struct vxlan_mdb_entry_key *group = &mdb_entry->key;
        struct vxlan_rdst *rd = rtnl_dereference(remote->rd);
        size_t nlmsg_size;

                     /* MDBA_MDB_ENTRY_INFO */
        nlmsg_size = nla_total_size(sizeof(struct br_mdb_entry)) +
                     /* MDBA_MDB_EATTR_TIMER */
                     nla_total_size(sizeof(u32));

        /* MDBA_MDB_EATTR_SOURCE */
        if (vxlan_mdb_is_sg(group))
                nlmsg_size += nla_total_size(vxlan_addr_size(&group->dst));
        /* MDBA_MDB_EATTR_RTPROT */
        nlmsg_size += nla_total_size(sizeof(u8));
        /* MDBA_MDB_EATTR_SRC_LIST */
        nlmsg_size += vxlan_mdb_nlmsg_src_list_size(group, remote);
        /* MDBA_MDB_EATTR_GROUP_MODE */
        nlmsg_size += nla_total_size(sizeof(u8));
        /* MDBA_MDB_EATTR_DST */
        nlmsg_size += nla_total_size(vxlan_addr_size(&rd->remote_ip));
        /* MDBA_MDB_EATTR_DST_PORT */
        if (rd->remote_port && rd->remote_port != vxlan->cfg.dst_port)
                nlmsg_size += nla_total_size(sizeof(u16));
        /* MDBA_MDB_EATTR_VNI */
        if (rd->remote_vni != vxlan->default_dst.remote_vni)
                nlmsg_size += nla_total_size(sizeof(u32));
        /* MDBA_MDB_EATTR_IFINDEX */
        if (rd->remote_ifindex)
                nlmsg_size += nla_total_size(sizeof(u32));
        /* MDBA_MDB_EATTR_SRC_VNI */
        if ((vxlan->cfg.flags & VXLAN_F_COLLECT_METADATA) && group->vni)
                nlmsg_size += nla_total_size(sizeof(u32));

        return nlmsg_size;
}

static size_t vxlan_mdb_nlmsg_size(const struct vxlan_dev *vxlan,
                                   const struct vxlan_mdb_entry *mdb_entry,
                                   const struct vxlan_mdb_remote *remote)
{
        return NLMSG_ALIGN(sizeof(struct br_port_msg)) +
               /* MDBA_MDB */
               nla_total_size(0) +
               /* MDBA_MDB_ENTRY */
               nla_total_size(0) +
               /* Remote entry */
               vxlan_mdb_nlmsg_remote_size(vxlan, mdb_entry, remote);
}

static int vxlan_mdb_nlmsg_fill(const struct vxlan_dev *vxlan,
                                struct sk_buff *skb,
                                const struct vxlan_mdb_entry *mdb_entry,
                                const struct vxlan_mdb_remote *remote,
                                int type)
{
        struct nlattr *mdb_nest, *mdb_entry_nest;
        struct br_port_msg *bpm;
        struct nlmsghdr *nlh;

        nlh = nlmsg_put(skb, 0, 0, type, sizeof(*bpm), 0);
        if (!nlh)
                return -EMSGSIZE;

        bpm = nlmsg_data(nlh);
        memset(bpm, 0, sizeof(*bpm));
        bpm->family  = AF_BRIDGE;
        bpm->ifindex = vxlan->dev->ifindex;

        mdb_nest = nla_nest_start_noflag(skb, MDBA_MDB);
        if (!mdb_nest)
                goto cancel;
        mdb_entry_nest = nla_nest_start_noflag(skb, MDBA_MDB_ENTRY);
        if (!mdb_entry_nest)
                goto cancel;

        if (vxlan_mdb_entry_info_fill(vxlan, skb, mdb_entry, remote))
                goto cancel;

        nla_nest_end(skb, mdb_entry_nest);
        nla_nest_end(skb, mdb_nest);
        nlmsg_end(skb, nlh);

        return 0;

cancel:
        nlmsg_cancel(skb, nlh);
        return -EMSGSIZE;
}

static void vxlan_mdb_remote_notify(const struct vxlan_dev *vxlan,
                                    const struct vxlan_mdb_entry *mdb_entry,
                                    const struct vxlan_mdb_remote *remote,
                                    int type)
{
        struct net *net = dev_net(vxlan->dev);
        struct sk_buff *skb;
        int err = -ENOBUFS;

        skb = nlmsg_new(vxlan_mdb_nlmsg_size(vxlan, mdb_entry, remote),
                        GFP_KERNEL);
        if (!skb)
                goto errout;

        err = vxlan_mdb_nlmsg_fill(vxlan, skb, mdb_entry, remote, type);
        if (err) {
                kfree_skb(skb);
                goto errout;
        }

        rtnl_notify(skb, net, 0, RTNLGRP_MDB, NULL, GFP_KERNEL);
        return;
errout:
        rtnl_set_sk_err(net, RTNLGRP_MDB, err);
}

static int
vxlan_mdb_remote_srcs_replace(const struct vxlan_mdb_config *cfg,
                              const struct vxlan_mdb_entry *mdb_entry,
                              struct vxlan_mdb_remote *remote,
                              struct netlink_ext_ack *extack)
{
        struct vxlan_dev *vxlan = cfg->vxlan;
        struct vxlan_mdb_src_entry *ent;
        struct hlist_node *tmp;
        int err;

        hlist_for_each_entry(ent, &remote->src_list, node)
                ent->flags |= VXLAN_SGRP_F_DELETE;

        err = vxlan_mdb_remote_srcs_add(cfg, remote, extack);
        if (err)
                goto err_clear_delete;

        hlist_for_each_entry_safe(ent, tmp, &remote->src_list, node) {
                if (ent->flags & VXLAN_SGRP_F_DELETE)
                        vxlan_mdb_remote_src_del(vxlan, &mdb_entry->key, remote,
                                                 ent);
        }

        return 0;

err_clear_delete:
        hlist_for_each_entry(ent, &remote->src_list, node)
                ent->flags &= ~VXLAN_SGRP_F_DELETE;
        return err;
}

static int vxlan_mdb_remote_replace(const struct vxlan_mdb_config *cfg,
                                    const struct vxlan_mdb_entry *mdb_entry,
                                    struct vxlan_mdb_remote *remote,
                                    struct netlink_ext_ack *extack)
{
        struct vxlan_rdst *new_rd, *old_rd = rtnl_dereference(remote->rd);
        struct vxlan_dev *vxlan = cfg->vxlan;
        int err;

        err = vxlan_mdb_remote_rdst_init(cfg, remote);
        if (err)
                return err;
        new_rd = rtnl_dereference(remote->rd);

        err = vxlan_mdb_remote_srcs_replace(cfg, mdb_entry, remote, extack);
        if (err)
                goto err_rdst_reset;

        WRITE_ONCE(remote->flags, cfg->flags);
        WRITE_ONCE(remote->filter_mode, cfg->filter_mode);
        remote->rt_protocol = cfg->rt_protocol;
        vxlan_mdb_remote_notify(vxlan, mdb_entry, remote, RTM_NEWMDB);

        vxlan_mdb_remote_rdst_fini(old_rd);

        return 0;

err_rdst_reset:
        rcu_assign_pointer(remote->rd, old_rd);
        vxlan_mdb_remote_rdst_fini(new_rd);
        return err;
}

static int vxlan_mdb_remote_add(const struct vxlan_mdb_config *cfg,
                                struct vxlan_mdb_entry *mdb_entry,
                                struct netlink_ext_ack *extack)
{
        struct vxlan_mdb_remote *remote;
        int err;

        remote = vxlan_mdb_remote_lookup(mdb_entry, &cfg->remote_ip);
        if (remote) {
                if (!(cfg->nlflags & NLM_F_REPLACE)) {
                        NL_SET_ERR_MSG_MOD(extack, "Replace not specified and MDB remote entry already exists");
                        return -EEXIST;
                }
                return vxlan_mdb_remote_replace(cfg, mdb_entry, remote, extack);
        }

        if (!(cfg->nlflags & NLM_F_CREATE)) {
                NL_SET_ERR_MSG_MOD(extack, "Create not specified and entry does not exist");
                return -ENOENT;
        }

        remote = kzalloc(sizeof(*remote), GFP_KERNEL);
        if (!remote)
                return -ENOMEM;

        err = vxlan_mdb_remote_init(cfg, remote);
        if (err) {
                NL_SET_ERR_MSG_MOD(extack, "Failed to initialize remote MDB entry");
                goto err_free_remote;
        }

        err = vxlan_mdb_remote_srcs_add(cfg, remote, extack);
        if (err)
                goto err_remote_fini;

        list_add_rcu(&remote->list, &mdb_entry->remotes);
        vxlan_mdb_remote_notify(cfg->vxlan, mdb_entry, remote, RTM_NEWMDB);

        return 0;

err_remote_fini:
        vxlan_mdb_remote_fini(cfg->vxlan, remote);
err_free_remote:
        kfree(remote);
        return err;
}

static void vxlan_mdb_remote_del(struct vxlan_dev *vxlan,
                                 struct vxlan_mdb_entry *mdb_entry,
                                 struct vxlan_mdb_remote *remote)
{
        vxlan_mdb_remote_notify(vxlan, mdb_entry, remote, RTM_DELMDB);
        list_del_rcu(&remote->list);
        vxlan_mdb_remote_srcs_del(vxlan, &mdb_entry->key, remote);
        vxlan_mdb_remote_fini(vxlan, remote);
        kfree_rcu(remote, rcu);
}

static struct vxlan_mdb_entry *
vxlan_mdb_entry_get(struct vxlan_dev *vxlan,
                    const struct vxlan_mdb_entry_key *group)
{
        struct vxlan_mdb_entry *mdb_entry;
        int err;

        mdb_entry = vxlan_mdb_entry_lookup(vxlan, group);
        if (mdb_entry)
                return mdb_entry;

        mdb_entry = kzalloc(sizeof(*mdb_entry), GFP_KERNEL);
        if (!mdb_entry)
                return ERR_PTR(-ENOMEM);

        INIT_LIST_HEAD(&mdb_entry->remotes);
        memcpy(&mdb_entry->key, group, sizeof(mdb_entry->key));
        hlist_add_head(&mdb_entry->mdb_node, &vxlan->mdb_list);

        err = rhashtable_lookup_insert_fast(&vxlan->mdb_tbl,
                                            &mdb_entry->rhnode,
                                            vxlan_mdb_rht_params);
        if (err)
                goto err_free_entry;

        if (hlist_is_singular_node(&mdb_entry->mdb_node, &vxlan->mdb_list))
                vxlan->cfg.flags |= VXLAN_F_MDB;

        return mdb_entry;

err_free_entry:
        hlist_del(&mdb_entry->mdb_node);
        kfree(mdb_entry);
        return ERR_PTR(err);
}

static void vxlan_mdb_entry_put(struct vxlan_dev *vxlan,
                                struct vxlan_mdb_entry *mdb_entry)
{
        if (!list_empty(&mdb_entry->remotes))
                return;

        if (hlist_is_singular_node(&mdb_entry->mdb_node, &vxlan->mdb_list))
                vxlan->cfg.flags &= ~VXLAN_F_MDB;

        rhashtable_remove_fast(&vxlan->mdb_tbl, &mdb_entry->rhnode,
                               vxlan_mdb_rht_params);
        hlist_del(&mdb_entry->mdb_node);
        kfree_rcu(mdb_entry, rcu);
}

static int __vxlan_mdb_add(const struct vxlan_mdb_config *cfg,
                           struct netlink_ext_ack *extack)
{
        struct vxlan_dev *vxlan = cfg->vxlan;
        struct vxlan_mdb_entry *mdb_entry;
        int err;

        mdb_entry = vxlan_mdb_entry_get(vxlan, &cfg->group);
        if (IS_ERR(mdb_entry))
                return PTR_ERR(mdb_entry);

        err = vxlan_mdb_remote_add(cfg, mdb_entry, extack);
        if (err)
                goto err_entry_put;

        vxlan->mdb_seq++;

        return 0;

err_entry_put:
        vxlan_mdb_entry_put(vxlan, mdb_entry);
        return err;
}

static int __vxlan_mdb_del(const struct vxlan_mdb_config *cfg,
                           struct netlink_ext_ack *extack)
{
        struct vxlan_dev *vxlan = cfg->vxlan;
        struct vxlan_mdb_entry *mdb_entry;
        struct vxlan_mdb_remote *remote;

        mdb_entry = vxlan_mdb_entry_lookup(vxlan, &cfg->group);
        if (!mdb_entry) {
                NL_SET_ERR_MSG_MOD(extack, "Did not find MDB entry");
                return -ENOENT;
        }

        remote = vxlan_mdb_remote_lookup(mdb_entry, &cfg->remote_ip);
        if (!remote) {
                NL_SET_ERR_MSG_MOD(extack, "Did not find MDB remote entry");
                return -ENOENT;
        }

        vxlan_mdb_remote_del(vxlan, mdb_entry, remote);
        vxlan_mdb_entry_put(vxlan, mdb_entry);

        vxlan->mdb_seq++;

        return 0;
}

int vxlan_mdb_add(struct net_device *dev, struct nlattr *tb[], u16 nlmsg_flags,
                  struct netlink_ext_ack *extack)
{
        struct vxlan_mdb_config cfg;
        int err;

        ASSERT_RTNL();

        err = vxlan_mdb_config_init(&cfg, dev, tb, nlmsg_flags, extack);
        if (err)
                return err;

        err = __vxlan_mdb_add(&cfg, extack);

        vxlan_mdb_config_fini(&cfg);
        return err;
}

int vxlan_mdb_del(struct net_device *dev, struct nlattr *tb[],
                  struct netlink_ext_ack *extack)
{
        struct vxlan_mdb_config cfg;
        int err;

        ASSERT_RTNL();

        err = vxlan_mdb_config_init(&cfg, dev, tb, 0, extack);
        if (err)
                return err;

        err = __vxlan_mdb_del(&cfg, extack);

        vxlan_mdb_config_fini(&cfg);
        return err;
}

static const struct nla_policy
vxlan_mdbe_attrs_del_bulk_pol[MDBE_ATTR_MAX + 1] = {
        [MDBE_ATTR_RTPROT] = NLA_POLICY_MIN(NLA_U8, RTPROT_STATIC),
        [MDBE_ATTR_DST] = NLA_POLICY_RANGE(NLA_BINARY,
                                           sizeof(struct in_addr),
                                           sizeof(struct in6_addr)),
        [MDBE_ATTR_DST_PORT] = { .type = NLA_U16 },
        [MDBE_ATTR_VNI] = NLA_POLICY_FULL_RANGE(NLA_U32, &vni_range),
        [MDBE_ATTR_SRC_VNI] = NLA_POLICY_FULL_RANGE(NLA_U32, &vni_range),
        [MDBE_ATTR_STATE_MASK] = NLA_POLICY_MASK(NLA_U8, MDB_PERMANENT),
};

static int vxlan_mdb_flush_desc_init(struct vxlan_dev *vxlan,
                                     struct vxlan_mdb_flush_desc *desc,
                                     struct nlattr *tb[],
                                     struct netlink_ext_ack *extack)
{
        struct br_mdb_entry *entry = nla_data(tb[MDBA_SET_ENTRY]);
        struct nlattr *mdbe_attrs[MDBE_ATTR_MAX + 1];
        int err;

        if (entry->ifindex && entry->ifindex != vxlan->dev->ifindex) {
                NL_SET_ERR_MSG_MOD(extack, "Invalid port net device");
                return -EINVAL;
        }

        if (entry->vid) {
                NL_SET_ERR_MSG_MOD(extack, "VID must not be specified");
                return -EINVAL;
        }

        if (!tb[MDBA_SET_ENTRY_ATTRS])
                return 0;

        err = nla_parse_nested(mdbe_attrs, MDBE_ATTR_MAX,
                               tb[MDBA_SET_ENTRY_ATTRS],
                               vxlan_mdbe_attrs_del_bulk_pol, extack);
        if (err)
                return err;

        if (mdbe_attrs[MDBE_ATTR_STATE_MASK]) {
                u8 state_mask = nla_get_u8(mdbe_attrs[MDBE_ATTR_STATE_MASK]);

                if ((state_mask & MDB_PERMANENT) && !(entry->state & MDB_PERMANENT)) {
                        NL_SET_ERR_MSG_MOD(extack, "Only permanent MDB entries are supported");
                        return -EINVAL;
                }
        }

        if (mdbe_attrs[MDBE_ATTR_RTPROT])
                desc->rt_protocol = nla_get_u8(mdbe_attrs[MDBE_ATTR_RTPROT]);

        if (mdbe_attrs[MDBE_ATTR_DST])
                vxlan_nla_get_addr(&desc->remote_ip, mdbe_attrs[MDBE_ATTR_DST]);

        if (mdbe_attrs[MDBE_ATTR_DST_PORT])
                desc->remote_port =
                        cpu_to_be16(nla_get_u16(mdbe_attrs[MDBE_ATTR_DST_PORT]));

        if (mdbe_attrs[MDBE_ATTR_VNI])
                desc->remote_vni =
                        cpu_to_be32(nla_get_u32(mdbe_attrs[MDBE_ATTR_VNI]));

        if (mdbe_attrs[MDBE_ATTR_SRC_VNI])
                desc->src_vni =
                        cpu_to_be32(nla_get_u32(mdbe_attrs[MDBE_ATTR_SRC_VNI]));

        return 0;
}

static void vxlan_mdb_remotes_flush(struct vxlan_dev *vxlan,
                                    struct vxlan_mdb_entry *mdb_entry,
                                    const struct vxlan_mdb_flush_desc *desc)
{
        struct vxlan_mdb_remote *remote, *tmp;

        list_for_each_entry_safe(remote, tmp, &mdb_entry->remotes, list) {
                struct vxlan_rdst *rd = rtnl_dereference(remote->rd);
                __be32 remote_vni;

                if (desc->remote_ip.sa.sa_family &&
                    !vxlan_addr_equal(&desc->remote_ip, &rd->remote_ip))
                        continue;

                /* Encapsulation is performed with source VNI if remote VNI
                 * is not set.
                 */
                remote_vni = rd->remote_vni ? : mdb_entry->key.vni;
                if (desc->remote_vni && desc->remote_vni != remote_vni)
                        continue;

                if (desc->remote_port && desc->remote_port != rd->remote_port)
                        continue;

                if (desc->rt_protocol &&
                    desc->rt_protocol != remote->rt_protocol)
                        continue;

                vxlan_mdb_remote_del(vxlan, mdb_entry, remote);
        }
}

static void vxlan_mdb_flush(struct vxlan_dev *vxlan,
                            const struct vxlan_mdb_flush_desc *desc)
{
        struct vxlan_mdb_entry *mdb_entry;
        struct hlist_node *tmp;

        /* The removal of an entry cannot trigger the removal of another entry
         * since entries are always added to the head of the list.
         */
        hlist_for_each_entry_safe(mdb_entry, tmp, &vxlan->mdb_list, mdb_node) {
                if (desc->src_vni && desc->src_vni != mdb_entry->key.vni)
                        continue;

                vxlan_mdb_remotes_flush(vxlan, mdb_entry, desc);
                /* Entry will only be removed if its remotes list is empty. */
                vxlan_mdb_entry_put(vxlan, mdb_entry);
        }
}

int vxlan_mdb_del_bulk(struct net_device *dev, struct nlattr *tb[],
                       struct netlink_ext_ack *extack)
{
        struct vxlan_dev *vxlan = netdev_priv(dev);
        struct vxlan_mdb_flush_desc desc = {};
        int err;

        ASSERT_RTNL();

        err = vxlan_mdb_flush_desc_init(vxlan, &desc, tb, extack);
        if (err)
                return err;

        vxlan_mdb_flush(vxlan, &desc);

        return 0;
}

static const struct nla_policy vxlan_mdbe_attrs_get_pol[MDBE_ATTR_MAX + 1] = {
        [MDBE_ATTR_SOURCE] = NLA_POLICY_RANGE(NLA_BINARY,
                                              sizeof(struct in_addr),
                                              sizeof(struct in6_addr)),
        [MDBE_ATTR_SRC_VNI] = NLA_POLICY_FULL_RANGE(NLA_U32, &vni_range),
};

static int vxlan_mdb_get_parse(struct net_device *dev, struct nlattr *tb[],
                               struct vxlan_mdb_entry_key *group,
                               struct netlink_ext_ack *extack)
{
        struct br_mdb_entry *entry = nla_data(tb[MDBA_GET_ENTRY]);
        struct nlattr *mdbe_attrs[MDBE_ATTR_MAX + 1];
        struct vxlan_dev *vxlan = netdev_priv(dev);
        int err;

        memset(group, 0, sizeof(*group));
        group->vni = vxlan->default_dst.remote_vni;

        if (!tb[MDBA_GET_ENTRY_ATTRS]) {
                vxlan_mdb_group_set(group, entry, NULL);
                return 0;
        }

        err = nla_parse_nested(mdbe_attrs, MDBE_ATTR_MAX,
                               tb[MDBA_GET_ENTRY_ATTRS],
                               vxlan_mdbe_attrs_get_pol, extack);
        if (err)
                return err;

        if (mdbe_attrs[MDBE_ATTR_SOURCE] &&
            !vxlan_mdb_is_valid_source(mdbe_attrs[MDBE_ATTR_SOURCE],
                                       entry->addr.proto, extack))
                return -EINVAL;

        vxlan_mdb_group_set(group, entry, mdbe_attrs[MDBE_ATTR_SOURCE]);

        if (mdbe_attrs[MDBE_ATTR_SRC_VNI])
                group->vni =
                        cpu_to_be32(nla_get_u32(mdbe_attrs[MDBE_ATTR_SRC_VNI]));

        return 0;
}

static struct sk_buff *
vxlan_mdb_get_reply_alloc(const struct vxlan_dev *vxlan,
                          const struct vxlan_mdb_entry *mdb_entry)
{
        struct vxlan_mdb_remote *remote;
        size_t nlmsg_size;

        nlmsg_size = NLMSG_ALIGN(sizeof(struct br_port_msg)) +
                     /* MDBA_MDB */
                     nla_total_size(0) +
                     /* MDBA_MDB_ENTRY */
                     nla_total_size(0);

        list_for_each_entry(remote, &mdb_entry->remotes, list)
                nlmsg_size += vxlan_mdb_nlmsg_remote_size(vxlan, mdb_entry,
                                                          remote);

        return nlmsg_new(nlmsg_size, GFP_KERNEL);
}

static int
vxlan_mdb_get_reply_fill(const struct vxlan_dev *vxlan,
                         struct sk_buff *skb,
                         const struct vxlan_mdb_entry *mdb_entry,
                         u32 portid, u32 seq)
{
        struct nlattr *mdb_nest, *mdb_entry_nest;
        struct vxlan_mdb_remote *remote;
        struct br_port_msg *bpm;
        struct nlmsghdr *nlh;
        int err;

        nlh = nlmsg_put(skb, portid, seq, RTM_NEWMDB, sizeof(*bpm), 0);
        if (!nlh)
                return -EMSGSIZE;

        bpm = nlmsg_data(nlh);
        memset(bpm, 0, sizeof(*bpm));
        bpm->family  = AF_BRIDGE;
        bpm->ifindex = vxlan->dev->ifindex;
        mdb_nest = nla_nest_start_noflag(skb, MDBA_MDB);
        if (!mdb_nest) {
                err = -EMSGSIZE;
                goto cancel;
        }
        mdb_entry_nest = nla_nest_start_noflag(skb, MDBA_MDB_ENTRY);
        if (!mdb_entry_nest) {
                err = -EMSGSIZE;
                goto cancel;
        }

        list_for_each_entry(remote, &mdb_entry->remotes, list) {
                err = vxlan_mdb_entry_info_fill(vxlan, skb, mdb_entry, remote);
                if (err)
                        goto cancel;
        }

        nla_nest_end(skb, mdb_entry_nest);
        nla_nest_end(skb, mdb_nest);
        nlmsg_end(skb, nlh);

        return 0;

cancel:
        nlmsg_cancel(skb, nlh);
        return err;
}

int vxlan_mdb_get(struct net_device *dev, struct nlattr *tb[], u32 portid,
                  u32 seq, struct netlink_ext_ack *extack)
{
        struct vxlan_dev *vxlan = netdev_priv(dev);
        struct vxlan_mdb_entry *mdb_entry;
        struct vxlan_mdb_entry_key group;
        struct sk_buff *skb;
        int err;

        ASSERT_RTNL();

        err = vxlan_mdb_get_parse(dev, tb, &group, extack);
        if (err)
                return err;

        mdb_entry = vxlan_mdb_entry_lookup(vxlan, &group);
        if (!mdb_entry) {
                NL_SET_ERR_MSG_MOD(extack, "MDB entry not found");
                return -ENOENT;
        }

        skb = vxlan_mdb_get_reply_alloc(vxlan, mdb_entry);
        if (!skb)
                return -ENOMEM;

        err = vxlan_mdb_get_reply_fill(vxlan, skb, mdb_entry, portid, seq);
        if (err) {
                NL_SET_ERR_MSG_MOD(extack, "Failed to fill MDB get reply");
                goto free;
        }

        return rtnl_unicast(skb, dev_net(dev), portid);

free:
        kfree_skb(skb);
        return err;
}

struct vxlan_mdb_entry *vxlan_mdb_entry_skb_get(struct vxlan_dev *vxlan,
                                                struct sk_buff *skb,
                                                __be32 src_vni)
{
        struct vxlan_mdb_entry *mdb_entry;
        struct vxlan_mdb_entry_key group;

        if (!is_multicast_ether_addr(eth_hdr(skb)->h_dest) ||
            is_broadcast_ether_addr(eth_hdr(skb)->h_dest))
                return NULL;

        /* When not in collect metadata mode, 'src_vni' is zero, but MDB
         * entries are stored with the VNI of the VXLAN device.
         */
        if (!(vxlan->cfg.flags & VXLAN_F_COLLECT_METADATA))
                src_vni = vxlan->default_dst.remote_vni;

        memset(&group, 0, sizeof(group));
        group.vni = src_vni;

        switch (skb->protocol) {
        case htons(ETH_P_IP):
                if (!pskb_may_pull(skb, sizeof(struct iphdr)))
                        return NULL;
                group.dst.sa.sa_family = AF_INET;
                group.dst.sin.sin_addr.s_addr = ip_hdr(skb)->daddr;
                group.src.sa.sa_family = AF_INET;
                group.src.sin.sin_addr.s_addr = ip_hdr(skb)->saddr;
                break;
#if IS_ENABLED(CONFIG_IPV6)
        case htons(ETH_P_IPV6):
                if (!pskb_may_pull(skb, sizeof(struct ipv6hdr)))
                        return NULL;
                group.dst.sa.sa_family = AF_INET6;
                group.dst.sin6.sin6_addr = ipv6_hdr(skb)->daddr;
                group.src.sa.sa_family = AF_INET6;
                group.src.sin6.sin6_addr = ipv6_hdr(skb)->saddr;
                break;
#endif
        default:
                return NULL;
        }

        mdb_entry = vxlan_mdb_entry_lookup(vxlan, &group);
        if (mdb_entry)
                return mdb_entry;

        memset(&group.src, 0, sizeof(group.src));
        mdb_entry = vxlan_mdb_entry_lookup(vxlan, &group);
        if (mdb_entry)
                return mdb_entry;

        /* No (S, G) or (*, G) found. Look up the all-zeros entry, but only if
         * the destination IP address is not link-local multicast since we want
         * to transmit such traffic together with broadcast and unknown unicast
         * traffic.
         */
        switch (skb->protocol) {
        case htons(ETH_P_IP):
                if (ipv4_is_local_multicast(group.dst.sin.sin_addr.s_addr))
                        return NULL;
                group.dst.sin.sin_addr.s_addr = 0;
                break;
#if IS_ENABLED(CONFIG_IPV6)
        case htons(ETH_P_IPV6):
                if (ipv6_addr_type(&group.dst.sin6.sin6_addr) &
                    IPV6_ADDR_LINKLOCAL)
                        return NULL;
                memset(&group.dst.sin6.sin6_addr, 0,
                       sizeof(group.dst.sin6.sin6_addr));
                break;
#endif
        default:
                return NULL;
        }

        return vxlan_mdb_entry_lookup(vxlan, &group);
}

netdev_tx_t vxlan_mdb_xmit(struct vxlan_dev *vxlan,
                           const struct vxlan_mdb_entry *mdb_entry,
                           struct sk_buff *skb)
{
        struct vxlan_mdb_remote *remote, *fremote = NULL;
        __be32 src_vni = mdb_entry->key.vni;

        list_for_each_entry_rcu(remote, &mdb_entry->remotes, list) {
                struct sk_buff *skb1;

                if ((vxlan_mdb_is_star_g(&mdb_entry->key) &&
                     READ_ONCE(remote->filter_mode) == MCAST_INCLUDE) ||
                    (READ_ONCE(remote->flags) & VXLAN_MDB_REMOTE_F_BLOCKED))
                        continue;

                if (!fremote) {
                        fremote = remote;
                        continue;
                }

                skb1 = skb_clone(skb, GFP_ATOMIC);
                if (skb1)
                        vxlan_xmit_one(skb1, vxlan->dev, src_vni,
                                       rcu_dereference(remote->rd), false);
        }

        if (fremote)
                vxlan_xmit_one(skb, vxlan->dev, src_vni,
                               rcu_dereference(fremote->rd), false);
        else
                kfree_skb_reason(skb, SKB_DROP_REASON_VXLAN_NO_REMOTE);

        return NETDEV_TX_OK;
}

static void vxlan_mdb_check_empty(void *ptr, void *arg)
{
        WARN_ON_ONCE(1);
}

int vxlan_mdb_init(struct vxlan_dev *vxlan)
{
        int err;

        err = rhashtable_init(&vxlan->mdb_tbl, &vxlan_mdb_rht_params);
        if (err)
                return err;

        INIT_HLIST_HEAD(&vxlan->mdb_list);

        return 0;
}

void vxlan_mdb_fini(struct vxlan_dev *vxlan)
{
        struct vxlan_mdb_flush_desc desc = {};

        vxlan_mdb_flush(vxlan, &desc);
        WARN_ON_ONCE(vxlan->cfg.flags & VXLAN_F_MDB);
        rhashtable_free_and_destroy(&vxlan->mdb_tbl, vxlan_mdb_check_empty,
                                    NULL);
}































































    4 









  307 






    1 

   34 












   34 





  274 





















   25 











    1 




















    1 


   18 




































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































  245 






































   13 







































































































































































































































































































































































































    3 





    1 































    9 
















   13 









































   18 
















    2 
























































































































































    1 












    6 
    1 

































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
/* SPDX-License-Identifier: GPL-2.0 */
/* IP Virtual Server
 * data structure and functionality definitions
 */

#ifndef _NET_IP_VS_H
#define _NET_IP_VS_H

#include <linux/ip_vs.h>                /* definitions shared with userland */

#include <asm/types.h>                  /* for __uXX types */

#include <linux/list.h>                 /* for struct list_head */
#include <linux/spinlock.h>             /* for struct rwlock_t */
#include <linux/atomic.h>               /* for struct atomic_t */
#include <linux/refcount.h>             /* for struct refcount_t */
#include <linux/workqueue.h>

#include <linux/compiler.h>
#include <linux/timer.h>
#include <linux/bug.h>

#include <net/checksum.h>
#include <linux/netfilter.h>                /* for union nf_inet_addr */
#include <linux/ip.h>
#include <linux/ipv6.h>                        /* for struct ipv6hdr */
#include <net/ipv6.h>
#if IS_ENABLED(CONFIG_NF_CONNTRACK)
#include <net/netfilter/nf_conntrack.h>
#endif
#include <net/net_namespace.h>                /* Netw namespace */
#include <linux/sched/isolation.h>

#define IP_VS_HDR_INVERSE        1
#define IP_VS_HDR_ICMP                2

/* Generic access of ipvs struct */
static inline struct netns_ipvs *net_ipvs(struct net* net)
{
        return net->ipvs;
}

/* Connections' size value needed by ip_vs_ctl.c */
extern int ip_vs_conn_tab_size;

extern struct mutex __ip_vs_mutex;

struct ip_vs_iphdr {
        int hdr_flags;        /* ipvs flags */
        __u32 off;        /* Where IP or IPv4 header starts */
        __u32 len;        /* IPv4 simply where L4 starts
                         * IPv6 where L4 Transport Header starts */
        __u16 fragoffs; /* IPv6 fragment offset, 0 if first frag (or not frag)*/
        __s16 protocol;
        __s32 flags;
        union nf_inet_addr saddr;
        union nf_inet_addr daddr;
};

static inline void *frag_safe_skb_hp(const struct sk_buff *skb, int offset,
                                      int len, void *buffer)
{
        return skb_header_pointer(skb, offset, len, buffer);
}

/* This function handles filling *ip_vs_iphdr, both for IPv4 and IPv6.
 * IPv6 requires some extra work, as finding proper header position,
 * depend on the IPv6 extension headers.
 */
static inline int
ip_vs_fill_iph_skb_off(int af, const struct sk_buff *skb, int offset,
                       int hdr_flags, struct ip_vs_iphdr *iphdr)
{
        iphdr->hdr_flags = hdr_flags;
        iphdr->off = offset;

#ifdef CONFIG_IP_VS_IPV6
        if (af == AF_INET6) {
                struct ipv6hdr _iph;
                const struct ipv6hdr *iph = skb_header_pointer(
                        skb, offset, sizeof(_iph), &_iph);
                if (!iph)
                        return 0;

                iphdr->saddr.in6 = iph->saddr;
                iphdr->daddr.in6 = iph->daddr;
                /* ipv6_find_hdr() updates len, flags */
                iphdr->len         = offset;
                iphdr->flags         = 0;
                iphdr->protocol  = ipv6_find_hdr(skb, &iphdr->len, -1,
                                                 &iphdr->fragoffs,
                                                 &iphdr->flags);
                if (iphdr->protocol < 0)
                        return 0;
        } else
#endif
        {
                struct iphdr _iph;
                const struct iphdr *iph = skb_header_pointer(
                        skb, offset, sizeof(_iph), &_iph);
                if (!iph)
                        return 0;

                iphdr->len        = offset + iph->ihl * 4;
                iphdr->fragoffs        = 0;
                iphdr->protocol        = iph->protocol;
                iphdr->saddr.ip        = iph->saddr;
                iphdr->daddr.ip        = iph->daddr;
        }

        return 1;
}

static inline int
ip_vs_fill_iph_skb_icmp(int af, const struct sk_buff *skb, int offset,
                        bool inverse, struct ip_vs_iphdr *iphdr)
{
        int hdr_flags = IP_VS_HDR_ICMP;

        if (inverse)
                hdr_flags |= IP_VS_HDR_INVERSE;

        return ip_vs_fill_iph_skb_off(af, skb, offset, hdr_flags, iphdr);
}

static inline int
ip_vs_fill_iph_skb(int af, const struct sk_buff *skb, bool inverse,
                   struct ip_vs_iphdr *iphdr)
{
        int hdr_flags = 0;

        if (inverse)
                hdr_flags |= IP_VS_HDR_INVERSE;

        return ip_vs_fill_iph_skb_off(af, skb, skb_network_offset(skb),
                                      hdr_flags, iphdr);
}

static inline bool
ip_vs_iph_inverse(const struct ip_vs_iphdr *iph)
{
        return !!(iph->hdr_flags & IP_VS_HDR_INVERSE);
}

static inline bool
ip_vs_iph_icmp(const struct ip_vs_iphdr *iph)
{
        return !!(iph->hdr_flags & IP_VS_HDR_ICMP);
}

static inline void ip_vs_addr_copy(int af, union nf_inet_addr *dst,
                                   const union nf_inet_addr *src)
{
#ifdef CONFIG_IP_VS_IPV6
        if (af == AF_INET6)
                dst->in6 = src->in6;
        else
#endif
        dst->ip = src->ip;
}

static inline void ip_vs_addr_set(int af, union nf_inet_addr *dst,
                                  const union nf_inet_addr *src)
{
#ifdef CONFIG_IP_VS_IPV6
        if (af == AF_INET6) {
                dst->in6 = src->in6;
                return;
        }
#endif
        dst->ip = src->ip;
        dst->all[1] = 0;
        dst->all[2] = 0;
        dst->all[3] = 0;
}

static inline int ip_vs_addr_equal(int af, const union nf_inet_addr *a,
                                   const union nf_inet_addr *b)
{
#ifdef CONFIG_IP_VS_IPV6
        if (af == AF_INET6)
                return ipv6_addr_equal(&a->in6, &b->in6);
#endif
        return a->ip == b->ip;
}

#ifdef CONFIG_IP_VS_DEBUG
#include <linux/net.h>

int ip_vs_get_debug_level(void);

static inline const char *ip_vs_dbg_addr(int af, char *buf, size_t buf_len,
                                         const union nf_inet_addr *addr,
                                         int *idx)
{
        int len;
#ifdef CONFIG_IP_VS_IPV6
        if (af == AF_INET6)
                len = snprintf(&buf[*idx], buf_len - *idx, "[%pI6c]",
                               &addr->in6) + 1;
        else
#endif
                len = snprintf(&buf[*idx], buf_len - *idx, "%pI4",
                               &addr->ip) + 1;

        *idx += len;
        BUG_ON(*idx > buf_len + 1);
        return &buf[*idx - len];
}

#define IP_VS_DBG_BUF(level, msg, ...)                                        \
        do {                                                                \
                char ip_vs_dbg_buf[160];                                \
                int ip_vs_dbg_idx = 0;                                        \
                if (level <= ip_vs_get_debug_level())                        \
                        printk(KERN_DEBUG pr_fmt(msg), ##__VA_ARGS__);        \
        } while (0)
#define IP_VS_ERR_BUF(msg...)                                                \
        do {                                                                \
                char ip_vs_dbg_buf[160];                                \
                int ip_vs_dbg_idx = 0;                                        \
                pr_err(msg);                                                \
        } while (0)

/* Only use from within IP_VS_DBG_BUF() or IP_VS_ERR_BUF macros */
#define IP_VS_DBG_ADDR(af, addr)                                        \
        ip_vs_dbg_addr(af, ip_vs_dbg_buf,                                \
                       sizeof(ip_vs_dbg_buf), addr,                        \
                       &ip_vs_dbg_idx)

#define IP_VS_DBG(level, msg, ...)                                        \
        do {                                                                \
                if (level <= ip_vs_get_debug_level())                        \
                        printk(KERN_DEBUG pr_fmt(msg), ##__VA_ARGS__);        \
        } while (0)
#define IP_VS_DBG_RL(msg, ...)                                                \
        do {                                                                \
                if (net_ratelimit())                                        \
                        printk(KERN_DEBUG pr_fmt(msg), ##__VA_ARGS__);        \
        } while (0)
#define IP_VS_DBG_PKT(level, af, pp, skb, ofs, msg)                        \
        do {                                                                \
                if (level <= ip_vs_get_debug_level())                        \
                        pp->debug_packet(af, pp, skb, ofs, msg);        \
        } while (0)
#define IP_VS_DBG_RL_PKT(level, af, pp, skb, ofs, msg)                        \
        do {                                                                \
                if (level <= ip_vs_get_debug_level() &&                        \
                    net_ratelimit())                                        \
                        pp->debug_packet(af, pp, skb, ofs, msg);        \
        } while (0)
#else        /* NO DEBUGGING at ALL */
#define IP_VS_DBG_BUF(level, msg...)  do {} while (0)
#define IP_VS_ERR_BUF(msg...)  do {} while (0)
#define IP_VS_DBG(level, msg...)  do {} while (0)
#define IP_VS_DBG_RL(msg...)  do {} while (0)
#define IP_VS_DBG_PKT(level, af, pp, skb, ofs, msg)        do {} while (0)
#define IP_VS_DBG_RL_PKT(level, af, pp, skb, ofs, msg)        do {} while (0)
#endif

#define IP_VS_BUG() BUG()
#define IP_VS_ERR_RL(msg, ...)                                                \
        do {                                                                \
                if (net_ratelimit())                                        \
                        pr_err(msg, ##__VA_ARGS__);                        \
        } while (0)

/* The port number of FTP service (in network order). */
#define FTPPORT  cpu_to_be16(21)
#define FTPDATA  cpu_to_be16(20)

/* TCP State Values */
enum {
        IP_VS_TCP_S_NONE = 0,
        IP_VS_TCP_S_ESTABLISHED,
        IP_VS_TCP_S_SYN_SENT,
        IP_VS_TCP_S_SYN_RECV,
        IP_VS_TCP_S_FIN_WAIT,
        IP_VS_TCP_S_TIME_WAIT,
        IP_VS_TCP_S_CLOSE,
        IP_VS_TCP_S_CLOSE_WAIT,
        IP_VS_TCP_S_LAST_ACK,
        IP_VS_TCP_S_LISTEN,
        IP_VS_TCP_S_SYNACK,
        IP_VS_TCP_S_LAST
};

/* UDP State Values */
enum {
        IP_VS_UDP_S_NORMAL,
        IP_VS_UDP_S_LAST,
};

/* ICMP State Values */
enum {
        IP_VS_ICMP_S_NORMAL,
        IP_VS_ICMP_S_LAST,
};

/* SCTP State Values */
enum ip_vs_sctp_states {
        IP_VS_SCTP_S_NONE,
        IP_VS_SCTP_S_INIT1,
        IP_VS_SCTP_S_INIT,
        IP_VS_SCTP_S_COOKIE_SENT,
        IP_VS_SCTP_S_COOKIE_REPLIED,
        IP_VS_SCTP_S_COOKIE_WAIT,
        IP_VS_SCTP_S_COOKIE,
        IP_VS_SCTP_S_COOKIE_ECHOED,
        IP_VS_SCTP_S_ESTABLISHED,
        IP_VS_SCTP_S_SHUTDOWN_SENT,
        IP_VS_SCTP_S_SHUTDOWN_RECEIVED,
        IP_VS_SCTP_S_SHUTDOWN_ACK_SENT,
        IP_VS_SCTP_S_REJECTED,
        IP_VS_SCTP_S_CLOSED,
        IP_VS_SCTP_S_LAST
};

/* Connection templates use bits from state */
#define IP_VS_CTPL_S_NONE                0x0000
#define IP_VS_CTPL_S_ASSURED                0x0001
#define IP_VS_CTPL_S_LAST                0x0002

/* Delta sequence info structure
 * Each ip_vs_conn has 2 (output AND input seq. changes).
 * Only used in the VS/NAT.
 */
struct ip_vs_seq {
        __u32                        init_seq;        /* Add delta from this seq */
        __u32                        delta;                /* Delta in sequence numbers */
        __u32                        previous_delta;        /* Delta in sequence numbers
                                                 * before last resized pkt */
};

/* counters per cpu */
struct ip_vs_counters {
        u64_stats_t        conns;                /* connections scheduled */
        u64_stats_t        inpkts;                /* incoming packets */
        u64_stats_t        outpkts;        /* outgoing packets */
        u64_stats_t        inbytes;        /* incoming bytes */
        u64_stats_t        outbytes;        /* outgoing bytes */
};
/* Stats per cpu */
struct ip_vs_cpu_stats {
        struct ip_vs_counters   cnt;
        struct u64_stats_sync   syncp;
};

/* Default nice for estimator kthreads */
#define IPVS_EST_NICE                0

/* IPVS statistics objects */
struct ip_vs_estimator {
        struct hlist_node        list;

        u64                        last_inbytes;
        u64                        last_outbytes;
        u64                        last_conns;
        u64                        last_inpkts;
        u64                        last_outpkts;

        u64                        cps;
        u64                        inpps;
        u64                        outpps;
        u64                        inbps;
        u64                        outbps;

        s32                        ktid:16,        /* kthread ID, -1=temp list */
                                ktrow:8,        /* row/tick ID for kthread */
                                ktcid:8;        /* chain ID for kthread tick */
};

/*
 * IPVS statistics object, 64-bit kernel version of struct ip_vs_stats_user
 */
struct ip_vs_kstats {
        u64                        conns;                /* connections scheduled */
        u64                        inpkts;                /* incoming packets */
        u64                        outpkts;        /* outgoing packets */
        u64                        inbytes;        /* incoming bytes */
        u64                        outbytes;        /* outgoing bytes */

        u64                        cps;                /* current connection rate */
        u64                        inpps;                /* current in packet rate */
        u64                        outpps;                /* current out packet rate */
        u64                        inbps;                /* current in byte rate */
        u64                        outbps;                /* current out byte rate */
};

struct ip_vs_stats {
        struct ip_vs_kstats        kstats;                /* kernel statistics */
        struct ip_vs_estimator        est;                /* estimator */
        struct ip_vs_cpu_stats __percpu        *cpustats;        /* per cpu counters */
        spinlock_t                lock;                /* spin lock */
        struct ip_vs_kstats        kstats0;        /* reset values */
};

struct ip_vs_stats_rcu {
        struct ip_vs_stats        s;
        struct rcu_head                rcu_head;
};

int ip_vs_stats_init_alloc(struct ip_vs_stats *s);
struct ip_vs_stats *ip_vs_stats_alloc(void);
void ip_vs_stats_release(struct ip_vs_stats *stats);
void ip_vs_stats_free(struct ip_vs_stats *stats);

/* Process estimators in multiple timer ticks (20/50/100, see ktrow) */
#define IPVS_EST_NTICKS                50
/* Estimation uses a 2-second period containing ticks (in jiffies) */
#define IPVS_EST_TICK                ((2 * HZ) / IPVS_EST_NTICKS)

/* Limit of CPU load per kthread (8 for 12.5%), ratio of CPU capacity (1/C).
 * Value of 4 and above ensures kthreads will take work without exceeding
 * the CPU capacity under different circumstances.
 */
#define IPVS_EST_LOAD_DIVISOR        8

/* Kthreads should not have work that exceeds the CPU load above 50% */
#define IPVS_EST_CPU_KTHREADS        (IPVS_EST_LOAD_DIVISOR / 2)

/* Desired number of chains per timer tick (chain load factor in 100us units),
 * 48=4.8ms of 40ms tick (12% CPU usage):
 * 2 sec * 1000 ms in sec * 10 (100us in ms) / 8 (12.5%) / 50
 */
#define IPVS_EST_CHAIN_FACTOR        \
        ALIGN_DOWN(2 * 1000 * 10 / IPVS_EST_LOAD_DIVISOR / IPVS_EST_NTICKS, 8)

/* Compiled number of chains per tick
 * The defines should match cond_resched_rcu
 */
#if defined(CONFIG_DEBUG_ATOMIC_SLEEP) || !defined(CONFIG_PREEMPT_RCU)
#define IPVS_EST_TICK_CHAINS        IPVS_EST_CHAIN_FACTOR
#else
#define IPVS_EST_TICK_CHAINS        1
#endif

#if IPVS_EST_NTICKS > 127
#error Too many timer ticks for ktrow
#endif

/* Multiple chains processed in same tick */
struct ip_vs_est_tick_data {
        struct rcu_head                rcu_head;
        struct hlist_head        chains[IPVS_EST_TICK_CHAINS];
        DECLARE_BITMAP(present, IPVS_EST_TICK_CHAINS);
        DECLARE_BITMAP(full, IPVS_EST_TICK_CHAINS);
        int                        chain_len[IPVS_EST_TICK_CHAINS];
};

/* Context for estimation kthread */
struct ip_vs_est_kt_data {
        struct netns_ipvs        *ipvs;
        struct task_struct        *task;                /* task if running */
        struct ip_vs_est_tick_data __rcu *ticks[IPVS_EST_NTICKS];
        DECLARE_BITMAP(avail, IPVS_EST_NTICKS);        /* tick has space for ests */
        unsigned long                est_timer;        /* estimation timer (jiffies) */
        struct ip_vs_stats        *calc_stats;        /* Used for calculation */
        int                        tick_len[IPVS_EST_NTICKS];        /* est count */
        int                        id;                /* ktid per netns */
        int                        chain_max;        /* max ests per tick chain */
        int                        tick_max;        /* max ests per tick */
        int                        est_count;        /* attached ests to kthread */
        int                        est_max_count;        /* max ests per kthread */
        int                        add_row;        /* row for new ests */
        int                        est_row;        /* estimated row */
};

struct dst_entry;
struct iphdr;
struct ip_vs_conn;
struct ip_vs_app;
struct sk_buff;
struct ip_vs_proto_data;

struct ip_vs_protocol {
        struct ip_vs_protocol        *next;
        char                        *name;
        u16                        protocol;
        u16                        num_states;
        int                        dont_defrag;

        void (*init)(struct ip_vs_protocol *pp);

        void (*exit)(struct ip_vs_protocol *pp);

        int (*init_netns)(struct netns_ipvs *ipvs, struct ip_vs_proto_data *pd);

        void (*exit_netns)(struct netns_ipvs *ipvs, struct ip_vs_proto_data *pd);

        int (*conn_schedule)(struct netns_ipvs *ipvs,
                             int af, struct sk_buff *skb,
                             struct ip_vs_proto_data *pd,
                             int *verdict, struct ip_vs_conn **cpp,
                             struct ip_vs_iphdr *iph);

        struct ip_vs_conn *
        (*conn_in_get)(struct netns_ipvs *ipvs,
                       int af,
                       const struct sk_buff *skb,
                       const struct ip_vs_iphdr *iph);

        struct ip_vs_conn *
        (*conn_out_get)(struct netns_ipvs *ipvs,
                        int af,
                        const struct sk_buff *skb,
                        const struct ip_vs_iphdr *iph);

        int (*snat_handler)(struct sk_buff *skb, struct ip_vs_protocol *pp,
                            struct ip_vs_conn *cp, struct ip_vs_iphdr *iph);

        int (*dnat_handler)(struct sk_buff *skb, struct ip_vs_protocol *pp,
                            struct ip_vs_conn *cp, struct ip_vs_iphdr *iph);

        const char *(*state_name)(int state);

        void (*state_transition)(struct ip_vs_conn *cp, int direction,
                                 const struct sk_buff *skb,
                                 struct ip_vs_proto_data *pd);

        int (*register_app)(struct netns_ipvs *ipvs, struct ip_vs_app *inc);

        void (*unregister_app)(struct netns_ipvs *ipvs, struct ip_vs_app *inc);

        int (*app_conn_bind)(struct ip_vs_conn *cp);

        void (*debug_packet)(int af, struct ip_vs_protocol *pp,
                             const struct sk_buff *skb,
                             int offset,
                             const char *msg);

        void (*timeout_change)(struct ip_vs_proto_data *pd, int flags);
};

/* protocol data per netns */
struct ip_vs_proto_data {
        struct ip_vs_proto_data        *next;
        struct ip_vs_protocol        *pp;
        int                        *timeout_table;        /* protocol timeout table */
        atomic_t                appcnt;                /* counter of proto app incs. */
        struct tcp_states_t        *tcp_state_table;
};

struct ip_vs_protocol   *ip_vs_proto_get(unsigned short proto);
struct ip_vs_proto_data *ip_vs_proto_data_get(struct netns_ipvs *ipvs,
                                              unsigned short proto);

struct ip_vs_conn_param {
        struct netns_ipvs                *ipvs;
        const union nf_inet_addr        *caddr;
        const union nf_inet_addr        *vaddr;
        __be16                                cport;
        __be16                                vport;
        __u16                                protocol;
        u16                                af;

        const struct ip_vs_pe                *pe;
        char                                *pe_data;
        __u8                                pe_data_len;
};

/* IP_VS structure allocated for each dynamically scheduled connection */
struct ip_vs_conn {
        struct hlist_node        c_list;         /* hashed list heads */
        /* Protocol, addresses and port numbers */
        __be16                  cport;
        __be16                  dport;
        __be16                  vport;
        u16                        af;                /* address family */
        union nf_inet_addr      caddr;          /* client address */
        union nf_inet_addr      vaddr;          /* virtual address */
        union nf_inet_addr      daddr;          /* destination address */
        volatile __u32          flags;          /* status flags */
        __u16                   protocol;       /* Which protocol (TCP/UDP) */
        __u16                        daf;                /* Address family of the dest */
        struct netns_ipvs        *ipvs;

        /* counter and timer */
        refcount_t                refcnt;                /* reference count */
        struct timer_list        timer;                /* Expiration timer */
        volatile unsigned long        timeout;        /* timeout */

        /* Flags and state transition */
        spinlock_t              lock;           /* lock for state transition */
        volatile __u16          state;          /* state info */
        volatile __u16          old_state;      /* old state, to be used for
                                                 * state transition triggered
                                                 * synchronization
                                                 */
        __u32                        fwmark;                /* Fire wall mark from skb */
        unsigned long                sync_endtime;        /* jiffies + sent_retries */

        /* Control members */
        struct ip_vs_conn       *control;       /* Master control connection */
        atomic_t                n_control;      /* Number of controlled ones */
        struct ip_vs_dest       *dest;          /* real server */
        atomic_t                in_pkts;        /* incoming packet counter */

        /* Packet transmitter for different forwarding methods.  If it
         * mangles the packet, it must return NF_DROP or better NF_STOLEN,
         * otherwise this must be changed to a sk_buff **.
         * NF_ACCEPT can be returned when destination is local.
         */
        int (*packet_xmit)(struct sk_buff *skb, struct ip_vs_conn *cp,
                           struct ip_vs_protocol *pp, struct ip_vs_iphdr *iph);

        /* Note: we can group the following members into a structure,
         * in order to save more space, and the following members are
         * only used in VS/NAT anyway
         */
        struct ip_vs_app        *app;           /* bound ip_vs_app object */
        void                    *app_data;      /* Application private data */
        struct_group(sync_conn_opt,
                struct ip_vs_seq  in_seq;       /* incoming seq. struct */
                struct ip_vs_seq  out_seq;      /* outgoing seq. struct */
        );

        const struct ip_vs_pe        *pe;
        char                        *pe_data;
        __u8                        pe_data_len;

        struct rcu_head                rcu_head;
};

/* Extended internal versions of struct ip_vs_service_user and ip_vs_dest_user
 * for IPv6 support.
 *
 * We need these to conveniently pass around service and destination
 * options, but unfortunately, we also need to keep the old definitions to
 * maintain userspace backwards compatibility for the setsockopt interface.
 */
struct ip_vs_service_user_kern {
        /* virtual service addresses */
        u16                        af;
        u16                        protocol;
        union nf_inet_addr        addr;                /* virtual ip address */
        __be16                        port;
        u32                        fwmark;                /* firewall mark of service */

        /* virtual service options */
        char                        *sched_name;
        char                        *pe_name;
        unsigned int                flags;                /* virtual service flags */
        unsigned int                timeout;        /* persistent timeout in sec */
        __be32                        netmask;        /* persistent netmask or plen */
};


struct ip_vs_dest_user_kern {
        /* destination server address */
        union nf_inet_addr        addr;
        __be16                        port;

        /* real server options */
        unsigned int                conn_flags;        /* connection flags */
        int                        weight;                /* destination weight */

        /* thresholds for active connections */
        u32                        u_threshold;        /* upper threshold */
        u32                        l_threshold;        /* lower threshold */

        /* Address family of addr */
        u16                        af;

        u16                        tun_type;        /* tunnel type */
        __be16                        tun_port;        /* tunnel port */
        u16                        tun_flags;        /* tunnel flags */
};


/*
 * The information about the virtual service offered to the net and the
 * forwarding entries.
 */
struct ip_vs_service {
        struct hlist_node        s_list;   /* for normal service table */
        struct hlist_node        f_list;   /* for fwmark-based service table */
        atomic_t                refcnt;   /* reference counter */

        u16                        af;       /* address family */
        __u16                        protocol; /* which protocol (TCP/UDP) */
        union nf_inet_addr        addr;          /* IP address for virtual service */
        __be16                        port;          /* port number for the service */
        __u32                   fwmark;   /* firewall mark of the service */
        unsigned int                flags;          /* service status flags */
        unsigned int                timeout;  /* persistent timeout in ticks */
        __be32                        netmask;  /* grouping granularity, mask/plen */
        struct netns_ipvs        *ipvs;

        struct list_head        destinations;  /* real server d-linked list */
        __u32                        num_dests;     /* number of servers */
        struct ip_vs_stats      stats;         /* statistics for the service */

        /* for scheduling */
        struct ip_vs_scheduler __rcu *scheduler; /* bound scheduler object */
        spinlock_t                sched_lock;    /* lock sched_data */
        void                        *sched_data;   /* scheduler application data */

        /* alternate persistence engine */
        struct ip_vs_pe __rcu        *pe;
        int                        conntrack_afmask;

        struct rcu_head                rcu_head;
};

/* Information for cached dst */
struct ip_vs_dest_dst {
        struct dst_entry        *dst_cache;        /* destination cache entry */
        u32                        dst_cookie;
        union nf_inet_addr        dst_saddr;
        struct rcu_head                rcu_head;
};

/* The real server destination forwarding entry with ip address, port number,
 * and so on.
 */
struct ip_vs_dest {
        struct list_head        n_list;   /* for the dests in the service */
        struct hlist_node        d_list;   /* for table with all the dests */

        u16                        af;                /* address family */
        __be16                        port;                /* port number of the server */
        union nf_inet_addr        addr;                /* IP address of the server */
        volatile unsigned int        flags;                /* dest status flags */
        atomic_t                conn_flags;        /* flags to copy to conn */
        atomic_t                weight;                /* server weight */
        atomic_t                last_weight;        /* server latest weight */
        __u16                        tun_type;        /* tunnel type */
        __be16                        tun_port;        /* tunnel port */
        __u16                        tun_flags;        /* tunnel flags */

        refcount_t                refcnt;                /* reference counter */
        struct ip_vs_stats      stats;          /* statistics */
        unsigned long                idle_start;        /* start time, jiffies */

        /* connection counters and thresholds */
        atomic_t                activeconns;        /* active connections */
        atomic_t                inactconns;        /* inactive connections */
        atomic_t                persistconns;        /* persistent connections */
        __u32                        u_threshold;        /* upper threshold */
        __u32                        l_threshold;        /* lower threshold */

        /* for destination cache */
        spinlock_t                dst_lock;        /* lock of dst_cache */
        struct ip_vs_dest_dst __rcu *dest_dst;        /* cached dst info */

        /* for virtual service */
        struct ip_vs_service __rcu *svc;        /* service it belongs to */
        __u16                        protocol;        /* which protocol (TCP/UDP) */
        __be16                        vport;                /* virtual port number */
        union nf_inet_addr        vaddr;                /* virtual IP address */
        __u32                        vfwmark;        /* firewall mark of service */

        struct rcu_head                rcu_head;
        struct list_head        t_list;                /* in dest_trash */
        unsigned int                in_rs_table:1;        /* we are in rs_table */
};

/* The scheduler object */
struct ip_vs_scheduler {
        struct list_head        n_list;                /* d-linked list head */
        char                        *name;                /* scheduler name */
        atomic_t                refcnt;                /* reference counter */
        struct module                *module;        /* THIS_MODULE/NULL */

        /* scheduler initializing service */
        int (*init_service)(struct ip_vs_service *svc);
        /* scheduling service finish */
        void (*done_service)(struct ip_vs_service *svc);
        /* dest is linked */
        int (*add_dest)(struct ip_vs_service *svc, struct ip_vs_dest *dest);
        /* dest is unlinked */
        int (*del_dest)(struct ip_vs_service *svc, struct ip_vs_dest *dest);
        /* dest is updated */
        int (*upd_dest)(struct ip_vs_service *svc, struct ip_vs_dest *dest);

        /* selecting a server from the given service */
        struct ip_vs_dest* (*schedule)(struct ip_vs_service *svc,
                                       const struct sk_buff *skb,
                                       struct ip_vs_iphdr *iph);
};

/* The persistence engine object */
struct ip_vs_pe {
        struct list_head        n_list;                /* d-linked list head */
        char                        *name;                /* scheduler name */
        atomic_t                refcnt;                /* reference counter */
        struct module                *module;        /* THIS_MODULE/NULL */

        /* get the connection template, if any */
        int (*fill_param)(struct ip_vs_conn_param *p, struct sk_buff *skb);
        bool (*ct_match)(const struct ip_vs_conn_param *p,
                         struct ip_vs_conn *ct);
        u32 (*hashkey_raw)(const struct ip_vs_conn_param *p, u32 initval,
                           bool inverse);
        int (*show_pe_data)(const struct ip_vs_conn *cp, char *buf);
        /* create connections for real-server outgoing packets */
        struct ip_vs_conn* (*conn_out)(struct ip_vs_service *svc,
                                       struct ip_vs_dest *dest,
                                       struct sk_buff *skb,
                                       const struct ip_vs_iphdr *iph,
                                       __be16 dport, __be16 cport);
};

/* The application module object (a.k.a. app incarnation) */
struct ip_vs_app {
        struct list_head        a_list;                /* member in app list */
        int                        type;                /* IP_VS_APP_TYPE_xxx */
        char                        *name;                /* application module name */
        __u16                        protocol;
        struct module                *module;        /* THIS_MODULE/NULL */
        struct list_head        incs_list;        /* list of incarnations */

        /* members for application incarnations */
        struct list_head        p_list;                /* member in proto app list */
        struct ip_vs_app        *app;                /* its real application */
        __be16                        port;                /* port number in net order */
        atomic_t                usecnt;                /* usage counter */
        struct rcu_head                rcu_head;

        /* output hook: Process packet in inout direction, diff set for TCP.
         * Return: 0=Error, 1=Payload Not Mangled/Mangled but checksum is ok,
         *           2=Mangled but checksum was not updated
         */
        int (*pkt_out)(struct ip_vs_app *, struct ip_vs_conn *,
                       struct sk_buff *, int *diff, struct ip_vs_iphdr *ipvsh);

        /* input hook: Process packet in outin direction, diff set for TCP.
         * Return: 0=Error, 1=Payload Not Mangled/Mangled but checksum is ok,
         *           2=Mangled but checksum was not updated
         */
        int (*pkt_in)(struct ip_vs_app *, struct ip_vs_conn *,
                      struct sk_buff *, int *diff, struct ip_vs_iphdr *ipvsh);

        /* ip_vs_app initializer */
        int (*init_conn)(struct ip_vs_app *, struct ip_vs_conn *);

        /* ip_vs_app finish */
        int (*done_conn)(struct ip_vs_app *, struct ip_vs_conn *);


        /* not used now */
        int (*bind_conn)(struct ip_vs_app *, struct ip_vs_conn *,
                         struct ip_vs_protocol *);

        void (*unbind_conn)(struct ip_vs_app *, struct ip_vs_conn *);

        int *                        timeout_table;
        int *                        timeouts;
        int                        timeouts_size;

        int (*conn_schedule)(struct sk_buff *skb, struct ip_vs_app *app,
                             int *verdict, struct ip_vs_conn **cpp);

        struct ip_vs_conn *
        (*conn_in_get)(const struct sk_buff *skb, struct ip_vs_app *app,
                       const struct iphdr *iph, int inverse);

        struct ip_vs_conn *
        (*conn_out_get)(const struct sk_buff *skb, struct ip_vs_app *app,
                        const struct iphdr *iph, int inverse);

        int (*state_transition)(struct ip_vs_conn *cp, int direction,
                                const struct sk_buff *skb,
                                struct ip_vs_app *app);

        void (*timeout_change)(struct ip_vs_app *app, int flags);
};

struct ipvs_master_sync_state {
        struct list_head        sync_queue;
        struct ip_vs_sync_buff        *sync_buff;
        unsigned long                sync_queue_len;
        unsigned int                sync_queue_delay;
        struct delayed_work        master_wakeup_work;
        struct netns_ipvs        *ipvs;
};

struct ip_vs_sync_thread_data;

/* How much time to keep dests in trash */
#define IP_VS_DEST_TRASH_PERIOD                (120 * HZ)

struct ipvs_sync_daemon_cfg {
        union nf_inet_addr        mcast_group;
        int                        syncid;
        u16                        sync_maxlen;
        u16                        mcast_port;
        u8                        mcast_af;
        u8                        mcast_ttl;
        /* multicast interface name */
        char                        mcast_ifn[IP_VS_IFNAME_MAXLEN];
};

/* IPVS in network namespace */
struct netns_ipvs {
        int                        gen;                /* Generation */
        int                        enable;                /* enable like nf_hooks do */
        /* Hash table: for real service lookups */
        #define IP_VS_RTAB_BITS 4
        #define IP_VS_RTAB_SIZE (1 << IP_VS_RTAB_BITS)
        #define IP_VS_RTAB_MASK (IP_VS_RTAB_SIZE - 1)

        struct hlist_head        rs_table[IP_VS_RTAB_SIZE];
        /* ip_vs_app */
        struct list_head        app_list;
        /* ip_vs_proto */
        #define IP_VS_PROTO_TAB_SIZE        32        /* must be power of 2 */
        struct ip_vs_proto_data *proto_data_table[IP_VS_PROTO_TAB_SIZE];
        /* ip_vs_proto_tcp */
#ifdef CONFIG_IP_VS_PROTO_TCP
        #define        TCP_APP_TAB_BITS        4
        #define        TCP_APP_TAB_SIZE        (1 << TCP_APP_TAB_BITS)
        #define        TCP_APP_TAB_MASK        (TCP_APP_TAB_SIZE - 1)
        struct list_head        tcp_apps[TCP_APP_TAB_SIZE];
#endif
        /* ip_vs_proto_udp */
#ifdef CONFIG_IP_VS_PROTO_UDP
        #define        UDP_APP_TAB_BITS        4
        #define        UDP_APP_TAB_SIZE        (1 << UDP_APP_TAB_BITS)
        #define        UDP_APP_TAB_MASK        (UDP_APP_TAB_SIZE - 1)
        struct list_head        udp_apps[UDP_APP_TAB_SIZE];
#endif
        /* ip_vs_proto_sctp */
#ifdef CONFIG_IP_VS_PROTO_SCTP
        #define SCTP_APP_TAB_BITS        4
        #define SCTP_APP_TAB_SIZE        (1 << SCTP_APP_TAB_BITS)
        #define SCTP_APP_TAB_MASK        (SCTP_APP_TAB_SIZE - 1)
        /* Hash table for SCTP application incarnations         */
        struct list_head        sctp_apps[SCTP_APP_TAB_SIZE];
#endif
        /* ip_vs_conn */
        atomic_t                conn_count;      /* connection counter */

        /* ip_vs_ctl */
        struct ip_vs_stats_rcu        *tot_stats;      /* Statistics & est. */

        int                        num_services;    /* no of virtual services */
        int                        num_services6;   /* IPv6 virtual services */

        /* Trash for destinations */
        struct list_head        dest_trash;
        spinlock_t                dest_trash_lock;
        struct timer_list        dest_trash_timer; /* expiration timer */
        /* Service counters */
        atomic_t                ftpsvc_counter;
        atomic_t                nullsvc_counter;
        atomic_t                conn_out_counter;

#ifdef CONFIG_SYSCTL
        /* delayed work for expiring no dest connections */
        struct delayed_work        expire_nodest_conn_work;
        /* 1/rate drop and drop-entry variables */
        struct delayed_work        defense_work;   /* Work handler */
        int                        drop_rate;
        int                        drop_counter;
        int                        old_secure_tcp;
        atomic_t                dropentry;
        /* locks in ctl.c */
        spinlock_t                dropentry_lock;  /* drop entry handling */
        spinlock_t                droppacket_lock; /* drop packet handling */
        spinlock_t                securetcp_lock;  /* state and timeout tables */

        /* sys-ctl struct */
        struct ctl_table_header        *sysctl_hdr;
        struct ctl_table        *sysctl_tbl;
#endif

        /* sysctl variables */
        int                        sysctl_amemthresh;
        int                        sysctl_am_droprate;
        int                        sysctl_drop_entry;
        int                        sysctl_drop_packet;
        int                        sysctl_secure_tcp;
#ifdef CONFIG_IP_VS_NFCT
        int                        sysctl_conntrack;
#endif
        int                        sysctl_snat_reroute;
        int                        sysctl_sync_ver;
        int                        sysctl_sync_ports;
        int                        sysctl_sync_persist_mode;
        unsigned long                sysctl_sync_qlen_max;
        int                        sysctl_sync_sock_size;
        int                        sysctl_cache_bypass;
        int                        sysctl_expire_nodest_conn;
        int                        sysctl_sloppy_tcp;
        int                        sysctl_sloppy_sctp;
        int                        sysctl_expire_quiescent_template;
        int                        sysctl_sync_threshold[2];
        unsigned int                sysctl_sync_refresh_period;
        int                        sysctl_sync_retries;
        int                        sysctl_nat_icmp_send;
        int                        sysctl_pmtu_disc;
        int                        sysctl_backup_only;
        int                        sysctl_conn_reuse_mode;
        int                        sysctl_schedule_icmp;
        int                        sysctl_ignore_tunneled;
        int                        sysctl_run_estimation;
#ifdef CONFIG_SYSCTL
        cpumask_var_t                sysctl_est_cpulist;        /* kthread cpumask */
        int                        est_cpulist_valid;        /* cpulist set */
        int                        sysctl_est_nice;        /* kthread nice */
        int                        est_stopped;                /* stop tasks */
#endif

        /* ip_vs_lblc */
        int                        sysctl_lblc_expiration;
        struct ctl_table_header        *lblc_ctl_header;
        struct ctl_table        *lblc_ctl_table;
        /* ip_vs_lblcr */
        int                        sysctl_lblcr_expiration;
        struct ctl_table_header        *lblcr_ctl_header;
        struct ctl_table        *lblcr_ctl_table;
        /* ip_vs_est */
        struct delayed_work        est_reload_work;/* Reload kthread tasks */
        struct mutex                est_mutex;        /* protect kthread tasks */
        struct hlist_head        est_temp_list;        /* Ests during calc phase */
        struct ip_vs_est_kt_data **est_kt_arr;        /* Array of kthread data ptrs */
        unsigned long                est_max_threads;/* Hard limit of kthreads */
        int                        est_calc_phase;        /* Calculation phase */
        int                        est_chain_max;        /* Calculated chain_max */
        int                        est_kt_count;        /* Allocated ptrs */
        int                        est_add_ktid;        /* ktid where to add ests */
        atomic_t                est_genid;        /* kthreads reload genid */
        atomic_t                est_genid_done;        /* applied genid */
        /* ip_vs_sync */
        spinlock_t                sync_lock;
        struct ipvs_master_sync_state *ms;
        spinlock_t                sync_buff_lock;
        struct ip_vs_sync_thread_data *master_tinfo;
        struct ip_vs_sync_thread_data *backup_tinfo;
        int                        threads_mask;
        volatile int                sync_state;
        struct mutex                sync_mutex;
        struct ipvs_sync_daemon_cfg        mcfg;        /* Master Configuration */
        struct ipvs_sync_daemon_cfg        bcfg;        /* Backup Configuration */
        /* net name space ptr */
        struct net                *net;            /* Needed by timer routines */
        /* Number of heterogeneous destinations, needed because heterogeneous
         * are not supported when synchronization is enabled.
         */
        unsigned int                mixed_address_family_dests;
        unsigned int                hooks_afmask;        /* &1=AF_INET, &2=AF_INET6 */
};

#define DEFAULT_SYNC_THRESHOLD        3
#define DEFAULT_SYNC_PERIOD        50
#define DEFAULT_SYNC_VER        1
#define DEFAULT_SLOPPY_TCP        0
#define DEFAULT_SLOPPY_SCTP        0
#define DEFAULT_SYNC_REFRESH_PERIOD        (0U * HZ)
#define DEFAULT_SYNC_RETRIES                0
#define IPVS_SYNC_WAKEUP_RATE        8
#define IPVS_SYNC_QLEN_MAX        (IPVS_SYNC_WAKEUP_RATE * 4)
#define IPVS_SYNC_SEND_DELAY        (HZ / 50)
#define IPVS_SYNC_CHECK_PERIOD        HZ
#define IPVS_SYNC_FLUSH_TIME        (HZ * 2)
#define IPVS_SYNC_PORTS_MAX        (1 << 6)

#ifdef CONFIG_SYSCTL

static inline int sysctl_sync_threshold(struct netns_ipvs *ipvs)
{
        return ipvs->sysctl_sync_threshold[0];
}

static inline int sysctl_sync_period(struct netns_ipvs *ipvs)
{
        return READ_ONCE(ipvs->sysctl_sync_threshold[1]);
}

static inline unsigned int sysctl_sync_refresh_period(struct netns_ipvs *ipvs)
{
        return READ_ONCE(ipvs->sysctl_sync_refresh_period);
}

static inline int sysctl_sync_retries(struct netns_ipvs *ipvs)
{
        return ipvs->sysctl_sync_retries;
}

static inline int sysctl_sync_ver(struct netns_ipvs *ipvs)
{
        return ipvs->sysctl_sync_ver;
}

static inline int sysctl_sloppy_tcp(struct netns_ipvs *ipvs)
{
        return ipvs->sysctl_sloppy_tcp;
}

static inline int sysctl_sloppy_sctp(struct netns_ipvs *ipvs)
{
        return ipvs->sysctl_sloppy_sctp;
}

static inline int sysctl_sync_ports(struct netns_ipvs *ipvs)
{
        return READ_ONCE(ipvs->sysctl_sync_ports);
}

static inline int sysctl_sync_persist_mode(struct netns_ipvs *ipvs)
{
        return ipvs->sysctl_sync_persist_mode;
}

static inline unsigned long sysctl_sync_qlen_max(struct netns_ipvs *ipvs)
{
        return ipvs->sysctl_sync_qlen_max;
}

static inline int sysctl_sync_sock_size(struct netns_ipvs *ipvs)
{
        return ipvs->sysctl_sync_sock_size;
}

static inline int sysctl_pmtu_disc(struct netns_ipvs *ipvs)
{
        return ipvs->sysctl_pmtu_disc;
}

static inline int sysctl_backup_only(struct netns_ipvs *ipvs)
{
        return ipvs->sync_state & IP_VS_STATE_BACKUP &&
               ipvs->sysctl_backup_only;
}

static inline int sysctl_conn_reuse_mode(struct netns_ipvs *ipvs)
{
        return ipvs->sysctl_conn_reuse_mode;
}

static inline int sysctl_expire_nodest_conn(struct netns_ipvs *ipvs)
{
        return ipvs->sysctl_expire_nodest_conn;
}

static inline int sysctl_schedule_icmp(struct netns_ipvs *ipvs)
{
        return ipvs->sysctl_schedule_icmp;
}

static inline int sysctl_ignore_tunneled(struct netns_ipvs *ipvs)
{
        return ipvs->sysctl_ignore_tunneled;
}

static inline int sysctl_cache_bypass(struct netns_ipvs *ipvs)
{
        return ipvs->sysctl_cache_bypass;
}

static inline int sysctl_run_estimation(struct netns_ipvs *ipvs)
{
        return ipvs->sysctl_run_estimation;
}

static inline const struct cpumask *sysctl_est_cpulist(struct netns_ipvs *ipvs)
{
        if (ipvs->est_cpulist_valid)
                return ipvs->sysctl_est_cpulist;
        else
                return housekeeping_cpumask(HK_TYPE_KTHREAD);
}

static inline int sysctl_est_nice(struct netns_ipvs *ipvs)
{
        return ipvs->sysctl_est_nice;
}

#else

static inline int sysctl_sync_threshold(struct netns_ipvs *ipvs)
{
        return DEFAULT_SYNC_THRESHOLD;
}

static inline int sysctl_sync_period(struct netns_ipvs *ipvs)
{
        return DEFAULT_SYNC_PERIOD;
}

static inline unsigned int sysctl_sync_refresh_period(struct netns_ipvs *ipvs)
{
        return DEFAULT_SYNC_REFRESH_PERIOD;
}

static inline int sysctl_sync_retries(struct netns_ipvs *ipvs)
{
        return DEFAULT_SYNC_RETRIES & 3;
}

static inline int sysctl_sync_ver(struct netns_ipvs *ipvs)
{
        return DEFAULT_SYNC_VER;
}

static inline int sysctl_sloppy_tcp(struct netns_ipvs *ipvs)
{
        return DEFAULT_SLOPPY_TCP;
}

static inline int sysctl_sloppy_sctp(struct netns_ipvs *ipvs)
{
        return DEFAULT_SLOPPY_SCTP;
}

static inline int sysctl_sync_ports(struct netns_ipvs *ipvs)
{
        return 1;
}

static inline int sysctl_sync_persist_mode(struct netns_ipvs *ipvs)
{
        return 0;
}

static inline unsigned long sysctl_sync_qlen_max(struct netns_ipvs *ipvs)
{
        return IPVS_SYNC_QLEN_MAX;
}

static inline int sysctl_sync_sock_size(struct netns_ipvs *ipvs)
{
        return 0;
}

static inline int sysctl_pmtu_disc(struct netns_ipvs *ipvs)
{
        return 1;
}

static inline int sysctl_backup_only(struct netns_ipvs *ipvs)
{
        return 0;
}

static inline int sysctl_conn_reuse_mode(struct netns_ipvs *ipvs)
{
        return 1;
}

static inline int sysctl_expire_nodest_conn(struct netns_ipvs *ipvs)
{
        return 0;
}

static inline int sysctl_schedule_icmp(struct netns_ipvs *ipvs)
{
        return 0;
}

static inline int sysctl_ignore_tunneled(struct netns_ipvs *ipvs)
{
        return 0;
}

static inline int sysctl_cache_bypass(struct netns_ipvs *ipvs)
{
        return 0;
}

static inline int sysctl_run_estimation(struct netns_ipvs *ipvs)
{
        return 1;
}

static inline const struct cpumask *sysctl_est_cpulist(struct netns_ipvs *ipvs)
{
        return housekeeping_cpumask(HK_TYPE_KTHREAD);
}

static inline int sysctl_est_nice(struct netns_ipvs *ipvs)
{
        return IPVS_EST_NICE;
}

#endif

/* IPVS core functions
 * (from ip_vs_core.c)
 */
const char *ip_vs_proto_name(unsigned int proto);
void ip_vs_init_hash_table(struct list_head *table, int rows);
struct ip_vs_conn *ip_vs_new_conn_out(struct ip_vs_service *svc,
                                      struct ip_vs_dest *dest,
                                      struct sk_buff *skb,
                                      const struct ip_vs_iphdr *iph,
                                      __be16 dport,
                                      __be16 cport);
#define IP_VS_INIT_HASH_TABLE(t) ip_vs_init_hash_table((t), ARRAY_SIZE((t)))

#define IP_VS_APP_TYPE_FTP        1

/* ip_vs_conn handling functions
 * (from ip_vs_conn.c)
 */
enum {
        IP_VS_DIR_INPUT = 0,
        IP_VS_DIR_OUTPUT,
        IP_VS_DIR_INPUT_ONLY,
        IP_VS_DIR_LAST,
};

static inline void ip_vs_conn_fill_param(struct netns_ipvs *ipvs, int af, int protocol,
                                         const union nf_inet_addr *caddr,
                                         __be16 cport,
                                         const union nf_inet_addr *vaddr,
                                         __be16 vport,
                                         struct ip_vs_conn_param *p)
{
        p->ipvs = ipvs;
        p->af = af;
        p->protocol = protocol;
        p->caddr = caddr;
        p->cport = cport;
        p->vaddr = vaddr;
        p->vport = vport;
        p->pe = NULL;
        p->pe_data = NULL;
}

struct ip_vs_conn *ip_vs_conn_in_get(const struct ip_vs_conn_param *p);
struct ip_vs_conn *ip_vs_ct_in_get(const struct ip_vs_conn_param *p);

struct ip_vs_conn * ip_vs_conn_in_get_proto(struct netns_ipvs *ipvs, int af,
                                            const struct sk_buff *skb,
                                            const struct ip_vs_iphdr *iph);

struct ip_vs_conn *ip_vs_conn_out_get(const struct ip_vs_conn_param *p);

struct ip_vs_conn * ip_vs_conn_out_get_proto(struct netns_ipvs *ipvs, int af,
                                             const struct sk_buff *skb,
                                             const struct ip_vs_iphdr *iph);

/* Get reference to gain full access to conn.
 * By default, RCU read-side critical sections have access only to
 * conn fields and its PE data, see ip_vs_conn_rcu_free() for reference.
 */
static inline bool __ip_vs_conn_get(struct ip_vs_conn *cp)
{
        return refcount_inc_not_zero(&cp->refcnt);
}

/* put back the conn without restarting its timer */
static inline void __ip_vs_conn_put(struct ip_vs_conn *cp)
{
        smp_mb__before_atomic();
        refcount_dec(&cp->refcnt);
}
void ip_vs_conn_put(struct ip_vs_conn *cp);
void ip_vs_conn_fill_cport(struct ip_vs_conn *cp, __be16 cport);

struct ip_vs_conn *ip_vs_conn_new(const struct ip_vs_conn_param *p, int dest_af,
                                  const union nf_inet_addr *daddr,
                                  __be16 dport, unsigned int flags,
                                  struct ip_vs_dest *dest, __u32 fwmark);
void ip_vs_conn_expire_now(struct ip_vs_conn *cp);

const char *ip_vs_state_name(const struct ip_vs_conn *cp);

void ip_vs_tcp_conn_listen(struct ip_vs_conn *cp);
int ip_vs_check_template(struct ip_vs_conn *ct, struct ip_vs_dest *cdest);
void ip_vs_random_dropentry(struct netns_ipvs *ipvs);
int ip_vs_conn_init(void);
void ip_vs_conn_cleanup(void);

static inline void ip_vs_control_del(struct ip_vs_conn *cp)
{
        struct ip_vs_conn *ctl_cp = cp->control;
        if (!ctl_cp) {
                IP_VS_ERR_BUF("request control DEL for uncontrolled: "
                              "%s:%d to %s:%d\n",
                              IP_VS_DBG_ADDR(cp->af, &cp->caddr),
                              ntohs(cp->cport),
                              IP_VS_DBG_ADDR(cp->af, &cp->vaddr),
                              ntohs(cp->vport));

                return;
        }

        IP_VS_DBG_BUF(7, "DELeting control for: "
                      "cp.dst=%s:%d ctl_cp.dst=%s:%d\n",
                      IP_VS_DBG_ADDR(cp->af, &cp->caddr),
                      ntohs(cp->cport),
                      IP_VS_DBG_ADDR(cp->af, &ctl_cp->caddr),
                      ntohs(ctl_cp->cport));

        cp->control = NULL;
        if (atomic_read(&ctl_cp->n_control) == 0) {
                IP_VS_ERR_BUF("BUG control DEL with n=0 : "
                              "%s:%d to %s:%d\n",
                              IP_VS_DBG_ADDR(cp->af, &cp->caddr),
                              ntohs(cp->cport),
                              IP_VS_DBG_ADDR(cp->af, &cp->vaddr),
                              ntohs(cp->vport));

                return;
        }
        atomic_dec(&ctl_cp->n_control);
}

static inline void
ip_vs_control_add(struct ip_vs_conn *cp, struct ip_vs_conn *ctl_cp)
{
        if (cp->control) {
                IP_VS_ERR_BUF("request control ADD for already controlled: "
                              "%s:%d to %s:%d\n",
                              IP_VS_DBG_ADDR(cp->af, &cp->caddr),
                              ntohs(cp->cport),
                              IP_VS_DBG_ADDR(cp->af, &cp->vaddr),
                              ntohs(cp->vport));

                ip_vs_control_del(cp);
        }

        IP_VS_DBG_BUF(7, "ADDing control for: "
                      "cp.dst=%s:%d ctl_cp.dst=%s:%d\n",
                      IP_VS_DBG_ADDR(cp->af, &cp->caddr),
                      ntohs(cp->cport),
                      IP_VS_DBG_ADDR(cp->af, &ctl_cp->caddr),
                      ntohs(ctl_cp->cport));

        cp->control = ctl_cp;
        atomic_inc(&ctl_cp->n_control);
}

/* Mark our template as assured */
static inline void
ip_vs_control_assure_ct(struct ip_vs_conn *cp)
{
        struct ip_vs_conn *ct = cp->control;

        if (ct && !(ct->state & IP_VS_CTPL_S_ASSURED) &&
            (ct->flags & IP_VS_CONN_F_TEMPLATE))
                ct->state |= IP_VS_CTPL_S_ASSURED;
}

/* IPVS netns init & cleanup functions */
int ip_vs_estimator_net_init(struct netns_ipvs *ipvs);
int ip_vs_control_net_init(struct netns_ipvs *ipvs);
int ip_vs_protocol_net_init(struct netns_ipvs *ipvs);
int ip_vs_app_net_init(struct netns_ipvs *ipvs);
int ip_vs_conn_net_init(struct netns_ipvs *ipvs);
int ip_vs_sync_net_init(struct netns_ipvs *ipvs);
void ip_vs_conn_net_cleanup(struct netns_ipvs *ipvs);
void ip_vs_app_net_cleanup(struct netns_ipvs *ipvs);
void ip_vs_protocol_net_cleanup(struct netns_ipvs *ipvs);
void ip_vs_control_net_cleanup(struct netns_ipvs *ipvs);
void ip_vs_estimator_net_cleanup(struct netns_ipvs *ipvs);
void ip_vs_sync_net_cleanup(struct netns_ipvs *ipvs);
void ip_vs_service_nets_cleanup(struct list_head *net_list);

/* IPVS application functions
 * (from ip_vs_app.c)
 */
#define IP_VS_APP_MAX_PORTS  8
struct ip_vs_app *register_ip_vs_app(struct netns_ipvs *ipvs, struct ip_vs_app *app);
void unregister_ip_vs_app(struct netns_ipvs *ipvs, struct ip_vs_app *app);
int ip_vs_bind_app(struct ip_vs_conn *cp, struct ip_vs_protocol *pp);
void ip_vs_unbind_app(struct ip_vs_conn *cp);
int register_ip_vs_app_inc(struct netns_ipvs *ipvs, struct ip_vs_app *app, __u16 proto,
                           __u16 port);
int ip_vs_app_inc_get(struct ip_vs_app *inc);
void ip_vs_app_inc_put(struct ip_vs_app *inc);

int ip_vs_app_pkt_out(struct ip_vs_conn *, struct sk_buff *skb,
                      struct ip_vs_iphdr *ipvsh);
int ip_vs_app_pkt_in(struct ip_vs_conn *, struct sk_buff *skb,
                     struct ip_vs_iphdr *ipvsh);

int register_ip_vs_pe(struct ip_vs_pe *pe);
int unregister_ip_vs_pe(struct ip_vs_pe *pe);
struct ip_vs_pe *ip_vs_pe_getbyname(const char *name);
struct ip_vs_pe *__ip_vs_pe_getbyname(const char *pe_name);

/* Use a #define to avoid all of module.h just for these trivial ops */
#define ip_vs_pe_get(pe)                        \
        if (pe && pe->module)                        \
                __module_get(pe->module);

#define ip_vs_pe_put(pe)                        \
        if (pe && pe->module)                        \
                module_put(pe->module);

/* IPVS protocol functions (from ip_vs_proto.c) */
int ip_vs_protocol_init(void);
void ip_vs_protocol_cleanup(void);
void ip_vs_protocol_timeout_change(struct netns_ipvs *ipvs, int flags);
int *ip_vs_create_timeout_table(int *table, int size);
void ip_vs_tcpudp_debug_packet(int af, struct ip_vs_protocol *pp,
                               const struct sk_buff *skb, int offset,
                               const char *msg);

extern struct ip_vs_protocol ip_vs_protocol_tcp;
extern struct ip_vs_protocol ip_vs_protocol_udp;
extern struct ip_vs_protocol ip_vs_protocol_icmp;
extern struct ip_vs_protocol ip_vs_protocol_esp;
extern struct ip_vs_protocol ip_vs_protocol_ah;
extern struct ip_vs_protocol ip_vs_protocol_sctp;

/* Registering/unregistering scheduler functions
 * (from ip_vs_sched.c)
 */
int register_ip_vs_scheduler(struct ip_vs_scheduler *scheduler);
int unregister_ip_vs_scheduler(struct ip_vs_scheduler *scheduler);
int ip_vs_bind_scheduler(struct ip_vs_service *svc,
                         struct ip_vs_scheduler *scheduler);
void ip_vs_unbind_scheduler(struct ip_vs_service *svc,
                            struct ip_vs_scheduler *sched);
struct ip_vs_scheduler *ip_vs_scheduler_get(const char *sched_name);
void ip_vs_scheduler_put(struct ip_vs_scheduler *scheduler);
struct ip_vs_conn *
ip_vs_schedule(struct ip_vs_service *svc, struct sk_buff *skb,
               struct ip_vs_proto_data *pd, int *ignored,
               struct ip_vs_iphdr *iph);
int ip_vs_leave(struct ip_vs_service *svc, struct sk_buff *skb,
                struct ip_vs_proto_data *pd, struct ip_vs_iphdr *iph);

void ip_vs_scheduler_err(struct ip_vs_service *svc, const char *msg);

/* IPVS control data and functions (from ip_vs_ctl.c) */
extern struct ip_vs_stats ip_vs_stats;
extern int sysctl_ip_vs_sync_ver;

struct ip_vs_service *
ip_vs_service_find(struct netns_ipvs *ipvs, int af, __u32 fwmark, __u16 protocol,
                  const union nf_inet_addr *vaddr, __be16 vport);

bool ip_vs_has_real_service(struct netns_ipvs *ipvs, int af, __u16 protocol,
                            const union nf_inet_addr *daddr, __be16 dport);

struct ip_vs_dest *
ip_vs_find_real_service(struct netns_ipvs *ipvs, int af, __u16 protocol,
                        const union nf_inet_addr *daddr, __be16 dport);
struct ip_vs_dest *ip_vs_find_tunnel(struct netns_ipvs *ipvs, int af,
                                     const union nf_inet_addr *daddr,
                                     __be16 tun_port);

int ip_vs_use_count_inc(void);
void ip_vs_use_count_dec(void);
int ip_vs_register_nl_ioctl(void);
void ip_vs_unregister_nl_ioctl(void);
int ip_vs_control_init(void);
void ip_vs_control_cleanup(void);
struct ip_vs_dest *
ip_vs_find_dest(struct netns_ipvs *ipvs, int svc_af, int dest_af,
                const union nf_inet_addr *daddr, __be16 dport,
                const union nf_inet_addr *vaddr, __be16 vport,
                __u16 protocol, __u32 fwmark, __u32 flags);
void ip_vs_try_bind_dest(struct ip_vs_conn *cp);

static inline void ip_vs_dest_hold(struct ip_vs_dest *dest)
{
        refcount_inc(&dest->refcnt);
}

static inline void ip_vs_dest_put(struct ip_vs_dest *dest)
{
        smp_mb__before_atomic();
        refcount_dec(&dest->refcnt);
}

static inline void ip_vs_dest_put_and_free(struct ip_vs_dest *dest)
{
        if (refcount_dec_and_test(&dest->refcnt))
                kfree(dest);
}

/* IPVS sync daemon data and function prototypes
 * (from ip_vs_sync.c)
 */
int start_sync_thread(struct netns_ipvs *ipvs, struct ipvs_sync_daemon_cfg *cfg,
                      int state);
int stop_sync_thread(struct netns_ipvs *ipvs, int state);
void ip_vs_sync_conn(struct netns_ipvs *ipvs, struct ip_vs_conn *cp, int pkts);

/* IPVS rate estimator prototypes (from ip_vs_est.c) */
int ip_vs_start_estimator(struct netns_ipvs *ipvs, struct ip_vs_stats *stats);
void ip_vs_stop_estimator(struct netns_ipvs *ipvs, struct ip_vs_stats *stats);
void ip_vs_zero_estimator(struct ip_vs_stats *stats);
void ip_vs_read_estimator(struct ip_vs_kstats *dst, struct ip_vs_stats *stats);
void ip_vs_est_reload_start(struct netns_ipvs *ipvs);
int ip_vs_est_kthread_start(struct netns_ipvs *ipvs,
                            struct ip_vs_est_kt_data *kd);
void ip_vs_est_kthread_stop(struct ip_vs_est_kt_data *kd);

static inline void ip_vs_est_stopped_recalc(struct netns_ipvs *ipvs)
{
#ifdef CONFIG_SYSCTL
        /* Stop tasks while cpulist is empty or if disabled with flag */
        ipvs->est_stopped = !sysctl_run_estimation(ipvs) ||
                            (ipvs->est_cpulist_valid &&
                             cpumask_empty(sysctl_est_cpulist(ipvs)));
#endif
}

static inline bool ip_vs_est_stopped(struct netns_ipvs *ipvs)
{
#ifdef CONFIG_SYSCTL
        return ipvs->est_stopped;
#else
        return false;
#endif
}

static inline int ip_vs_est_max_threads(struct netns_ipvs *ipvs)
{
        unsigned int limit = IPVS_EST_CPU_KTHREADS *
                             cpumask_weight(sysctl_est_cpulist(ipvs));

        return max(1U, limit);
}

/* Various IPVS packet transmitters (from ip_vs_xmit.c) */
int ip_vs_null_xmit(struct sk_buff *skb, struct ip_vs_conn *cp,
                    struct ip_vs_protocol *pp, struct ip_vs_iphdr *iph);
int ip_vs_bypass_xmit(struct sk_buff *skb, struct ip_vs_conn *cp,
                      struct ip_vs_protocol *pp, struct ip_vs_iphdr *iph);
int ip_vs_nat_xmit(struct sk_buff *skb, struct ip_vs_conn *cp,
                   struct ip_vs_protocol *pp, struct ip_vs_iphdr *iph);
int ip_vs_tunnel_xmit(struct sk_buff *skb, struct ip_vs_conn *cp,
                      struct ip_vs_protocol *pp, struct ip_vs_iphdr *iph);
int ip_vs_dr_xmit(struct sk_buff *skb, struct ip_vs_conn *cp,
                  struct ip_vs_protocol *pp, struct ip_vs_iphdr *iph);
int ip_vs_icmp_xmit(struct sk_buff *skb, struct ip_vs_conn *cp,
                    struct ip_vs_protocol *pp, int offset,
                    unsigned int hooknum, struct ip_vs_iphdr *iph);
void ip_vs_dest_dst_rcu_free(struct rcu_head *head);

#ifdef CONFIG_IP_VS_IPV6
int ip_vs_bypass_xmit_v6(struct sk_buff *skb, struct ip_vs_conn *cp,
                         struct ip_vs_protocol *pp, struct ip_vs_iphdr *iph);
int ip_vs_nat_xmit_v6(struct sk_buff *skb, struct ip_vs_conn *cp,
                      struct ip_vs_protocol *pp, struct ip_vs_iphdr *iph);
int ip_vs_tunnel_xmit_v6(struct sk_buff *skb, struct ip_vs_conn *cp,
                         struct ip_vs_protocol *pp, struct ip_vs_iphdr *iph);
int ip_vs_dr_xmit_v6(struct sk_buff *skb, struct ip_vs_conn *cp,
                     struct ip_vs_protocol *pp, struct ip_vs_iphdr *iph);
int ip_vs_icmp_xmit_v6(struct sk_buff *skb, struct ip_vs_conn *cp,
                       struct ip_vs_protocol *pp, int offset,
                       unsigned int hooknum, struct ip_vs_iphdr *iph);
#endif

#ifdef CONFIG_SYSCTL
/* This is a simple mechanism to ignore packets when
 * we are loaded. Just set ip_vs_drop_rate to 'n' and
 * we start to drop 1/rate of the packets
 */
static inline int ip_vs_todrop(struct netns_ipvs *ipvs)
{
        if (!ipvs->drop_rate)
                return 0;
        if (--ipvs->drop_counter > 0)
                return 0;
        ipvs->drop_counter = ipvs->drop_rate;
        return 1;
}
#else
static inline int ip_vs_todrop(struct netns_ipvs *ipvs) { return 0; }
#endif

#ifdef CONFIG_SYSCTL
/* Enqueue delayed work for expiring no dest connections
 * Only run when sysctl_expire_nodest=1
 */
static inline void ip_vs_enqueue_expire_nodest_conns(struct netns_ipvs *ipvs)
{
        if (sysctl_expire_nodest_conn(ipvs))
                queue_delayed_work(system_long_wq,
                                   &ipvs->expire_nodest_conn_work, 1);
}

void ip_vs_expire_nodest_conn_flush(struct netns_ipvs *ipvs);
#else
static inline void ip_vs_enqueue_expire_nodest_conns(struct netns_ipvs *ipvs) {}
#endif

#define IP_VS_DFWD_METHOD(dest) (atomic_read(&(dest)->conn_flags) & \
                                 IP_VS_CONN_F_FWD_MASK)

/* ip_vs_fwd_tag returns the forwarding tag of the connection */
#define IP_VS_FWD_METHOD(cp)  (cp->flags & IP_VS_CONN_F_FWD_MASK)

static inline char ip_vs_fwd_tag(struct ip_vs_conn *cp)
{
        char fwd;

        switch (IP_VS_FWD_METHOD(cp)) {
        case IP_VS_CONN_F_MASQ:
                fwd = 'M'; break;
        case IP_VS_CONN_F_LOCALNODE:
                fwd = 'L'; break;
        case IP_VS_CONN_F_TUNNEL:
                fwd = 'T'; break;
        case IP_VS_CONN_F_DROUTE:
                fwd = 'R'; break;
        case IP_VS_CONN_F_BYPASS:
                fwd = 'B'; break;
        default:
                fwd = '?'; break;
        }
        return fwd;
}

void ip_vs_nat_icmp(struct sk_buff *skb, struct ip_vs_protocol *pp,
                    struct ip_vs_conn *cp, int dir);

#ifdef CONFIG_IP_VS_IPV6
void ip_vs_nat_icmp_v6(struct sk_buff *skb, struct ip_vs_protocol *pp,
                       struct ip_vs_conn *cp, int dir);
#endif

__sum16 ip_vs_checksum_complete(struct sk_buff *skb, int offset);

static inline __wsum ip_vs_check_diff4(__be32 old, __be32 new, __wsum oldsum)
{
        __be32 diff[2] = { ~old, new };

        return csum_partial(diff, sizeof(diff), oldsum);
}

#ifdef CONFIG_IP_VS_IPV6
static inline __wsum ip_vs_check_diff16(const __be32 *old, const __be32 *new,
                                        __wsum oldsum)
{
        __be32 diff[8] = { ~old[3], ~old[2], ~old[1], ~old[0],
                            new[3],  new[2],  new[1],  new[0] };

        return csum_partial(diff, sizeof(diff), oldsum);
}
#endif

static inline __wsum ip_vs_check_diff2(__be16 old, __be16 new, __wsum oldsum)
{
        __be16 diff[2] = { ~old, new };

        return csum_partial(diff, sizeof(diff), oldsum);
}

/* Forget current conntrack (unconfirmed) and attach notrack entry */
static inline void ip_vs_notrack(struct sk_buff *skb)
{
#if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
        enum ip_conntrack_info ctinfo;
        struct nf_conn *ct = nf_ct_get(skb, &ctinfo);

        if (ct) {
                nf_conntrack_put(&ct->ct_general);
                nf_ct_set(skb, NULL, IP_CT_UNTRACKED);
        }
#endif
}

#ifdef CONFIG_IP_VS_NFCT
/* Netfilter connection tracking
 * (from ip_vs_nfct.c)
 */
static inline int ip_vs_conntrack_enabled(struct netns_ipvs *ipvs)
{
#ifdef CONFIG_SYSCTL
        return ipvs->sysctl_conntrack;
#else
        return 0;
#endif
}

void ip_vs_update_conntrack(struct sk_buff *skb, struct ip_vs_conn *cp,
                            int outin);
int ip_vs_confirm_conntrack(struct sk_buff *skb);
void ip_vs_nfct_expect_related(struct sk_buff *skb, struct nf_conn *ct,
                               struct ip_vs_conn *cp, u_int8_t proto,
                               const __be16 port, int from_rs);
void ip_vs_conn_drop_conntrack(struct ip_vs_conn *cp);

#else

static inline int ip_vs_conntrack_enabled(struct netns_ipvs *ipvs)
{
        return 0;
}

static inline void ip_vs_update_conntrack(struct sk_buff *skb,
                                          struct ip_vs_conn *cp, int outin)
{
}

static inline int ip_vs_confirm_conntrack(struct sk_buff *skb)
{
        return NF_ACCEPT;
}

static inline void ip_vs_conn_drop_conntrack(struct ip_vs_conn *cp)
{
}
#endif /* CONFIG_IP_VS_NFCT */

/* Using old conntrack that can not be redirected to another real server? */
static inline bool ip_vs_conn_uses_old_conntrack(struct ip_vs_conn *cp,
                                                 struct sk_buff *skb)
{
#ifdef CONFIG_IP_VS_NFCT
        enum ip_conntrack_info ctinfo;
        struct nf_conn *ct;

        ct = nf_ct_get(skb, &ctinfo);
        if (ct && nf_ct_is_confirmed(ct))
                return true;
#endif
        return false;
}

static inline int ip_vs_register_conntrack(struct ip_vs_service *svc)
{
#if IS_ENABLED(CONFIG_NF_CONNTRACK)
        int afmask = (svc->af == AF_INET6) ? 2 : 1;
        int ret = 0;

        if (!(svc->conntrack_afmask & afmask)) {
                ret = nf_ct_netns_get(svc->ipvs->net, svc->af);
                if (ret >= 0)
                        svc->conntrack_afmask |= afmask;
        }
        return ret;
#else
        return 0;
#endif
}

static inline void ip_vs_unregister_conntrack(struct ip_vs_service *svc)
{
#if IS_ENABLED(CONFIG_NF_CONNTRACK)
        int afmask = (svc->af == AF_INET6) ? 2 : 1;

        if (svc->conntrack_afmask & afmask) {
                nf_ct_netns_put(svc->ipvs->net, svc->af);
                svc->conntrack_afmask &= ~afmask;
        }
#endif
}

int ip_vs_register_hooks(struct netns_ipvs *ipvs, unsigned int af);
void ip_vs_unregister_hooks(struct netns_ipvs *ipvs, unsigned int af);

static inline int
ip_vs_dest_conn_overhead(struct ip_vs_dest *dest)
{
        /* We think the overhead of processing active connections is 256
         * times higher than that of inactive connections in average. (This
         * 256 times might not be accurate, we will change it later) We
         * use the following formula to estimate the overhead now:
         *                  dest->activeconns*256 + dest->inactconns
         */
        return (atomic_read(&dest->activeconns) << 8) +
                atomic_read(&dest->inactconns);
}

#ifdef CONFIG_IP_VS_PROTO_TCP
INDIRECT_CALLABLE_DECLARE(int
        tcp_snat_handler(struct sk_buff *skb, struct ip_vs_protocol *pp,
                         struct ip_vs_conn *cp, struct ip_vs_iphdr *iph));
#endif

#ifdef CONFIG_IP_VS_PROTO_UDP
INDIRECT_CALLABLE_DECLARE(int
        udp_snat_handler(struct sk_buff *skb, struct ip_vs_protocol *pp,
                         struct ip_vs_conn *cp, struct ip_vs_iphdr *iph));
#endif
#endif        /* _NET_IP_VS_H */





























































    9 



    9 


    1 




    1 



    5 

    2 















































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
// SPDX-License-Identifier: GPL-2.0-only
/* iptables module for using new netfilter netlink queue
 *
 * (C) 2005 by Harald Welte <laforge@netfilter.org>
 */

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#include <linux/module.h>
#include <linux/skbuff.h>

#include <linux/netfilter.h>
#include <linux/netfilter_arp.h>
#include <linux/netfilter/x_tables.h>
#include <linux/netfilter/xt_NFQUEUE.h>

#include <net/netfilter/nf_queue.h>

MODULE_AUTHOR("Harald Welte <laforge@netfilter.org>");
MODULE_DESCRIPTION("Xtables: packet forwarding to netlink");
MODULE_LICENSE("GPL");
MODULE_ALIAS("ipt_NFQUEUE");
MODULE_ALIAS("ip6t_NFQUEUE");
MODULE_ALIAS("arpt_NFQUEUE");

static u32 jhash_initval __read_mostly;

static unsigned int
nfqueue_tg(struct sk_buff *skb, const struct xt_action_param *par)
{
        const struct xt_NFQ_info *tinfo = par->targinfo;

        return NF_QUEUE_NR(tinfo->queuenum);
}

static unsigned int
nfqueue_tg_v1(struct sk_buff *skb, const struct xt_action_param *par)
{
        const struct xt_NFQ_info_v1 *info = par->targinfo;
        u32 queue = info->queuenum;

        if (info->queues_total > 1) {
                queue = nfqueue_hash(skb, queue, info->queues_total,
                                     xt_family(par), jhash_initval);
        }
        return NF_QUEUE_NR(queue);
}

static unsigned int
nfqueue_tg_v2(struct sk_buff *skb, const struct xt_action_param *par)
{
        const struct xt_NFQ_info_v2 *info = par->targinfo;
        unsigned int ret = nfqueue_tg_v1(skb, par);

        if (info->bypass)
                ret |= NF_VERDICT_FLAG_QUEUE_BYPASS;
        return ret;
}

static int nfqueue_tg_check(const struct xt_tgchk_param *par)
{
        const struct xt_NFQ_info_v3 *info = par->targinfo;
        u32 maxid;

        init_hashrandom(&jhash_initval);

        if (info->queues_total == 0) {
                pr_info_ratelimited("number of total queues is 0\n");
                return -EINVAL;
        }
        maxid = info->queues_total - 1 + info->queuenum;
        if (maxid > 0xffff) {
                pr_info_ratelimited("number of queues (%u) out of range (got %u)\n",
                                    info->queues_total, maxid);
                return -ERANGE;
        }
        if (par->target->revision == 2 && info->flags > 1)
                return -EINVAL;
        if (par->target->revision == 3 && info->flags & ~NFQ_FLAG_MASK)
                return -EINVAL;

        return 0;
}

static unsigned int
nfqueue_tg_v3(struct sk_buff *skb, const struct xt_action_param *par)
{
        const struct xt_NFQ_info_v3 *info = par->targinfo;
        u32 queue = info->queuenum;
        int ret;

        if (info->queues_total > 1) {
                if (info->flags & NFQ_FLAG_CPU_FANOUT) {
                        int cpu = smp_processor_id();

                        queue = info->queuenum + cpu % info->queues_total;
                } else {
                        queue = nfqueue_hash(skb, queue, info->queues_total,
                                             xt_family(par), jhash_initval);
                }
        }

        ret = NF_QUEUE_NR(queue);
        if (info->flags & NFQ_FLAG_BYPASS)
                ret |= NF_VERDICT_FLAG_QUEUE_BYPASS;

        return ret;
}

static struct xt_target nfqueue_tg_reg[] __read_mostly = {
        {
                .name                = "NFQUEUE",
                .family                = NFPROTO_UNSPEC,
                .target                = nfqueue_tg,
                .targetsize        = sizeof(struct xt_NFQ_info),
                .me                = THIS_MODULE,
        },
        {
                .name                = "NFQUEUE",
                .revision        = 1,
                .family                = NFPROTO_UNSPEC,
                .checkentry        = nfqueue_tg_check,
                .target                = nfqueue_tg_v1,
                .targetsize        = sizeof(struct xt_NFQ_info_v1),
                .me                = THIS_MODULE,
        },
        {
                .name                = "NFQUEUE",
                .revision        = 2,
                .family                = NFPROTO_UNSPEC,
                .checkentry        = nfqueue_tg_check,
                .target                = nfqueue_tg_v2,
                .targetsize        = sizeof(struct xt_NFQ_info_v2),
                .me                = THIS_MODULE,
        },
        {
                .name                = "NFQUEUE",
                .revision        = 3,
                .family                = NFPROTO_UNSPEC,
                .checkentry        = nfqueue_tg_check,
                .target                = nfqueue_tg_v3,
                .targetsize        = sizeof(struct xt_NFQ_info_v3),
                .me                = THIS_MODULE,
        },
};

static int __init nfqueue_tg_init(void)
{
        return xt_register_targets(nfqueue_tg_reg, ARRAY_SIZE(nfqueue_tg_reg));
}

static void __exit nfqueue_tg_exit(void)
{
        xt_unregister_targets(nfqueue_tg_reg, ARRAY_SIZE(nfqueue_tg_reg));
}

module_init(nfqueue_tg_init);
module_exit(nfqueue_tg_exit);








































    1 


    2 
    1 

    3 






































    1 



    2 
























    1 





















    3 






    1 


    2 

    2 





















    2 









    2 

    2 
























    3 






    1 



    2 


    2 




























    4 






    1 


    2 

    2 




    1 


















    3 






    1 


    1 
    1 


    2 


























    4 






    1 


    1 
    1 
    1 

    3 
















   32 






    1 
   31 


    6 



    2 

    1 


   25 
    4 





   22 
    3 





   19 
    1 

    1 



   16 



    3 



   15 


    2 



   12 



    3 



    8 


    4 



    5 


    3 



    1 



    4 



































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
// SPDX-License-Identifier: GPL-2.0-or-later

#include <net/genetlink.h>

#include <uapi/linux/mrp_bridge.h>
#include "br_private.h"
#include "br_private_mrp.h"

static const struct nla_policy br_mrp_policy[IFLA_BRIDGE_MRP_MAX + 1] = {
        [IFLA_BRIDGE_MRP_UNSPEC]        = { .type = NLA_REJECT },
        [IFLA_BRIDGE_MRP_INSTANCE]        = { .type = NLA_NESTED },
        [IFLA_BRIDGE_MRP_PORT_STATE]        = { .type = NLA_NESTED },
        [IFLA_BRIDGE_MRP_PORT_ROLE]        = { .type = NLA_NESTED },
        [IFLA_BRIDGE_MRP_RING_STATE]        = { .type = NLA_NESTED },
        [IFLA_BRIDGE_MRP_RING_ROLE]        = { .type = NLA_NESTED },
        [IFLA_BRIDGE_MRP_START_TEST]        = { .type = NLA_NESTED },
        [IFLA_BRIDGE_MRP_IN_ROLE]        = { .type = NLA_NESTED },
        [IFLA_BRIDGE_MRP_IN_STATE]        = { .type = NLA_NESTED },
        [IFLA_BRIDGE_MRP_START_IN_TEST]        = { .type = NLA_NESTED },
};

static const struct nla_policy
br_mrp_instance_policy[IFLA_BRIDGE_MRP_INSTANCE_MAX + 1] = {
        [IFLA_BRIDGE_MRP_INSTANCE_UNSPEC]        = { .type = NLA_REJECT },
        [IFLA_BRIDGE_MRP_INSTANCE_RING_ID]        = { .type = NLA_U32 },
        [IFLA_BRIDGE_MRP_INSTANCE_P_IFINDEX]        = { .type = NLA_U32 },
        [IFLA_BRIDGE_MRP_INSTANCE_S_IFINDEX]        = { .type = NLA_U32 },
        [IFLA_BRIDGE_MRP_INSTANCE_PRIO]                = { .type = NLA_U16 },
};

static int br_mrp_instance_parse(struct net_bridge *br, struct nlattr *attr,
                                 int cmd, struct netlink_ext_ack *extack)
{
        struct nlattr *tb[IFLA_BRIDGE_MRP_INSTANCE_MAX + 1];
        struct br_mrp_instance inst;
        int err;

        err = nla_parse_nested(tb, IFLA_BRIDGE_MRP_INSTANCE_MAX, attr,
                               br_mrp_instance_policy, extack);
        if (err)
                return err;

        if (!tb[IFLA_BRIDGE_MRP_INSTANCE_RING_ID] ||
            !tb[IFLA_BRIDGE_MRP_INSTANCE_P_IFINDEX] ||
            !tb[IFLA_BRIDGE_MRP_INSTANCE_S_IFINDEX]) {
                NL_SET_ERR_MSG_MOD(extack,
                                   "Missing attribute: RING_ID or P_IFINDEX or S_IFINDEX");
                return -EINVAL;
        }

        memset(&inst, 0, sizeof(inst));

        inst.ring_id = nla_get_u32(tb[IFLA_BRIDGE_MRP_INSTANCE_RING_ID]);
        inst.p_ifindex = nla_get_u32(tb[IFLA_BRIDGE_MRP_INSTANCE_P_IFINDEX]);
        inst.s_ifindex = nla_get_u32(tb[IFLA_BRIDGE_MRP_INSTANCE_S_IFINDEX]);
        inst.prio = MRP_DEFAULT_PRIO;

        if (tb[IFLA_BRIDGE_MRP_INSTANCE_PRIO])
                inst.prio = nla_get_u16(tb[IFLA_BRIDGE_MRP_INSTANCE_PRIO]);

        if (cmd == RTM_SETLINK)
                return br_mrp_add(br, &inst);
        else
                return br_mrp_del(br, &inst);

        return 0;
}

static const struct nla_policy
br_mrp_port_state_policy[IFLA_BRIDGE_MRP_PORT_STATE_MAX + 1] = {
        [IFLA_BRIDGE_MRP_PORT_STATE_UNSPEC]        = { .type = NLA_REJECT },
        [IFLA_BRIDGE_MRP_PORT_STATE_STATE]        = { .type = NLA_U32 },
};

static int br_mrp_port_state_parse(struct net_bridge_port *p,
                                   struct nlattr *attr,
                                   struct netlink_ext_ack *extack)
{
        struct nlattr *tb[IFLA_BRIDGE_MRP_PORT_STATE_MAX + 1];
        enum br_mrp_port_state_type state;
        int err;

        err = nla_parse_nested(tb, IFLA_BRIDGE_MRP_PORT_STATE_MAX, attr,
                               br_mrp_port_state_policy, extack);
        if (err)
                return err;

        if (!tb[IFLA_BRIDGE_MRP_PORT_STATE_STATE]) {
                NL_SET_ERR_MSG_MOD(extack, "Missing attribute: STATE");
                return -EINVAL;
        }

        state = nla_get_u32(tb[IFLA_BRIDGE_MRP_PORT_STATE_STATE]);

        return br_mrp_set_port_state(p, state);
}

static const struct nla_policy
br_mrp_port_role_policy[IFLA_BRIDGE_MRP_PORT_ROLE_MAX + 1] = {
        [IFLA_BRIDGE_MRP_PORT_ROLE_UNSPEC]        = { .type = NLA_REJECT },
        [IFLA_BRIDGE_MRP_PORT_ROLE_ROLE]        = { .type = NLA_U32 },
};

static int br_mrp_port_role_parse(struct net_bridge_port *p,
                                  struct nlattr *attr,
                                  struct netlink_ext_ack *extack)
{
        struct nlattr *tb[IFLA_BRIDGE_MRP_PORT_ROLE_MAX + 1];
        enum br_mrp_port_role_type role;
        int err;

        err = nla_parse_nested(tb, IFLA_BRIDGE_MRP_PORT_ROLE_MAX, attr,
                               br_mrp_port_role_policy, extack);
        if (err)
                return err;

        if (!tb[IFLA_BRIDGE_MRP_PORT_ROLE_ROLE]) {
                NL_SET_ERR_MSG_MOD(extack, "Missing attribute: ROLE");
                return -EINVAL;
        }

        role = nla_get_u32(tb[IFLA_BRIDGE_MRP_PORT_ROLE_ROLE]);

        return br_mrp_set_port_role(p, role);
}

static const struct nla_policy
br_mrp_ring_state_policy[IFLA_BRIDGE_MRP_RING_STATE_MAX + 1] = {
        [IFLA_BRIDGE_MRP_RING_STATE_UNSPEC]        = { .type = NLA_REJECT },
        [IFLA_BRIDGE_MRP_RING_STATE_RING_ID]        = { .type = NLA_U32 },
        [IFLA_BRIDGE_MRP_RING_STATE_STATE]        = { .type = NLA_U32 },
};

static int br_mrp_ring_state_parse(struct net_bridge *br, struct nlattr *attr,
                                   struct netlink_ext_ack *extack)
{
        struct nlattr *tb[IFLA_BRIDGE_MRP_RING_STATE_MAX + 1];
        struct br_mrp_ring_state state;
        int err;

        err = nla_parse_nested(tb, IFLA_BRIDGE_MRP_RING_STATE_MAX, attr,
                               br_mrp_ring_state_policy, extack);
        if (err)
                return err;

        if (!tb[IFLA_BRIDGE_MRP_RING_STATE_RING_ID] ||
            !tb[IFLA_BRIDGE_MRP_RING_STATE_STATE]) {
                NL_SET_ERR_MSG_MOD(extack,
                                   "Missing attribute: RING_ID or STATE");
                return -EINVAL;
        }

        memset(&state, 0x0, sizeof(state));

        state.ring_id = nla_get_u32(tb[IFLA_BRIDGE_MRP_RING_STATE_RING_ID]);
        state.ring_state = nla_get_u32(tb[IFLA_BRIDGE_MRP_RING_STATE_STATE]);

        return br_mrp_set_ring_state(br, &state);
}

static const struct nla_policy
br_mrp_ring_role_policy[IFLA_BRIDGE_MRP_RING_ROLE_MAX + 1] = {
        [IFLA_BRIDGE_MRP_RING_ROLE_UNSPEC]        = { .type = NLA_REJECT },
        [IFLA_BRIDGE_MRP_RING_ROLE_RING_ID]        = { .type = NLA_U32 },
        [IFLA_BRIDGE_MRP_RING_ROLE_ROLE]        = { .type = NLA_U32 },
};

static int br_mrp_ring_role_parse(struct net_bridge *br, struct nlattr *attr,
                                  struct netlink_ext_ack *extack)
{
        struct nlattr *tb[IFLA_BRIDGE_MRP_RING_ROLE_MAX + 1];
        struct br_mrp_ring_role role;
        int err;

        err = nla_parse_nested(tb, IFLA_BRIDGE_MRP_RING_ROLE_MAX, attr,
                               br_mrp_ring_role_policy, extack);
        if (err)
                return err;

        if (!tb[IFLA_BRIDGE_MRP_RING_ROLE_RING_ID] ||
            !tb[IFLA_BRIDGE_MRP_RING_ROLE_ROLE]) {
                NL_SET_ERR_MSG_MOD(extack,
                                   "Missing attribute: RING_ID or ROLE");
                return -EINVAL;
        }

        memset(&role, 0x0, sizeof(role));

        role.ring_id = nla_get_u32(tb[IFLA_BRIDGE_MRP_RING_ROLE_RING_ID]);
        role.ring_role = nla_get_u32(tb[IFLA_BRIDGE_MRP_RING_ROLE_ROLE]);

        return br_mrp_set_ring_role(br, &role);
}

static const struct nla_policy
br_mrp_start_test_policy[IFLA_BRIDGE_MRP_START_TEST_MAX + 1] = {
        [IFLA_BRIDGE_MRP_START_TEST_UNSPEC]        = { .type = NLA_REJECT },
        [IFLA_BRIDGE_MRP_START_TEST_RING_ID]        = { .type = NLA_U32 },
        [IFLA_BRIDGE_MRP_START_TEST_INTERVAL]        = { .type = NLA_U32 },
        [IFLA_BRIDGE_MRP_START_TEST_MAX_MISS]        = { .type = NLA_U32 },
        [IFLA_BRIDGE_MRP_START_TEST_PERIOD]        = { .type = NLA_U32 },
        [IFLA_BRIDGE_MRP_START_TEST_MONITOR]        = { .type = NLA_U32 },
};

static int br_mrp_start_test_parse(struct net_bridge *br, struct nlattr *attr,
                                   struct netlink_ext_ack *extack)
{
        struct nlattr *tb[IFLA_BRIDGE_MRP_START_TEST_MAX + 1];
        struct br_mrp_start_test test;
        int err;

        err = nla_parse_nested(tb, IFLA_BRIDGE_MRP_START_TEST_MAX, attr,
                               br_mrp_start_test_policy, extack);
        if (err)
                return err;

        if (!tb[IFLA_BRIDGE_MRP_START_TEST_RING_ID] ||
            !tb[IFLA_BRIDGE_MRP_START_TEST_INTERVAL] ||
            !tb[IFLA_BRIDGE_MRP_START_TEST_MAX_MISS] ||
            !tb[IFLA_BRIDGE_MRP_START_TEST_PERIOD]) {
                NL_SET_ERR_MSG_MOD(extack,
                                   "Missing attribute: RING_ID or INTERVAL or MAX_MISS or PERIOD");
                return -EINVAL;
        }

        memset(&test, 0x0, sizeof(test));

        test.ring_id = nla_get_u32(tb[IFLA_BRIDGE_MRP_START_TEST_RING_ID]);
        test.interval = nla_get_u32(tb[IFLA_BRIDGE_MRP_START_TEST_INTERVAL]);
        test.max_miss = nla_get_u32(tb[IFLA_BRIDGE_MRP_START_TEST_MAX_MISS]);
        test.period = nla_get_u32(tb[IFLA_BRIDGE_MRP_START_TEST_PERIOD]);
        test.monitor = false;

        if (tb[IFLA_BRIDGE_MRP_START_TEST_MONITOR])
                test.monitor =
                        nla_get_u32(tb[IFLA_BRIDGE_MRP_START_TEST_MONITOR]);

        return br_mrp_start_test(br, &test);
}

static const struct nla_policy
br_mrp_in_state_policy[IFLA_BRIDGE_MRP_IN_STATE_MAX + 1] = {
        [IFLA_BRIDGE_MRP_IN_STATE_UNSPEC]        = { .type = NLA_REJECT },
        [IFLA_BRIDGE_MRP_IN_STATE_IN_ID]        = { .type = NLA_U32 },
        [IFLA_BRIDGE_MRP_IN_STATE_STATE]        = { .type = NLA_U32 },
};

static int br_mrp_in_state_parse(struct net_bridge *br, struct nlattr *attr,
                                 struct netlink_ext_ack *extack)
{
        struct nlattr *tb[IFLA_BRIDGE_MRP_IN_STATE_MAX + 1];
        struct br_mrp_in_state state;
        int err;

        err = nla_parse_nested(tb, IFLA_BRIDGE_MRP_IN_STATE_MAX, attr,
                               br_mrp_in_state_policy, extack);
        if (err)
                return err;

        if (!tb[IFLA_BRIDGE_MRP_IN_STATE_IN_ID] ||
            !tb[IFLA_BRIDGE_MRP_IN_STATE_STATE]) {
                NL_SET_ERR_MSG_MOD(extack,
                                   "Missing attribute: IN_ID or STATE");
                return -EINVAL;
        }

        memset(&state, 0x0, sizeof(state));

        state.in_id = nla_get_u32(tb[IFLA_BRIDGE_MRP_IN_STATE_IN_ID]);
        state.in_state = nla_get_u32(tb[IFLA_BRIDGE_MRP_IN_STATE_STATE]);

        return br_mrp_set_in_state(br, &state);
}

static const struct nla_policy
br_mrp_in_role_policy[IFLA_BRIDGE_MRP_IN_ROLE_MAX + 1] = {
        [IFLA_BRIDGE_MRP_IN_ROLE_UNSPEC]        = { .type = NLA_REJECT },
        [IFLA_BRIDGE_MRP_IN_ROLE_RING_ID]        = { .type = NLA_U32 },
        [IFLA_BRIDGE_MRP_IN_ROLE_IN_ID]                = { .type = NLA_U16 },
        [IFLA_BRIDGE_MRP_IN_ROLE_ROLE]                = { .type = NLA_U32 },
        [IFLA_BRIDGE_MRP_IN_ROLE_I_IFINDEX]        = { .type = NLA_U32 },
};

static int br_mrp_in_role_parse(struct net_bridge *br, struct nlattr *attr,
                                struct netlink_ext_ack *extack)
{
        struct nlattr *tb[IFLA_BRIDGE_MRP_IN_ROLE_MAX + 1];
        struct br_mrp_in_role role;
        int err;

        err = nla_parse_nested(tb, IFLA_BRIDGE_MRP_IN_ROLE_MAX, attr,
                               br_mrp_in_role_policy, extack);
        if (err)
                return err;

        if (!tb[IFLA_BRIDGE_MRP_IN_ROLE_RING_ID] ||
            !tb[IFLA_BRIDGE_MRP_IN_ROLE_IN_ID] ||
            !tb[IFLA_BRIDGE_MRP_IN_ROLE_I_IFINDEX] ||
            !tb[IFLA_BRIDGE_MRP_IN_ROLE_ROLE]) {
                NL_SET_ERR_MSG_MOD(extack,
                                   "Missing attribute: RING_ID or ROLE or IN_ID or I_IFINDEX");
                return -EINVAL;
        }

        memset(&role, 0x0, sizeof(role));

        role.ring_id = nla_get_u32(tb[IFLA_BRIDGE_MRP_IN_ROLE_RING_ID]);
        role.in_id = nla_get_u16(tb[IFLA_BRIDGE_MRP_IN_ROLE_IN_ID]);
        role.i_ifindex = nla_get_u32(tb[IFLA_BRIDGE_MRP_IN_ROLE_I_IFINDEX]);
        role.in_role = nla_get_u32(tb[IFLA_BRIDGE_MRP_IN_ROLE_ROLE]);

        return br_mrp_set_in_role(br, &role);
}

static const struct nla_policy
br_mrp_start_in_test_policy[IFLA_BRIDGE_MRP_START_IN_TEST_MAX + 1] = {
        [IFLA_BRIDGE_MRP_START_IN_TEST_UNSPEC]        = { .type = NLA_REJECT },
        [IFLA_BRIDGE_MRP_START_IN_TEST_IN_ID]        = { .type = NLA_U32 },
        [IFLA_BRIDGE_MRP_START_IN_TEST_INTERVAL]        = { .type = NLA_U32 },
        [IFLA_BRIDGE_MRP_START_IN_TEST_MAX_MISS]        = { .type = NLA_U32 },
        [IFLA_BRIDGE_MRP_START_IN_TEST_PERIOD]        = { .type = NLA_U32 },
};

static int br_mrp_start_in_test_parse(struct net_bridge *br,
                                      struct nlattr *attr,
                                      struct netlink_ext_ack *extack)
{
        struct nlattr *tb[IFLA_BRIDGE_MRP_START_IN_TEST_MAX + 1];
        struct br_mrp_start_in_test test;
        int err;

        err = nla_parse_nested(tb, IFLA_BRIDGE_MRP_START_IN_TEST_MAX, attr,
                               br_mrp_start_in_test_policy, extack);
        if (err)
                return err;

        if (!tb[IFLA_BRIDGE_MRP_START_IN_TEST_IN_ID] ||
            !tb[IFLA_BRIDGE_MRP_START_IN_TEST_INTERVAL] ||
            !tb[IFLA_BRIDGE_MRP_START_IN_TEST_MAX_MISS] ||
            !tb[IFLA_BRIDGE_MRP_START_IN_TEST_PERIOD]) {
                NL_SET_ERR_MSG_MOD(extack,
                                   "Missing attribute: RING_ID or INTERVAL or MAX_MISS or PERIOD");
                return -EINVAL;
        }

        memset(&test, 0x0, sizeof(test));

        test.in_id = nla_get_u32(tb[IFLA_BRIDGE_MRP_START_IN_TEST_IN_ID]);
        test.interval = nla_get_u32(tb[IFLA_BRIDGE_MRP_START_IN_TEST_INTERVAL]);
        test.max_miss = nla_get_u32(tb[IFLA_BRIDGE_MRP_START_IN_TEST_MAX_MISS]);
        test.period = nla_get_u32(tb[IFLA_BRIDGE_MRP_START_IN_TEST_PERIOD]);

        return br_mrp_start_in_test(br, &test);
}

int br_mrp_parse(struct net_bridge *br, struct net_bridge_port *p,
                 struct nlattr *attr, int cmd, struct netlink_ext_ack *extack)
{
        struct nlattr *tb[IFLA_BRIDGE_MRP_MAX + 1];
        int err;

        /* When this function is called for a port then the br pointer is
         * invalid, therefor set the br to point correctly
         */
        if (p)
                br = p->br;

        if (br->stp_enabled != BR_NO_STP) {
                NL_SET_ERR_MSG_MOD(extack, "MRP can't be enabled if STP is already enabled");
                return -EINVAL;
        }

        err = nla_parse_nested(tb, IFLA_BRIDGE_MRP_MAX, attr,
                               br_mrp_policy, extack);
        if (err)
                return err;

        if (tb[IFLA_BRIDGE_MRP_INSTANCE]) {
                err = br_mrp_instance_parse(br, tb[IFLA_BRIDGE_MRP_INSTANCE],
                                            cmd, extack);
                if (err)
                        return err;
        }

        if (tb[IFLA_BRIDGE_MRP_PORT_STATE]) {
                err = br_mrp_port_state_parse(p, tb[IFLA_BRIDGE_MRP_PORT_STATE],
                                              extack);
                if (err)
                        return err;
        }

        if (tb[IFLA_BRIDGE_MRP_PORT_ROLE]) {
                err = br_mrp_port_role_parse(p, tb[IFLA_BRIDGE_MRP_PORT_ROLE],
                                             extack);
                if (err)
                        return err;
        }

        if (tb[IFLA_BRIDGE_MRP_RING_STATE]) {
                err = br_mrp_ring_state_parse(br,
                                              tb[IFLA_BRIDGE_MRP_RING_STATE],
                                              extack);
                if (err)
                        return err;
        }

        if (tb[IFLA_BRIDGE_MRP_RING_ROLE]) {
                err = br_mrp_ring_role_parse(br, tb[IFLA_BRIDGE_MRP_RING_ROLE],
                                             extack);
                if (err)
                        return err;
        }

        if (tb[IFLA_BRIDGE_MRP_START_TEST]) {
                err = br_mrp_start_test_parse(br,
                                              tb[IFLA_BRIDGE_MRP_START_TEST],
                                              extack);
                if (err)
                        return err;
        }

        if (tb[IFLA_BRIDGE_MRP_IN_STATE]) {
                err = br_mrp_in_state_parse(br, tb[IFLA_BRIDGE_MRP_IN_STATE],
                                            extack);
                if (err)
                        return err;
        }

        if (tb[IFLA_BRIDGE_MRP_IN_ROLE]) {
                err = br_mrp_in_role_parse(br, tb[IFLA_BRIDGE_MRP_IN_ROLE],
                                           extack);
                if (err)
                        return err;
        }

        if (tb[IFLA_BRIDGE_MRP_START_IN_TEST]) {
                err = br_mrp_start_in_test_parse(br,
                                                 tb[IFLA_BRIDGE_MRP_START_IN_TEST],
                                                 extack);
                if (err)
                        return err;
        }

        return 0;
}

int br_mrp_fill_info(struct sk_buff *skb, struct net_bridge *br)
{
        struct nlattr *tb, *mrp_tb;
        struct br_mrp *mrp;

        mrp_tb = nla_nest_start_noflag(skb, IFLA_BRIDGE_MRP);
        if (!mrp_tb)
                return -EMSGSIZE;

        hlist_for_each_entry_rcu(mrp, &br->mrp_list, list) {
                struct net_bridge_port *p;

                tb = nla_nest_start_noflag(skb, IFLA_BRIDGE_MRP_INFO);
                if (!tb)
                        goto nla_info_failure;

                if (nla_put_u32(skb, IFLA_BRIDGE_MRP_INFO_RING_ID,
                                mrp->ring_id))
                        goto nla_put_failure;

                p = rcu_dereference(mrp->p_port);
                if (p && nla_put_u32(skb, IFLA_BRIDGE_MRP_INFO_P_IFINDEX,
                                     p->dev->ifindex))
                        goto nla_put_failure;

                p = rcu_dereference(mrp->s_port);
                if (p && nla_put_u32(skb, IFLA_BRIDGE_MRP_INFO_S_IFINDEX,
                                     p->dev->ifindex))
                        goto nla_put_failure;

                p = rcu_dereference(mrp->i_port);
                if (p && nla_put_u32(skb, IFLA_BRIDGE_MRP_INFO_I_IFINDEX,
                                     p->dev->ifindex))
                        goto nla_put_failure;

                if (nla_put_u16(skb, IFLA_BRIDGE_MRP_INFO_PRIO,
                                mrp->prio))
                        goto nla_put_failure;
                if (nla_put_u32(skb, IFLA_BRIDGE_MRP_INFO_RING_STATE,
                                mrp->ring_state))
                        goto nla_put_failure;
                if (nla_put_u32(skb, IFLA_BRIDGE_MRP_INFO_RING_ROLE,
                                mrp->ring_role))
                        goto nla_put_failure;
                if (nla_put_u32(skb, IFLA_BRIDGE_MRP_INFO_TEST_INTERVAL,
                                mrp->test_interval))
                        goto nla_put_failure;
                if (nla_put_u32(skb, IFLA_BRIDGE_MRP_INFO_TEST_MAX_MISS,
                                mrp->test_max_miss))
                        goto nla_put_failure;
                if (nla_put_u32(skb, IFLA_BRIDGE_MRP_INFO_TEST_MONITOR,
                                mrp->test_monitor))
                        goto nla_put_failure;

                if (nla_put_u32(skb, IFLA_BRIDGE_MRP_INFO_IN_STATE,
                                mrp->in_state))
                        goto nla_put_failure;
                if (nla_put_u32(skb, IFLA_BRIDGE_MRP_INFO_IN_ROLE,
                                mrp->in_role))
                        goto nla_put_failure;
                if (nla_put_u32(skb, IFLA_BRIDGE_MRP_INFO_IN_TEST_INTERVAL,
                                mrp->in_test_interval))
                        goto nla_put_failure;
                if (nla_put_u32(skb, IFLA_BRIDGE_MRP_INFO_IN_TEST_MAX_MISS,
                                mrp->in_test_max_miss))
                        goto nla_put_failure;

                nla_nest_end(skb, tb);
        }
        nla_nest_end(skb, mrp_tb);

        return 0;

nla_put_failure:
        nla_nest_cancel(skb, tb);

nla_info_failure:
        nla_nest_cancel(skb, mrp_tb);

        return -EMSGSIZE;
}

int br_mrp_ring_port_open(struct net_device *dev, u8 loc)
{
        struct net_bridge_port *p;
        int err = 0;

        p = br_port_get_rcu(dev);
        if (!p) {
                err = -EINVAL;
                goto out;
        }

        if (loc)
                p->flags |= BR_MRP_LOST_CONT;
        else
                p->flags &= ~BR_MRP_LOST_CONT;

        br_ifinfo_notify(RTM_NEWLINK, NULL, p);

out:
        return err;
}

int br_mrp_in_port_open(struct net_device *dev, u8 loc)
{
        struct net_bridge_port *p;
        int err = 0;

        p = br_port_get_rcu(dev);
        if (!p) {
                err = -EINVAL;
                goto out;
        }

        if (loc)
                p->flags |= BR_MRP_LOST_IN_CONT;
        else
                p->flags &= ~BR_MRP_LOST_IN_CONT;

        br_ifinfo_notify(RTM_NEWLINK, NULL, p);

out:
        return err;
}

































































































































































































   26 






















   32 






    4 























  705 
















































   96 




  858 




















































































































































   42 


























































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_HUGE_MM_H
#define _LINUX_HUGE_MM_H

#include <linux/sched/coredump.h>
#include <linux/mm_types.h>

#include <linux/fs.h> /* only for vma_is_dax() */
#include <linux/kobject.h>

vm_fault_t do_huge_pmd_anonymous_page(struct vm_fault *vmf);
int copy_huge_pmd(struct mm_struct *dst_mm, struct mm_struct *src_mm,
                  pmd_t *dst_pmd, pmd_t *src_pmd, unsigned long addr,
                  struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma);
void huge_pmd_set_accessed(struct vm_fault *vmf);
int copy_huge_pud(struct mm_struct *dst_mm, struct mm_struct *src_mm,
                  pud_t *dst_pud, pud_t *src_pud, unsigned long addr,
                  struct vm_area_struct *vma);

#ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
void huge_pud_set_accessed(struct vm_fault *vmf, pud_t orig_pud);
#else
static inline void huge_pud_set_accessed(struct vm_fault *vmf, pud_t orig_pud)
{
}
#endif

vm_fault_t do_huge_pmd_wp_page(struct vm_fault *vmf);
bool madvise_free_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
                           pmd_t *pmd, unsigned long addr, unsigned long next);
int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, pmd_t *pmd,
                 unsigned long addr);
int zap_huge_pud(struct mmu_gather *tlb, struct vm_area_struct *vma, pud_t *pud,
                 unsigned long addr);
bool move_huge_pmd(struct vm_area_struct *vma, unsigned long old_addr,
                   unsigned long new_addr, pmd_t *old_pmd, pmd_t *new_pmd);
int change_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
                    pmd_t *pmd, unsigned long addr, pgprot_t newprot,
                    unsigned long cp_flags);

vm_fault_t vmf_insert_pfn_pmd(struct vm_fault *vmf, pfn_t pfn, bool write);
vm_fault_t vmf_insert_pfn_pud(struct vm_fault *vmf, pfn_t pfn, bool write);

enum transparent_hugepage_flag {
        TRANSPARENT_HUGEPAGE_UNSUPPORTED,
        TRANSPARENT_HUGEPAGE_FLAG,
        TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG,
        TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG,
        TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG,
        TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG,
        TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG,
        TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG,
        TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG,
};

struct kobject;
struct kobj_attribute;

ssize_t single_hugepage_flag_store(struct kobject *kobj,
                                   struct kobj_attribute *attr,
                                   const char *buf, size_t count,
                                   enum transparent_hugepage_flag flag);
ssize_t single_hugepage_flag_show(struct kobject *kobj,
                                  struct kobj_attribute *attr, char *buf,
                                  enum transparent_hugepage_flag flag);
extern struct kobj_attribute shmem_enabled_attr;
extern struct kobj_attribute thpsize_shmem_enabled_attr;

/*
 * Mask of all large folio orders supported for anonymous THP; all orders up to
 * and including PMD_ORDER, except order-0 (which is not "huge") and order-1
 * (which is a limitation of the THP implementation).
 */
#define THP_ORDERS_ALL_ANON        ((BIT(PMD_ORDER + 1) - 1) & ~(BIT(0) | BIT(1)))

/*
 * Mask of all large folio orders supported for file THP. Folios in a DAX
 * file is never split and the MAX_PAGECACHE_ORDER limit does not apply to
 * it.  Same to PFNMAPs where there's neither page* nor pagecache.
 */
#define THP_ORDERS_ALL_SPECIAL                \
        (BIT(PMD_ORDER) | BIT(PUD_ORDER))
#define THP_ORDERS_ALL_FILE_DEFAULT        \
        ((BIT(MAX_PAGECACHE_ORDER + 1) - 1) & ~BIT(0))

/*
 * Mask of all large folio orders supported for THP.
 */
#define THP_ORDERS_ALL        \
        (THP_ORDERS_ALL_ANON | THP_ORDERS_ALL_SPECIAL | THP_ORDERS_ALL_FILE_DEFAULT)

#define TVA_SMAPS                (1 << 0)        /* Will be used for procfs */
#define TVA_IN_PF                (1 << 1)        /* Page fault handler */
#define TVA_ENFORCE_SYSFS        (1 << 2)        /* Obey sysfs configuration */

#define thp_vma_allowable_order(vma, vm_flags, tva_flags, order) \
        (!!thp_vma_allowable_orders(vma, vm_flags, tva_flags, BIT(order)))

#define split_folio(f) split_folio_to_list(f, NULL)

#ifdef CONFIG_PGTABLE_HAS_HUGE_LEAVES
#define HPAGE_PMD_SHIFT PMD_SHIFT
#define HPAGE_PUD_SHIFT PUD_SHIFT
#else
#define HPAGE_PMD_SHIFT ({ BUILD_BUG(); 0; })
#define HPAGE_PUD_SHIFT ({ BUILD_BUG(); 0; })
#endif

#define HPAGE_PMD_ORDER (HPAGE_PMD_SHIFT-PAGE_SHIFT)
#define HPAGE_PMD_NR (1<<HPAGE_PMD_ORDER)
#define HPAGE_PMD_MASK        (~(HPAGE_PMD_SIZE - 1))
#define HPAGE_PMD_SIZE        ((1UL) << HPAGE_PMD_SHIFT)

#define HPAGE_PUD_ORDER (HPAGE_PUD_SHIFT-PAGE_SHIFT)
#define HPAGE_PUD_NR (1<<HPAGE_PUD_ORDER)
#define HPAGE_PUD_MASK        (~(HPAGE_PUD_SIZE - 1))
#define HPAGE_PUD_SIZE        ((1UL) << HPAGE_PUD_SHIFT)

enum mthp_stat_item {
        MTHP_STAT_ANON_FAULT_ALLOC,
        MTHP_STAT_ANON_FAULT_FALLBACK,
        MTHP_STAT_ANON_FAULT_FALLBACK_CHARGE,
        MTHP_STAT_SWPOUT,
        MTHP_STAT_SWPOUT_FALLBACK,
        MTHP_STAT_SHMEM_ALLOC,
        MTHP_STAT_SHMEM_FALLBACK,
        MTHP_STAT_SHMEM_FALLBACK_CHARGE,
        MTHP_STAT_SPLIT,
        MTHP_STAT_SPLIT_FAILED,
        MTHP_STAT_SPLIT_DEFERRED,
        MTHP_STAT_NR_ANON,
        MTHP_STAT_NR_ANON_PARTIALLY_MAPPED,
        __MTHP_STAT_COUNT
};

#if defined(CONFIG_TRANSPARENT_HUGEPAGE) && defined(CONFIG_SYSFS)
struct mthp_stat {
        unsigned long stats[ilog2(MAX_PTRS_PER_PTE) + 1][__MTHP_STAT_COUNT];
};

DECLARE_PER_CPU(struct mthp_stat, mthp_stats);

static inline void mod_mthp_stat(int order, enum mthp_stat_item item, int delta)
{
        if (order <= 0 || order > PMD_ORDER)
                return;

        this_cpu_add(mthp_stats.stats[order][item], delta);
}

static inline void count_mthp_stat(int order, enum mthp_stat_item item)
{
        mod_mthp_stat(order, item, 1);
}

#else
static inline void mod_mthp_stat(int order, enum mthp_stat_item item, int delta)
{
}

static inline void count_mthp_stat(int order, enum mthp_stat_item item)
{
}
#endif

#ifdef CONFIG_TRANSPARENT_HUGEPAGE

extern unsigned long transparent_hugepage_flags;
extern unsigned long huge_anon_orders_always;
extern unsigned long huge_anon_orders_madvise;
extern unsigned long huge_anon_orders_inherit;

static inline bool hugepage_global_enabled(void)
{
        return transparent_hugepage_flags &
                        ((1<<TRANSPARENT_HUGEPAGE_FLAG) |
                        (1<<TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG));
}

static inline bool hugepage_global_always(void)
{
        return transparent_hugepage_flags &
                        (1<<TRANSPARENT_HUGEPAGE_FLAG);
}

static inline int highest_order(unsigned long orders)
{
        return fls_long(orders) - 1;
}

static inline int next_order(unsigned long *orders, int prev)
{
        *orders &= ~BIT(prev);
        return highest_order(*orders);
}

/*
 * Do the below checks:
 *   - For file vma, check if the linear page offset of vma is
 *     order-aligned within the file.  The hugepage is
 *     guaranteed to be order-aligned within the file, but we must
 *     check that the order-aligned addresses in the VMA map to
 *     order-aligned offsets within the file, else the hugepage will
 *     not be mappable.
 *   - For all vmas, check if the haddr is in an aligned hugepage
 *     area.
 */
static inline bool thp_vma_suitable_order(struct vm_area_struct *vma,
                unsigned long addr, int order)
{
        unsigned long hpage_size = PAGE_SIZE << order;
        unsigned long haddr;

        /* Don't have to check pgoff for anonymous vma */
        if (!vma_is_anonymous(vma)) {
                if (!IS_ALIGNED((vma->vm_start >> PAGE_SHIFT) - vma->vm_pgoff,
                                hpage_size >> PAGE_SHIFT))
                        return false;
        }

        haddr = ALIGN_DOWN(addr, hpage_size);

        if (haddr < vma->vm_start || haddr + hpage_size > vma->vm_end)
                return false;
        return true;
}

/*
 * Filter the bitfield of input orders to the ones suitable for use in the vma.
 * See thp_vma_suitable_order().
 * All orders that pass the checks are returned as a bitfield.
 */
static inline unsigned long thp_vma_suitable_orders(struct vm_area_struct *vma,
                unsigned long addr, unsigned long orders)
{
        int order;

        /*
         * Iterate over orders, highest to lowest, removing orders that don't
         * meet alignment requirements from the set. Exit loop at first order
         * that meets requirements, since all lower orders must also meet
         * requirements.
         */

        order = highest_order(orders);

        while (orders) {
                if (thp_vma_suitable_order(vma, addr, order))
                        break;
                order = next_order(&orders, order);
        }

        return orders;
}

static inline bool file_thp_enabled(struct vm_area_struct *vma)
{
        struct inode *inode;

        if (!vma->vm_file)
                return false;

        inode = vma->vm_file->f_inode;

        return (IS_ENABLED(CONFIG_READ_ONLY_THP_FOR_FS)) &&
               !inode_is_open_for_write(inode) && S_ISREG(inode->i_mode);
}

unsigned long __thp_vma_allowable_orders(struct vm_area_struct *vma,
                                         unsigned long vm_flags,
                                         unsigned long tva_flags,
                                         unsigned long orders);

/**
 * thp_vma_allowable_orders - determine hugepage orders that are allowed for vma
 * @vma:  the vm area to check
 * @vm_flags: use these vm_flags instead of vma->vm_flags
 * @tva_flags: Which TVA flags to honour
 * @orders: bitfield of all orders to consider
 *
 * Calculates the intersection of the requested hugepage orders and the allowed
 * hugepage orders for the provided vma. Permitted orders are encoded as a set
 * bit at the corresponding bit position (bit-2 corresponds to order-2, bit-3
 * corresponds to order-3, etc). Order-0 is never considered a hugepage order.
 *
 * Return: bitfield of orders allowed for hugepage in the vma. 0 if no hugepage
 * orders are allowed.
 */
static inline
unsigned long thp_vma_allowable_orders(struct vm_area_struct *vma,
                                       unsigned long vm_flags,
                                       unsigned long tva_flags,
                                       unsigned long orders)
{
        /* Optimization to check if required orders are enabled early. */
        if ((tva_flags & TVA_ENFORCE_SYSFS) && vma_is_anonymous(vma)) {
                unsigned long mask = READ_ONCE(huge_anon_orders_always);

                if (vm_flags & VM_HUGEPAGE)
                        mask |= READ_ONCE(huge_anon_orders_madvise);
                if (hugepage_global_always() ||
                    ((vm_flags & VM_HUGEPAGE) && hugepage_global_enabled()))
                        mask |= READ_ONCE(huge_anon_orders_inherit);

                orders &= mask;
                if (!orders)
                        return 0;
        }

        return __thp_vma_allowable_orders(vma, vm_flags, tva_flags, orders);
}

struct thpsize {
        struct kobject kobj;
        struct list_head node;
        int order;
};

#define to_thpsize(kobj) container_of(kobj, struct thpsize, kobj)

#define transparent_hugepage_use_zero_page()                                \
        (transparent_hugepage_flags &                                        \
         (1<<TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG))

static inline bool vma_thp_disabled(struct vm_area_struct *vma,
                unsigned long vm_flags)
{
        /*
         * Explicitly disabled through madvise or prctl, or some
         * architectures may disable THP for some mappings, for
         * example, s390 kvm.
         */
        return (vm_flags & VM_NOHUGEPAGE) ||
               test_bit(MMF_DISABLE_THP, &vma->vm_mm->flags);
}

static inline bool thp_disabled_by_hw(void)
{
        /* If the hardware/firmware marked hugepage support disabled. */
        return transparent_hugepage_flags & (1 << TRANSPARENT_HUGEPAGE_UNSUPPORTED);
}

unsigned long thp_get_unmapped_area(struct file *filp, unsigned long addr,
                unsigned long len, unsigned long pgoff, unsigned long flags);
unsigned long thp_get_unmapped_area_vmflags(struct file *filp, unsigned long addr,
                unsigned long len, unsigned long pgoff, unsigned long flags,
                vm_flags_t vm_flags);

bool can_split_folio(struct folio *folio, int caller_pins, int *pextra_pins);
int split_huge_page_to_list_to_order(struct page *page, struct list_head *list,
                unsigned int new_order);
int min_order_for_split(struct folio *folio);
int split_folio_to_list(struct folio *folio, struct list_head *list);
static inline int split_huge_page(struct page *page)
{
        struct folio *folio = page_folio(page);
        int ret = min_order_for_split(folio);

        if (ret < 0)
                return ret;

        /*
         * split_huge_page() locks the page before splitting and
         * expects the same page that has been split to be locked when
         * returned. split_folio(page_folio(page)) cannot be used here
         * because it converts the page to folio and passes the head
         * page to be split.
         */
        return split_huge_page_to_list_to_order(page, NULL, ret);
}
void deferred_split_folio(struct folio *folio, bool partially_mapped);

void __split_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,
                unsigned long address, bool freeze, struct folio *folio);

#define split_huge_pmd(__vma, __pmd, __address)                                \
        do {                                                                \
                pmd_t *____pmd = (__pmd);                                \
                if (is_swap_pmd(*____pmd) || pmd_trans_huge(*____pmd)        \
                                        || pmd_devmap(*____pmd))        \
                        __split_huge_pmd(__vma, __pmd, __address,        \
                                                false, NULL);                \
        }  while (0)


void split_huge_pmd_address(struct vm_area_struct *vma, unsigned long address,
                bool freeze, struct folio *folio);

void __split_huge_pud(struct vm_area_struct *vma, pud_t *pud,
                unsigned long address);

#ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
int change_huge_pud(struct mmu_gather *tlb, struct vm_area_struct *vma,
                    pud_t *pudp, unsigned long addr, pgprot_t newprot,
                    unsigned long cp_flags);
#else
static inline int
change_huge_pud(struct mmu_gather *tlb, struct vm_area_struct *vma,
                pud_t *pudp, unsigned long addr, pgprot_t newprot,
                unsigned long cp_flags) { return 0; }
#endif

#define split_huge_pud(__vma, __pud, __address)                                \
        do {                                                                \
                pud_t *____pud = (__pud);                                \
                if (pud_trans_huge(*____pud)                                \
                                        || pud_devmap(*____pud))        \
                        __split_huge_pud(__vma, __pud, __address);        \
        }  while (0)

int hugepage_madvise(struct vm_area_struct *vma, unsigned long *vm_flags,
                     int advice);
int madvise_collapse(struct vm_area_struct *vma,
                     struct vm_area_struct **prev,
                     unsigned long start, unsigned long end);
void vma_adjust_trans_huge(struct vm_area_struct *vma, unsigned long start,
                           unsigned long end, long adjust_next);
spinlock_t *__pmd_trans_huge_lock(pmd_t *pmd, struct vm_area_struct *vma);
spinlock_t *__pud_trans_huge_lock(pud_t *pud, struct vm_area_struct *vma);

static inline int is_swap_pmd(pmd_t pmd)
{
        return !pmd_none(pmd) && !pmd_present(pmd);
}

/* mmap_lock must be held on entry */
static inline spinlock_t *pmd_trans_huge_lock(pmd_t *pmd,
                struct vm_area_struct *vma)
{
        if (is_swap_pmd(*pmd) || pmd_trans_huge(*pmd) || pmd_devmap(*pmd))
                return __pmd_trans_huge_lock(pmd, vma);
        else
                return NULL;
}
static inline spinlock_t *pud_trans_huge_lock(pud_t *pud,
                struct vm_area_struct *vma)
{
        if (pud_trans_huge(*pud) || pud_devmap(*pud))
                return __pud_trans_huge_lock(pud, vma);
        else
                return NULL;
}

/**
 * folio_test_pmd_mappable - Can we map this folio with a PMD?
 * @folio: The folio to test
 */
static inline bool folio_test_pmd_mappable(struct folio *folio)
{
        return folio_order(folio) >= HPAGE_PMD_ORDER;
}

struct page *follow_devmap_pmd(struct vm_area_struct *vma, unsigned long addr,
                pmd_t *pmd, int flags, struct dev_pagemap **pgmap);

vm_fault_t do_huge_pmd_numa_page(struct vm_fault *vmf);

extern struct folio *huge_zero_folio;
extern unsigned long huge_zero_pfn;

static inline bool is_huge_zero_folio(const struct folio *folio)
{
        return READ_ONCE(huge_zero_folio) == folio;
}

static inline bool is_huge_zero_pmd(pmd_t pmd)
{
        return pmd_present(pmd) && READ_ONCE(huge_zero_pfn) == pmd_pfn(pmd);
}

struct folio *mm_get_huge_zero_folio(struct mm_struct *mm);
void mm_put_huge_zero_folio(struct mm_struct *mm);

#define mk_huge_pmd(page, prot) pmd_mkhuge(mk_pmd(page, prot))

static inline bool thp_migration_supported(void)
{
        return IS_ENABLED(CONFIG_ARCH_ENABLE_THP_MIGRATION);
}

void split_huge_pmd_locked(struct vm_area_struct *vma, unsigned long address,
                           pmd_t *pmd, bool freeze, struct folio *folio);
bool unmap_huge_pmd_locked(struct vm_area_struct *vma, unsigned long addr,
                           pmd_t *pmdp, struct folio *folio);

#else /* CONFIG_TRANSPARENT_HUGEPAGE */

static inline bool folio_test_pmd_mappable(struct folio *folio)
{
        return false;
}

static inline bool thp_vma_suitable_order(struct vm_area_struct *vma,
                unsigned long addr, int order)
{
        return false;
}

static inline unsigned long thp_vma_suitable_orders(struct vm_area_struct *vma,
                unsigned long addr, unsigned long orders)
{
        return 0;
}

static inline unsigned long thp_vma_allowable_orders(struct vm_area_struct *vma,
                                        unsigned long vm_flags,
                                        unsigned long tva_flags,
                                        unsigned long orders)
{
        return 0;
}

#define transparent_hugepage_flags 0UL

#define thp_get_unmapped_area        NULL

static inline unsigned long
thp_get_unmapped_area_vmflags(struct file *filp, unsigned long addr,
                              unsigned long len, unsigned long pgoff,
                              unsigned long flags, vm_flags_t vm_flags)
{
        return 0;
}

static inline bool
can_split_folio(struct folio *folio, int caller_pins, int *pextra_pins)
{
        return false;
}
static inline int
split_huge_page_to_list_to_order(struct page *page, struct list_head *list,
                unsigned int new_order)
{
        return 0;
}
static inline int split_huge_page(struct page *page)
{
        return 0;
}

static inline int split_folio_to_list(struct folio *folio, struct list_head *list)
{
        return 0;
}

static inline void deferred_split_folio(struct folio *folio, bool partially_mapped) {}
#define split_huge_pmd(__vma, __pmd, __address)        \
        do { } while (0)

static inline void __split_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,
                unsigned long address, bool freeze, struct folio *folio) {}
static inline void split_huge_pmd_address(struct vm_area_struct *vma,
                unsigned long address, bool freeze, struct folio *folio) {}
static inline void split_huge_pmd_locked(struct vm_area_struct *vma,
                                         unsigned long address, pmd_t *pmd,
                                         bool freeze, struct folio *folio) {}

static inline bool unmap_huge_pmd_locked(struct vm_area_struct *vma,
                                         unsigned long addr, pmd_t *pmdp,
                                         struct folio *folio)
{
        return false;
}

#define split_huge_pud(__vma, __pmd, __address)        \
        do { } while (0)

static inline int hugepage_madvise(struct vm_area_struct *vma,
                                   unsigned long *vm_flags, int advice)
{
        return -EINVAL;
}

static inline int madvise_collapse(struct vm_area_struct *vma,
                                   struct vm_area_struct **prev,
                                   unsigned long start, unsigned long end)
{
        return -EINVAL;
}

static inline void vma_adjust_trans_huge(struct vm_area_struct *vma,
                                         unsigned long start,
                                         unsigned long end,
                                         long adjust_next)
{
}
static inline int is_swap_pmd(pmd_t pmd)
{
        return 0;
}
static inline spinlock_t *pmd_trans_huge_lock(pmd_t *pmd,
                struct vm_area_struct *vma)
{
        return NULL;
}
static inline spinlock_t *pud_trans_huge_lock(pud_t *pud,
                struct vm_area_struct *vma)
{
        return NULL;
}

static inline vm_fault_t do_huge_pmd_numa_page(struct vm_fault *vmf)
{
        return 0;
}

static inline bool is_huge_zero_folio(const struct folio *folio)
{
        return false;
}

static inline bool is_huge_zero_pmd(pmd_t pmd)
{
        return false;
}

static inline void mm_put_huge_zero_folio(struct mm_struct *mm)
{
        return;
}

static inline struct page *follow_devmap_pmd(struct vm_area_struct *vma,
        unsigned long addr, pmd_t *pmd, int flags, struct dev_pagemap **pgmap)
{
        return NULL;
}

static inline bool thp_migration_supported(void)
{
        return false;
}

static inline int highest_order(unsigned long orders)
{
        return 0;
}

static inline int next_order(unsigned long *orders, int prev)
{
        return 0;
}

static inline void __split_huge_pud(struct vm_area_struct *vma, pud_t *pud,
                                    unsigned long address)
{
}

static inline int change_huge_pud(struct mmu_gather *tlb,
                                  struct vm_area_struct *vma, pud_t *pudp,
                                  unsigned long addr, pgprot_t newprot,
                                  unsigned long cp_flags)
{
        return 0;
}
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */

static inline int split_folio_to_list_to_order(struct folio *folio,
                struct list_head *list, int new_order)
{
        return split_huge_page_to_list_to_order(&folio->page, list, new_order);
}

static inline int split_folio_to_order(struct folio *folio, int new_order)
{
        return split_folio_to_list_to_order(folio, NULL, new_order);
}

#endif /* _LINUX_HUGE_MM_H */


































    1 









   10 



















    1 





    5 
    5 

    4 




    1 









    5 
    2 




    2 



    5 





















    7 





    1 







    3 










    4 






    1 




   10 














   10 













    8 

    1 

    1 





    1 




    1 

    1 


    7 



    7 


    7 

    1 


    1 







    1 

    1 





    2 










    4 










    1 


















    2 









    2 






    1 
    3 


    4 













    1 
    1 
    1 



    1 








    1 










    1 


    1 











    1 















   57 
   57 









   57 



    5 



   16 





    4 



   24 
   30 








   49 







   48 





    4 




























   14 










    6 



   11 





    8 
    1 

















    8 

    1 











    9 








    4 







    4 













    3 

    3 

























   11 







   12 

   11 











    3 

























   12 













    4 







    3 




    3 

















   14 








   11 
    2 












    4 












    4 




    1 

    4 











   12 



   11 













    2 
    2 
    2 













   14 










    2 





















































   12 


















   43 

   43 






   30 


   14 













   14 
   14 

   12 



   12 















   14 



   12 





   14 










   27 








    1 


   26 




























    2 
    2 













   17 





    8 

    3 




































    3 




    4 


    6 



















    7 

























    8 


    7 


    8 

    6 
    5 



    8 

    7 
    2 





    8 





   14 





   11 


   15 








    3 





    3 



    3 







   45 

   45 










   25 

















   22 


    8 
    3 










    3 






    3 



    3 


    3 




    3 




    3 






    3 







    8 







    6 
    1 











    4 






































   57 











   49 












   25 










    5 








    5 








































   58 








































    6 
    2 



















    8 




    6 












    2 











   48 

    1 

    8 

















   48 





    1 




    8 










    1 










    4 













    4 
























    3 




    1 





    1 
    3 


























































   11 







    2 



   57 






    2 












    1 
   57 



    1 




    1 









1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
// SPDX-License-Identifier: GPL-2.0
/* Multipath TCP
 *
 * Copyright (c) 2017 - 2019, Intel Corporation.
 */

#define pr_fmt(fmt) "MPTCP: " fmt

#include <linux/kernel.h>
#include <crypto/sha2.h>
#include <net/tcp.h>
#include <net/mptcp.h>
#include "protocol.h"
#include "mib.h"

#include <trace/events/mptcp.h>

static bool mptcp_cap_flag_sha256(u8 flags)
{
        return (flags & MPTCP_CAP_FLAG_MASK) == MPTCP_CAP_HMAC_SHA256;
}

static void mptcp_parse_option(const struct sk_buff *skb,
                               const unsigned char *ptr, int opsize,
                               struct mptcp_options_received *mp_opt)
{
        u8 subtype = *ptr >> 4;
        int expected_opsize;
        u16 subopt;
        u8 version;
        u8 flags;
        u8 i;

        switch (subtype) {
        case MPTCPOPT_MP_CAPABLE:
                /* strict size checking */
                if (!(TCP_SKB_CB(skb)->tcp_flags & TCPHDR_SYN)) {
                        if (skb->len > tcp_hdr(skb)->doff << 2)
                                expected_opsize = TCPOLEN_MPTCP_MPC_ACK_DATA;
                        else
                                expected_opsize = TCPOLEN_MPTCP_MPC_ACK;
                        subopt = OPTION_MPTCP_MPC_ACK;
                } else {
                        if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_ACK) {
                                expected_opsize = TCPOLEN_MPTCP_MPC_SYNACK;
                                subopt = OPTION_MPTCP_MPC_SYNACK;
                        } else {
                                expected_opsize = TCPOLEN_MPTCP_MPC_SYN;
                                subopt = OPTION_MPTCP_MPC_SYN;
                        }
                }

                /* Cfr RFC 8684 Section 3.3.0:
                 * If a checksum is present but its use had
                 * not been negotiated in the MP_CAPABLE handshake, the receiver MUST
                 * close the subflow with a RST, as it is not behaving as negotiated.
                 * If a checksum is not present when its use has been negotiated, the
                 * receiver MUST close the subflow with a RST, as it is considered
                 * broken
                 * We parse even option with mismatching csum presence, so that
                 * later in subflow_data_ready we can trigger the reset.
                 */
                if (opsize != expected_opsize &&
                    (expected_opsize != TCPOLEN_MPTCP_MPC_ACK_DATA ||
                     opsize != TCPOLEN_MPTCP_MPC_ACK_DATA_CSUM))
                        break;

                /* try to be gentle vs future versions on the initial syn */
                version = *ptr++ & MPTCP_VERSION_MASK;
                if (opsize != TCPOLEN_MPTCP_MPC_SYN) {
                        if (version != MPTCP_SUPPORTED_VERSION)
                                break;
                } else if (version < MPTCP_SUPPORTED_VERSION) {
                        break;
                }

                flags = *ptr++;
                if (!mptcp_cap_flag_sha256(flags) ||
                    (flags & MPTCP_CAP_EXTENSIBILITY))
                        break;

                /* RFC 6824, Section 3.1:
                 * "For the Checksum Required bit (labeled "A"), if either
                 * host requires the use of checksums, checksums MUST be used.
                 * In other words, the only way for checksums not to be used
                 * is if both hosts in their SYNs set A=0."
                 */
                if (flags & MPTCP_CAP_CHECKSUM_REQD)
                        mp_opt->suboptions |= OPTION_MPTCP_CSUMREQD;

                mp_opt->deny_join_id0 = !!(flags & MPTCP_CAP_DENY_JOIN_ID0);

                mp_opt->suboptions |= subopt;
                if (opsize >= TCPOLEN_MPTCP_MPC_SYNACK) {
                        mp_opt->sndr_key = get_unaligned_be64(ptr);
                        ptr += 8;
                }
                if (opsize >= TCPOLEN_MPTCP_MPC_ACK) {
                        mp_opt->rcvr_key = get_unaligned_be64(ptr);
                        ptr += 8;
                }
                if (opsize >= TCPOLEN_MPTCP_MPC_ACK_DATA) {
                        /* Section 3.1.:
                         * "the data parameters in a MP_CAPABLE are semantically
                         * equivalent to those in a DSS option and can be used
                         * interchangeably."
                         */
                        mp_opt->suboptions |= OPTION_MPTCP_DSS;
                        mp_opt->use_map = 1;
                        mp_opt->mpc_map = 1;
                        mp_opt->use_ack = 0;
                        mp_opt->data_len = get_unaligned_be16(ptr);
                        ptr += 2;
                }
                if (opsize == TCPOLEN_MPTCP_MPC_ACK_DATA_CSUM) {
                        mp_opt->csum = get_unaligned((__force __sum16 *)ptr);
                        mp_opt->suboptions |= OPTION_MPTCP_CSUMREQD;
                        ptr += 2;
                }
                pr_debug("MP_CAPABLE version=%x, flags=%x, optlen=%d sndr=%llu, rcvr=%llu len=%d csum=%u\n",
                         version, flags, opsize, mp_opt->sndr_key,
                         mp_opt->rcvr_key, mp_opt->data_len, mp_opt->csum);
                break;

        case MPTCPOPT_MP_JOIN:
                if (opsize == TCPOLEN_MPTCP_MPJ_SYN) {
                        mp_opt->suboptions |= OPTION_MPTCP_MPJ_SYN;
                        mp_opt->backup = *ptr++ & MPTCPOPT_BACKUP;
                        mp_opt->join_id = *ptr++;
                        mp_opt->token = get_unaligned_be32(ptr);
                        ptr += 4;
                        mp_opt->nonce = get_unaligned_be32(ptr);
                        ptr += 4;
                        pr_debug("MP_JOIN bkup=%u, id=%u, token=%u, nonce=%u\n",
                                 mp_opt->backup, mp_opt->join_id,
                                 mp_opt->token, mp_opt->nonce);
                } else if (opsize == TCPOLEN_MPTCP_MPJ_SYNACK) {
                        mp_opt->suboptions |= OPTION_MPTCP_MPJ_SYNACK;
                        mp_opt->backup = *ptr++ & MPTCPOPT_BACKUP;
                        mp_opt->join_id = *ptr++;
                        mp_opt->thmac = get_unaligned_be64(ptr);
                        ptr += 8;
                        mp_opt->nonce = get_unaligned_be32(ptr);
                        ptr += 4;
                        pr_debug("MP_JOIN bkup=%u, id=%u, thmac=%llu, nonce=%u\n",
                                 mp_opt->backup, mp_opt->join_id,
                                 mp_opt->thmac, mp_opt->nonce);
                } else if (opsize == TCPOLEN_MPTCP_MPJ_ACK) {
                        mp_opt->suboptions |= OPTION_MPTCP_MPJ_ACK;
                        ptr += 2;
                        memcpy(mp_opt->hmac, ptr, MPTCPOPT_HMAC_LEN);
                        pr_debug("MP_JOIN hmac\n");
                }
                break;

        case MPTCPOPT_DSS:
                pr_debug("DSS\n");
                ptr++;

                /* we must clear 'mpc_map' be able to detect MP_CAPABLE
                 * map vs DSS map in mptcp_incoming_options(), and reconstruct
                 * map info accordingly
                 */
                mp_opt->mpc_map = 0;
                flags = (*ptr++) & MPTCP_DSS_FLAG_MASK;
                mp_opt->data_fin = (flags & MPTCP_DSS_DATA_FIN) != 0;
                mp_opt->dsn64 = (flags & MPTCP_DSS_DSN64) != 0;
                mp_opt->use_map = (flags & MPTCP_DSS_HAS_MAP) != 0;
                mp_opt->ack64 = (flags & MPTCP_DSS_ACK64) != 0;
                mp_opt->use_ack = (flags & MPTCP_DSS_HAS_ACK);

                pr_debug("data_fin=%d dsn64=%d use_map=%d ack64=%d use_ack=%d\n",
                         mp_opt->data_fin, mp_opt->dsn64,
                         mp_opt->use_map, mp_opt->ack64,
                         mp_opt->use_ack);

                expected_opsize = TCPOLEN_MPTCP_DSS_BASE;

                if (mp_opt->use_ack) {
                        if (mp_opt->ack64)
                                expected_opsize += TCPOLEN_MPTCP_DSS_ACK64;
                        else
                                expected_opsize += TCPOLEN_MPTCP_DSS_ACK32;
                }

                if (mp_opt->use_map) {
                        if (mp_opt->dsn64)
                                expected_opsize += TCPOLEN_MPTCP_DSS_MAP64;
                        else
                                expected_opsize += TCPOLEN_MPTCP_DSS_MAP32;
                }

                /* Always parse any csum presence combination, we will enforce
                 * RFC 8684 Section 3.3.0 checks later in subflow_data_ready
                 */
                if (opsize != expected_opsize &&
                    opsize != expected_opsize + TCPOLEN_MPTCP_DSS_CHECKSUM)
                        break;

                mp_opt->suboptions |= OPTION_MPTCP_DSS;
                if (mp_opt->use_ack) {
                        if (mp_opt->ack64) {
                                mp_opt->data_ack = get_unaligned_be64(ptr);
                                ptr += 8;
                        } else {
                                mp_opt->data_ack = get_unaligned_be32(ptr);
                                ptr += 4;
                        }

                        pr_debug("data_ack=%llu\n", mp_opt->data_ack);
                }

                if (mp_opt->use_map) {
                        if (mp_opt->dsn64) {
                                mp_opt->data_seq = get_unaligned_be64(ptr);
                                ptr += 8;
                        } else {
                                mp_opt->data_seq = get_unaligned_be32(ptr);
                                ptr += 4;
                        }

                        mp_opt->subflow_seq = get_unaligned_be32(ptr);
                        ptr += 4;

                        mp_opt->data_len = get_unaligned_be16(ptr);
                        ptr += 2;

                        if (opsize == expected_opsize + TCPOLEN_MPTCP_DSS_CHECKSUM) {
                                mp_opt->suboptions |= OPTION_MPTCP_CSUMREQD;
                                mp_opt->csum = get_unaligned((__force __sum16 *)ptr);
                                ptr += 2;
                        }

                        pr_debug("data_seq=%llu subflow_seq=%u data_len=%u csum=%d:%u\n",
                                 mp_opt->data_seq, mp_opt->subflow_seq,
                                 mp_opt->data_len, !!(mp_opt->suboptions & OPTION_MPTCP_CSUMREQD),
                                 mp_opt->csum);
                }

                break;

        case MPTCPOPT_ADD_ADDR:
                mp_opt->echo = (*ptr++) & MPTCP_ADDR_ECHO;
                if (!mp_opt->echo) {
                        if (opsize == TCPOLEN_MPTCP_ADD_ADDR ||
                            opsize == TCPOLEN_MPTCP_ADD_ADDR_PORT)
                                mp_opt->addr.family = AF_INET;
#if IS_ENABLED(CONFIG_MPTCP_IPV6)
                        else if (opsize == TCPOLEN_MPTCP_ADD_ADDR6 ||
                                 opsize == TCPOLEN_MPTCP_ADD_ADDR6_PORT)
                                mp_opt->addr.family = AF_INET6;
#endif
                        else
                                break;
                } else {
                        if (opsize == TCPOLEN_MPTCP_ADD_ADDR_BASE ||
                            opsize == TCPOLEN_MPTCP_ADD_ADDR_BASE_PORT)
                                mp_opt->addr.family = AF_INET;
#if IS_ENABLED(CONFIG_MPTCP_IPV6)
                        else if (opsize == TCPOLEN_MPTCP_ADD_ADDR6_BASE ||
                                 opsize == TCPOLEN_MPTCP_ADD_ADDR6_BASE_PORT)
                                mp_opt->addr.family = AF_INET6;
#endif
                        else
                                break;
                }

                mp_opt->suboptions |= OPTION_MPTCP_ADD_ADDR;
                mp_opt->addr.id = *ptr++;
                mp_opt->addr.port = 0;
                mp_opt->ahmac = 0;
                if (mp_opt->addr.family == AF_INET) {
                        memcpy((u8 *)&mp_opt->addr.addr.s_addr, (u8 *)ptr, 4);
                        ptr += 4;
                        if (opsize == TCPOLEN_MPTCP_ADD_ADDR_PORT ||
                            opsize == TCPOLEN_MPTCP_ADD_ADDR_BASE_PORT) {
                                mp_opt->addr.port = htons(get_unaligned_be16(ptr));
                                ptr += 2;
                        }
                }
#if IS_ENABLED(CONFIG_MPTCP_IPV6)
                else {
                        memcpy(mp_opt->addr.addr6.s6_addr, (u8 *)ptr, 16);
                        ptr += 16;
                        if (opsize == TCPOLEN_MPTCP_ADD_ADDR6_PORT ||
                            opsize == TCPOLEN_MPTCP_ADD_ADDR6_BASE_PORT) {
                                mp_opt->addr.port = htons(get_unaligned_be16(ptr));
                                ptr += 2;
                        }
                }
#endif
                if (!mp_opt->echo) {
                        mp_opt->ahmac = get_unaligned_be64(ptr);
                        ptr += 8;
                }
                pr_debug("ADD_ADDR%s: id=%d, ahmac=%llu, echo=%d, port=%d\n",
                         (mp_opt->addr.family == AF_INET6) ? "6" : "",
                         mp_opt->addr.id, mp_opt->ahmac, mp_opt->echo, ntohs(mp_opt->addr.port));
                break;

        case MPTCPOPT_RM_ADDR:
                if (opsize < TCPOLEN_MPTCP_RM_ADDR_BASE + 1 ||
                    opsize > TCPOLEN_MPTCP_RM_ADDR_BASE + MPTCP_RM_IDS_MAX)
                        break;

                ptr++;

                mp_opt->suboptions |= OPTION_MPTCP_RM_ADDR;
                mp_opt->rm_list.nr = opsize - TCPOLEN_MPTCP_RM_ADDR_BASE;
                for (i = 0; i < mp_opt->rm_list.nr; i++)
                        mp_opt->rm_list.ids[i] = *ptr++;
                pr_debug("RM_ADDR: rm_list_nr=%d\n", mp_opt->rm_list.nr);
                break;

        case MPTCPOPT_MP_PRIO:
                if (opsize != TCPOLEN_MPTCP_PRIO)
                        break;

                mp_opt->suboptions |= OPTION_MPTCP_PRIO;
                mp_opt->backup = *ptr++ & MPTCP_PRIO_BKUP;
                pr_debug("MP_PRIO: prio=%d\n", mp_opt->backup);
                break;

        case MPTCPOPT_MP_FASTCLOSE:
                if (opsize != TCPOLEN_MPTCP_FASTCLOSE)
                        break;

                ptr += 2;
                mp_opt->rcvr_key = get_unaligned_be64(ptr);
                ptr += 8;
                mp_opt->suboptions |= OPTION_MPTCP_FASTCLOSE;
                pr_debug("MP_FASTCLOSE: recv_key=%llu\n", mp_opt->rcvr_key);
                break;

        case MPTCPOPT_RST:
                if (opsize != TCPOLEN_MPTCP_RST)
                        break;

                if (!(TCP_SKB_CB(skb)->tcp_flags & TCPHDR_RST))
                        break;

                mp_opt->suboptions |= OPTION_MPTCP_RST;
                flags = *ptr++;
                mp_opt->reset_transient = flags & MPTCP_RST_TRANSIENT;
                mp_opt->reset_reason = *ptr;
                pr_debug("MP_RST: transient=%u reason=%u\n",
                         mp_opt->reset_transient, mp_opt->reset_reason);
                break;

        case MPTCPOPT_MP_FAIL:
                if (opsize != TCPOLEN_MPTCP_FAIL)
                        break;

                ptr += 2;
                mp_opt->suboptions |= OPTION_MPTCP_FAIL;
                mp_opt->fail_seq = get_unaligned_be64(ptr);
                pr_debug("MP_FAIL: data_seq=%llu\n", mp_opt->fail_seq);
                break;

        default:
                break;
        }
}

void mptcp_get_options(const struct sk_buff *skb,
                       struct mptcp_options_received *mp_opt)
{
        const struct tcphdr *th = tcp_hdr(skb);
        const unsigned char *ptr;
        int length;

        /* initialize option status */
        mp_opt->suboptions = 0;

        length = (th->doff * 4) - sizeof(struct tcphdr);
        ptr = (const unsigned char *)(th + 1);

        while (length > 0) {
                int opcode = *ptr++;
                int opsize;

                switch (opcode) {
                case TCPOPT_EOL:
                        return;
                case TCPOPT_NOP:        /* Ref: RFC 793 section 3.1 */
                        length--;
                        continue;
                default:
                        if (length < 2)
                                return;
                        opsize = *ptr++;
                        if (opsize < 2) /* "silly options" */
                                return;
                        if (opsize > length)
                                return;        /* don't parse partial options */
                        if (opcode == TCPOPT_MPTCP)
                                mptcp_parse_option(skb, ptr, opsize, mp_opt);
                        ptr += opsize - 2;
                        length -= opsize;
                }
        }
}

bool mptcp_syn_options(struct sock *sk, const struct sk_buff *skb,
                       unsigned int *size, struct mptcp_out_options *opts)
{
        struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);

        /* we will use snd_isn to detect first pkt [re]transmission
         * in mptcp_established_options_mp()
         */
        subflow->snd_isn = TCP_SKB_CB(skb)->end_seq;
        if (subflow->request_mptcp) {
                opts->suboptions = OPTION_MPTCP_MPC_SYN;
                opts->csum_reqd = mptcp_is_checksum_enabled(sock_net(sk));
                opts->allow_join_id0 = mptcp_allow_join_id0(sock_net(sk));
                *size = TCPOLEN_MPTCP_MPC_SYN;
                return true;
        } else if (subflow->request_join) {
                pr_debug("remote_token=%u, nonce=%u\n", subflow->remote_token,
                         subflow->local_nonce);
                opts->suboptions = OPTION_MPTCP_MPJ_SYN;
                opts->join_id = subflow->local_id;
                opts->token = subflow->remote_token;
                opts->nonce = subflow->local_nonce;
                opts->backup = subflow->request_bkup;
                *size = TCPOLEN_MPTCP_MPJ_SYN;
                return true;
        }
        return false;
}

static void clear_3rdack_retransmission(struct sock *sk)
{
        struct inet_connection_sock *icsk = inet_csk(sk);

        sk_stop_timer(sk, &icsk->icsk_delack_timer);
        icsk->icsk_ack.timeout = 0;
        icsk->icsk_ack.ato = 0;
        icsk->icsk_ack.pending &= ~(ICSK_ACK_SCHED | ICSK_ACK_TIMER);
}

static bool mptcp_established_options_mp(struct sock *sk, struct sk_buff *skb,
                                         bool snd_data_fin_enable,
                                         unsigned int *size,
                                         struct mptcp_out_options *opts)
{
        struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
        struct mptcp_sock *msk = mptcp_sk(subflow->conn);
        struct mptcp_ext *mpext;
        unsigned int data_len;
        u8 len;

        /* When skb is not available, we better over-estimate the emitted
         * options len. A full DSS option (28 bytes) is longer than
         * TCPOLEN_MPTCP_MPC_ACK_DATA(22) or TCPOLEN_MPTCP_MPJ_ACK(24), so
         * tell the caller to defer the estimate to
         * mptcp_established_options_dss(), which will reserve enough space.
         */
        if (!skb)
                return false;

        /* MPC/MPJ needed only on 3rd ack packet, DATA_FIN and TCP shutdown take precedence */
        if (READ_ONCE(subflow->fully_established) || snd_data_fin_enable ||
            subflow->snd_isn != TCP_SKB_CB(skb)->seq ||
            sk->sk_state != TCP_ESTABLISHED)
                return false;

        if (subflow->mp_capable) {
                mpext = mptcp_get_ext(skb);
                data_len = mpext ? mpext->data_len : 0;

                /* we will check ops->data_len in mptcp_write_options() to
                 * discriminate between TCPOLEN_MPTCP_MPC_ACK_DATA and
                 * TCPOLEN_MPTCP_MPC_ACK
                 */
                opts->data_len = data_len;
                opts->suboptions = OPTION_MPTCP_MPC_ACK;
                opts->sndr_key = subflow->local_key;
                opts->rcvr_key = subflow->remote_key;
                opts->csum_reqd = READ_ONCE(msk->csum_enabled);
                opts->allow_join_id0 = mptcp_allow_join_id0(sock_net(sk));

                /* Section 3.1.
                 * The MP_CAPABLE option is carried on the SYN, SYN/ACK, and ACK
                 * packets that start the first subflow of an MPTCP connection,
                 * as well as the first packet that carries data
                 */
                if (data_len > 0) {
                        len = TCPOLEN_MPTCP_MPC_ACK_DATA;
                        if (opts->csum_reqd) {
                                /* we need to propagate more info to csum the pseudo hdr */
                                opts->data_seq = mpext->data_seq;
                                opts->subflow_seq = mpext->subflow_seq;
                                opts->csum = mpext->csum;
                                len += TCPOLEN_MPTCP_DSS_CHECKSUM;
                        }
                        *size = ALIGN(len, 4);
                } else {
                        *size = TCPOLEN_MPTCP_MPC_ACK;
                }

                pr_debug("subflow=%p, local_key=%llu, remote_key=%llu map_len=%d\n",
                         subflow, subflow->local_key, subflow->remote_key,
                         data_len);

                return true;
        } else if (subflow->mp_join) {
                opts->suboptions = OPTION_MPTCP_MPJ_ACK;
                memcpy(opts->hmac, subflow->hmac, MPTCPOPT_HMAC_LEN);
                *size = TCPOLEN_MPTCP_MPJ_ACK;
                pr_debug("subflow=%p\n", subflow);

                /* we can use the full delegate action helper only from BH context
                 * If we are in process context - sk is flushing the backlog at
                 * socket lock release time - just set the appropriate flag, will
                 * be handled by the release callback
                 */
                if (sock_owned_by_user(sk))
                        set_bit(MPTCP_DELEGATE_ACK, &subflow->delegated_status);
                else
                        mptcp_subflow_delegate(subflow, MPTCP_DELEGATE_ACK);
                return true;
        }
        return false;
}

static void mptcp_write_data_fin(struct mptcp_subflow_context *subflow,
                                 struct sk_buff *skb, struct mptcp_ext *ext)
{
        /* The write_seq value has already been incremented, so the actual
         * sequence number for the DATA_FIN is one less.
         */
        u64 data_fin_tx_seq = READ_ONCE(mptcp_sk(subflow->conn)->write_seq) - 1;

        if (!ext->use_map || !skb->len) {
                /* RFC6824 requires a DSS mapping with specific values
                 * if DATA_FIN is set but no data payload is mapped
                 */
                ext->data_fin = 1;
                ext->use_map = 1;
                ext->dsn64 = 1;
                ext->data_seq = data_fin_tx_seq;
                ext->subflow_seq = 0;
                ext->data_len = 1;
        } else if (ext->data_seq + ext->data_len == data_fin_tx_seq) {
                /* If there's an existing DSS mapping and it is the
                 * final mapping, DATA_FIN consumes 1 additional byte of
                 * mapping space.
                 */
                ext->data_fin = 1;
                ext->data_len++;
        }
}

static bool mptcp_established_options_dss(struct sock *sk, struct sk_buff *skb,
                                          bool snd_data_fin_enable,
                                          unsigned int *size,
                                          struct mptcp_out_options *opts)
{
        struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
        struct mptcp_sock *msk = mptcp_sk(subflow->conn);
        unsigned int dss_size = 0;
        struct mptcp_ext *mpext;
        unsigned int ack_size;
        bool ret = false;
        u64 ack_seq;

        opts->csum_reqd = READ_ONCE(msk->csum_enabled);
        mpext = skb ? mptcp_get_ext(skb) : NULL;

        if (!skb || (mpext && mpext->use_map) || snd_data_fin_enable) {
                unsigned int map_size = TCPOLEN_MPTCP_DSS_BASE + TCPOLEN_MPTCP_DSS_MAP64;

                if (mpext) {
                        if (opts->csum_reqd)
                                map_size += TCPOLEN_MPTCP_DSS_CHECKSUM;

                        opts->ext_copy = *mpext;
                }

                dss_size = map_size;
                if (skb && snd_data_fin_enable)
                        mptcp_write_data_fin(subflow, skb, &opts->ext_copy);
                opts->suboptions = OPTION_MPTCP_DSS;
                ret = true;
        }

        /* passive sockets msk will set the 'can_ack' after accept(), even
         * if the first subflow may have the already the remote key handy
         */
        opts->ext_copy.use_ack = 0;
        if (!READ_ONCE(msk->can_ack)) {
                *size = ALIGN(dss_size, 4);
                return ret;
        }

        ack_seq = READ_ONCE(msk->ack_seq);
        if (READ_ONCE(msk->use_64bit_ack)) {
                ack_size = TCPOLEN_MPTCP_DSS_ACK64;
                opts->ext_copy.data_ack = ack_seq;
                opts->ext_copy.ack64 = 1;
        } else {
                ack_size = TCPOLEN_MPTCP_DSS_ACK32;
                opts->ext_copy.data_ack32 = (uint32_t)ack_seq;
                opts->ext_copy.ack64 = 0;
        }
        opts->ext_copy.use_ack = 1;
        opts->suboptions = OPTION_MPTCP_DSS;
        WRITE_ONCE(msk->old_wspace, __mptcp_space((struct sock *)msk));

        /* Add kind/length/subtype/flag overhead if mapping is not populated */
        if (dss_size == 0)
                ack_size += TCPOLEN_MPTCP_DSS_BASE;

        dss_size += ack_size;

        *size = ALIGN(dss_size, 4);
        return true;
}

static u64 add_addr_generate_hmac(u64 key1, u64 key2,
                                  struct mptcp_addr_info *addr)
{
        u16 port = ntohs(addr->port);
        u8 hmac[SHA256_DIGEST_SIZE];
        u8 msg[19];
        int i = 0;

        msg[i++] = addr->id;
        if (addr->family == AF_INET) {
                memcpy(&msg[i], &addr->addr.s_addr, 4);
                i += 4;
        }
#if IS_ENABLED(CONFIG_MPTCP_IPV6)
        else if (addr->family == AF_INET6) {
                memcpy(&msg[i], &addr->addr6.s6_addr, 16);
                i += 16;
        }
#endif
        msg[i++] = port >> 8;
        msg[i++] = port & 0xFF;

        mptcp_crypto_hmac_sha(key1, key2, msg, i, hmac);

        return get_unaligned_be64(&hmac[SHA256_DIGEST_SIZE - sizeof(u64)]);
}

static bool mptcp_established_options_add_addr(struct sock *sk, struct sk_buff *skb,
                                               unsigned int *size,
                                               unsigned int remaining,
                                               struct mptcp_out_options *opts)
{
        struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
        struct mptcp_sock *msk = mptcp_sk(subflow->conn);
        bool drop_other_suboptions = false;
        unsigned int opt_size = *size;
        bool echo;
        int len;

        /* add addr will strip the existing options, be sure to avoid breaking
         * MPC/MPJ handshakes
         */
        if (!mptcp_pm_should_add_signal(msk) ||
            (opts->suboptions & (OPTION_MPTCP_MPJ_ACK | OPTION_MPTCP_MPC_ACK)) ||
            !mptcp_pm_add_addr_signal(msk, skb, opt_size, remaining, &opts->addr,
                    &echo, &drop_other_suboptions))
                return false;

        if (drop_other_suboptions)
                remaining += opt_size;
        len = mptcp_add_addr_len(opts->addr.family, echo, !!opts->addr.port);
        if (remaining < len)
                return false;

        *size = len;
        if (drop_other_suboptions) {
                pr_debug("drop other suboptions\n");
                opts->suboptions = 0;

                /* note that e.g. DSS could have written into the memory
                 * aliased by ahmac, we must reset the field here
                 * to avoid appending the hmac even for ADD_ADDR echo
                 * options
                 */
                opts->ahmac = 0;
                *size -= opt_size;
        }
        opts->suboptions |= OPTION_MPTCP_ADD_ADDR;
        if (!echo) {
                MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_ADDADDRTX);
                opts->ahmac = add_addr_generate_hmac(READ_ONCE(msk->local_key),
                                                     READ_ONCE(msk->remote_key),
                                                     &opts->addr);
        } else {
                MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_ECHOADDTX);
        }
        pr_debug("addr_id=%d, ahmac=%llu, echo=%d, port=%d\n",
                 opts->addr.id, opts->ahmac, echo, ntohs(opts->addr.port));

        return true;
}

static bool mptcp_established_options_rm_addr(struct sock *sk,
                                              unsigned int *size,
                                              unsigned int remaining,
                                              struct mptcp_out_options *opts)
{
        struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
        struct mptcp_sock *msk = mptcp_sk(subflow->conn);
        struct mptcp_rm_list rm_list;
        int i, len;

        if (!mptcp_pm_should_rm_signal(msk) ||
            !(mptcp_pm_rm_addr_signal(msk, remaining, &rm_list)))
                return false;

        len = mptcp_rm_addr_len(&rm_list);
        if (len < 0)
                return false;
        if (remaining < len)
                return false;

        *size = len;
        opts->suboptions |= OPTION_MPTCP_RM_ADDR;
        opts->rm_list = rm_list;

        for (i = 0; i < opts->rm_list.nr; i++)
                pr_debug("rm_list_ids[%d]=%d\n", i, opts->rm_list.ids[i]);
        MPTCP_ADD_STATS(sock_net(sk), MPTCP_MIB_RMADDRTX, opts->rm_list.nr);
        return true;
}

static bool mptcp_established_options_mp_prio(struct sock *sk,
                                              unsigned int *size,
                                              unsigned int remaining,
                                              struct mptcp_out_options *opts)
{
        struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);

        /* can't send MP_PRIO with MPC, as they share the same option space:
         * 'backup'. Also it makes no sense at all
         */
        if (!subflow->send_mp_prio || (opts->suboptions & OPTIONS_MPTCP_MPC))
                return false;

        /* account for the trailing 'nop' option */
        if (remaining < TCPOLEN_MPTCP_PRIO_ALIGN)
                return false;

        *size = TCPOLEN_MPTCP_PRIO_ALIGN;
        opts->suboptions |= OPTION_MPTCP_PRIO;
        opts->backup = subflow->request_bkup;

        pr_debug("prio=%d\n", opts->backup);

        return true;
}

static noinline bool mptcp_established_options_rst(struct sock *sk, struct sk_buff *skb,
                                                   unsigned int *size,
                                                   unsigned int remaining,
                                                   struct mptcp_out_options *opts)
{
        const struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);

        if (remaining < TCPOLEN_MPTCP_RST)
                return false;

        *size = TCPOLEN_MPTCP_RST;
        opts->suboptions |= OPTION_MPTCP_RST;
        opts->reset_transient = subflow->reset_transient;
        opts->reset_reason = subflow->reset_reason;
        MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_MPRSTTX);

        return true;
}

static bool mptcp_established_options_fastclose(struct sock *sk,
                                                unsigned int *size,
                                                unsigned int remaining,
                                                struct mptcp_out_options *opts)
{
        struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
        struct mptcp_sock *msk = mptcp_sk(subflow->conn);

        if (likely(!subflow->send_fastclose))
                return false;

        if (remaining < TCPOLEN_MPTCP_FASTCLOSE)
                return false;

        *size = TCPOLEN_MPTCP_FASTCLOSE;
        opts->suboptions |= OPTION_MPTCP_FASTCLOSE;
        opts->rcvr_key = READ_ONCE(msk->remote_key);

        pr_debug("FASTCLOSE key=%llu\n", opts->rcvr_key);
        MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_MPFASTCLOSETX);
        return true;
}

static bool mptcp_established_options_mp_fail(struct sock *sk,
                                              unsigned int *size,
                                              unsigned int remaining,
                                              struct mptcp_out_options *opts)
{
        struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);

        if (likely(!subflow->send_mp_fail))
                return false;

        if (remaining < TCPOLEN_MPTCP_FAIL)
                return false;

        *size = TCPOLEN_MPTCP_FAIL;
        opts->suboptions |= OPTION_MPTCP_FAIL;
        opts->fail_seq = subflow->map_seq;

        pr_debug("MP_FAIL fail_seq=%llu\n", opts->fail_seq);
        MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_MPFAILTX);

        return true;
}

bool mptcp_established_options(struct sock *sk, struct sk_buff *skb,
                               unsigned int *size, unsigned int remaining,
                               struct mptcp_out_options *opts)
{
        struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
        struct mptcp_sock *msk = mptcp_sk(subflow->conn);
        unsigned int opt_size = 0;
        bool snd_data_fin;
        bool ret = false;

        opts->suboptions = 0;

        if (unlikely(__mptcp_check_fallback(msk) && !mptcp_check_infinite_map(skb)))
                return false;

        if (unlikely(skb && TCP_SKB_CB(skb)->tcp_flags & TCPHDR_RST)) {
                if (mptcp_established_options_fastclose(sk, &opt_size, remaining, opts) ||
                    mptcp_established_options_mp_fail(sk, &opt_size, remaining, opts)) {
                        *size += opt_size;
                        remaining -= opt_size;
                }
                /* MP_RST can be used with MP_FASTCLOSE and MP_FAIL if there is room */
                if (mptcp_established_options_rst(sk, skb, &opt_size, remaining, opts)) {
                        *size += opt_size;
                        remaining -= opt_size;
                }
                return true;
        }

        snd_data_fin = mptcp_data_fin_enabled(msk);
        if (mptcp_established_options_mp(sk, skb, snd_data_fin, &opt_size, opts))
                ret = true;
        else if (mptcp_established_options_dss(sk, skb, snd_data_fin, &opt_size, opts)) {
                unsigned int mp_fail_size;

                ret = true;
                if (mptcp_established_options_mp_fail(sk, &mp_fail_size,
                                                      remaining - opt_size, opts)) {
                        *size += opt_size + mp_fail_size;
                        remaining -= opt_size - mp_fail_size;
                        return true;
                }
        }

        /* we reserved enough space for the above options, and exceeding the
         * TCP option space would be fatal
         */
        if (WARN_ON_ONCE(opt_size > remaining))
                return false;

        *size += opt_size;
        remaining -= opt_size;
        if (mptcp_established_options_add_addr(sk, skb, &opt_size, remaining, opts)) {
                *size += opt_size;
                remaining -= opt_size;
                ret = true;
        } else if (mptcp_established_options_rm_addr(sk, &opt_size, remaining, opts)) {
                *size += opt_size;
                remaining -= opt_size;
                ret = true;
        }

        if (mptcp_established_options_mp_prio(sk, &opt_size, remaining, opts)) {
                *size += opt_size;
                remaining -= opt_size;
                ret = true;
        }

        return ret;
}

bool mptcp_synack_options(const struct request_sock *req, unsigned int *size,
                          struct mptcp_out_options *opts)
{
        struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req);

        if (subflow_req->mp_capable) {
                opts->suboptions = OPTION_MPTCP_MPC_SYNACK;
                opts->sndr_key = subflow_req->local_key;
                opts->csum_reqd = subflow_req->csum_reqd;
                opts->allow_join_id0 = subflow_req->allow_join_id0;
                *size = TCPOLEN_MPTCP_MPC_SYNACK;
                pr_debug("subflow_req=%p, local_key=%llu\n",
                         subflow_req, subflow_req->local_key);
                return true;
        } else if (subflow_req->mp_join) {
                opts->suboptions = OPTION_MPTCP_MPJ_SYNACK;
                opts->backup = subflow_req->request_bkup;
                opts->join_id = subflow_req->local_id;
                opts->thmac = subflow_req->thmac;
                opts->nonce = subflow_req->local_nonce;
                pr_debug("req=%p, bkup=%u, id=%u, thmac=%llu, nonce=%u\n",
                         subflow_req, opts->backup, opts->join_id,
                         opts->thmac, opts->nonce);
                *size = TCPOLEN_MPTCP_MPJ_SYNACK;
                return true;
        }
        return false;
}

static bool check_fully_established(struct mptcp_sock *msk, struct sock *ssk,
                                    struct mptcp_subflow_context *subflow,
                                    struct sk_buff *skb,
                                    struct mptcp_options_received *mp_opt)
{
        /* here we can process OoO, in-window pkts, only in-sequence 4th ack
         * will make the subflow fully established
         */
        if (likely(READ_ONCE(subflow->fully_established))) {
                /* on passive sockets, check for 3rd ack retransmission
                 * note that msk is always set by subflow_syn_recv_sock()
                 * for mp_join subflows
                 */
                if (TCP_SKB_CB(skb)->seq == subflow->ssn_offset + 1 &&
                    TCP_SKB_CB(skb)->end_seq == TCP_SKB_CB(skb)->seq &&
                    subflow->mp_join && (mp_opt->suboptions & OPTIONS_MPTCP_MPJ) &&
                    !subflow->request_join)
                        tcp_send_ack(ssk);
                goto check_notify;
        }

        /* we must process OoO packets before the first subflow is fully
         * established. OoO packets are instead a protocol violation
         * for MP_JOIN subflows as the peer must not send any data
         * before receiving the forth ack - cfr. RFC 8684 section 3.2.
         */
        if (TCP_SKB_CB(skb)->seq != subflow->ssn_offset + 1) {
                if (subflow->mp_join)
                        goto reset;
                if (subflow->is_mptfo && mp_opt->suboptions & OPTION_MPTCP_MPC_ACK)
                        goto set_fully_established;
                return subflow->mp_capable;
        }

        if (subflow->remote_key_valid &&
            (((mp_opt->suboptions & OPTION_MPTCP_DSS) && mp_opt->use_ack) ||
             ((mp_opt->suboptions & OPTION_MPTCP_ADD_ADDR) &&
              (!mp_opt->echo || subflow->mp_join)))) {
                /* subflows are fully established as soon as we get any
                 * additional ack, including ADD_ADDR.
                 */
                goto set_fully_established;
        }

        /* If the first established packet does not contain MP_CAPABLE + data
         * then fallback to TCP. Fallback scenarios requires a reset for
         * MP_JOIN subflows.
         */
        if (!(mp_opt->suboptions & OPTIONS_MPTCP_MPC)) {
                if (subflow->mp_join)
                        goto reset;
                subflow->mp_capable = 0;
                pr_fallback(msk);
                mptcp_do_fallback(ssk);
                return false;
        }

        if (mp_opt->deny_join_id0)
                WRITE_ONCE(msk->pm.remote_deny_join_id0, true);

        if (unlikely(!READ_ONCE(msk->pm.server_side)))
                pr_warn_once("bogus mpc option on established client sk");

set_fully_established:
        mptcp_data_lock((struct sock *)msk);
        __mptcp_subflow_fully_established(msk, subflow, mp_opt);
        mptcp_data_unlock((struct sock *)msk);

check_notify:
        /* if the subflow is not already linked into the conn_list, we can't
         * notify the PM: this subflow is still on the listener queue
         * and the PM possibly acquiring the subflow lock could race with
         * the listener close
         */
        if (likely(subflow->pm_notified) || list_empty(&subflow->node))
                return true;

        subflow->pm_notified = 1;
        if (subflow->mp_join) {
                clear_3rdack_retransmission(ssk);
                mptcp_pm_subflow_established(msk);
        } else {
                mptcp_pm_fully_established(msk, ssk);
        }
        return true;

reset:
        mptcp_subflow_reset(ssk);
        return false;
}

u64 __mptcp_expand_seq(u64 old_seq, u64 cur_seq)
{
        u32 old_seq32, cur_seq32;

        old_seq32 = (u32)old_seq;
        cur_seq32 = (u32)cur_seq;
        cur_seq = (old_seq & GENMASK_ULL(63, 32)) + cur_seq32;
        if (unlikely(cur_seq32 < old_seq32 && before(old_seq32, cur_seq32)))
                return cur_seq + (1LL << 32);

        /* reverse wrap could happen, too */
        if (unlikely(cur_seq32 > old_seq32 && after(old_seq32, cur_seq32)))
                return cur_seq - (1LL << 32);
        return cur_seq;
}

static void __mptcp_snd_una_update(struct mptcp_sock *msk, u64 new_snd_una)
{
        msk->bytes_acked += new_snd_una - msk->snd_una;
        WRITE_ONCE(msk->snd_una, new_snd_una);
}

static void ack_update_msk(struct mptcp_sock *msk,
                           struct sock *ssk,
                           struct mptcp_options_received *mp_opt)
{
        u64 new_wnd_end, new_snd_una, snd_nxt = READ_ONCE(msk->snd_nxt);
        struct sock *sk = (struct sock *)msk;
        u64 old_snd_una;

        mptcp_data_lock(sk);

        /* avoid ack expansion on update conflict, to reduce the risk of
         * wrongly expanding to a future ack sequence number, which is way
         * more dangerous than missing an ack
         */
        old_snd_una = msk->snd_una;
        new_snd_una = mptcp_expand_seq(old_snd_una, mp_opt->data_ack, mp_opt->ack64);

        /* ACK for data not even sent yet? Ignore.*/
        if (unlikely(after64(new_snd_una, snd_nxt)))
                new_snd_una = old_snd_una;

        new_wnd_end = new_snd_una + tcp_sk(ssk)->snd_wnd;

        if (after64(new_wnd_end, msk->wnd_end))
                WRITE_ONCE(msk->wnd_end, new_wnd_end);

        /* this assumes mptcp_incoming_options() is invoked after tcp_ack() */
        if (after64(msk->wnd_end, snd_nxt))
                __mptcp_check_push(sk, ssk);

        if (after64(new_snd_una, old_snd_una)) {
                __mptcp_snd_una_update(msk, new_snd_una);
                __mptcp_data_acked(sk);
        }
        msk->last_ack_recv = tcp_jiffies32;
        mptcp_data_unlock(sk);

        trace_ack_update_msk(mp_opt->data_ack,
                             old_snd_una, new_snd_una,
                             new_wnd_end, READ_ONCE(msk->wnd_end));
}

bool mptcp_update_rcv_data_fin(struct mptcp_sock *msk, u64 data_fin_seq, bool use_64bit)
{
        /* Skip if DATA_FIN was already received.
         * If updating simultaneously with the recvmsg loop, values
         * should match. If they mismatch, the peer is misbehaving and
         * we will prefer the most recent information.
         */
        if (READ_ONCE(msk->rcv_data_fin))
                return false;

        WRITE_ONCE(msk->rcv_data_fin_seq,
                   mptcp_expand_seq(READ_ONCE(msk->ack_seq), data_fin_seq, use_64bit));
        WRITE_ONCE(msk->rcv_data_fin, 1);

        return true;
}

static bool add_addr_hmac_valid(struct mptcp_sock *msk,
                                struct mptcp_options_received *mp_opt)
{
        u64 hmac = 0;

        if (mp_opt->echo)
                return true;

        hmac = add_addr_generate_hmac(READ_ONCE(msk->remote_key),
                                      READ_ONCE(msk->local_key),
                                      &mp_opt->addr);

        pr_debug("msk=%p, ahmac=%llu, mp_opt->ahmac=%llu\n",
                 msk, hmac, mp_opt->ahmac);

        return hmac == mp_opt->ahmac;
}

/* Return false if a subflow has been reset, else return true */
bool mptcp_incoming_options(struct sock *sk, struct sk_buff *skb)
{
        struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
        struct mptcp_sock *msk = mptcp_sk(subflow->conn);
        struct mptcp_options_received mp_opt;
        struct mptcp_ext *mpext;

        if (__mptcp_check_fallback(msk)) {
                /* Keep it simple and unconditionally trigger send data cleanup and
                 * pending queue spooling. We will need to acquire the data lock
                 * for more accurate checks, and once the lock is acquired, such
                 * helpers are cheap.
                 */
                mptcp_data_lock(subflow->conn);
                if (sk_stream_memory_free(sk))
                        __mptcp_check_push(subflow->conn, sk);

                /* on fallback we just need to ignore the msk-level snd_una, as
                 * this is really plain TCP
                 */
                __mptcp_snd_una_update(msk, READ_ONCE(msk->snd_nxt));

                __mptcp_data_acked(subflow->conn);
                mptcp_data_unlock(subflow->conn);
                return true;
        }

        mptcp_get_options(skb, &mp_opt);

        /* The subflow can be in close state only if check_fully_established()
         * just sent a reset. If so, tell the caller to ignore the current packet.
         */
        if (!check_fully_established(msk, sk, subflow, skb, &mp_opt))
                return sk->sk_state != TCP_CLOSE;

        if (unlikely(mp_opt.suboptions != OPTION_MPTCP_DSS)) {
                if ((mp_opt.suboptions & OPTION_MPTCP_FASTCLOSE) &&
                    READ_ONCE(msk->local_key) == mp_opt.rcvr_key) {
                        WRITE_ONCE(msk->rcv_fastclose, true);
                        mptcp_schedule_work((struct sock *)msk);
                        MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_MPFASTCLOSERX);
                }

                if ((mp_opt.suboptions & OPTION_MPTCP_ADD_ADDR) &&
                    add_addr_hmac_valid(msk, &mp_opt)) {
                        if (!mp_opt.echo) {
                                mptcp_pm_add_addr_received(sk, &mp_opt.addr);
                                MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_ADDADDR);
                        } else {
                                mptcp_pm_add_addr_echoed(msk, &mp_opt.addr);
                                mptcp_pm_del_add_timer(msk, &mp_opt.addr, true);
                                MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_ECHOADD);
                        }

                        if (mp_opt.addr.port)
                                MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_PORTADD);
                }

                if (mp_opt.suboptions & OPTION_MPTCP_RM_ADDR)
                        mptcp_pm_rm_addr_received(msk, &mp_opt.rm_list);

                if (mp_opt.suboptions & OPTION_MPTCP_PRIO) {
                        mptcp_pm_mp_prio_received(sk, mp_opt.backup);
                        MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_MPPRIORX);
                }

                if (mp_opt.suboptions & OPTION_MPTCP_FAIL) {
                        mptcp_pm_mp_fail_received(sk, mp_opt.fail_seq);
                        MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_MPFAILRX);
                }

                if (mp_opt.suboptions & OPTION_MPTCP_RST) {
                        subflow->reset_seen = 1;
                        subflow->reset_reason = mp_opt.reset_reason;
                        subflow->reset_transient = mp_opt.reset_transient;
                        MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_MPRSTRX);
                }

                if (!(mp_opt.suboptions & OPTION_MPTCP_DSS))
                        return true;
        }

        /* we can't wait for recvmsg() to update the ack_seq, otherwise
         * monodirectional flows will stuck
         */
        if (mp_opt.use_ack)
                ack_update_msk(msk, sk, &mp_opt);

        /* Zero-data-length packets are dropped by the caller and not
         * propagated to the MPTCP layer, so the skb extension does not
         * need to be allocated or populated. DATA_FIN information, if
         * present, needs to be updated here before the skb is freed.
         */
        if (TCP_SKB_CB(skb)->seq == TCP_SKB_CB(skb)->end_seq) {
                if (mp_opt.data_fin && mp_opt.data_len == 1 &&
                    mptcp_update_rcv_data_fin(msk, mp_opt.data_seq, mp_opt.dsn64))
                        mptcp_schedule_work((struct sock *)msk);

                return true;
        }

        mpext = skb_ext_add(skb, SKB_EXT_MPTCP);
        if (!mpext)
                return true;

        memset(mpext, 0, sizeof(*mpext));

        if (likely(mp_opt.use_map)) {
                if (mp_opt.mpc_map) {
                        /* this is an MP_CAPABLE carrying MPTCP data
                         * we know this map the first chunk of data
                         */
                        mptcp_crypto_key_sha(subflow->remote_key, NULL,
                                             &mpext->data_seq);
                        mpext->data_seq++;
                        mpext->subflow_seq = 1;
                        mpext->dsn64 = 1;
                        mpext->mpc_map = 1;
                        mpext->data_fin = 0;
                } else {
                        mpext->data_seq = mp_opt.data_seq;
                        mpext->subflow_seq = mp_opt.subflow_seq;
                        mpext->dsn64 = mp_opt.dsn64;
                        mpext->data_fin = mp_opt.data_fin;
                }
                mpext->data_len = mp_opt.data_len;
                mpext->use_map = 1;
                mpext->csum_reqd = !!(mp_opt.suboptions & OPTION_MPTCP_CSUMREQD);

                if (mpext->csum_reqd)
                        mpext->csum = mp_opt.csum;
        }

        return true;
}

static void mptcp_set_rwin(struct tcp_sock *tp, struct tcphdr *th)
{
        const struct sock *ssk = (const struct sock *)tp;
        struct mptcp_subflow_context *subflow;
        u64 ack_seq, rcv_wnd_old, rcv_wnd_new;
        struct mptcp_sock *msk;
        u32 new_win;
        u64 win;

        subflow = mptcp_subflow_ctx(ssk);
        msk = mptcp_sk(subflow->conn);

        ack_seq = READ_ONCE(msk->ack_seq);
        rcv_wnd_new = ack_seq + tp->rcv_wnd;

        rcv_wnd_old = atomic64_read(&msk->rcv_wnd_sent);
        if (after64(rcv_wnd_new, rcv_wnd_old)) {
                u64 rcv_wnd;

                for (;;) {
                        rcv_wnd = atomic64_cmpxchg(&msk->rcv_wnd_sent, rcv_wnd_old, rcv_wnd_new);

                        if (rcv_wnd == rcv_wnd_old)
                                break;

                        rcv_wnd_old = rcv_wnd;
                        if (before64(rcv_wnd_new, rcv_wnd_old)) {
                                MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_RCVWNDCONFLICTUPDATE);
                                goto raise_win;
                        }
                        MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_RCVWNDCONFLICT);
                }
                return;
        }

        if (rcv_wnd_new != rcv_wnd_old) {
raise_win:
                win = rcv_wnd_old - ack_seq;
                tp->rcv_wnd = min_t(u64, win, U32_MAX);
                new_win = tp->rcv_wnd;

                /* Make sure we do not exceed the maximum possible
                 * scaled window.
                 */
                if (unlikely(th->syn))
                        new_win = min(new_win, 65535U) << tp->rx_opt.rcv_wscale;
                if (!tp->rx_opt.rcv_wscale &&
                    READ_ONCE(sock_net(ssk)->ipv4.sysctl_tcp_workaround_signed_windows))
                        new_win = min(new_win, MAX_TCP_WINDOW);
                else
                        new_win = min(new_win, (65535U << tp->rx_opt.rcv_wscale));

                /* RFC1323 scaling applied */
                new_win >>= tp->rx_opt.rcv_wscale;
                th->window = htons(new_win);
                MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_RCVWNDSHARED);
        }
}

__sum16 __mptcp_make_csum(u64 data_seq, u32 subflow_seq, u16 data_len, __wsum sum)
{
        struct csum_pseudo_header header;
        __wsum csum;

        /* cfr RFC 8684 3.3.1.:
         * the data sequence number used in the pseudo-header is
         * always the 64-bit value, irrespective of what length is used in the
         * DSS option itself.
         */
        header.data_seq = cpu_to_be64(data_seq);
        header.subflow_seq = htonl(subflow_seq);
        header.data_len = htons(data_len);
        header.csum = 0;

        csum = csum_partial(&header, sizeof(header), sum);
        return csum_fold(csum);
}

static __sum16 mptcp_make_csum(const struct mptcp_ext *mpext)
{
        return __mptcp_make_csum(mpext->data_seq, mpext->subflow_seq, mpext->data_len,
                                 ~csum_unfold(mpext->csum));
}

static void put_len_csum(u16 len, __sum16 csum, void *data)
{
        __sum16 *sumptr = data + 2;
        __be16 *ptr = data;

        put_unaligned_be16(len, ptr);

        put_unaligned(csum, sumptr);
}

void mptcp_write_options(struct tcphdr *th, __be32 *ptr, struct tcp_sock *tp,
                         struct mptcp_out_options *opts)
{
        const struct sock *ssk = (const struct sock *)tp;
        struct mptcp_subflow_context *subflow;

        /* Which options can be used together?
         *
         * X: mutually exclusive
         * O: often used together
         * C: can be used together in some cases
         * P: could be used together but we prefer not to (optimisations)
         *
         *  Opt: | MPC  | MPJ  | DSS  | ADD  |  RM  | PRIO | FAIL |  FC  |
         * ------|------|------|------|------|------|------|------|------|
         *  MPC  |------|------|------|------|------|------|------|------|
         *  MPJ  |  X   |------|------|------|------|------|------|------|
         *  DSS  |  X   |  X   |------|------|------|------|------|------|
         *  ADD  |  X   |  X   |  P   |------|------|------|------|------|
         *  RM   |  C   |  C   |  C   |  P   |------|------|------|------|
         *  PRIO |  X   |  C   |  C   |  C   |  C   |------|------|------|
         *  FAIL |  X   |  X   |  C   |  X   |  X   |  X   |------|------|
         *  FC   |  X   |  X   |  X   |  X   |  X   |  X   |  X   |------|
         *  RST  |  X   |  X   |  X   |  X   |  X   |  X   |  O   |  O   |
         * ------|------|------|------|------|------|------|------|------|
         *
         * The same applies in mptcp_established_options() function.
         */
        if (likely(OPTION_MPTCP_DSS & opts->suboptions)) {
                struct mptcp_ext *mpext = &opts->ext_copy;
                u8 len = TCPOLEN_MPTCP_DSS_BASE;
                u8 flags = 0;

                if (mpext->use_ack) {
                        flags = MPTCP_DSS_HAS_ACK;
                        if (mpext->ack64) {
                                len += TCPOLEN_MPTCP_DSS_ACK64;
                                flags |= MPTCP_DSS_ACK64;
                        } else {
                                len += TCPOLEN_MPTCP_DSS_ACK32;
                        }
                }

                if (mpext->use_map) {
                        len += TCPOLEN_MPTCP_DSS_MAP64;

                        /* Use only 64-bit mapping flags for now, add
                         * support for optional 32-bit mappings later.
                         */
                        flags |= MPTCP_DSS_HAS_MAP | MPTCP_DSS_DSN64;
                        if (mpext->data_fin)
                                flags |= MPTCP_DSS_DATA_FIN;

                        if (opts->csum_reqd)
                                len += TCPOLEN_MPTCP_DSS_CHECKSUM;
                }

                *ptr++ = mptcp_option(MPTCPOPT_DSS, len, 0, flags);

                if (mpext->use_ack) {
                        if (mpext->ack64) {
                                put_unaligned_be64(mpext->data_ack, ptr);
                                ptr += 2;
                        } else {
                                put_unaligned_be32(mpext->data_ack32, ptr);
                                ptr += 1;
                        }
                }

                if (mpext->use_map) {
                        put_unaligned_be64(mpext->data_seq, ptr);
                        ptr += 2;
                        put_unaligned_be32(mpext->subflow_seq, ptr);
                        ptr += 1;
                        if (opts->csum_reqd) {
                                /* data_len == 0 is reserved for the infinite mapping,
                                 * the checksum will also be set to 0.
                                 */
                                put_len_csum(mpext->data_len,
                                             (mpext->data_len ? mptcp_make_csum(mpext) : 0),
                                             ptr);
                        } else {
                                put_unaligned_be32(mpext->data_len << 16 |
                                                   TCPOPT_NOP << 8 | TCPOPT_NOP, ptr);
                        }
                        ptr += 1;
                }

                /* We might need to add MP_FAIL options in rare cases */
                if (unlikely(OPTION_MPTCP_FAIL & opts->suboptions))
                        goto mp_fail;
        } else if (OPTIONS_MPTCP_MPC & opts->suboptions) {
                u8 len, flag = MPTCP_CAP_HMAC_SHA256;

                if (OPTION_MPTCP_MPC_SYN & opts->suboptions) {
                        len = TCPOLEN_MPTCP_MPC_SYN;
                } else if (OPTION_MPTCP_MPC_SYNACK & opts->suboptions) {
                        len = TCPOLEN_MPTCP_MPC_SYNACK;
                } else if (opts->data_len) {
                        len = TCPOLEN_MPTCP_MPC_ACK_DATA;
                        if (opts->csum_reqd)
                                len += TCPOLEN_MPTCP_DSS_CHECKSUM;
                } else {
                        len = TCPOLEN_MPTCP_MPC_ACK;
                }

                if (opts->csum_reqd)
                        flag |= MPTCP_CAP_CHECKSUM_REQD;

                if (!opts->allow_join_id0)
                        flag |= MPTCP_CAP_DENY_JOIN_ID0;

                *ptr++ = mptcp_option(MPTCPOPT_MP_CAPABLE, len,
                                      MPTCP_SUPPORTED_VERSION,
                                      flag);

                if (!((OPTION_MPTCP_MPC_SYNACK | OPTION_MPTCP_MPC_ACK) &
                    opts->suboptions))
                        goto mp_capable_done;

                put_unaligned_be64(opts->sndr_key, ptr);
                ptr += 2;
                if (!((OPTION_MPTCP_MPC_ACK) & opts->suboptions))
                        goto mp_capable_done;

                put_unaligned_be64(opts->rcvr_key, ptr);
                ptr += 2;
                if (!opts->data_len)
                        goto mp_capable_done;

                if (opts->csum_reqd) {
                        put_len_csum(opts->data_len,
                                     __mptcp_make_csum(opts->data_seq,
                                                       opts->subflow_seq,
                                                       opts->data_len,
                                                       ~csum_unfold(opts->csum)),
                                     ptr);
                } else {
                        put_unaligned_be32(opts->data_len << 16 |
                                           TCPOPT_NOP << 8 | TCPOPT_NOP, ptr);
                }
                ptr += 1;

                /* MPC is additionally mutually exclusive with MP_PRIO */
                goto mp_capable_done;
        } else if (OPTIONS_MPTCP_MPJ & opts->suboptions) {
                if (OPTION_MPTCP_MPJ_SYN & opts->suboptions) {
                        *ptr++ = mptcp_option(MPTCPOPT_MP_JOIN,
                                              TCPOLEN_MPTCP_MPJ_SYN,
                                              opts->backup, opts->join_id);
                        put_unaligned_be32(opts->token, ptr);
                        ptr += 1;
                        put_unaligned_be32(opts->nonce, ptr);
                        ptr += 1;
                } else if (OPTION_MPTCP_MPJ_SYNACK & opts->suboptions) {
                        *ptr++ = mptcp_option(MPTCPOPT_MP_JOIN,
                                              TCPOLEN_MPTCP_MPJ_SYNACK,
                                              opts->backup, opts->join_id);
                        put_unaligned_be64(opts->thmac, ptr);
                        ptr += 2;
                        put_unaligned_be32(opts->nonce, ptr);
                        ptr += 1;
                } else {
                        *ptr++ = mptcp_option(MPTCPOPT_MP_JOIN,
                                              TCPOLEN_MPTCP_MPJ_ACK, 0, 0);
                        memcpy(ptr, opts->hmac, MPTCPOPT_HMAC_LEN);
                        ptr += 5;
                }
        } else if (OPTION_MPTCP_ADD_ADDR & opts->suboptions) {
                u8 len = TCPOLEN_MPTCP_ADD_ADDR_BASE;
                u8 echo = MPTCP_ADDR_ECHO;

#if IS_ENABLED(CONFIG_MPTCP_IPV6)
                if (opts->addr.family == AF_INET6)
                        len = TCPOLEN_MPTCP_ADD_ADDR6_BASE;
#endif

                if (opts->addr.port)
                        len += TCPOLEN_MPTCP_PORT_LEN;

                if (opts->ahmac) {
                        len += sizeof(opts->ahmac);
                        echo = 0;
                }

                *ptr++ = mptcp_option(MPTCPOPT_ADD_ADDR,
                                      len, echo, opts->addr.id);
                if (opts->addr.family == AF_INET) {
                        memcpy((u8 *)ptr, (u8 *)&opts->addr.addr.s_addr, 4);
                        ptr += 1;
                }
#if IS_ENABLED(CONFIG_MPTCP_IPV6)
                else if (opts->addr.family == AF_INET6) {
                        memcpy((u8 *)ptr, opts->addr.addr6.s6_addr, 16);
                        ptr += 4;
                }
#endif

                if (!opts->addr.port) {
                        if (opts->ahmac) {
                                put_unaligned_be64(opts->ahmac, ptr);
                                ptr += 2;
                        }
                } else {
                        u16 port = ntohs(opts->addr.port);

                        if (opts->ahmac) {
                                u8 *bptr = (u8 *)ptr;

                                put_unaligned_be16(port, bptr);
                                bptr += 2;
                                put_unaligned_be64(opts->ahmac, bptr);
                                bptr += 8;
                                put_unaligned_be16(TCPOPT_NOP << 8 |
                                                   TCPOPT_NOP, bptr);

                                ptr += 3;
                        } else {
                                put_unaligned_be32(port << 16 |
                                                   TCPOPT_NOP << 8 |
                                                   TCPOPT_NOP, ptr);
                                ptr += 1;
                        }
                }
        } else if (unlikely(OPTION_MPTCP_FASTCLOSE & opts->suboptions)) {
                /* FASTCLOSE is mutually exclusive with others except RST */
                *ptr++ = mptcp_option(MPTCPOPT_MP_FASTCLOSE,
                                      TCPOLEN_MPTCP_FASTCLOSE,
                                      0, 0);
                put_unaligned_be64(opts->rcvr_key, ptr);
                ptr += 2;

                if (OPTION_MPTCP_RST & opts->suboptions)
                        goto mp_rst;
                return;
        } else if (unlikely(OPTION_MPTCP_FAIL & opts->suboptions)) {
mp_fail:
                /* MP_FAIL is mutually exclusive with others except RST */
                subflow = mptcp_subflow_ctx(ssk);
                subflow->send_mp_fail = 0;

                *ptr++ = mptcp_option(MPTCPOPT_MP_FAIL,
                                      TCPOLEN_MPTCP_FAIL,
                                      0, 0);
                put_unaligned_be64(opts->fail_seq, ptr);
                ptr += 2;

                if (OPTION_MPTCP_RST & opts->suboptions)
                        goto mp_rst;
                return;
        } else if (unlikely(OPTION_MPTCP_RST & opts->suboptions)) {
mp_rst:
                *ptr++ = mptcp_option(MPTCPOPT_RST,
                                      TCPOLEN_MPTCP_RST,
                                      opts->reset_transient,
                                      opts->reset_reason);
                return;
        }

        if (OPTION_MPTCP_PRIO & opts->suboptions) {
                subflow = mptcp_subflow_ctx(ssk);
                subflow->send_mp_prio = 0;

                *ptr++ = mptcp_option(MPTCPOPT_MP_PRIO,
                                      TCPOLEN_MPTCP_PRIO,
                                      opts->backup, TCPOPT_NOP);

                MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_MPPRIOTX);
        }

mp_capable_done:
        if (OPTION_MPTCP_RM_ADDR & opts->suboptions) {
                u8 i = 1;

                *ptr++ = mptcp_option(MPTCPOPT_RM_ADDR,
                                      TCPOLEN_MPTCP_RM_ADDR_BASE + opts->rm_list.nr,
                                      0, opts->rm_list.ids[0]);

                while (i < opts->rm_list.nr) {
                        u8 id1, id2, id3, id4;

                        id1 = opts->rm_list.ids[i];
                        id2 = i + 1 < opts->rm_list.nr ? opts->rm_list.ids[i + 1] : TCPOPT_NOP;
                        id3 = i + 2 < opts->rm_list.nr ? opts->rm_list.ids[i + 2] : TCPOPT_NOP;
                        id4 = i + 3 < opts->rm_list.nr ? opts->rm_list.ids[i + 3] : TCPOPT_NOP;
                        put_unaligned_be32(id1 << 24 | id2 << 16 | id3 << 8 | id4, ptr);
                        ptr += 1;
                        i += 4;
                }
        }

        if (tp)
                mptcp_set_rwin(tp, th);
}

__be32 mptcp_get_reset_option(const struct sk_buff *skb)
{
        const struct mptcp_ext *ext = mptcp_get_ext(skb);
        u8 flags, reason;

        if (ext) {
                flags = ext->reset_transient;
                reason = ext->reset_reason;

                return mptcp_option(MPTCPOPT_RST, TCPOLEN_MPTCP_RST,
                                    flags, reason);
        }

        return htonl(0u);
}
EXPORT_SYMBOL_GPL(mptcp_get_reset_option);






























































   14 




   14 
   14 

   14 






























   14 





























    3 






    3 




















    3 




















    2 
















    2 
    3 





















    5 




















    4 
















    1 
    6 




















   14 





   10 


    3 



    3 

    3 






    3 




    1 





    3 


    3 
    1 














   14 


   11 
    3 

   11 









   13 




















    2 
    1 

    1 








    1 








    5 











    5 









    9 















    9 













    9 












































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * The AEGIS-128 Authenticated-Encryption Algorithm
 *
 * Copyright (c) 2017-2018 Ondrej Mosnacek <omosnacek@gmail.com>
 * Copyright (C) 2017-2018 Red Hat, Inc. All rights reserved.
 */

#include <crypto/algapi.h>
#include <crypto/internal/aead.h>
#include <crypto/internal/simd.h>
#include <crypto/internal/skcipher.h>
#include <crypto/scatterwalk.h>
#include <linux/err.h>
#include <linux/init.h>
#include <linux/jump_label.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/scatterlist.h>

#include <asm/simd.h>

#include "aegis.h"

#define AEGIS128_NONCE_SIZE 16
#define AEGIS128_STATE_BLOCKS 5
#define AEGIS128_KEY_SIZE 16
#define AEGIS128_MIN_AUTH_SIZE 8
#define AEGIS128_MAX_AUTH_SIZE 16

struct aegis_state {
        union aegis_block blocks[AEGIS128_STATE_BLOCKS];
};

struct aegis_ctx {
        union aegis_block key;
};

static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_simd);

static const union aegis_block crypto_aegis_const[2] = {
        { .words64 = {
                cpu_to_le64(U64_C(0x0d08050302010100)),
                cpu_to_le64(U64_C(0x6279e99059372215)),
        } },
        { .words64 = {
                cpu_to_le64(U64_C(0xf12fc26d55183ddb)),
                cpu_to_le64(U64_C(0xdd28b57342311120)),
        } },
};

static bool aegis128_do_simd(void)
{
#ifdef CONFIG_CRYPTO_AEGIS128_SIMD
        if (static_branch_likely(&have_simd))
                return crypto_simd_usable();
#endif
        return false;
}

static void crypto_aegis128_update(struct aegis_state *state)
{
        union aegis_block tmp;
        unsigned int i;

        tmp = state->blocks[AEGIS128_STATE_BLOCKS - 1];
        for (i = AEGIS128_STATE_BLOCKS - 1; i > 0; i--)
                crypto_aegis_aesenc(&state->blocks[i], &state->blocks[i - 1],
                                    &state->blocks[i]);
        crypto_aegis_aesenc(&state->blocks[0], &tmp, &state->blocks[0]);
}

static void crypto_aegis128_update_a(struct aegis_state *state,
                                     const union aegis_block *msg,
                                     bool do_simd)
{
        if (IS_ENABLED(CONFIG_CRYPTO_AEGIS128_SIMD) && do_simd) {
                crypto_aegis128_update_simd(state, msg);
                return;
        }

        crypto_aegis128_update(state);
        crypto_aegis_block_xor(&state->blocks[0], msg);
}

static void crypto_aegis128_update_u(struct aegis_state *state, const void *msg,
                                     bool do_simd)
{
        if (IS_ENABLED(CONFIG_CRYPTO_AEGIS128_SIMD) && do_simd) {
                crypto_aegis128_update_simd(state, msg);
                return;
        }

        crypto_aegis128_update(state);
        crypto_xor(state->blocks[0].bytes, msg, AEGIS_BLOCK_SIZE);
}

static void crypto_aegis128_init(struct aegis_state *state,
                                 const union aegis_block *key,
                                 const u8 *iv)
{
        union aegis_block key_iv;
        unsigned int i;

        key_iv = *key;
        crypto_xor(key_iv.bytes, iv, AEGIS_BLOCK_SIZE);

        state->blocks[0] = key_iv;
        state->blocks[1] = crypto_aegis_const[1];
        state->blocks[2] = crypto_aegis_const[0];
        state->blocks[3] = *key;
        state->blocks[4] = *key;

        crypto_aegis_block_xor(&state->blocks[3], &crypto_aegis_const[0]);
        crypto_aegis_block_xor(&state->blocks[4], &crypto_aegis_const[1]);

        for (i = 0; i < 5; i++) {
                crypto_aegis128_update_a(state, key, false);
                crypto_aegis128_update_a(state, &key_iv, false);
        }
}

static void crypto_aegis128_ad(struct aegis_state *state,
                               const u8 *src, unsigned int size,
                               bool do_simd)
{
        if (AEGIS_ALIGNED(src)) {
                const union aegis_block *src_blk =
                                (const union aegis_block *)src;

                while (size >= AEGIS_BLOCK_SIZE) {
                        crypto_aegis128_update_a(state, src_blk, do_simd);

                        size -= AEGIS_BLOCK_SIZE;
                        src_blk++;
                }
        } else {
                while (size >= AEGIS_BLOCK_SIZE) {
                        crypto_aegis128_update_u(state, src, do_simd);

                        size -= AEGIS_BLOCK_SIZE;
                        src += AEGIS_BLOCK_SIZE;
                }
        }
}

static void crypto_aegis128_wipe_chunk(struct aegis_state *state, u8 *dst,
                                       const u8 *src, unsigned int size)
{
        memzero_explicit(dst, size);
}

static void crypto_aegis128_encrypt_chunk(struct aegis_state *state, u8 *dst,
                                          const u8 *src, unsigned int size)
{
        union aegis_block tmp;

        if (AEGIS_ALIGNED(src) && AEGIS_ALIGNED(dst)) {
                while (size >= AEGIS_BLOCK_SIZE) {
                        union aegis_block *dst_blk =
                                        (union aegis_block *)dst;
                        const union aegis_block *src_blk =
                                        (const union aegis_block *)src;

                        tmp = state->blocks[2];
                        crypto_aegis_block_and(&tmp, &state->blocks[3]);
                        crypto_aegis_block_xor(&tmp, &state->blocks[4]);
                        crypto_aegis_block_xor(&tmp, &state->blocks[1]);
                        crypto_aegis_block_xor(&tmp, src_blk);

                        crypto_aegis128_update_a(state, src_blk, false);

                        *dst_blk = tmp;

                        size -= AEGIS_BLOCK_SIZE;
                        src += AEGIS_BLOCK_SIZE;
                        dst += AEGIS_BLOCK_SIZE;
                }
        } else {
                while (size >= AEGIS_BLOCK_SIZE) {
                        tmp = state->blocks[2];
                        crypto_aegis_block_and(&tmp, &state->blocks[3]);
                        crypto_aegis_block_xor(&tmp, &state->blocks[4]);
                        crypto_aegis_block_xor(&tmp, &state->blocks[1]);
                        crypto_xor(tmp.bytes, src, AEGIS_BLOCK_SIZE);

                        crypto_aegis128_update_u(state, src, false);

                        memcpy(dst, tmp.bytes, AEGIS_BLOCK_SIZE);

                        size -= AEGIS_BLOCK_SIZE;
                        src += AEGIS_BLOCK_SIZE;
                        dst += AEGIS_BLOCK_SIZE;
                }
        }

        if (size > 0) {
                union aegis_block msg = {};
                memcpy(msg.bytes, src, size);

                tmp = state->blocks[2];
                crypto_aegis_block_and(&tmp, &state->blocks[3]);
                crypto_aegis_block_xor(&tmp, &state->blocks[4]);
                crypto_aegis_block_xor(&tmp, &state->blocks[1]);

                crypto_aegis128_update_a(state, &msg, false);

                crypto_aegis_block_xor(&msg, &tmp);

                memcpy(dst, msg.bytes, size);
        }
}

static void crypto_aegis128_decrypt_chunk(struct aegis_state *state, u8 *dst,
                                          const u8 *src, unsigned int size)
{
        union aegis_block tmp;

        if (AEGIS_ALIGNED(src) && AEGIS_ALIGNED(dst)) {
                while (size >= AEGIS_BLOCK_SIZE) {
                        union aegis_block *dst_blk =
                                        (union aegis_block *)dst;
                        const union aegis_block *src_blk =
                                        (const union aegis_block *)src;

                        tmp = state->blocks[2];
                        crypto_aegis_block_and(&tmp, &state->blocks[3]);
                        crypto_aegis_block_xor(&tmp, &state->blocks[4]);
                        crypto_aegis_block_xor(&tmp, &state->blocks[1]);
                        crypto_aegis_block_xor(&tmp, src_blk);

                        crypto_aegis128_update_a(state, &tmp, false);

                        *dst_blk = tmp;

                        size -= AEGIS_BLOCK_SIZE;
                        src += AEGIS_BLOCK_SIZE;
                        dst += AEGIS_BLOCK_SIZE;
                }
        } else {
                while (size >= AEGIS_BLOCK_SIZE) {
                        tmp = state->blocks[2];
                        crypto_aegis_block_and(&tmp, &state->blocks[3]);
                        crypto_aegis_block_xor(&tmp, &state->blocks[4]);
                        crypto_aegis_block_xor(&tmp, &state->blocks[1]);
                        crypto_xor(tmp.bytes, src, AEGIS_BLOCK_SIZE);

                        crypto_aegis128_update_a(state, &tmp, false);

                        memcpy(dst, tmp.bytes, AEGIS_BLOCK_SIZE);

                        size -= AEGIS_BLOCK_SIZE;
                        src += AEGIS_BLOCK_SIZE;
                        dst += AEGIS_BLOCK_SIZE;
                }
        }

        if (size > 0) {
                union aegis_block msg = {};
                memcpy(msg.bytes, src, size);

                tmp = state->blocks[2];
                crypto_aegis_block_and(&tmp, &state->blocks[3]);
                crypto_aegis_block_xor(&tmp, &state->blocks[4]);
                crypto_aegis_block_xor(&tmp, &state->blocks[1]);
                crypto_aegis_block_xor(&msg, &tmp);

                memset(msg.bytes + size, 0, AEGIS_BLOCK_SIZE - size);

                crypto_aegis128_update_a(state, &msg, false);

                memcpy(dst, msg.bytes, size);
        }
}

static void crypto_aegis128_process_ad(struct aegis_state *state,
                                       struct scatterlist *sg_src,
                                       unsigned int assoclen,
                                       bool do_simd)
{
        struct scatter_walk walk;
        union aegis_block buf;
        unsigned int pos = 0;

        scatterwalk_start(&walk, sg_src);
        while (assoclen != 0) {
                unsigned int size = scatterwalk_clamp(&walk, assoclen);
                unsigned int left = size;
                void *mapped = scatterwalk_map(&walk);
                const u8 *src = (const u8 *)mapped;

                if (pos + size >= AEGIS_BLOCK_SIZE) {
                        if (pos > 0) {
                                unsigned int fill = AEGIS_BLOCK_SIZE - pos;
                                memcpy(buf.bytes + pos, src, fill);
                                crypto_aegis128_update_a(state, &buf, do_simd);
                                pos = 0;
                                left -= fill;
                                src += fill;
                        }

                        crypto_aegis128_ad(state, src, left, do_simd);
                        src += left & ~(AEGIS_BLOCK_SIZE - 1);
                        left &= AEGIS_BLOCK_SIZE - 1;
                }

                memcpy(buf.bytes + pos, src, left);

                pos += left;
                assoclen -= size;
                scatterwalk_unmap(mapped);
                scatterwalk_advance(&walk, size);
                scatterwalk_done(&walk, 0, assoclen);
        }

        if (pos > 0) {
                memset(buf.bytes + pos, 0, AEGIS_BLOCK_SIZE - pos);
                crypto_aegis128_update_a(state, &buf, do_simd);
        }
}

static __always_inline
int crypto_aegis128_process_crypt(struct aegis_state *state,
                                  struct skcipher_walk *walk,
                                  void (*crypt)(struct aegis_state *state,
                                                u8 *dst,
                                                const u8 *src,
                                                unsigned int size))
{
        int err = 0;

        while (walk->nbytes) {
                unsigned int nbytes = walk->nbytes;

                if (nbytes < walk->total)
                        nbytes = round_down(nbytes, walk->stride);

                crypt(state, walk->dst.virt.addr, walk->src.virt.addr, nbytes);

                err = skcipher_walk_done(walk, walk->nbytes - nbytes);
        }
        return err;
}

static void crypto_aegis128_final(struct aegis_state *state,
                                  union aegis_block *tag_xor,
                                  u64 assoclen, u64 cryptlen)
{
        u64 assocbits = assoclen * 8;
        u64 cryptbits = cryptlen * 8;

        union aegis_block tmp;
        unsigned int i;

        tmp.words64[0] = cpu_to_le64(assocbits);
        tmp.words64[1] = cpu_to_le64(cryptbits);

        crypto_aegis_block_xor(&tmp, &state->blocks[3]);

        for (i = 0; i < 7; i++)
                crypto_aegis128_update_a(state, &tmp, false);

        for (i = 0; i < AEGIS128_STATE_BLOCKS; i++)
                crypto_aegis_block_xor(tag_xor, &state->blocks[i]);
}

static int crypto_aegis128_setkey(struct crypto_aead *aead, const u8 *key,
                                  unsigned int keylen)
{
        struct aegis_ctx *ctx = crypto_aead_ctx(aead);

        if (keylen != AEGIS128_KEY_SIZE)
                return -EINVAL;

        memcpy(ctx->key.bytes, key, AEGIS128_KEY_SIZE);
        return 0;
}

static int crypto_aegis128_setauthsize(struct crypto_aead *tfm,
                                       unsigned int authsize)
{
        if (authsize > AEGIS128_MAX_AUTH_SIZE)
                return -EINVAL;
        if (authsize < AEGIS128_MIN_AUTH_SIZE)
                return -EINVAL;
        return 0;
}

static int crypto_aegis128_encrypt_generic(struct aead_request *req)
{
        struct crypto_aead *tfm = crypto_aead_reqtfm(req);
        union aegis_block tag = {};
        unsigned int authsize = crypto_aead_authsize(tfm);
        struct aegis_ctx *ctx = crypto_aead_ctx(tfm);
        unsigned int cryptlen = req->cryptlen;
        struct skcipher_walk walk;
        struct aegis_state state;

        skcipher_walk_aead_encrypt(&walk, req, false);
        crypto_aegis128_init(&state, &ctx->key, req->iv);
        crypto_aegis128_process_ad(&state, req->src, req->assoclen, false);
        crypto_aegis128_process_crypt(&state, &walk,
                                      crypto_aegis128_encrypt_chunk);
        crypto_aegis128_final(&state, &tag, req->assoclen, cryptlen);

        scatterwalk_map_and_copy(tag.bytes, req->dst, req->assoclen + cryptlen,
                                 authsize, 1);
        return 0;
}

static int crypto_aegis128_decrypt_generic(struct aead_request *req)
{
        static const u8 zeros[AEGIS128_MAX_AUTH_SIZE] = {};
        struct crypto_aead *tfm = crypto_aead_reqtfm(req);
        union aegis_block tag;
        unsigned int authsize = crypto_aead_authsize(tfm);
        unsigned int cryptlen = req->cryptlen - authsize;
        struct aegis_ctx *ctx = crypto_aead_ctx(tfm);
        struct skcipher_walk walk;
        struct aegis_state state;

        scatterwalk_map_and_copy(tag.bytes, req->src, req->assoclen + cryptlen,
                                 authsize, 0);

        skcipher_walk_aead_decrypt(&walk, req, false);
        crypto_aegis128_init(&state, &ctx->key, req->iv);
        crypto_aegis128_process_ad(&state, req->src, req->assoclen, false);
        crypto_aegis128_process_crypt(&state, &walk,
                                      crypto_aegis128_decrypt_chunk);
        crypto_aegis128_final(&state, &tag, req->assoclen, cryptlen);

        if (unlikely(crypto_memneq(tag.bytes, zeros, authsize))) {
                /*
                 * From Chapter 4. 'Security Analysis' of the AEGIS spec [0]
                 *
                 * "3. If verification fails, the decrypted plaintext and the
                 *     wrong authentication tag should not be given as output."
                 *
                 * [0] https://competitions.cr.yp.to/round3/aegisv11.pdf
                 */
                skcipher_walk_aead_decrypt(&walk, req, false);
                crypto_aegis128_process_crypt(NULL, &walk,
                                              crypto_aegis128_wipe_chunk);
                memzero_explicit(&tag, sizeof(tag));
                return -EBADMSG;
        }
        return 0;
}

static int crypto_aegis128_encrypt_simd(struct aead_request *req)
{
        struct crypto_aead *tfm = crypto_aead_reqtfm(req);
        union aegis_block tag = {};
        unsigned int authsize = crypto_aead_authsize(tfm);
        struct aegis_ctx *ctx = crypto_aead_ctx(tfm);
        unsigned int cryptlen = req->cryptlen;
        struct skcipher_walk walk;
        struct aegis_state state;

        if (!aegis128_do_simd())
                return crypto_aegis128_encrypt_generic(req);

        skcipher_walk_aead_encrypt(&walk, req, false);
        crypto_aegis128_init_simd(&state, &ctx->key, req->iv);
        crypto_aegis128_process_ad(&state, req->src, req->assoclen, true);
        crypto_aegis128_process_crypt(&state, &walk,
                                      crypto_aegis128_encrypt_chunk_simd);
        crypto_aegis128_final_simd(&state, &tag, req->assoclen, cryptlen, 0);

        scatterwalk_map_and_copy(tag.bytes, req->dst, req->assoclen + cryptlen,
                                 authsize, 1);
        return 0;
}

static int crypto_aegis128_decrypt_simd(struct aead_request *req)
{
        struct crypto_aead *tfm = crypto_aead_reqtfm(req);
        union aegis_block tag;
        unsigned int authsize = crypto_aead_authsize(tfm);
        unsigned int cryptlen = req->cryptlen - authsize;
        struct aegis_ctx *ctx = crypto_aead_ctx(tfm);
        struct skcipher_walk walk;
        struct aegis_state state;

        if (!aegis128_do_simd())
                return crypto_aegis128_decrypt_generic(req);

        scatterwalk_map_and_copy(tag.bytes, req->src, req->assoclen + cryptlen,
                                 authsize, 0);

        skcipher_walk_aead_decrypt(&walk, req, false);
        crypto_aegis128_init_simd(&state, &ctx->key, req->iv);
        crypto_aegis128_process_ad(&state, req->src, req->assoclen, true);
        crypto_aegis128_process_crypt(&state, &walk,
                                      crypto_aegis128_decrypt_chunk_simd);

        if (unlikely(crypto_aegis128_final_simd(&state, &tag, req->assoclen,
                                                cryptlen, authsize))) {
                skcipher_walk_aead_decrypt(&walk, req, false);
                crypto_aegis128_process_crypt(NULL, &walk,
                                              crypto_aegis128_wipe_chunk);
                return -EBADMSG;
        }
        return 0;
}

static struct aead_alg crypto_aegis128_alg_generic = {
        .setkey                        = crypto_aegis128_setkey,
        .setauthsize                = crypto_aegis128_setauthsize,
        .encrypt                = crypto_aegis128_encrypt_generic,
        .decrypt                = crypto_aegis128_decrypt_generic,

        .ivsize                        = AEGIS128_NONCE_SIZE,
        .maxauthsize                = AEGIS128_MAX_AUTH_SIZE,
        .chunksize                = AEGIS_BLOCK_SIZE,

        .base.cra_blocksize        = 1,
        .base.cra_ctxsize        = sizeof(struct aegis_ctx),
        .base.cra_alignmask        = 0,
        .base.cra_priority        = 100,
        .base.cra_name                = "aegis128",
        .base.cra_driver_name        = "aegis128-generic",
        .base.cra_module        = THIS_MODULE,
};

static struct aead_alg crypto_aegis128_alg_simd = {
        .setkey                        = crypto_aegis128_setkey,
        .setauthsize                = crypto_aegis128_setauthsize,
        .encrypt                = crypto_aegis128_encrypt_simd,
        .decrypt                = crypto_aegis128_decrypt_simd,

        .ivsize                        = AEGIS128_NONCE_SIZE,
        .maxauthsize                = AEGIS128_MAX_AUTH_SIZE,
        .chunksize                = AEGIS_BLOCK_SIZE,

        .base.cra_blocksize        = 1,
        .base.cra_ctxsize        = sizeof(struct aegis_ctx),
        .base.cra_alignmask        = 0,
        .base.cra_priority        = 200,
        .base.cra_name                = "aegis128",
        .base.cra_driver_name        = "aegis128-simd",
        .base.cra_module        = THIS_MODULE,
};

static int __init crypto_aegis128_module_init(void)
{
        int ret;

        ret = crypto_register_aead(&crypto_aegis128_alg_generic);
        if (ret)
                return ret;

        if (IS_ENABLED(CONFIG_CRYPTO_AEGIS128_SIMD) &&
            crypto_aegis128_have_simd()) {
                ret = crypto_register_aead(&crypto_aegis128_alg_simd);
                if (ret) {
                        crypto_unregister_aead(&crypto_aegis128_alg_generic);
                        return ret;
                }
                static_branch_enable(&have_simd);
        }
        return 0;
}

static void __exit crypto_aegis128_module_exit(void)
{
        if (IS_ENABLED(CONFIG_CRYPTO_AEGIS128_SIMD) &&
            crypto_aegis128_have_simd())
                crypto_unregister_aead(&crypto_aegis128_alg_simd);

        crypto_unregister_aead(&crypto_aegis128_alg_generic);
}

subsys_initcall(crypto_aegis128_module_init);
module_exit(crypto_aegis128_module_exit);

MODULE_LICENSE("GPL");
MODULE_AUTHOR("Ondrej Mosnacek <omosnacek@gmail.com>");
MODULE_DESCRIPTION("AEGIS-128 AEAD algorithm");
MODULE_ALIAS_CRYPTO("aegis128");
MODULE_ALIAS_CRYPTO("aegis128-generic");
MODULE_ALIAS_CRYPTO("aegis128-simd");





















   14 




   12 



























































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
// SPDX-License-Identifier: GPL-2.0-or-later
/* Null security operations.
 *
 * Copyright (C) 2016 Red Hat, Inc. All Rights Reserved.
 * Written by David Howells (dhowells@redhat.com)
 */

#include <net/af_rxrpc.h>
#include "ar-internal.h"

static int none_init_connection_security(struct rxrpc_connection *conn,
                                         struct rxrpc_key_token *token)
{
        return 0;
}

/*
 * Allocate an appropriately sized buffer for the amount of data remaining.
 */
static struct rxrpc_txbuf *none_alloc_txbuf(struct rxrpc_call *call, size_t remain, gfp_t gfp)
{
        return rxrpc_alloc_data_txbuf(call, min_t(size_t, remain, RXRPC_JUMBO_DATALEN), 1, gfp);
}

static int none_secure_packet(struct rxrpc_call *call, struct rxrpc_txbuf *txb)
{
        return 0;
}

static int none_verify_packet(struct rxrpc_call *call, struct sk_buff *skb)
{
        struct rxrpc_skb_priv *sp = rxrpc_skb(skb);

        sp->flags |= RXRPC_RX_VERIFIED;
        return 0;
}

static void none_free_call_crypto(struct rxrpc_call *call)
{
}

static int none_respond_to_challenge(struct rxrpc_connection *conn,
                                     struct sk_buff *skb)
{
        return rxrpc_abort_conn(conn, skb, RX_PROTOCOL_ERROR, -EPROTO,
                                rxrpc_eproto_rxnull_challenge);
}

static int none_verify_response(struct rxrpc_connection *conn,
                                struct sk_buff *skb)
{
        return rxrpc_abort_conn(conn, skb, RX_PROTOCOL_ERROR, -EPROTO,
                                rxrpc_eproto_rxnull_response);
}

static void none_clear(struct rxrpc_connection *conn)
{
}

static int none_init(void)
{
        return 0;
}

static void none_exit(void)
{
}

/*
 * RxRPC Kerberos-based security
 */
const struct rxrpc_security rxrpc_no_security = {
        .name                                = "none",
        .security_index                        = RXRPC_SECURITY_NONE,
        .init                                = none_init,
        .exit                                = none_exit,
        .init_connection_security        = none_init_connection_security,
        .free_call_crypto                = none_free_call_crypto,
        .alloc_txbuf                        = none_alloc_txbuf,
        .secure_packet                        = none_secure_packet,
        .verify_packet                        = none_verify_packet,
        .respond_to_challenge                = none_respond_to_challenge,
        .verify_response                = none_verify_response,
        .clear                                = none_clear,
};









































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
/* SPDX-License-Identifier: GPL-2.0 */
/*
 * Shared Memory Communications over RDMA (SMC-R) and RoCE
 *
 * Manage send buffer
 *
 * Copyright IBM Corp. 2016
 *
 * Author(s):  Ursula Braun <ubraun@linux.vnet.ibm.com>
 */

#ifndef SMC_TX_H
#define SMC_TX_H

#include <linux/socket.h>
#include <linux/types.h>

#include "smc.h"
#include "smc_cdc.h"

static inline int smc_tx_prepared_sends(struct smc_connection *conn)
{
        union smc_host_cursor sent, prep;

        smc_curs_copy(&sent, &conn->tx_curs_sent, conn);
        smc_curs_copy(&prep, &conn->tx_curs_prep, conn);
        return smc_curs_diff(conn->sndbuf_desc->len, &sent, &prep);
}

void smc_tx_pending(struct smc_connection *conn);
void smc_tx_work(struct work_struct *work);
void smc_tx_init(struct smc_sock *smc);
int smc_tx_sendmsg(struct smc_sock *smc, struct msghdr *msg, size_t len);
int smc_tx_sndbuf_nonempty(struct smc_connection *conn);
void smc_tx_sndbuf_nonfull(struct smc_sock *smc);
void smc_tx_consumer_update(struct smc_connection *conn, bool force);
int smcd_tx_ism_write(struct smc_connection *conn, void *data, size_t len,
                      u32 offset, int signal);

#endif /* SMC_TX_H */























































   29 
   29 
































































































   48 




   19 





































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 *        Bridge per vlan tunnel port dst_metadata handling code
 *
 *        Authors:
 *        Roopa Prabhu                <roopa@cumulusnetworks.com>
 */

#include <linux/kernel.h>
#include <linux/netdevice.h>
#include <linux/rtnetlink.h>
#include <linux/slab.h>
#include <net/switchdev.h>
#include <net/dst_metadata.h>

#include "br_private.h"
#include "br_private_tunnel.h"

static inline int br_vlan_tunid_cmp(struct rhashtable_compare_arg *arg,
                                    const void *ptr)
{
        const struct net_bridge_vlan *vle = ptr;
        __be64 tunid = *(__be64 *)arg->key;

        return vle->tinfo.tunnel_id != tunid;
}

static const struct rhashtable_params br_vlan_tunnel_rht_params = {
        .head_offset = offsetof(struct net_bridge_vlan, tnode),
        .key_offset = offsetof(struct net_bridge_vlan, tinfo.tunnel_id),
        .key_len = sizeof(__be64),
        .nelem_hint = 3,
        .obj_cmpfn = br_vlan_tunid_cmp,
        .automatic_shrinking = true,
};

static struct net_bridge_vlan *br_vlan_tunnel_lookup(struct rhashtable *tbl,
                                                     __be64 tunnel_id)
{
        return rhashtable_lookup_fast(tbl, &tunnel_id,
                                      br_vlan_tunnel_rht_params);
}

static void vlan_tunnel_info_release(struct net_bridge_vlan *vlan)
{
        struct metadata_dst *tdst = rtnl_dereference(vlan->tinfo.tunnel_dst);

        WRITE_ONCE(vlan->tinfo.tunnel_id, 0);
        RCU_INIT_POINTER(vlan->tinfo.tunnel_dst, NULL);
        dst_release(&tdst->dst);
}

void vlan_tunnel_info_del(struct net_bridge_vlan_group *vg,
                          struct net_bridge_vlan *vlan)
{
        if (!rcu_access_pointer(vlan->tinfo.tunnel_dst))
                return;
        rhashtable_remove_fast(&vg->tunnel_hash, &vlan->tnode,
                               br_vlan_tunnel_rht_params);
        vlan_tunnel_info_release(vlan);
}

static int __vlan_tunnel_info_add(struct net_bridge_vlan_group *vg,
                                  struct net_bridge_vlan *vlan, u32 tun_id)
{
        struct metadata_dst *metadata = rtnl_dereference(vlan->tinfo.tunnel_dst);
        __be64 key = key32_to_tunnel_id(cpu_to_be32(tun_id));
        IP_TUNNEL_DECLARE_FLAGS(flags) = { };
        int err;

        if (metadata)
                return -EEXIST;

        __set_bit(IP_TUNNEL_KEY_BIT, flags);
        metadata = __ip_tun_set_dst(0, 0, 0, 0, 0, flags, key, 0);
        if (!metadata)
                return -EINVAL;

        metadata->u.tun_info.mode |= IP_TUNNEL_INFO_TX | IP_TUNNEL_INFO_BRIDGE;
        rcu_assign_pointer(vlan->tinfo.tunnel_dst, metadata);
        WRITE_ONCE(vlan->tinfo.tunnel_id, key);

        err = rhashtable_lookup_insert_fast(&vg->tunnel_hash, &vlan->tnode,
                                            br_vlan_tunnel_rht_params);
        if (err)
                goto out;

        return 0;
out:
        vlan_tunnel_info_release(vlan);

        return err;
}

/* Must be protected by RTNL.
 * Must be called with vid in range from 1 to 4094 inclusive.
 */
int nbp_vlan_tunnel_info_add(const struct net_bridge_port *port, u16 vid,
                             u32 tun_id)
{
        struct net_bridge_vlan_group *vg;
        struct net_bridge_vlan *vlan;

        ASSERT_RTNL();

        vg = nbp_vlan_group(port);
        vlan = br_vlan_find(vg, vid);
        if (!vlan)
                return -EINVAL;

        return __vlan_tunnel_info_add(vg, vlan, tun_id);
}

/* Must be protected by RTNL.
 * Must be called with vid in range from 1 to 4094 inclusive.
 */
int nbp_vlan_tunnel_info_delete(const struct net_bridge_port *port, u16 vid)
{
        struct net_bridge_vlan_group *vg;
        struct net_bridge_vlan *v;

        ASSERT_RTNL();

        vg = nbp_vlan_group(port);
        v = br_vlan_find(vg, vid);
        if (!v)
                return -ENOENT;

        vlan_tunnel_info_del(vg, v);

        return 0;
}

static void __vlan_tunnel_info_flush(struct net_bridge_vlan_group *vg)
{
        struct net_bridge_vlan *vlan, *tmp;

        list_for_each_entry_safe(vlan, tmp, &vg->vlan_list, vlist)
                vlan_tunnel_info_del(vg, vlan);
}

void nbp_vlan_tunnel_info_flush(struct net_bridge_port *port)
{
        struct net_bridge_vlan_group *vg;

        ASSERT_RTNL();

        vg = nbp_vlan_group(port);
        __vlan_tunnel_info_flush(vg);
}

int vlan_tunnel_init(struct net_bridge_vlan_group *vg)
{
        return rhashtable_init(&vg->tunnel_hash, &br_vlan_tunnel_rht_params);
}

void vlan_tunnel_deinit(struct net_bridge_vlan_group *vg)
{
        rhashtable_destroy(&vg->tunnel_hash);
}

void br_handle_ingress_vlan_tunnel(struct sk_buff *skb,
                                   struct net_bridge_port *p,
                                   struct net_bridge_vlan_group *vg)
{
        struct ip_tunnel_info *tinfo = skb_tunnel_info(skb);
        struct net_bridge_vlan *vlan;

        if (!vg || !tinfo)
                return;

        /* if already tagged, ignore */
        if (skb_vlan_tagged(skb))
                return;

        /* lookup vid, given tunnel id */
        vlan = br_vlan_tunnel_lookup(&vg->tunnel_hash, tinfo->key.tun_id);
        if (!vlan)
                return;

        skb_dst_drop(skb);

        __vlan_hwaccel_put_tag(skb, p->br->vlan_proto, vlan->vid);
}

int br_handle_egress_vlan_tunnel(struct sk_buff *skb,
                                 struct net_bridge_vlan *vlan)
{
        IP_TUNNEL_DECLARE_FLAGS(flags) = { };
        struct metadata_dst *tunnel_dst;
        __be64 tunnel_id;
        int err;

        if (!vlan)
                return 0;

        tunnel_id = READ_ONCE(vlan->tinfo.tunnel_id);
        if (!tunnel_id || unlikely(!skb_vlan_tag_present(skb)))
                return 0;

        skb_dst_drop(skb);
        err = skb_vlan_pop(skb);
        if (err)
                return err;

        if (BR_INPUT_SKB_CB(skb)->backup_nhid) {
                __set_bit(IP_TUNNEL_KEY_BIT, flags);
                tunnel_dst = __ip_tun_set_dst(0, 0, 0, 0, 0, flags,
                                              tunnel_id, 0);
                if (!tunnel_dst)
                        return -ENOMEM;

                tunnel_dst->u.tun_info.mode |= IP_TUNNEL_INFO_TX |
                                               IP_TUNNEL_INFO_BRIDGE;
                tunnel_dst->u.tun_info.key.nhid =
                        BR_INPUT_SKB_CB(skb)->backup_nhid;
                skb_dst_set(skb, &tunnel_dst->dst);

                return 0;
        }

        tunnel_dst = rcu_dereference(vlan->tinfo.tunnel_dst);
        if (tunnel_dst && dst_hold_safe(&tunnel_dst->dst))
                skb_dst_set(skb, &tunnel_dst->dst);

        return 0;
}


















































































































































































    1 





































































































    3 















































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
/* SPDX-License-Identifier: GPL-2.0 */
/*
 *        Declarations of AX.25 type objects.
 *
 *        Alan Cox (GW4PTS)         10/11/93
 */
#ifndef _AX25_H
#define _AX25_H 

#include <linux/ax25.h>
#include <linux/spinlock.h>
#include <linux/timer.h>
#include <linux/list.h>
#include <linux/slab.h>
#include <linux/refcount.h>
#include <net/neighbour.h>
#include <net/sock.h>
#include <linux/seq_file.h>

#define        AX25_T1CLAMPLO                  1
#define        AX25_T1CLAMPHI                         (30 * HZ)

#define        AX25_BPQ_HEADER_LEN                16
#define        AX25_KISS_HEADER_LEN                1

#define        AX25_HEADER_LEN                        17
#define        AX25_ADDR_LEN                        7
#define        AX25_DIGI_HEADER_LEN                (AX25_MAX_DIGIS * AX25_ADDR_LEN)
#define        AX25_MAX_HEADER_LEN                (AX25_HEADER_LEN + AX25_DIGI_HEADER_LEN)

/* AX.25 Protocol IDs */
#define AX25_P_ROSE                        0x01
#define AX25_P_VJCOMP                        0x06        /* Compressed TCP/IP packet   */
                                                /* Van Jacobsen (RFC 1144)    */
#define AX25_P_VJUNCOMP                        0x07        /* Uncompressed TCP/IP packet */
                                                /* Van Jacobsen (RFC 1144)    */
#define        AX25_P_SEGMENT                        0x08        /* Segmentation fragment      */
#define AX25_P_TEXNET                        0xc3        /* TEXTNET datagram protocol  */
#define AX25_P_LQ                        0xc4        /* Link Quality Protocol      */
#define AX25_P_ATALK                        0xca        /* Appletalk                  */
#define AX25_P_ATALK_ARP                0xcb        /* Appletalk ARP              */
#define AX25_P_IP                        0xcc        /* ARPA Internet Protocol     */
#define AX25_P_ARP                        0xcd        /* ARPA Address Resolution    */
#define AX25_P_FLEXNET                        0xce        /* FlexNet                    */
#define AX25_P_NETROM                         0xcf        /* NET/ROM                    */
#define AX25_P_TEXT                         0xF0        /* No layer 3 protocol impl.  */

/* AX.25 Segment control values */
#define        AX25_SEG_REM                        0x7F
#define        AX25_SEG_FIRST                        0x80

#define AX25_CBIT                        0x80        /* Command/Response bit */
#define AX25_EBIT                        0x01        /* HDLC Address Extension bit */
#define AX25_HBIT                        0x80        /* Has been repeated bit */

#define AX25_SSSID_SPARE                0x60        /* Unused bits in SSID for standard AX.25 */
#define AX25_ESSID_SPARE                0x20        /* Unused bits in SSID for extended AX.25 */
#define AX25_DAMA_FLAG                        0x20        /* Well, it is *NOT* unused! (dl1bke 951121 */

#define        AX25_COND_ACK_PENDING                0x01
#define        AX25_COND_REJECT                0x02
#define        AX25_COND_PEER_RX_BUSY                0x04
#define        AX25_COND_OWN_RX_BUSY                0x08
#define        AX25_COND_DAMA_MODE                0x10

#ifndef _LINUX_NETDEVICE_H
#include <linux/netdevice.h>
#endif

/* Upper sub-layer (LAPB) definitions */

/* Control field templates */
#define        AX25_I                        0x00        /* Information frames */
#define        AX25_S                        0x01        /* Supervisory frames */
#define        AX25_RR                        0x01        /* Receiver ready */
#define        AX25_RNR                0x05        /* Receiver not ready */
#define        AX25_REJ                0x09        /* Reject */
#define        AX25_U                        0x03        /* Unnumbered frames */
#define        AX25_SABM                0x2f        /* Set Asynchronous Balanced Mode */
#define        AX25_SABME                0x6f        /* Set Asynchronous Balanced Mode Extended */
#define        AX25_DISC                0x43        /* Disconnect */
#define        AX25_DM                        0x0f        /* Disconnected mode */
#define        AX25_UA                        0x63        /* Unnumbered acknowledge */
#define        AX25_FRMR                0x87        /* Frame reject */
#define        AX25_UI                        0x03        /* Unnumbered information */
#define        AX25_XID                0xaf        /* Exchange information */
#define        AX25_TEST                0xe3        /* Test */

#define        AX25_PF                        0x10        /* Poll/final bit for standard AX.25 */
#define        AX25_EPF                0x01        /* Poll/final bit for extended AX.25 */

#define AX25_ILLEGAL                0x100        /* Impossible to be a real frame type */

#define        AX25_POLLOFF                0
#define        AX25_POLLON                1

/* AX25 L2 C-bit */
#define AX25_COMMAND                1
#define AX25_RESPONSE                2

/* Define Link State constants. */

enum { 
        AX25_STATE_0,                        /* Listening */
        AX25_STATE_1,                        /* SABM sent */
        AX25_STATE_2,                        /* DISC sent */
        AX25_STATE_3,                        /* Established */
        AX25_STATE_4                        /* Recovery */
};

#define AX25_MODULUS                 8        /*  Standard AX.25 modulus */
#define        AX25_EMODULUS                128        /*  Extended AX.25 modulus */

enum {
        AX25_PROTO_STD_SIMPLEX,
        AX25_PROTO_STD_DUPLEX,
#ifdef CONFIG_AX25_DAMA_SLAVE
        AX25_PROTO_DAMA_SLAVE,
#ifdef CONFIG_AX25_DAMA_MASTER
        AX25_PROTO_DAMA_MASTER,
#define AX25_PROTO_MAX AX25_PROTO_DAMA_MASTER
#endif
#endif
        __AX25_PROTO_MAX,
        AX25_PROTO_MAX = __AX25_PROTO_MAX -1
};

enum {
        AX25_VALUES_IPDEFMODE,        /* 0=DG 1=VC */
        AX25_VALUES_AXDEFMODE,        /* 0=Normal 1=Extended Seq Nos */
        AX25_VALUES_BACKOFF,        /* 0=None 1=Linear 2=Exponential */
        AX25_VALUES_CONMODE,        /* Allow connected modes - 0=No 1=no "PID text" 2=all PIDs */
        AX25_VALUES_WINDOW,        /* Default window size for standard AX.25 */
        AX25_VALUES_EWINDOW,        /* Default window size for extended AX.25 */
        AX25_VALUES_T1,                /* Default T1 timeout value */
        AX25_VALUES_T2,                /* Default T2 timeout value */
        AX25_VALUES_T3,                /* Default T3 timeout value */
        AX25_VALUES_IDLE,        /* Connected mode idle timer */
        AX25_VALUES_N2,                /* Default N2 value */
        AX25_VALUES_PACLEN,        /* AX.25 MTU */
        AX25_VALUES_PROTOCOL,        /* Std AX.25, DAMA Slave, DAMA Master */
#ifdef CONFIG_AX25_DAMA_SLAVE
        AX25_VALUES_DS_TIMEOUT,        /* DAMA Slave timeout */
#endif
        AX25_MAX_VALUES                /* THIS MUST REMAIN THE LAST ENTRY OF THIS LIST */
};

#define        AX25_DEF_IPDEFMODE        0                        /* Datagram */
#define        AX25_DEF_AXDEFMODE        0                        /* Normal */
#define        AX25_DEF_BACKOFF        1                        /* Linear backoff */
#define        AX25_DEF_CONMODE        2                        /* Connected mode allowed */
#define        AX25_DEF_WINDOW                2                        /* Window=2 */
#define        AX25_DEF_EWINDOW        32                        /* Module-128 Window=32 */
#define        AX25_DEF_T1                10000                        /* T1=10s */
#define        AX25_DEF_T2                3000                        /* T2=3s  */
#define        AX25_DEF_T3                300000                        /* T3=300s */
#define        AX25_DEF_N2                10                        /* N2=10 */
#define AX25_DEF_IDLE                0                        /* Idle=None */
#define AX25_DEF_PACLEN                256                        /* Paclen=256 */
#define        AX25_DEF_PROTOCOL        AX25_PROTO_STD_SIMPLEX        /* Standard AX.25 */
#define AX25_DEF_DS_TIMEOUT        180000                        /* DAMA timeout 3 minutes */

typedef struct ax25_uid_assoc {
        struct hlist_node        uid_node;
        refcount_t                refcount;
        kuid_t                        uid;
        ax25_address                call;
} ax25_uid_assoc;

#define ax25_uid_for_each(__ax25, list) \
        hlist_for_each_entry(__ax25, list, uid_node)

#define ax25_uid_hold(ax25) \
        refcount_inc(&((ax25)->refcount))

static inline void ax25_uid_put(ax25_uid_assoc *assoc)
{
        if (refcount_dec_and_test(&assoc->refcount)) {
                kfree(assoc);
        }
}

typedef struct {
        ax25_address                calls[AX25_MAX_DIGIS];
        unsigned char                repeated[AX25_MAX_DIGIS];
        unsigned char                ndigi;
        signed char                lastrepeat;
} ax25_digi;

typedef struct ax25_route {
        struct ax25_route        *next;
        ax25_address                callsign;
        struct net_device        *dev;
        ax25_digi                *digipeat;
        char                        ip_mode;
} ax25_route;

void __ax25_put_route(ax25_route *ax25_rt);

extern rwlock_t ax25_route_lock;

static inline void ax25_route_lock_use(void)
{
        read_lock(&ax25_route_lock);
}

static inline void ax25_route_lock_unuse(void)
{
        read_unlock(&ax25_route_lock);
}

typedef struct {
        char                        slave;                        /* slave_mode?   */
        struct timer_list        slave_timer;                /* timeout timer */
        unsigned short                slave_timeout;                /* when? */
} ax25_dama_info;

struct ctl_table;

typedef struct ax25_dev {
        struct list_head        list;

        struct net_device        *dev;
        netdevice_tracker        dev_tracker;

        struct net_device        *forward;
        struct ctl_table_header *sysheader;
        int                        values[AX25_MAX_VALUES];
#if defined(CONFIG_AX25_DAMA_SLAVE) || defined(CONFIG_AX25_DAMA_MASTER)
        ax25_dama_info                dama;
#endif
        refcount_t                refcount;
        bool device_up;
} ax25_dev;

typedef struct ax25_cb {
        struct hlist_node        ax25_node;
        ax25_address                source_addr, dest_addr;
        ax25_digi                *digipeat;
        ax25_dev                *ax25_dev;
        netdevice_tracker        dev_tracker;
        unsigned char                iamdigi;
        unsigned char                state, modulus, pidincl;
        unsigned short                vs, vr, va;
        unsigned char                condition, backoff;
        unsigned char                n2, n2count;
        struct timer_list        t1timer, t2timer, t3timer, idletimer;
        unsigned long                t1, t2, t3, idle, rtt;
        unsigned short                paclen, fragno, fraglen;
        struct sk_buff_head        write_queue;
        struct sk_buff_head        reseq_queue;
        struct sk_buff_head        ack_queue;
        struct sk_buff_head        frag_queue;
        unsigned char                window;
        struct timer_list        timer, dtimer;
        struct sock                *sk;                /* Backlink to socket */
        refcount_t                refcount;
} ax25_cb;

struct ax25_sock {
        struct sock                sk;
        struct ax25_cb                *cb;
};

#define ax25_sk(ptr) container_of_const(ptr, struct ax25_sock, sk)

static inline struct ax25_cb *sk_to_ax25(const struct sock *sk)
{
        return ax25_sk(sk)->cb;
}

#define ax25_for_each(__ax25, list) \
        hlist_for_each_entry(__ax25, list, ax25_node)

#define ax25_cb_hold(__ax25) \
        refcount_inc(&((__ax25)->refcount))

static __inline__ void ax25_cb_put(ax25_cb *ax25)
{
        if (refcount_dec_and_test(&ax25->refcount)) {
                kfree(ax25->digipeat);
                kfree(ax25);
        }
}

static inline void ax25_dev_hold(ax25_dev *ax25_dev)
{
        refcount_inc(&ax25_dev->refcount);
}

static inline void ax25_dev_put(ax25_dev *ax25_dev)
{
        if (refcount_dec_and_test(&ax25_dev->refcount)) {
                kfree(ax25_dev);
        }
}
static inline __be16 ax25_type_trans(struct sk_buff *skb, struct net_device *dev)
{
        skb->dev      = dev;
        skb_reset_mac_header(skb);
        skb->pkt_type = PACKET_HOST;
        return htons(ETH_P_AX25);
}

/* af_ax25.c */
extern struct hlist_head ax25_list;
extern spinlock_t ax25_list_lock;
void ax25_cb_add(ax25_cb *);
struct sock *ax25_find_listener(ax25_address *, int, struct net_device *, int);
struct sock *ax25_get_socket(ax25_address *, ax25_address *, int);
ax25_cb *ax25_find_cb(const ax25_address *, ax25_address *, ax25_digi *,
                      struct net_device *);
void ax25_send_to_raw(ax25_address *, struct sk_buff *, int);
void ax25_destroy_socket(ax25_cb *);
ax25_cb * __must_check ax25_create_cb(void);
void ax25_fillin_cb(ax25_cb *, ax25_dev *);
struct sock *ax25_make_new(struct sock *, struct ax25_dev *);

/* ax25_addr.c */
extern const ax25_address ax25_bcast;
extern const ax25_address ax25_defaddr;
extern const ax25_address null_ax25_address;
char *ax2asc(char *buf, const ax25_address *);
void asc2ax(ax25_address *addr, const char *callsign);
int ax25cmp(const ax25_address *, const ax25_address *);
int ax25digicmp(const ax25_digi *, const ax25_digi *);
const unsigned char *ax25_addr_parse(const unsigned char *, int,
        ax25_address *, ax25_address *, ax25_digi *, int *, int *);
int ax25_addr_build(unsigned char *, const ax25_address *,
                    const ax25_address *, const ax25_digi *, int, int);
int ax25_addr_size(const ax25_digi *);
void ax25_digi_invert(const ax25_digi *, ax25_digi *);

/* ax25_dev.c */
extern spinlock_t ax25_dev_lock;

#if IS_ENABLED(CONFIG_AX25)
static inline ax25_dev *ax25_dev_ax25dev(struct net_device *dev)
{
        return dev->ax25_ptr;
}
#endif

ax25_dev *ax25_addr_ax25dev(ax25_address *);
void ax25_dev_device_up(struct net_device *);
void ax25_dev_device_down(struct net_device *);
int ax25_fwd_ioctl(unsigned int, struct ax25_fwd_struct *);
struct net_device *ax25_fwd_dev(struct net_device *);
void ax25_dev_free(void);

/* ax25_ds_in.c */
int ax25_ds_frame_in(ax25_cb *, struct sk_buff *, int);

/* ax25_ds_subr.c */
void ax25_ds_nr_error_recovery(ax25_cb *);
void ax25_ds_enquiry_response(ax25_cb *);
void ax25_ds_establish_data_link(ax25_cb *);
void ax25_dev_dama_off(ax25_dev *);
void ax25_dama_on(ax25_cb *);
void ax25_dama_off(ax25_cb *);

/* ax25_ds_timer.c */
void ax25_ds_setup_timer(ax25_dev *);
void ax25_ds_set_timer(ax25_dev *);
void ax25_ds_del_timer(ax25_dev *);
void ax25_ds_timer(ax25_cb *);
void ax25_ds_t1_timeout(ax25_cb *);
void ax25_ds_heartbeat_expiry(ax25_cb *);
void ax25_ds_t3timer_expiry(ax25_cb *);
void ax25_ds_idletimer_expiry(ax25_cb *);

/* ax25_iface.c */

struct ax25_protocol {
        struct ax25_protocol *next;
        unsigned int pid;
        int (*func)(struct sk_buff *, ax25_cb *);
};

void ax25_register_pid(struct ax25_protocol *ap);
void ax25_protocol_release(unsigned int);

struct ax25_linkfail {
        struct hlist_node lf_node;
        void (*func)(ax25_cb *, int);
};

void ax25_linkfail_register(struct ax25_linkfail *lf);
void ax25_linkfail_release(struct ax25_linkfail *lf);
int __must_check ax25_listen_register(const ax25_address *,
                                      struct net_device *);
void ax25_listen_release(const ax25_address *, struct net_device *);
int(*ax25_protocol_function(unsigned int))(struct sk_buff *, ax25_cb *);
int ax25_listen_mine(const ax25_address *, struct net_device *);
void ax25_link_failed(ax25_cb *, int);
int ax25_protocol_is_registered(unsigned int);

/* ax25_in.c */
int ax25_rx_iframe(ax25_cb *, struct sk_buff *);
int ax25_kiss_rcv(struct sk_buff *, struct net_device *, struct packet_type *,
                  struct net_device *);

/* ax25_ip.c */
netdev_tx_t ax25_ip_xmit(struct sk_buff *skb);
extern const struct header_ops ax25_header_ops;

/* ax25_out.c */
ax25_cb *ax25_send_frame(struct sk_buff *, int, const ax25_address *,
                         ax25_address *, ax25_digi *, struct net_device *);
void ax25_output(ax25_cb *, int, struct sk_buff *);
void ax25_kick(ax25_cb *);
void ax25_transmit_buffer(ax25_cb *, struct sk_buff *, int);
void ax25_queue_xmit(struct sk_buff *skb, struct net_device *dev);
int ax25_check_iframes_acked(ax25_cb *, unsigned short);

/* ax25_route.c */
void ax25_rt_device_down(struct net_device *);
int ax25_rt_ioctl(unsigned int, void __user *);
extern const struct seq_operations ax25_rt_seqops;
ax25_route *ax25_get_route(ax25_address *addr, struct net_device *dev);
int ax25_rt_autobind(ax25_cb *, ax25_address *);
struct sk_buff *ax25_rt_build_path(struct sk_buff *, ax25_address *,
                                   ax25_address *, ax25_digi *);
void ax25_rt_free(void);

/* ax25_std_in.c */
int ax25_std_frame_in(ax25_cb *, struct sk_buff *, int);

/* ax25_std_subr.c */
void ax25_std_nr_error_recovery(ax25_cb *);
void ax25_std_establish_data_link(ax25_cb *);
void ax25_std_transmit_enquiry(ax25_cb *);
void ax25_std_enquiry_response(ax25_cb *);
void ax25_std_timeout_response(ax25_cb *);

/* ax25_std_timer.c */
void ax25_std_heartbeat_expiry(ax25_cb *);
void ax25_std_t1timer_expiry(ax25_cb *);
void ax25_std_t2timer_expiry(ax25_cb *);
void ax25_std_t3timer_expiry(ax25_cb *);
void ax25_std_idletimer_expiry(ax25_cb *);

/* ax25_subr.c */
void ax25_clear_queues(ax25_cb *);
void ax25_frames_acked(ax25_cb *, unsigned short);
void ax25_requeue_frames(ax25_cb *);
int ax25_validate_nr(ax25_cb *, unsigned short);
int ax25_decode(ax25_cb *, struct sk_buff *, int *, int *, int *);
void ax25_send_control(ax25_cb *, int, int, int);
void ax25_return_dm(struct net_device *, ax25_address *, ax25_address *,
                    ax25_digi *);
void ax25_calculate_t1(ax25_cb *);
void ax25_calculate_rtt(ax25_cb *);
void ax25_disconnect(ax25_cb *, int);

/* ax25_timer.c */
void ax25_setup_timers(ax25_cb *);
void ax25_start_heartbeat(ax25_cb *);
void ax25_start_t1timer(ax25_cb *);
void ax25_start_t2timer(ax25_cb *);
void ax25_start_t3timer(ax25_cb *);
void ax25_start_idletimer(ax25_cb *);
void ax25_stop_heartbeat(ax25_cb *);
void ax25_stop_t1timer(ax25_cb *);
void ax25_stop_t2timer(ax25_cb *);
void ax25_stop_t3timer(ax25_cb *);
void ax25_stop_idletimer(ax25_cb *);
int ax25_t1timer_running(ax25_cb *);
unsigned long ax25_display_timer(struct timer_list *);

/* ax25_uid.c */
extern int  ax25_uid_policy;
ax25_uid_assoc *ax25_findbyuid(kuid_t);
int __must_check ax25_uid_ioctl(int, struct sockaddr_ax25 *);
extern const struct seq_operations ax25_uid_seqops;
void ax25_uid_free(void);

/* sysctl_net_ax25.c */
#ifdef CONFIG_SYSCTL
int ax25_register_dev_sysctl(ax25_dev *ax25_dev);
void ax25_unregister_dev_sysctl(ax25_dev *ax25_dev);
#else
static inline int ax25_register_dev_sysctl(ax25_dev *ax25_dev) { return 0; }
static inline void ax25_unregister_dev_sysctl(ax25_dev *ax25_dev) {}
#endif /* CONFIG_SYSCTL */

#endif


















    3 
    3 







    1 





    2 





    2 
    2 
    2 










    2 

    2 












    2 







    1 
    1 






    2 
    2 
    2 
    2 

    2 




























    2 






   10 









    1 






    9 









    1 







    7 





    1 











    5 






    4 
    2 


    1 



    5 
































    5 





    4 
























































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 *        IPV4 GSO/GRO offload support
 *        Linux INET implementation
 *
 *        GRE GSO support
 */

#include <linux/skbuff.h>
#include <linux/init.h>
#include <net/protocol.h>
#include <net/gre.h>
#include <net/gro.h>
#include <net/gso.h>

static struct sk_buff *gre_gso_segment(struct sk_buff *skb,
                                       netdev_features_t features)
{
        int tnl_hlen = skb_inner_mac_header(skb) - skb_transport_header(skb);
        bool need_csum, offload_csum, gso_partial, need_ipsec;
        struct sk_buff *segs = ERR_PTR(-EINVAL);
        u16 mac_offset = skb->mac_header;
        __be16 protocol = skb->protocol;
        u16 mac_len = skb->mac_len;
        int gre_offset, outer_hlen;

        if (!skb->encapsulation)
                goto out;

        if (unlikely(tnl_hlen < sizeof(struct gre_base_hdr)))
                goto out;

        if (unlikely(!pskb_may_pull(skb, tnl_hlen)))
                goto out;

        /* setup inner skb. */
        skb->encapsulation = 0;
        SKB_GSO_CB(skb)->encap_level = 0;
        __skb_pull(skb, tnl_hlen);
        skb_reset_mac_header(skb);
        skb_set_network_header(skb, skb_inner_network_offset(skb));
        skb->mac_len = skb_inner_network_offset(skb);
        skb->protocol = skb->inner_protocol;

        need_csum = !!(skb_shinfo(skb)->gso_type & SKB_GSO_GRE_CSUM);
        skb->encap_hdr_csum = need_csum;

        features &= skb->dev->hw_enc_features;
        if (need_csum)
                features &= ~NETIF_F_SCTP_CRC;

        need_ipsec = skb_dst(skb) && dst_xfrm(skb_dst(skb));
        /* Try to offload checksum if possible */
        offload_csum = !!(need_csum && !need_ipsec &&
                          (skb->dev->features & NETIF_F_HW_CSUM));

        /* segment inner packet. */
        segs = skb_mac_gso_segment(skb, features);
        if (IS_ERR_OR_NULL(segs)) {
                skb_gso_error_unwind(skb, protocol, tnl_hlen, mac_offset,
                                     mac_len);
                goto out;
        }

        gso_partial = !!(skb_shinfo(segs)->gso_type & SKB_GSO_PARTIAL);

        outer_hlen = skb_tnl_header_len(skb);
        gre_offset = outer_hlen - tnl_hlen;
        skb = segs;
        do {
                struct gre_base_hdr *greh;
                __sum16 *pcsum;

                /* Set up inner headers if we are offloading inner checksum */
                if (skb->ip_summed == CHECKSUM_PARTIAL) {
                        skb_reset_inner_headers(skb);
                        skb->encapsulation = 1;
                }

                skb->mac_len = mac_len;
                skb->protocol = protocol;

                __skb_push(skb, outer_hlen);
                skb_reset_mac_header(skb);
                skb_set_network_header(skb, mac_len);
                skb_set_transport_header(skb, gre_offset);

                if (!need_csum)
                        continue;

                greh = (struct gre_base_hdr *)skb_transport_header(skb);
                pcsum = (__sum16 *)(greh + 1);

                if (gso_partial && skb_is_gso(skb)) {
                        unsigned int partial_adj;

                        /* Adjust checksum to account for the fact that
                         * the partial checksum is based on actual size
                         * whereas headers should be based on MSS size.
                         */
                        partial_adj = skb->len + skb_headroom(skb) -
                                      SKB_GSO_CB(skb)->data_offset -
                                      skb_shinfo(skb)->gso_size;
                        *pcsum = ~csum_fold((__force __wsum)htonl(partial_adj));
                } else {
                        *pcsum = 0;
                }

                *(pcsum + 1) = 0;
                if (skb->encapsulation || !offload_csum) {
                        *pcsum = gso_make_checksum(skb, 0);
                } else {
                        skb->ip_summed = CHECKSUM_PARTIAL;
                        skb->csum_start = skb_transport_header(skb) - skb->head;
                        skb->csum_offset = sizeof(*greh);
                }
        } while ((skb = skb->next));
out:
        return segs;
}

static struct sk_buff *gre_gro_receive(struct list_head *head,
                                       struct sk_buff *skb)
{
        struct sk_buff *pp = NULL;
        struct sk_buff *p;
        const struct gre_base_hdr *greh;
        unsigned int hlen, grehlen;
        unsigned int off;
        int flush = 1;
        struct packet_offload *ptype;
        __be16 type;

        if (NAPI_GRO_CB(skb)->encap_mark)
                goto out;

        NAPI_GRO_CB(skb)->encap_mark = 1;

        off = skb_gro_offset(skb);
        hlen = off + sizeof(*greh);
        greh = skb_gro_header(skb, hlen, off);
        if (unlikely(!greh))
                goto out;

        /* Only support version 0 and K (key), C (csum) flags. Note that
         * although the support for the S (seq#) flag can be added easily
         * for GRO, this is problematic for GSO hence can not be enabled
         * here because a GRO pkt may end up in the forwarding path, thus
         * requiring GSO support to break it up correctly.
         */
        if ((greh->flags & ~(GRE_KEY|GRE_CSUM)) != 0)
                goto out;

        /* We can only support GRE_CSUM if we can track the location of
         * the GRE header.  In the case of FOU/GUE we cannot because the
         * outer UDP header displaces the GRE header leaving us in a state
         * of limbo.
         */
        if ((greh->flags & GRE_CSUM) && NAPI_GRO_CB(skb)->is_fou)
                goto out;

        type = greh->protocol;

        ptype = gro_find_receive_by_type(type);
        if (!ptype)
                goto out;

        grehlen = GRE_HEADER_SECTION;

        if (greh->flags & GRE_KEY)
                grehlen += GRE_HEADER_SECTION;

        if (greh->flags & GRE_CSUM)
                grehlen += GRE_HEADER_SECTION;

        hlen = off + grehlen;
        if (!skb_gro_may_pull(skb, hlen)) {
                greh = skb_gro_header_slow(skb, hlen, off);
                if (unlikely(!greh))
                        goto out;
        }

        /* Don't bother verifying checksum if we're going to flush anyway. */
        if ((greh->flags & GRE_CSUM) && !NAPI_GRO_CB(skb)->flush) {
                if (skb_gro_checksum_simple_validate(skb))
                        goto out;

                skb_gro_checksum_try_convert(skb, IPPROTO_GRE,
                                             null_compute_pseudo);
        }

        list_for_each_entry(p, head, list) {
                const struct gre_base_hdr *greh2;

                if (!NAPI_GRO_CB(p)->same_flow)
                        continue;

                /* The following checks are needed to ensure only pkts
                 * from the same tunnel are considered for aggregation.
                 * The criteria for "the same tunnel" includes:
                 * 1) same version (we only support version 0 here)
                 * 2) same protocol (we only support ETH_P_IP for now)
                 * 3) same set of flags
                 * 4) same key if the key field is present.
                 */
                greh2 = (struct gre_base_hdr *)(p->data + off);

                if (greh2->flags != greh->flags ||
                    greh2->protocol != greh->protocol) {
                        NAPI_GRO_CB(p)->same_flow = 0;
                        continue;
                }
                if (greh->flags & GRE_KEY) {
                        /* compare keys */
                        if (*(__be32 *)(greh2+1) != *(__be32 *)(greh+1)) {
                                NAPI_GRO_CB(p)->same_flow = 0;
                                continue;
                        }
                }
        }

        skb_gro_pull(skb, grehlen);

        /* Adjusted NAPI_GRO_CB(skb)->csum after skb_gro_pull()*/
        skb_gro_postpull_rcsum(skb, greh, grehlen);

        pp = call_gro_receive(ptype->callbacks.gro_receive, head, skb);
        flush = 0;

out:
        skb_gro_flush_final(skb, pp, flush);

        return pp;
}

static int gre_gro_complete(struct sk_buff *skb, int nhoff)
{
        struct gre_base_hdr *greh = (struct gre_base_hdr *)(skb->data + nhoff);
        struct packet_offload *ptype;
        unsigned int grehlen = sizeof(*greh);
        int err = -ENOENT;
        __be16 type;

        skb->encapsulation = 1;
        skb_shinfo(skb)->gso_type = SKB_GSO_GRE;

        type = greh->protocol;
        if (greh->flags & GRE_KEY)
                grehlen += GRE_HEADER_SECTION;

        if (greh->flags & GRE_CSUM)
                grehlen += GRE_HEADER_SECTION;

        ptype = gro_find_complete_by_type(type);
        if (ptype)
                err = ptype->callbacks.gro_complete(skb, nhoff + grehlen);

        skb_set_inner_mac_header(skb, nhoff + grehlen);

        return err;
}

static const struct net_offload gre_offload = {
        .callbacks = {
                .gso_segment = gre_gso_segment,
                .gro_receive = gre_gro_receive,
                .gro_complete = gre_gro_complete,
        },
};

static int __init gre_offload_init(void)
{
        int err;

        err = inet_add_offload(&gre_offload, IPPROTO_GRE);
#if IS_ENABLED(CONFIG_IPV6)
        if (err)
                return err;

        err = inet6_add_offload(&gre_offload, IPPROTO_GRE);
        if (err)
                inet_del_offload(&gre_offload, IPPROTO_GRE);
#endif

        return err;
}
device_initcall(gre_offload_init);















































































































































































































































































































































































































































    2 











































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
/* SPDX-License-Identifier: GPL-2.0 */
/*
 * This file define the new driver API for Wireless Extensions
 *
 * Version :        8        16.3.07
 *
 * Authors :        Jean Tourrilhes - HPL - <jt@hpl.hp.com>
 * Copyright (c) 2001-2007 Jean Tourrilhes, All Rights Reserved.
 */

#ifndef _IW_HANDLER_H
#define _IW_HANDLER_H

/************************** DOCUMENTATION **************************/
/*
 * Initial driver API (1996 -> onward) :
 * -----------------------------------
 * The initial API just sends the IOCTL request received from user space
 * to the driver (via the driver ioctl handler). The driver has to
 * handle all the rest...
 *
 * The initial API also defines a specific handler in struct net_device
 * to handle wireless statistics.
 *
 * The initial APIs served us well and has proven a reasonably good design.
 * However, there are a few shortcomings :
 *        o No events, everything is a request to the driver.
 *        o Large ioctl function in driver with gigantic switch statement
 *          (i.e. spaghetti code).
 *        o Driver has to mess up with copy_to/from_user, and in many cases
 *          does it unproperly. Common mistakes are :
 *                * buffer overflows (no checks or off by one checks)
 *                * call copy_to/from_user with irq disabled
 *        o The user space interface is tied to ioctl because of the use
 *          copy_to/from_user.
 *
 * New driver API (2002 -> onward) :
 * -------------------------------
 * The new driver API is just a bunch of standard functions (handlers),
 * each handling a specific Wireless Extension. The driver just export
 * the list of handler it supports, and those will be called appropriately.
 *
 * I tried to keep the main advantage of the previous API (simplicity,
 * efficiency and light weight), and also I provide a good dose of backward
 * compatibility (most structures are the same, driver can use both API
 * simultaneously, ...).
 * Hopefully, I've also addressed the shortcoming of the initial API.
 *
 * The advantage of the new API are :
 *        o Handling of Extensions in driver broken in small contained functions
 *        o Tighter checks of ioctl before calling the driver
 *        o Flexible commit strategy (at least, the start of it)
 *        o Backward compatibility (can be mixed with old API)
 *        o Driver doesn't have to worry about memory and user-space issues
 * The last point is important for the following reasons :
 *        o You are now able to call the new driver API from any API you
 *                want (including from within other parts of the kernel).
 *        o Common mistakes are avoided (buffer overflow, user space copy
 *                with irq disabled and so on).
 *
 * The Drawback of the new API are :
 *        o bloat (especially kernel)
 *        o need to migrate existing drivers to new API
 * My initial testing shows that the new API adds around 3kB to the kernel
 * and save between 0 and 5kB from a typical driver.
 * Also, as all structures and data types are unchanged, the migration is
 * quite straightforward (but tedious).
 *
 * ---
 *
 * The new driver API is defined below in this file. User space should
 * not be aware of what's happening down there...
 *
 * A new kernel wrapper is in charge of validating the IOCTLs and calling
 * the appropriate driver handler. This is implemented in :
 *        # net/core/wireless.c
 *
 * The driver export the list of handlers in :
 *        # include/linux/netdevice.h (one place)
 *
 * The new driver API is available for WIRELESS_EXT >= 13.
 * Good luck with migration to the new API ;-)
 */

/* ---------------------- THE IMPLEMENTATION ---------------------- */
/*
 * Some of the choice I've made are pretty controversial. Defining an
 * API is very much weighting compromises. This goes into some of the
 * details and the thinking behind the implementation.
 *
 * Implementation goals :
 * --------------------
 * The implementation goals were as follow :
 *        o Obvious : you should not need a PhD to understand what's happening,
 *                the benefit is easier maintenance.
 *        o Flexible : it should accommodate a wide variety of driver
 *                implementations and be as flexible as the old API.
 *        o Lean : it should be efficient memory wise to minimise the impact
 *                on kernel footprint.
 *        o Transparent to user space : the large number of user space
 *                applications that use Wireless Extensions should not need
 *                any modifications.
 *
 * Array of functions versus Struct of functions
 * ---------------------------------------------
 * 1) Having an array of functions allow the kernel code to access the
 * handler in a single lookup, which is much more efficient (think hash
 * table here).
 * 2) The only drawback is that driver writer may put their handler in
 * the wrong slot. This is trivial to test (I set the frequency, the
 * bitrate changes). Once the handler is in the proper slot, it will be
 * there forever, because the array is only extended at the end.
 * 3) Backward/forward compatibility : adding new handler just require
 * extending the array, so you can put newer driver in older kernel
 * without having to patch the kernel code (and vice versa).
 *
 * All handler are of the same generic type
 * ----------------------------------------
 * That's a feature !!!
 * 1) Having a generic handler allow to have generic code, which is more
 * efficient. If each of the handler was individually typed I would need
 * to add a big switch in the kernel (== more bloat). This solution is
 * more scalable, adding new Wireless Extensions doesn't add new code.
 * 2) You can use the same handler in different slots of the array. For
 * hardware, it may be more efficient or logical to handle multiple
 * Wireless Extensions with a single function, and the API allow you to
 * do that. (An example would be a single record on the card to control
 * both bitrate and frequency, the handler would read the old record,
 * modify it according to info->cmd and rewrite it).
 *
 * Functions prototype uses union iwreq_data
 * -----------------------------------------
 * Some would have preferred functions defined this way :
 *        static int mydriver_ioctl_setrate(struct net_device *dev, 
 *                                          long rate, int auto)
 * 1) The kernel code doesn't "validate" the content of iwreq_data, and
 * can't do it (different hardware may have different notion of what a
 * valid frequency is), so we don't pretend that we do it.
 * 2) The above form is not extendable. If I want to add a flag (for
 * example to distinguish setting max rate and basic rate), I would
 * break the prototype. Using iwreq_data is more flexible.
 * 3) Also, the above form is not generic (see above).
 * 4) I don't expect driver developer using the wrong field of the
 * union (Doh !), so static typechecking doesn't add much value.
 * 5) Lastly, you can skip the union by doing :
 *        static int mydriver_ioctl_setrate(struct net_device *dev,
 *                                          struct iw_request_info *info,
 *                                          struct iw_param *rrq,
 *                                          char *extra)
 * And then adding the handler in the array like this :
 *        (iw_handler) mydriver_ioctl_setrate,             // SIOCSIWRATE
 *
 * Using functions and not a registry
 * ----------------------------------
 * Another implementation option would have been for every instance to
 * define a registry (a struct containing all the Wireless Extensions)
 * and only have a function to commit the registry to the hardware.
 * 1) This approach can be emulated by the current code, but not
 * vice versa.
 * 2) Some drivers don't keep any configuration in the driver, for them
 * adding such a registry would be a significant bloat.
 * 3) The code to translate from Wireless Extension to native format is
 * needed anyway, so it would not reduce significantely the amount of code.
 * 4) The current approach only selectively translate Wireless Extensions
 * to native format and only selectively set, whereas the registry approach
 * would require to translate all WE and set all parameters for any single
 * change.
 * 5) For many Wireless Extensions, the GET operation return the current
 * dynamic value, not the value that was set.
 *
 * This header is <net/iw_handler.h>
 * ---------------------------------
 * 1) This header is kernel space only and should not be exported to
 * user space. Headers in "include/linux/" are exported, headers in
 * "include/net/" are not.
 *
 * Mixed 32/64 bit issues
 * ----------------------
 * The Wireless Extensions are designed to be 64 bit clean, by using only
 * datatypes with explicit storage size.
 * There are some issues related to kernel and user space using different
 * memory model, and in particular 64bit kernel with 32bit user space.
 * The problem is related to struct iw_point, that contains a pointer
 * that *may* need to be translated.
 * This is quite messy. The new API doesn't solve this problem (it can't),
 * but is a step in the right direction :
 * 1) Meta data about each ioctl is easily available, so we know what type
 * of translation is needed.
 * 2) The move of data between kernel and user space is only done in a single
 * place in the kernel, so adding specific hooks in there is possible.
 * 3) In the long term, it allows to move away from using ioctl as the
 * user space API.
 *
 * So many comments and so few code
 * --------------------------------
 * That's a feature. Comments won't bloat the resulting kernel binary.
 */

/***************************** INCLUDES *****************************/

#include <linux/wireless.h>                /* IOCTL user space API */
#include <linux/if_ether.h>

/***************************** VERSION *****************************/
/*
 * This constant is used to know which version of the driver API is
 * available. Hopefully, this will be pretty stable and no changes
 * will be needed...
 * I just plan to increment with each new version.
 */
#define IW_HANDLER_VERSION        8

/*
 * Changes :
 *
 * V2 to V3
 * --------
 *        - Move event definition in <linux/wireless.h>
 *        - Add Wireless Event support :
 *                o wireless_send_event() prototype
 *                o iwe_stream_add_event/point() inline functions
 * V3 to V4
 * --------
 *        - Reshuffle IW_HEADER_TYPE_XXX to map IW_PRIV_TYPE_XXX changes
 *
 * V4 to V5
 * --------
 *        - Add new spy support : struct iw_spy_data & prototypes
 *
 * V5 to V6
 * --------
 *        - Change the way we get to spy_data method for added safety
 *        - Remove spy #ifdef, they are always on -> cleaner code
 *        - Add IW_DESCR_FLAG_NOMAX flag for very large requests
 *        - Start migrating get_wireless_stats to struct iw_handler_def
 *
 * V6 to V7
 * --------
 *        - Add struct ieee80211_device pointer in struct iw_public_data
 *        - Remove (struct iw_point *)->pointer from events and streams
 *        - Remove spy_offset from struct iw_handler_def
 *        - Add "check" version of event macros for ieee802.11 stack
 *
 * V7 to V8
 * ----------
 *        - Prevent leaking of kernel space in stream on 64 bits.
 */

/**************************** CONSTANTS ****************************/

/* Enhanced spy support available */
#define IW_WIRELESS_SPY
#define IW_WIRELESS_THRSPY

/* Special error message for the driver to indicate that we
 * should do a commit after return from the iw_handler */
#define EIWCOMMIT        EINPROGRESS

/* Flags available in struct iw_request_info */
#define IW_REQUEST_FLAG_COMPAT        0x0001        /* Compat ioctl call */

/* Type of headers we know about (basically union iwreq_data) */
#define IW_HEADER_TYPE_NULL        0        /* Not available */
#define IW_HEADER_TYPE_CHAR        2        /* char [IFNAMSIZ] */
#define IW_HEADER_TYPE_UINT        4        /* __u32 */
#define IW_HEADER_TYPE_FREQ        5        /* struct iw_freq */
#define IW_HEADER_TYPE_ADDR        6        /* struct sockaddr */
#define IW_HEADER_TYPE_POINT        8        /* struct iw_point */
#define IW_HEADER_TYPE_PARAM        9        /* struct iw_param */
#define IW_HEADER_TYPE_QUAL        10        /* struct iw_quality */

/* Handling flags */
/* Most are not implemented. I just use them as a reminder of some
 * cool features we might need one day ;-) */
#define IW_DESCR_FLAG_NONE        0x0000        /* Obvious */
/* Wrapper level flags */
#define IW_DESCR_FLAG_DUMP        0x0001        /* Not part of the dump command */
#define IW_DESCR_FLAG_EVENT        0x0002        /* Generate an event on SET */
#define IW_DESCR_FLAG_RESTRICT        0x0004        /* GET : request is ROOT only */
                                /* SET : Omit payload from generated iwevent */
#define IW_DESCR_FLAG_NOMAX        0x0008        /* GET : no limit on request size */

/****************************** TYPES ******************************/

/* ----------------------- WIRELESS HANDLER ----------------------- */
/*
 * A wireless handler is just a standard function, that looks like the
 * ioctl handler.
 * We also define there how a handler list look like... As the Wireless
 * Extension space is quite dense, we use a simple array, which is faster
 * (that's the perfect hash table ;-).
 */

/*
 * Meta data about the request passed to the iw_handler.
 * Most handlers can safely ignore what's in there.
 * The 'cmd' field might come handy if you want to use the same handler
 * for multiple command...
 * This struct is also my long term insurance. I can add new fields here
 * without breaking the prototype of iw_handler...
 */
struct iw_request_info {
        __u16                cmd;                /* Wireless Extension command */
        __u16                flags;                /* More to come ;-) */
};

struct net_device;

/*
 * This is how a function handling a Wireless Extension should look
 * like (both get and set, standard and private).
 */
typedef int (*iw_handler)(struct net_device *dev, struct iw_request_info *info,
                          union iwreq_data *wrqu, char *extra);

/*
 * This define all the handler that the driver export.
 * As you need only one per driver type, please use a static const
 * shared by all driver instances... Same for the members...
 * This will be linked from net_device in <linux/netdevice.h>
 */
struct iw_handler_def {

        /* Array of handlers for standard ioctls
         * We will call dev->wireless_handlers->standard[ioctl - SIOCIWFIRST]
         */
        const iw_handler *        standard;
        /* Number of handlers defined (more precisely, index of the
         * last defined handler + 1) */
        __u16                        num_standard;

#ifdef CONFIG_WEXT_PRIV
        __u16                        num_private;
        /* Number of private arg description */
        __u16                        num_private_args;
        /* Array of handlers for private ioctls
         * Will call dev->wireless_handlers->private[ioctl - SIOCIWFIRSTPRIV]
         */
        const iw_handler *        private;

        /* Arguments of private handler. This one is just a list, so you
         * can put it in any order you want and should not leave holes...
         * We will automatically export that to user space... */
        const struct iw_priv_args *        private_args;
#endif

        /* New location of get_wireless_stats, to de-bloat struct net_device.
         * The old pointer in struct net_device will be gradually phased
         * out, and drivers are encouraged to use this one... */
        struct iw_statistics*        (*get_wireless_stats)(struct net_device *dev);
};

/* ---------------------- IOCTL DESCRIPTION ---------------------- */
/*
 * One of the main goal of the new interface is to deal entirely with
 * user space/kernel space memory move.
 * For that, we need to know :
 *        o if iwreq is a pointer or contain the full data
 *        o what is the size of the data to copy
 *
 * For private IOCTLs, we use the same rules as used by iwpriv and
 * defined in struct iw_priv_args.
 *
 * For standard IOCTLs, things are quite different and we need to
 * use the structures below. Actually, this struct is also more
 * efficient, but that's another story...
 */

/*
 * Describe how a standard IOCTL looks like.
 */
struct iw_ioctl_description {
        __u8        header_type;                /* NULL, iw_point or other */
        __u8        flags;                        /* Special handling of the request */
        __u16        token_size;                /* Granularity of payload */
        __u16        min_tokens;                /* Min acceptable token number */
        __u16        max_tokens;                /* Max acceptable token number */
};

/* Need to think of short header translation table. Later. */

/* --------------------- ENHANCED SPY SUPPORT --------------------- */
/*
 * In the old days, the driver was handling spy support all by itself.
 * Now, the driver can delegate this task to Wireless Extensions.
 * It needs to include this struct in its private part and use the
 * standard spy iw_handler.
 */

/*
 * Instance specific spy data, i.e. addresses spied and quality for them.
 */
struct iw_spy_data {
        /* --- Standard spy support --- */
        int                        spy_number;
        u_char                        spy_address[IW_MAX_SPY][ETH_ALEN];
        struct iw_quality        spy_stat[IW_MAX_SPY];
        /* --- Enhanced spy support (event) */
        struct iw_quality        spy_thr_low;        /* Low threshold */
        struct iw_quality        spy_thr_high;        /* High threshold */
        u_char                        spy_thr_under[IW_MAX_SPY];
};

/**************************** PROTOTYPES ****************************/
/*
 * Functions part of the Wireless Extensions (defined in net/wireless/wext-core.c).
 * Those may be called by driver modules.
 */

/* Send a single event to user space */
void wireless_send_event(struct net_device *dev, unsigned int cmd,
                         union iwreq_data *wrqu, const char *extra);
#ifdef CONFIG_WEXT_CORE
/* flush all previous wext events - if work is done from netdev notifiers */
void wireless_nlevent_flush(void);
#else
static inline void wireless_nlevent_flush(void) {}
#endif

/* We may need a function to send a stream of events to user space.
 * More on that later... */

/************************* INLINE FUNCTIONS *************************/
/*
 * Function that are so simple that it's more efficient inlining them
 */

static inline int iwe_stream_lcp_len(struct iw_request_info *info)
{
#ifdef CONFIG_COMPAT
        if (info->flags & IW_REQUEST_FLAG_COMPAT)
                return IW_EV_COMPAT_LCP_LEN;
#endif
        return IW_EV_LCP_LEN;
}

static inline int iwe_stream_point_len(struct iw_request_info *info)
{
#ifdef CONFIG_COMPAT
        if (info->flags & IW_REQUEST_FLAG_COMPAT)
                return IW_EV_COMPAT_POINT_LEN;
#endif
        return IW_EV_POINT_LEN;
}

static inline int iwe_stream_event_len_adjust(struct iw_request_info *info,
                                              int event_len)
{
#ifdef CONFIG_COMPAT
        if (info->flags & IW_REQUEST_FLAG_COMPAT) {
                event_len -= IW_EV_LCP_LEN;
                event_len += IW_EV_COMPAT_LCP_LEN;
        }
#endif

        return event_len;
}

/*------------------------------------------------------------------*/
/*
 * Wrapper to add an Wireless Event to a stream of events.
 */
char *iwe_stream_add_event(struct iw_request_info *info, char *stream,
                           char *ends, struct iw_event *iwe, int event_len);

static inline char *
iwe_stream_add_event_check(struct iw_request_info *info, char *stream,
                           char *ends, struct iw_event *iwe, int event_len)
{
        char *res = iwe_stream_add_event(info, stream, ends, iwe, event_len);

        if (res == stream)
                return ERR_PTR(-E2BIG);
        return res;
}

/*------------------------------------------------------------------*/
/*
 * Wrapper to add an short Wireless Event containing a pointer to a
 * stream of events.
 */
char *iwe_stream_add_point(struct iw_request_info *info, char *stream,
                           char *ends, struct iw_event *iwe, char *extra);

static inline char *
iwe_stream_add_point_check(struct iw_request_info *info, char *stream,
                           char *ends, struct iw_event *iwe, char *extra)
{
        char *res = iwe_stream_add_point(info, stream, ends, iwe, extra);

        if (res == stream)
                return ERR_PTR(-E2BIG);
        return res;
}

/*------------------------------------------------------------------*/
/*
 * Wrapper to add a value to a Wireless Event in a stream of events.
 * Be careful, this one is tricky to use properly :
 * At the first run, you need to have (value = event + IW_EV_LCP_LEN).
 */
char *iwe_stream_add_value(struct iw_request_info *info, char *event,
                           char *value, char *ends, struct iw_event *iwe,
                           int event_len);

#endif        /* _IW_HANDLER_H */












































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































    2 







    2 


















































    2 






    2 

    2 

    2 


































    2 
    2 








    2 







    2 










    2 
    2 



    2 




    1 


    1 






    6 





    6 




    1 


    1 















    1 







    1 






























    2 
















































































































































































































































































































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
// SPDX-License-Identifier: GPL-2.0-only
/*
 * mac80211 TDLS handling code
 *
 * Copyright 2006-2010        Johannes Berg <johannes@sipsolutions.net>
 * Copyright 2014, Intel Corporation
 * Copyright 2014  Intel Mobile Communications GmbH
 * Copyright 2015 - 2016 Intel Deutschland GmbH
 * Copyright (C) 2019, 2021-2024 Intel Corporation
 */

#include <linux/ieee80211.h>
#include <linux/log2.h>
#include <net/cfg80211.h>
#include <linux/rtnetlink.h>
#include "ieee80211_i.h"
#include "driver-ops.h"
#include "rate.h"
#include "wme.h"

/* give usermode some time for retries in setting up the TDLS session */
#define TDLS_PEER_SETUP_TIMEOUT        (15 * HZ)

void ieee80211_tdls_peer_del_work(struct wiphy *wiphy, struct wiphy_work *wk)
{
        struct ieee80211_sub_if_data *sdata;
        struct ieee80211_local *local;

        sdata = container_of(wk, struct ieee80211_sub_if_data,
                             u.mgd.tdls_peer_del_work.work);
        local = sdata->local;

        lockdep_assert_wiphy(local->hw.wiphy);

        if (!is_zero_ether_addr(sdata->u.mgd.tdls_peer)) {
                tdls_dbg(sdata, "TDLS del peer %pM\n", sdata->u.mgd.tdls_peer);
                sta_info_destroy_addr(sdata, sdata->u.mgd.tdls_peer);
                eth_zero_addr(sdata->u.mgd.tdls_peer);
        }
}

static void ieee80211_tdls_add_ext_capab(struct ieee80211_link_data *link,
                                         struct sk_buff *skb)
{
        struct ieee80211_sub_if_data *sdata = link->sdata;
        struct ieee80211_local *local = sdata->local;
        struct ieee80211_if_managed *ifmgd = &sdata->u.mgd;
        bool chan_switch = local->hw.wiphy->features &
                           NL80211_FEATURE_TDLS_CHANNEL_SWITCH;
        bool wider_band = ieee80211_hw_check(&local->hw, TDLS_WIDER_BW) &&
                          !ifmgd->tdls_wider_bw_prohibited;
        bool buffer_sta = ieee80211_hw_check(&local->hw,
                                             SUPPORTS_TDLS_BUFFER_STA);
        struct ieee80211_supported_band *sband = ieee80211_get_link_sband(link);
        bool vht = sband && sband->vht_cap.vht_supported;
        u8 *pos = skb_put(skb, 10);

        *pos++ = WLAN_EID_EXT_CAPABILITY;
        *pos++ = 8; /* len */
        *pos++ = 0x0;
        *pos++ = 0x0;
        *pos++ = 0x0;
        *pos++ = (chan_switch ? WLAN_EXT_CAPA4_TDLS_CHAN_SWITCH : 0) |
                 (buffer_sta ? WLAN_EXT_CAPA4_TDLS_BUFFER_STA : 0);
        *pos++ = WLAN_EXT_CAPA5_TDLS_ENABLED;
        *pos++ = 0;
        *pos++ = 0;
        *pos++ = (vht && wider_band) ? WLAN_EXT_CAPA8_TDLS_WIDE_BW_ENABLED : 0;
}

static u8
ieee80211_tdls_add_subband(struct ieee80211_sub_if_data *sdata,
                           struct sk_buff *skb, u16 start, u16 end,
                           u16 spacing)
{
        u8 subband_cnt = 0, ch_cnt = 0;
        struct ieee80211_channel *ch;
        struct cfg80211_chan_def chandef;
        int i, subband_start;
        struct wiphy *wiphy = sdata->local->hw.wiphy;

        for (i = start; i <= end; i += spacing) {
                if (!ch_cnt)
                        subband_start = i;

                ch = ieee80211_get_channel(sdata->local->hw.wiphy, i);
                if (ch) {
                        /* we will be active on the channel */
                        cfg80211_chandef_create(&chandef, ch,
                                                NL80211_CHAN_NO_HT);
                        if (cfg80211_reg_can_beacon_relax(wiphy, &chandef,
                                                          sdata->wdev.iftype)) {
                                ch_cnt++;
                                /*
                                 * check if the next channel is also part of
                                 * this allowed range
                                 */
                                continue;
                        }
                }

                /*
                 * we've reached the end of a range, with allowed channels
                 * found
                 */
                if (ch_cnt) {
                        u8 *pos = skb_put(skb, 2);
                        *pos++ = ieee80211_frequency_to_channel(subband_start);
                        *pos++ = ch_cnt;

                        subband_cnt++;
                        ch_cnt = 0;
                }
        }

        /* all channels in the requested range are allowed - add them here */
        if (ch_cnt) {
                u8 *pos = skb_put(skb, 2);
                *pos++ = ieee80211_frequency_to_channel(subband_start);
                *pos++ = ch_cnt;

                subband_cnt++;
        }

        return subband_cnt;
}

static void
ieee80211_tdls_add_supp_channels(struct ieee80211_sub_if_data *sdata,
                                 struct sk_buff *skb)
{
        /*
         * Add possible channels for TDLS. These are channels that are allowed
         * to be active.
         */
        u8 subband_cnt;
        u8 *pos = skb_put(skb, 2);

        *pos++ = WLAN_EID_SUPPORTED_CHANNELS;

        /*
         * 5GHz and 2GHz channels numbers can overlap. Ignore this for now, as
         * this doesn't happen in real world scenarios.
         */

        /* 2GHz, with 5MHz spacing */
        subband_cnt = ieee80211_tdls_add_subband(sdata, skb, 2412, 2472, 5);

        /* 5GHz, with 20MHz spacing */
        subband_cnt += ieee80211_tdls_add_subband(sdata, skb, 5000, 5825, 20);

        /* length */
        *pos = 2 * subband_cnt;
}

static void ieee80211_tdls_add_oper_classes(struct ieee80211_link_data *link,
                                            struct sk_buff *skb)
{
        u8 *pos;
        u8 op_class;

        if (!ieee80211_chandef_to_operating_class(&link->conf->chanreq.oper,
                                                  &op_class))
                return;

        pos = skb_put(skb, 4);
        *pos++ = WLAN_EID_SUPPORTED_REGULATORY_CLASSES;
        *pos++ = 2; /* len */

        *pos++ = op_class;
        *pos++ = op_class; /* give current operating class as alternate too */
}

static void ieee80211_tdls_add_bss_coex_ie(struct sk_buff *skb)
{
        u8 *pos = skb_put(skb, 3);

        *pos++ = WLAN_EID_BSS_COEX_2040;
        *pos++ = 1; /* len */

        *pos++ = WLAN_BSS_COEX_INFORMATION_REQUEST;
}

static u16 ieee80211_get_tdls_sta_capab(struct ieee80211_link_data *link,
                                        u16 status_code)
{
        struct ieee80211_supported_band *sband;

        /* The capability will be 0 when sending a failure code */
        if (status_code != 0)
                return 0;

        sband = ieee80211_get_link_sband(link);

        if (sband && sband->band == NL80211_BAND_2GHZ) {
                return WLAN_CAPABILITY_SHORT_SLOT_TIME |
                       WLAN_CAPABILITY_SHORT_PREAMBLE;
        }

        return 0;
}

static void ieee80211_tdls_add_link_ie(struct ieee80211_link_data *link,
                                       struct sk_buff *skb, const u8 *peer,
                                       bool initiator)
{
        struct ieee80211_sub_if_data *sdata = link->sdata;
        struct ieee80211_tdls_lnkie *lnkid;
        const u8 *init_addr, *rsp_addr;

        if (initiator) {
                init_addr = sdata->vif.addr;
                rsp_addr = peer;
        } else {
                init_addr = peer;
                rsp_addr = sdata->vif.addr;
        }

        lnkid = skb_put(skb, sizeof(struct ieee80211_tdls_lnkie));

        lnkid->ie_type = WLAN_EID_LINK_ID;
        lnkid->ie_len = sizeof(struct ieee80211_tdls_lnkie) - 2;

        memcpy(lnkid->bssid, link->u.mgd.bssid, ETH_ALEN);
        memcpy(lnkid->init_sta, init_addr, ETH_ALEN);
        memcpy(lnkid->resp_sta, rsp_addr, ETH_ALEN);
}

static void
ieee80211_tdls_add_aid(struct ieee80211_sub_if_data *sdata, struct sk_buff *skb)
{
        u8 *pos = skb_put(skb, 4);

        *pos++ = WLAN_EID_AID;
        *pos++ = 2; /* len */
        put_unaligned_le16(sdata->vif.cfg.aid, pos);
}

/* translate numbering in the WMM parameter IE to the mac80211 notation */
static enum ieee80211_ac_numbers ieee80211_ac_from_wmm(int ac)
{
        switch (ac) {
        default:
                WARN_ON_ONCE(1);
                fallthrough;
        case 0:
                return IEEE80211_AC_BE;
        case 1:
                return IEEE80211_AC_BK;
        case 2:
                return IEEE80211_AC_VI;
        case 3:
                return IEEE80211_AC_VO;
        }
}

static u8 ieee80211_wmm_aci_aifsn(int aifsn, bool acm, int aci)
{
        u8 ret;

        ret = aifsn & 0x0f;
        if (acm)
                ret |= 0x10;
        ret |= (aci << 5) & 0x60;
        return ret;
}

static u8 ieee80211_wmm_ecw(u16 cw_min, u16 cw_max)
{
        return ((ilog2(cw_min + 1) << 0x0) & 0x0f) |
               ((ilog2(cw_max + 1) << 0x4) & 0xf0);
}

static void ieee80211_tdls_add_wmm_param_ie(struct ieee80211_sub_if_data *sdata,
                                            struct sk_buff *skb)
{
        struct ieee80211_wmm_param_ie *wmm;
        struct ieee80211_tx_queue_params *txq;
        int i;

        wmm = skb_put_zero(skb, sizeof(*wmm));

        wmm->element_id = WLAN_EID_VENDOR_SPECIFIC;
        wmm->len = sizeof(*wmm) - 2;

        wmm->oui[0] = 0x00; /* Microsoft OUI 00:50:F2 */
        wmm->oui[1] = 0x50;
        wmm->oui[2] = 0xf2;
        wmm->oui_type = 2; /* WME */
        wmm->oui_subtype = 1; /* WME param */
        wmm->version = 1; /* WME ver */
        wmm->qos_info = 0; /* U-APSD not in use */

        /*
         * Use the EDCA parameters defined for the BSS, or default if the AP
         * doesn't support it, as mandated by 802.11-2012 section 10.22.4
         */
        for (i = 0; i < IEEE80211_NUM_ACS; i++) {
                txq = &sdata->deflink.tx_conf[ieee80211_ac_from_wmm(i)];
                wmm->ac[i].aci_aifsn = ieee80211_wmm_aci_aifsn(txq->aifs,
                                                               txq->acm, i);
                wmm->ac[i].cw = ieee80211_wmm_ecw(txq->cw_min, txq->cw_max);
                wmm->ac[i].txop_limit = cpu_to_le16(txq->txop);
        }
}

static void
ieee80211_tdls_chandef_vht_upgrade(struct ieee80211_sub_if_data *sdata,
                                   struct sta_info *sta)
{
        /* IEEE802.11ac-2013 Table E-4 */
        static const u16 centers_80mhz[] = { 5210, 5290, 5530, 5610, 5690, 5775 };
        struct cfg80211_chan_def uc = sta->tdls_chandef;
        enum nl80211_chan_width max_width =
                ieee80211_sta_cap_chan_bw(&sta->deflink);
        int i;

        /* only support upgrading non-narrow channels up to 80Mhz */
        if (max_width == NL80211_CHAN_WIDTH_5 ||
            max_width == NL80211_CHAN_WIDTH_10)
                return;

        if (max_width > NL80211_CHAN_WIDTH_80)
                max_width = NL80211_CHAN_WIDTH_80;

        if (uc.width >= max_width)
                return;
        /*
         * Channel usage constrains in the IEEE802.11ac-2013 specification only
         * allow expanding a 20MHz channel to 80MHz in a single way. In
         * addition, there are no 40MHz allowed channels that are not part of
         * the allowed 80MHz range in the 5GHz spectrum (the relevant one here).
         */
        for (i = 0; i < ARRAY_SIZE(centers_80mhz); i++)
                if (abs(uc.chan->center_freq - centers_80mhz[i]) <= 30) {
                        uc.center_freq1 = centers_80mhz[i];
                        uc.center_freq2 = 0;
                        uc.width = NL80211_CHAN_WIDTH_80;
                        break;
                }

        if (!uc.center_freq1)
                return;

        /* proceed to downgrade the chandef until usable or the same as AP BW */
        while (uc.width > max_width ||
               (uc.width > sta->tdls_chandef.width &&
                !cfg80211_reg_can_beacon_relax(sdata->local->hw.wiphy, &uc,
                                               sdata->wdev.iftype)))
                ieee80211_chandef_downgrade(&uc, NULL);

        if (!cfg80211_chandef_identical(&uc, &sta->tdls_chandef)) {
                tdls_dbg(sdata, "TDLS ch width upgraded %d -> %d\n",
                         sta->tdls_chandef.width, uc.width);

                /*
                 * the station is not yet authorized when BW upgrade is done,
                 * locking is not required
                 */
                sta->tdls_chandef = uc;
        }
}

static void
ieee80211_tdls_add_setup_start_ies(struct ieee80211_link_data *link,
                                   struct sk_buff *skb, const u8 *peer,
                                   u8 action_code, bool initiator,
                                   const u8 *extra_ies, size_t extra_ies_len)
{
        struct ieee80211_sub_if_data *sdata = link->sdata;
        struct ieee80211_supported_band *sband;
        struct ieee80211_local *local = sdata->local;
        struct ieee80211_sta_ht_cap ht_cap;
        struct ieee80211_sta_vht_cap vht_cap;
        const struct ieee80211_sta_he_cap *he_cap;
        const struct ieee80211_sta_eht_cap *eht_cap;
        struct sta_info *sta = NULL;
        size_t offset = 0, noffset;
        u8 *pos;

        sband = ieee80211_get_link_sband(link);
        if (WARN_ON_ONCE(!sband))
                return;

        ieee80211_put_srates_elem(skb, sband, 0, 0, 0, WLAN_EID_SUPP_RATES);
        ieee80211_put_srates_elem(skb, sband, 0, 0, 0, WLAN_EID_EXT_SUPP_RATES);
        ieee80211_tdls_add_supp_channels(sdata, skb);

        /* add any custom IEs that go before Extended Capabilities */
        if (extra_ies_len) {
                static const u8 before_ext_cap[] = {
                        WLAN_EID_SUPP_RATES,
                        WLAN_EID_COUNTRY,
                        WLAN_EID_EXT_SUPP_RATES,
                        WLAN_EID_SUPPORTED_CHANNELS,
                        WLAN_EID_RSN,
                };
                noffset = ieee80211_ie_split(extra_ies, extra_ies_len,
                                             before_ext_cap,
                                             ARRAY_SIZE(before_ext_cap),
                                             offset);
                skb_put_data(skb, extra_ies + offset, noffset - offset);
                offset = noffset;
        }

        ieee80211_tdls_add_ext_capab(link, skb);

        /* add the QoS element if we support it */
        if (local->hw.queues >= IEEE80211_NUM_ACS &&
            action_code != WLAN_PUB_ACTION_TDLS_DISCOVER_RES)
                ieee80211_add_wmm_info_ie(skb_put(skb, 9), 0); /* no U-APSD */

        /* add any custom IEs that go before HT capabilities */
        if (extra_ies_len) {
                static const u8 before_ht_cap[] = {
                        WLAN_EID_SUPP_RATES,
                        WLAN_EID_COUNTRY,
                        WLAN_EID_EXT_SUPP_RATES,
                        WLAN_EID_SUPPORTED_CHANNELS,
                        WLAN_EID_RSN,
                        WLAN_EID_EXT_CAPABILITY,
                        WLAN_EID_QOS_CAPA,
                        WLAN_EID_FAST_BSS_TRANSITION,
                        WLAN_EID_TIMEOUT_INTERVAL,
                        WLAN_EID_SUPPORTED_REGULATORY_CLASSES,
                };
                noffset = ieee80211_ie_split(extra_ies, extra_ies_len,
                                             before_ht_cap,
                                             ARRAY_SIZE(before_ht_cap),
                                             offset);
                skb_put_data(skb, extra_ies + offset, noffset - offset);
                offset = noffset;
        }

        /* we should have the peer STA if we're already responding */
        if (action_code == WLAN_TDLS_SETUP_RESPONSE) {
                sta = sta_info_get(sdata, peer);
                if (WARN_ON_ONCE(!sta))
                        return;

                sta->tdls_chandef = link->conf->chanreq.oper;
        }

        ieee80211_tdls_add_oper_classes(link, skb);

        /*
         * with TDLS we can switch channels, and HT-caps are not necessarily
         * the same on all bands. The specification limits the setup to a
         * single HT-cap, so use the current band for now.
         */
        memcpy(&ht_cap, &sband->ht_cap, sizeof(ht_cap));

        if ((action_code == WLAN_TDLS_SETUP_REQUEST ||
             action_code == WLAN_PUB_ACTION_TDLS_DISCOVER_RES) &&
            ht_cap.ht_supported) {
                ieee80211_apply_htcap_overrides(sdata, &ht_cap);

                /* disable SMPS in TDLS initiator */
                ht_cap.cap |= WLAN_HT_CAP_SM_PS_DISABLED
                                << IEEE80211_HT_CAP_SM_PS_SHIFT;

                pos = skb_put(skb, sizeof(struct ieee80211_ht_cap) + 2);
                ieee80211_ie_build_ht_cap(pos, &ht_cap, ht_cap.cap);
        } else if (action_code == WLAN_TDLS_SETUP_RESPONSE &&
                   ht_cap.ht_supported && sta->sta.deflink.ht_cap.ht_supported) {
                /* the peer caps are already intersected with our own */
                memcpy(&ht_cap, &sta->sta.deflink.ht_cap, sizeof(ht_cap));

                pos = skb_put(skb, sizeof(struct ieee80211_ht_cap) + 2);
                ieee80211_ie_build_ht_cap(pos, &ht_cap, ht_cap.cap);
        }

        if (ht_cap.ht_supported &&
            (ht_cap.cap & IEEE80211_HT_CAP_SUP_WIDTH_20_40))
                ieee80211_tdls_add_bss_coex_ie(skb);

        ieee80211_tdls_add_link_ie(link, skb, peer, initiator);

        /* add any custom IEs that go before VHT capabilities */
        if (extra_ies_len) {
                static const u8 before_vht_cap[] = {
                        WLAN_EID_SUPP_RATES,
                        WLAN_EID_COUNTRY,
                        WLAN_EID_EXT_SUPP_RATES,
                        WLAN_EID_SUPPORTED_CHANNELS,
                        WLAN_EID_RSN,
                        WLAN_EID_EXT_CAPABILITY,
                        WLAN_EID_QOS_CAPA,
                        WLAN_EID_FAST_BSS_TRANSITION,
                        WLAN_EID_TIMEOUT_INTERVAL,
                        WLAN_EID_SUPPORTED_REGULATORY_CLASSES,
                        WLAN_EID_MULTI_BAND,
                };
                noffset = ieee80211_ie_split(extra_ies, extra_ies_len,
                                             before_vht_cap,
                                             ARRAY_SIZE(before_vht_cap),
                                             offset);
                skb_put_data(skb, extra_ies + offset, noffset - offset);
                offset = noffset;
        }

        /* add AID if VHT, HE or EHT capabilities supported */
        memcpy(&vht_cap, &sband->vht_cap, sizeof(vht_cap));
        he_cap = ieee80211_get_he_iftype_cap_vif(sband, &sdata->vif);
        eht_cap = ieee80211_get_eht_iftype_cap_vif(sband, &sdata->vif);
        if ((vht_cap.vht_supported || he_cap || eht_cap) &&
            (action_code == WLAN_TDLS_SETUP_REQUEST ||
             action_code == WLAN_TDLS_SETUP_RESPONSE))
                ieee80211_tdls_add_aid(sdata, skb);

        /* build the VHT-cap similarly to the HT-cap */
        if ((action_code == WLAN_TDLS_SETUP_REQUEST ||
             action_code == WLAN_PUB_ACTION_TDLS_DISCOVER_RES) &&
            vht_cap.vht_supported) {
                ieee80211_apply_vhtcap_overrides(sdata, &vht_cap);

                pos = skb_put(skb, sizeof(struct ieee80211_vht_cap) + 2);
                ieee80211_ie_build_vht_cap(pos, &vht_cap, vht_cap.cap);
        } else if (action_code == WLAN_TDLS_SETUP_RESPONSE &&
                   vht_cap.vht_supported && sta->sta.deflink.vht_cap.vht_supported) {
                /* the peer caps are already intersected with our own */
                memcpy(&vht_cap, &sta->sta.deflink.vht_cap, sizeof(vht_cap));

                pos = skb_put(skb, sizeof(struct ieee80211_vht_cap) + 2);
                ieee80211_ie_build_vht_cap(pos, &vht_cap, vht_cap.cap);

                /*
                 * if both peers support WIDER_BW, we can expand the chandef to
                 * a wider compatible one, up to 80MHz
                 */
                if (test_sta_flag(sta, WLAN_STA_TDLS_WIDER_BW))
                        ieee80211_tdls_chandef_vht_upgrade(sdata, sta);
        }

        /* add any custom IEs that go before HE capabilities */
        if (extra_ies_len) {
                static const u8 before_he_cap[] = {
                        WLAN_EID_EXTENSION,
                        WLAN_EID_EXT_FILS_REQ_PARAMS,
                        WLAN_EID_AP_CSN,
                };
                noffset = ieee80211_ie_split(extra_ies, extra_ies_len,
                                             before_he_cap,
                                             ARRAY_SIZE(before_he_cap),
                                             offset);
                skb_put_data(skb, extra_ies + offset, noffset - offset);
                offset = noffset;
        }

        /* build the HE-cap from sband */
        if (action_code == WLAN_TDLS_SETUP_REQUEST ||
            action_code == WLAN_TDLS_SETUP_RESPONSE ||
            action_code == WLAN_PUB_ACTION_TDLS_DISCOVER_RES) {
                ieee80211_put_he_cap(skb, sdata, sband, NULL);

                /* Build HE 6Ghz capa IE from sband */
                if (sband->band == NL80211_BAND_6GHZ)
                        ieee80211_put_he_6ghz_cap(skb, sdata, link->smps_mode);
        }

        /* add any custom IEs that go before EHT capabilities */
        if (extra_ies_len) {
                static const u8 before_he_cap[] = {
                        WLAN_EID_EXTENSION,
                        WLAN_EID_EXT_FILS_REQ_PARAMS,
                        WLAN_EID_AP_CSN,
                };

                noffset = ieee80211_ie_split(extra_ies, extra_ies_len,
                                             before_he_cap,
                                             ARRAY_SIZE(before_he_cap),
                                             offset);
                skb_put_data(skb, extra_ies + offset, noffset - offset);
                offset = noffset;
        }

        /* build the EHT-cap from sband */
        if (action_code == WLAN_TDLS_SETUP_REQUEST ||
            action_code == WLAN_TDLS_SETUP_RESPONSE ||
            action_code == WLAN_PUB_ACTION_TDLS_DISCOVER_RES)
                ieee80211_put_eht_cap(skb, sdata, sband, NULL);

        /* add any remaining IEs */
        if (extra_ies_len) {
                noffset = extra_ies_len;
                skb_put_data(skb, extra_ies + offset, noffset - offset);
        }

}

static void
ieee80211_tdls_add_setup_cfm_ies(struct ieee80211_link_data *link,
                                 struct sk_buff *skb, const u8 *peer,
                                 bool initiator, const u8 *extra_ies,
                                 size_t extra_ies_len)
{
        struct ieee80211_sub_if_data *sdata = link->sdata;
        struct ieee80211_local *local = sdata->local;
        size_t offset = 0, noffset;
        struct sta_info *sta, *ap_sta;
        struct ieee80211_supported_band *sband;
        u8 *pos;

        sband = ieee80211_get_link_sband(link);
        if (WARN_ON_ONCE(!sband))
                return;

        sta = sta_info_get(sdata, peer);
        ap_sta = sta_info_get(sdata, sdata->vif.cfg.ap_addr);

        if (WARN_ON_ONCE(!sta || !ap_sta))
                return;

        sta->tdls_chandef = link->conf->chanreq.oper;

        /* add any custom IEs that go before the QoS IE */
        if (extra_ies_len) {
                static const u8 before_qos[] = {
                        WLAN_EID_RSN,
                };
                noffset = ieee80211_ie_split(extra_ies, extra_ies_len,
                                             before_qos,
                                             ARRAY_SIZE(before_qos),
                                             offset);
                skb_put_data(skb, extra_ies + offset, noffset - offset);
                offset = noffset;
        }

        /* add the QoS param IE if both the peer and we support it */
        if (local->hw.queues >= IEEE80211_NUM_ACS && sta->sta.wme)
                ieee80211_tdls_add_wmm_param_ie(sdata, skb);

        /* add any custom IEs that go before HT operation */
        if (extra_ies_len) {
                static const u8 before_ht_op[] = {
                        WLAN_EID_RSN,
                        WLAN_EID_QOS_CAPA,
                        WLAN_EID_FAST_BSS_TRANSITION,
                        WLAN_EID_TIMEOUT_INTERVAL,
                };
                noffset = ieee80211_ie_split(extra_ies, extra_ies_len,
                                             before_ht_op,
                                             ARRAY_SIZE(before_ht_op),
                                             offset);
                skb_put_data(skb, extra_ies + offset, noffset - offset);
                offset = noffset;
        }

        /*
         * if HT support is only added in TDLS, we need an HT-operation IE.
         * add the IE as required by IEEE802.11-2012 9.23.3.2.
         */
        if (!ap_sta->sta.deflink.ht_cap.ht_supported && sta->sta.deflink.ht_cap.ht_supported) {
                u16 prot = IEEE80211_HT_OP_MODE_PROTECTION_NONHT_MIXED |
                           IEEE80211_HT_OP_MODE_NON_GF_STA_PRSNT |
                           IEEE80211_HT_OP_MODE_NON_HT_STA_PRSNT;

                pos = skb_put(skb, 2 + sizeof(struct ieee80211_ht_operation));
                ieee80211_ie_build_ht_oper(pos, &sta->sta.deflink.ht_cap,
                                           &link->conf->chanreq.oper, prot,
                                           true);
        }

        ieee80211_tdls_add_link_ie(link, skb, peer, initiator);

        /* only include VHT-operation if not on the 2.4GHz band */
        if (sband->band != NL80211_BAND_2GHZ &&
            sta->sta.deflink.vht_cap.vht_supported) {
                /*
                 * if both peers support WIDER_BW, we can expand the chandef to
                 * a wider compatible one, up to 80MHz
                 */
                if (test_sta_flag(sta, WLAN_STA_TDLS_WIDER_BW))
                        ieee80211_tdls_chandef_vht_upgrade(sdata, sta);

                pos = skb_put(skb, 2 + sizeof(struct ieee80211_vht_operation));
                ieee80211_ie_build_vht_oper(pos, &sta->sta.deflink.vht_cap,
                                            &sta->tdls_chandef);
        }

        /* add any remaining IEs */
        if (extra_ies_len) {
                noffset = extra_ies_len;
                skb_put_data(skb, extra_ies + offset, noffset - offset);
        }
}

static void
ieee80211_tdls_add_chan_switch_req_ies(struct ieee80211_link_data *link,
                                       struct sk_buff *skb, const u8 *peer,
                                       bool initiator, const u8 *extra_ies,
                                       size_t extra_ies_len, u8 oper_class,
                                       struct cfg80211_chan_def *chandef)
{
        struct ieee80211_tdls_data *tf;
        size_t offset = 0, noffset;

        if (WARN_ON_ONCE(!chandef))
                return;

        tf = (void *)skb->data;
        tf->u.chan_switch_req.target_channel =
                ieee80211_frequency_to_channel(chandef->chan->center_freq);
        tf->u.chan_switch_req.oper_class = oper_class;

        if (extra_ies_len) {
                static const u8 before_lnkie[] = {
                        WLAN_EID_SECONDARY_CHANNEL_OFFSET,
                };
                noffset = ieee80211_ie_split(extra_ies, extra_ies_len,
                                             before_lnkie,
                                             ARRAY_SIZE(before_lnkie),
                                             offset);
                skb_put_data(skb, extra_ies + offset, noffset - offset);
                offset = noffset;
        }

        ieee80211_tdls_add_link_ie(link, skb, peer, initiator);

        /* add any remaining IEs */
        if (extra_ies_len) {
                noffset = extra_ies_len;
                skb_put_data(skb, extra_ies + offset, noffset - offset);
        }
}

static void
ieee80211_tdls_add_chan_switch_resp_ies(struct ieee80211_link_data *link,
                                        struct sk_buff *skb, const u8 *peer,
                                        u16 status_code, bool initiator,
                                        const u8 *extra_ies,
                                        size_t extra_ies_len)
{
        if (status_code == 0)
                ieee80211_tdls_add_link_ie(link, skb, peer, initiator);

        if (extra_ies_len)
                skb_put_data(skb, extra_ies, extra_ies_len);
}

static void ieee80211_tdls_add_ies(struct ieee80211_link_data *link,
                                   struct sk_buff *skb, const u8 *peer,
                                   u8 action_code, u16 status_code,
                                   bool initiator, const u8 *extra_ies,
                                   size_t extra_ies_len, u8 oper_class,
                                   struct cfg80211_chan_def *chandef)
{
        switch (action_code) {
        case WLAN_TDLS_SETUP_REQUEST:
        case WLAN_TDLS_SETUP_RESPONSE:
        case WLAN_PUB_ACTION_TDLS_DISCOVER_RES:
                if (status_code == 0)
                        ieee80211_tdls_add_setup_start_ies(link,
                                                           skb, peer,
                                                           action_code,
                                                           initiator,
                                                           extra_ies,
                                                           extra_ies_len);
                break;
        case WLAN_TDLS_SETUP_CONFIRM:
                if (status_code == 0)
                        ieee80211_tdls_add_setup_cfm_ies(link, skb, peer,
                                                         initiator, extra_ies,
                                                         extra_ies_len);
                break;
        case WLAN_TDLS_TEARDOWN:
        case WLAN_TDLS_DISCOVERY_REQUEST:
                if (extra_ies_len)
                        skb_put_data(skb, extra_ies, extra_ies_len);
                if (status_code == 0 || action_code == WLAN_TDLS_TEARDOWN)
                        ieee80211_tdls_add_link_ie(link, skb,
                                                   peer, initiator);
                break;
        case WLAN_TDLS_CHANNEL_SWITCH_REQUEST:
                ieee80211_tdls_add_chan_switch_req_ies(link, skb, peer,
                                                       initiator, extra_ies,
                                                       extra_ies_len,
                                                       oper_class, chandef);
                break;
        case WLAN_TDLS_CHANNEL_SWITCH_RESPONSE:
                ieee80211_tdls_add_chan_switch_resp_ies(link, skb, peer,
                                                        status_code,
                                                        initiator, extra_ies,
                                                        extra_ies_len);
                break;
        }

}

static int
ieee80211_prep_tdls_encap_data(struct wiphy *wiphy, struct net_device *dev,
                               struct ieee80211_link_data *link,
                               const u8 *peer, u8 action_code, u8 dialog_token,
                               u16 status_code, struct sk_buff *skb)
{
        struct ieee80211_sub_if_data *sdata = IEEE80211_DEV_TO_SUB_IF(dev);
        struct ieee80211_tdls_data *tf;

        tf = skb_put(skb, offsetof(struct ieee80211_tdls_data, u));

        memcpy(tf->da, peer, ETH_ALEN);
        memcpy(tf->sa, sdata->vif.addr, ETH_ALEN);
        tf->ether_type = cpu_to_be16(ETH_P_TDLS);
        tf->payload_type = WLAN_TDLS_SNAP_RFTYPE;

        /* network header is after the ethernet header */
        skb_set_network_header(skb, ETH_HLEN);

        switch (action_code) {
        case WLAN_TDLS_SETUP_REQUEST:
                tf->category = WLAN_CATEGORY_TDLS;
                tf->action_code = WLAN_TDLS_SETUP_REQUEST;

                skb_put(skb, sizeof(tf->u.setup_req));
                tf->u.setup_req.dialog_token = dialog_token;
                tf->u.setup_req.capability =
                        cpu_to_le16(ieee80211_get_tdls_sta_capab(link,
                                                                 status_code));
                break;
        case WLAN_TDLS_SETUP_RESPONSE:
                tf->category = WLAN_CATEGORY_TDLS;
                tf->action_code = WLAN_TDLS_SETUP_RESPONSE;

                skb_put(skb, sizeof(tf->u.setup_resp));
                tf->u.setup_resp.status_code = cpu_to_le16(status_code);
                tf->u.setup_resp.dialog_token = dialog_token;
                tf->u.setup_resp.capability =
                        cpu_to_le16(ieee80211_get_tdls_sta_capab(link,
                                                                 status_code));
                break;
        case WLAN_TDLS_SETUP_CONFIRM:
                tf->category = WLAN_CATEGORY_TDLS;
                tf->action_code = WLAN_TDLS_SETUP_CONFIRM;

                skb_put(skb, sizeof(tf->u.setup_cfm));
                tf->u.setup_cfm.status_code = cpu_to_le16(status_code);
                tf->u.setup_cfm.dialog_token = dialog_token;
                break;
        case WLAN_TDLS_TEARDOWN:
                tf->category = WLAN_CATEGORY_TDLS;
                tf->action_code = WLAN_TDLS_TEARDOWN;

                skb_put(skb, sizeof(tf->u.teardown));
                tf->u.teardown.reason_code = cpu_to_le16(status_code);
                break;
        case WLAN_TDLS_DISCOVERY_REQUEST:
                tf->category = WLAN_CATEGORY_TDLS;
                tf->action_code = WLAN_TDLS_DISCOVERY_REQUEST;

                skb_put(skb, sizeof(tf->u.discover_req));
                tf->u.discover_req.dialog_token = dialog_token;
                break;
        case WLAN_TDLS_CHANNEL_SWITCH_REQUEST:
                tf->category = WLAN_CATEGORY_TDLS;
                tf->action_code = WLAN_TDLS_CHANNEL_SWITCH_REQUEST;

                skb_put(skb, sizeof(tf->u.chan_switch_req));
                break;
        case WLAN_TDLS_CHANNEL_SWITCH_RESPONSE:
                tf->category = WLAN_CATEGORY_TDLS;
                tf->action_code = WLAN_TDLS_CHANNEL_SWITCH_RESPONSE;

                skb_put(skb, sizeof(tf->u.chan_switch_resp));
                tf->u.chan_switch_resp.status_code = cpu_to_le16(status_code);
                break;
        default:
                return -EINVAL;
        }

        return 0;
}

static int
ieee80211_prep_tdls_direct(struct wiphy *wiphy, struct net_device *dev,
                           const u8 *peer, struct ieee80211_link_data *link,
                           u8 action_code, u8 dialog_token,
                           u16 status_code, struct sk_buff *skb)
{
        struct ieee80211_sub_if_data *sdata = IEEE80211_DEV_TO_SUB_IF(dev);
        struct ieee80211_mgmt *mgmt;

        mgmt = skb_put_zero(skb, 24);
        memcpy(mgmt->da, peer, ETH_ALEN);
        memcpy(mgmt->sa, sdata->vif.addr, ETH_ALEN);
        memcpy(mgmt->bssid, link->u.mgd.bssid, ETH_ALEN);
        mgmt->frame_control = cpu_to_le16(IEEE80211_FTYPE_MGMT |
                                          IEEE80211_STYPE_ACTION);

        switch (action_code) {
        case WLAN_PUB_ACTION_TDLS_DISCOVER_RES:
                skb_put(skb, 1 + sizeof(mgmt->u.action.u.tdls_discover_resp));
                mgmt->u.action.category = WLAN_CATEGORY_PUBLIC;
                mgmt->u.action.u.tdls_discover_resp.action_code =
                        WLAN_PUB_ACTION_TDLS_DISCOVER_RES;
                mgmt->u.action.u.tdls_discover_resp.dialog_token =
                        dialog_token;
                mgmt->u.action.u.tdls_discover_resp.capability =
                        cpu_to_le16(ieee80211_get_tdls_sta_capab(link,
                                                                 status_code));
                break;
        default:
                return -EINVAL;
        }

        return 0;
}

static struct sk_buff *
ieee80211_tdls_build_mgmt_packet_data(struct ieee80211_sub_if_data *sdata,
                                      const u8 *peer, int link_id,
                                      u8 action_code, u8 dialog_token,
                                      u16 status_code, bool initiator,
                                      const u8 *extra_ies, size_t extra_ies_len,
                                      u8 oper_class,
                                      struct cfg80211_chan_def *chandef)
{
        struct ieee80211_local *local = sdata->local;
        struct sk_buff *skb;
        int ret;
        struct ieee80211_link_data *link;

        link_id = link_id >= 0 ? link_id : 0;
        rcu_read_lock();
        link = rcu_dereference(sdata->link[link_id]);
        if (WARN_ON(!link))
                goto unlock;

        skb = netdev_alloc_skb(sdata->dev,
                               local->hw.extra_tx_headroom +
                               max(sizeof(struct ieee80211_mgmt),
                                   sizeof(struct ieee80211_tdls_data)) +
                               50 + /* supported rates */
                               10 + /* ext capab */
                               26 + /* max(WMM-info, WMM-param) */
                               2 + max(sizeof(struct ieee80211_ht_cap),
                                       sizeof(struct ieee80211_ht_operation)) +
                               2 + max(sizeof(struct ieee80211_vht_cap),
                                       sizeof(struct ieee80211_vht_operation)) +
                               2 + 1 + sizeof(struct ieee80211_he_cap_elem) +
                                       sizeof(struct ieee80211_he_mcs_nss_supp) +
                                       IEEE80211_HE_PPE_THRES_MAX_LEN +
                               2 + 1 + sizeof(struct ieee80211_he_6ghz_capa) +
                               2 + 1 + sizeof(struct ieee80211_eht_cap_elem) +
                                       sizeof(struct ieee80211_eht_mcs_nss_supp) +
                                       IEEE80211_EHT_PPE_THRES_MAX_LEN +
                               50 + /* supported channels */
                               3 + /* 40/20 BSS coex */
                               4 + /* AID */
                               4 + /* oper classes */
                               extra_ies_len +
                               sizeof(struct ieee80211_tdls_lnkie));
        if (!skb)
                goto unlock;

        skb_reserve(skb, local->hw.extra_tx_headroom);

        switch (action_code) {
        case WLAN_TDLS_SETUP_REQUEST:
        case WLAN_TDLS_SETUP_RESPONSE:
        case WLAN_TDLS_SETUP_CONFIRM:
        case WLAN_TDLS_TEARDOWN:
        case WLAN_TDLS_DISCOVERY_REQUEST:
        case WLAN_TDLS_CHANNEL_SWITCH_REQUEST:
        case WLAN_TDLS_CHANNEL_SWITCH_RESPONSE:
                ret = ieee80211_prep_tdls_encap_data(local->hw.wiphy,
                                                     sdata->dev, link, peer,
                                                     action_code, dialog_token,
                                                     status_code, skb);
                break;
        case WLAN_PUB_ACTION_TDLS_DISCOVER_RES:
                ret = ieee80211_prep_tdls_direct(local->hw.wiphy, sdata->dev,
                                                 peer, link, action_code,
                                                 dialog_token, status_code,
                                                 skb);
                break;
        default:
                ret = -EOPNOTSUPP;
                break;
        }

        if (ret < 0)
                goto fail;

        ieee80211_tdls_add_ies(link, skb, peer, action_code, status_code,
                               initiator, extra_ies, extra_ies_len, oper_class,
                               chandef);
        rcu_read_unlock();
        return skb;

fail:
        dev_kfree_skb(skb);
unlock:
        rcu_read_unlock();
        return NULL;
}

static int
ieee80211_tdls_prep_mgmt_packet(struct wiphy *wiphy, struct net_device *dev,
                                const u8 *peer, int link_id,
                                u8 action_code, u8 dialog_token,
                                u16 status_code, u32 peer_capability,
                                bool initiator, const u8 *extra_ies,
                                size_t extra_ies_len, u8 oper_class,
                                struct cfg80211_chan_def *chandef)
{
        struct ieee80211_sub_if_data *sdata = IEEE80211_DEV_TO_SUB_IF(dev);
        struct sk_buff *skb = NULL;
        struct sta_info *sta;
        u32 flags = 0;
        int ret = 0;

        rcu_read_lock();
        sta = sta_info_get(sdata, peer);

        /* infer the initiator if we can, to support old userspace */
        switch (action_code) {
        case WLAN_TDLS_SETUP_REQUEST:
                if (sta) {
                        set_sta_flag(sta, WLAN_STA_TDLS_INITIATOR);
                        sta->sta.tdls_initiator = false;
                }
                fallthrough;
        case WLAN_TDLS_SETUP_CONFIRM:
        case WLAN_TDLS_DISCOVERY_REQUEST:
                initiator = true;
                break;
        case WLAN_TDLS_SETUP_RESPONSE:
                /*
                 * In some testing scenarios, we send a request and response.
                 * Make the last packet sent take effect for the initiator
                 * value.
                 */
                if (sta) {
                        clear_sta_flag(sta, WLAN_STA_TDLS_INITIATOR);
                        sta->sta.tdls_initiator = true;
                }
                fallthrough;
        case WLAN_PUB_ACTION_TDLS_DISCOVER_RES:
                initiator = false;
                break;
        case WLAN_TDLS_TEARDOWN:
        case WLAN_TDLS_CHANNEL_SWITCH_REQUEST:
        case WLAN_TDLS_CHANNEL_SWITCH_RESPONSE:
                /* any value is ok */
                break;
        default:
                ret = -EOPNOTSUPP;
                break;
        }

        if (sta && test_sta_flag(sta, WLAN_STA_TDLS_INITIATOR))
                initiator = true;

        rcu_read_unlock();
        if (ret < 0)
                goto fail;

        skb = ieee80211_tdls_build_mgmt_packet_data(sdata, peer,
                                                    link_id, action_code,
                                                    dialog_token, status_code,
                                                    initiator, extra_ies,
                                                    extra_ies_len, oper_class,
                                                    chandef);
        if (!skb) {
                ret = -EINVAL;
                goto fail;
        }

        if (action_code == WLAN_PUB_ACTION_TDLS_DISCOVER_RES) {
                ieee80211_tx_skb_tid(sdata, skb, 7, link_id);
                return 0;
        }

        /*
         * According to 802.11z: Setup req/resp are sent in AC_BK, otherwise
         * we should default to AC_VI.
         */
        switch (action_code) {
        case WLAN_TDLS_SETUP_REQUEST:
        case WLAN_TDLS_SETUP_RESPONSE:
                skb->priority = 256 + 2;
                break;
        default:
                skb->priority = 256 + 5;
                break;
        }

        /*
         * Set the WLAN_TDLS_TEARDOWN flag to indicate a teardown in progress.
         * Later, if no ACK is returned from peer, we will re-send the teardown
         * packet through the AP.
         */
        if ((action_code == WLAN_TDLS_TEARDOWN) &&
            ieee80211_hw_check(&sdata->local->hw, REPORTS_TX_ACK_STATUS)) {
                bool try_resend; /* Should we keep skb for possible resend */

                /* If not sending directly to peer - no point in keeping skb */
                rcu_read_lock();
                sta = sta_info_get(sdata, peer);
                try_resend = sta && test_sta_flag(sta, WLAN_STA_TDLS_PEER_AUTH);
                rcu_read_unlock();

                spin_lock_bh(&sdata->u.mgd.teardown_lock);
                if (try_resend && !sdata->u.mgd.teardown_skb) {
                        /* Mark it as requiring TX status callback  */
                        flags |= IEEE80211_TX_CTL_REQ_TX_STATUS |
                                 IEEE80211_TX_INTFL_MLME_CONN_TX;

                        /*
                         * skb is copied since mac80211 will later set
                         * properties that might not be the same as the AP,
                         * such as encryption, QoS, addresses, etc.
                         *
                         * No problem if skb_copy() fails, so no need to check.
                         */
                        sdata->u.mgd.teardown_skb = skb_copy(skb, GFP_ATOMIC);
                        sdata->u.mgd.orig_teardown_skb = skb;
                }
                spin_unlock_bh(&sdata->u.mgd.teardown_lock);
        }

        /* disable bottom halves when entering the Tx path */
        local_bh_disable();
        __ieee80211_subif_start_xmit(skb, dev, flags,
                                     IEEE80211_TX_CTRL_MLO_LINK_UNSPEC, NULL);
        local_bh_enable();

        return ret;

fail:
        dev_kfree_skb(skb);
        return ret;
}

static int
ieee80211_tdls_mgmt_setup(struct wiphy *wiphy, struct net_device *dev,
                          const u8 *peer, int link_id,
                          u8 action_code, u8 dialog_token,
                          u16 status_code, u32 peer_capability, bool initiator,
                          const u8 *extra_ies, size_t extra_ies_len)
{
        struct ieee80211_sub_if_data *sdata = IEEE80211_DEV_TO_SUB_IF(dev);
        struct ieee80211_local *local = sdata->local;
        enum ieee80211_smps_mode smps_mode =
                sdata->deflink.u.mgd.driver_smps_mode;
        int ret;

        /* don't support setup with forced SMPS mode that's not off */
        if (smps_mode != IEEE80211_SMPS_AUTOMATIC &&
            smps_mode != IEEE80211_SMPS_OFF) {
                tdls_dbg(sdata, "Aborting TDLS setup due to SMPS mode %d\n",
                         smps_mode);
                return -EOPNOTSUPP;
        }

        lockdep_assert_wiphy(local->hw.wiphy);

        /* we don't support concurrent TDLS peer setups */
        if (!is_zero_ether_addr(sdata->u.mgd.tdls_peer) &&
            !ether_addr_equal(sdata->u.mgd.tdls_peer, peer)) {
                ret = -EBUSY;
                goto out_unlock;
        }

        /*
         * make sure we have a STA representing the peer so we drop or buffer
         * non-TDLS-setup frames to the peer. We can't send other packets
         * during setup through the AP path.
         * Allow error packets to be sent - sometimes we don't even add a STA
         * before failing the setup.
         */
        if (status_code == 0) {
                rcu_read_lock();
                if (!sta_info_get(sdata, peer)) {
                        rcu_read_unlock();
                        ret = -ENOLINK;
                        goto out_unlock;
                }
                rcu_read_unlock();
        }

        ieee80211_flush_queues(local, sdata, false);
        memcpy(sdata->u.mgd.tdls_peer, peer, ETH_ALEN);

        /* we cannot take the mutex while preparing the setup packet */
        ret = ieee80211_tdls_prep_mgmt_packet(wiphy, dev, peer,
                                              link_id, action_code,
                                              dialog_token, status_code,
                                              peer_capability, initiator,
                                              extra_ies, extra_ies_len, 0,
                                              NULL);
        if (ret < 0) {
                eth_zero_addr(sdata->u.mgd.tdls_peer);
                return ret;
        }

        wiphy_delayed_work_queue(sdata->local->hw.wiphy,
                                 &sdata->u.mgd.tdls_peer_del_work,
                                 TDLS_PEER_SETUP_TIMEOUT);
        return 0;

out_unlock:
        return ret;
}

static int
ieee80211_tdls_mgmt_teardown(struct wiphy *wiphy, struct net_device *dev,
                             const u8 *peer, int link_id,
                             u8 action_code, u8 dialog_token,
                             u16 status_code, u32 peer_capability,
                             bool initiator, const u8 *extra_ies,
                             size_t extra_ies_len)
{
        struct ieee80211_sub_if_data *sdata = IEEE80211_DEV_TO_SUB_IF(dev);
        struct ieee80211_local *local = sdata->local;
        struct sta_info *sta;
        int ret;

        /*
         * No packets can be transmitted to the peer via the AP during setup -
         * the STA is set as a TDLS peer, but is not authorized.
         * During teardown, we prevent direct transmissions by stopping the
         * queues and flushing all direct packets.
         */
        ieee80211_stop_vif_queues(local, sdata,
                                  IEEE80211_QUEUE_STOP_REASON_TDLS_TEARDOWN);
        ieee80211_flush_queues(local, sdata, false);

        ret = ieee80211_tdls_prep_mgmt_packet(wiphy, dev, peer,
                                              link_id, action_code,
                                              dialog_token, status_code,
                                              peer_capability, initiator,
                                              extra_ies, extra_ies_len, 0,
                                              NULL);
        if (ret < 0)
                sdata_err(sdata, "Failed sending TDLS teardown packet %d\n",
                          ret);

        /*
         * Remove the STA AUTH flag to force further traffic through the AP. If
         * the STA was unreachable, it was already removed.
         */
        rcu_read_lock();
        sta = sta_info_get(sdata, peer);
        if (sta)
                clear_sta_flag(sta, WLAN_STA_TDLS_PEER_AUTH);
        rcu_read_unlock();

        ieee80211_wake_vif_queues(local, sdata,
                                  IEEE80211_QUEUE_STOP_REASON_TDLS_TEARDOWN);

        return 0;
}

int ieee80211_tdls_mgmt(struct wiphy *wiphy, struct net_device *dev,
                        const u8 *peer, int link_id,
                        u8 action_code, u8 dialog_token, u16 status_code,
                        u32 peer_capability, bool initiator,
                        const u8 *extra_ies, size_t extra_ies_len)
{
        struct ieee80211_sub_if_data *sdata = IEEE80211_DEV_TO_SUB_IF(dev);
        int ret;

        if (!(wiphy->flags & WIPHY_FLAG_SUPPORTS_TDLS))
                return -EOPNOTSUPP;

        /* make sure we are in managed mode, and associated */
        if (sdata->vif.type != NL80211_IFTYPE_STATION ||
            !sdata->u.mgd.associated)
                return -EINVAL;

        switch (action_code) {
        case WLAN_TDLS_SETUP_REQUEST:
        case WLAN_TDLS_SETUP_RESPONSE:
                ret = ieee80211_tdls_mgmt_setup(wiphy, dev, peer,
                                                link_id, action_code,
                                                dialog_token, status_code,
                                                peer_capability, initiator,
                                                extra_ies, extra_ies_len);
                break;
        case WLAN_TDLS_TEARDOWN:
                ret = ieee80211_tdls_mgmt_teardown(wiphy, dev, peer, link_id,
                                                   action_code, dialog_token,
                                                   status_code,
                                                   peer_capability, initiator,
                                                   extra_ies, extra_ies_len);
                break;
        case WLAN_TDLS_DISCOVERY_REQUEST:
                /*
                 * Protect the discovery so we can hear the TDLS discovery
                 * response frame. It is transmitted directly and not buffered
                 * by the AP.
                 */
                drv_mgd_protect_tdls_discover(sdata->local, sdata, link_id);
                fallthrough;
        case WLAN_TDLS_SETUP_CONFIRM:
        case WLAN_PUB_ACTION_TDLS_DISCOVER_RES:
                /* no special handling */
                ret = ieee80211_tdls_prep_mgmt_packet(wiphy, dev, peer,
                                                      link_id, action_code,
                                                      dialog_token,
                                                      status_code,
                                                      peer_capability,
                                                      initiator, extra_ies,
                                                      extra_ies_len, 0, NULL);
                break;
        default:
                ret = -EOPNOTSUPP;
                break;
        }

        tdls_dbg(sdata, "TDLS mgmt action %d peer %pM link_id %d status %d\n",
                 action_code, peer, link_id, ret);
        return ret;
}

static void iee80211_tdls_recalc_chanctx(struct ieee80211_sub_if_data *sdata,
                                         struct sta_info *sta)
{
        struct ieee80211_local *local = sdata->local;
        struct ieee80211_chanctx_conf *conf;
        struct ieee80211_chanctx *ctx;
        enum nl80211_chan_width width;
        struct ieee80211_supported_band *sband;

        lockdep_assert_wiphy(local->hw.wiphy);

        conf = rcu_dereference_protected(sdata->vif.bss_conf.chanctx_conf,
                                         lockdep_is_held(&local->hw.wiphy->mtx));
        if (conf) {
                width = conf->def.width;
                sband = local->hw.wiphy->bands[conf->def.chan->band];
                ctx = container_of(conf, struct ieee80211_chanctx, conf);
                ieee80211_recalc_chanctx_chantype(local, ctx);

                /* if width changed and a peer is given, update its BW */
                if (width != conf->def.width && sta &&
                    test_sta_flag(sta, WLAN_STA_TDLS_WIDER_BW)) {
                        enum ieee80211_sta_rx_bandwidth bw;

                        bw = ieee80211_chan_width_to_rx_bw(conf->def.width);
                        bw = min(bw, ieee80211_sta_cap_rx_bw(&sta->deflink));
                        if (bw != sta->sta.deflink.bandwidth) {
                                sta->sta.deflink.bandwidth = bw;
                                rate_control_rate_update(local, sband,
                                                         &sta->deflink,
                                                         IEEE80211_RC_BW_CHANGED);
                                /*
                                 * if a TDLS peer BW was updated, we need to
                                 * recalc the chandef width again, to get the
                                 * correct chanctx min_def
                                 */
                                ieee80211_recalc_chanctx_chantype(local, ctx);
                        }
                }

        }
}

static int iee80211_tdls_have_ht_peers(struct ieee80211_sub_if_data *sdata)
{
        struct sta_info *sta;
        bool result = false;

        rcu_read_lock();
        list_for_each_entry_rcu(sta, &sdata->local->sta_list, list) {
                if (!sta->sta.tdls || sta->sdata != sdata || !sta->uploaded ||
                    !test_sta_flag(sta, WLAN_STA_AUTHORIZED) ||
                    !test_sta_flag(sta, WLAN_STA_TDLS_PEER_AUTH) ||
                    !sta->sta.deflink.ht_cap.ht_supported)
                        continue;
                result = true;
                break;
        }
        rcu_read_unlock();

        return result;
}

static void
iee80211_tdls_recalc_ht_protection(struct ieee80211_sub_if_data *sdata,
                                   struct sta_info *sta)
{
        bool tdls_ht;
        u16 protection = IEEE80211_HT_OP_MODE_PROTECTION_NONHT_MIXED |
                         IEEE80211_HT_OP_MODE_NON_GF_STA_PRSNT |
                         IEEE80211_HT_OP_MODE_NON_HT_STA_PRSNT;
        u16 opmode;

        /* Nothing to do if the BSS connection uses (at least) HT */
        if (sdata->deflink.u.mgd.conn.mode >= IEEE80211_CONN_MODE_HT)
                return;

        tdls_ht = (sta && sta->sta.deflink.ht_cap.ht_supported) ||
                  iee80211_tdls_have_ht_peers(sdata);

        opmode = sdata->vif.bss_conf.ht_operation_mode;

        if (tdls_ht)
                opmode |= protection;
        else
                opmode &= ~protection;

        if (opmode == sdata->vif.bss_conf.ht_operation_mode)
                return;

        sdata->vif.bss_conf.ht_operation_mode = opmode;
        ieee80211_link_info_change_notify(sdata, &sdata->deflink,
                                          BSS_CHANGED_HT);
}

int ieee80211_tdls_oper(struct wiphy *wiphy, struct net_device *dev,
                        const u8 *peer, enum nl80211_tdls_operation oper)
{
        struct sta_info *sta;
        struct ieee80211_sub_if_data *sdata = IEEE80211_DEV_TO_SUB_IF(dev);
        struct ieee80211_local *local = sdata->local;
        int ret;

        lockdep_assert_wiphy(local->hw.wiphy);

        if (!(wiphy->flags & WIPHY_FLAG_SUPPORTS_TDLS))
                return -EOPNOTSUPP;

        if (sdata->vif.type != NL80211_IFTYPE_STATION)
                return -EINVAL;

        switch (oper) {
        case NL80211_TDLS_ENABLE_LINK:
        case NL80211_TDLS_DISABLE_LINK:
                break;
        case NL80211_TDLS_TEARDOWN:
        case NL80211_TDLS_SETUP:
        case NL80211_TDLS_DISCOVERY_REQ:
                /* We don't support in-driver setup/teardown/discovery */
                return -EOPNOTSUPP;
        }

        /* protect possible bss_conf changes and avoid concurrency in
         * ieee80211_bss_info_change_notify()
         */
        tdls_dbg(sdata, "TDLS oper %d peer %pM\n", oper, peer);

        switch (oper) {
        case NL80211_TDLS_ENABLE_LINK:
                if (sdata->vif.bss_conf.csa_active) {
                        tdls_dbg(sdata, "TDLS: disallow link during CSA\n");
                        return -EBUSY;
                }

                sta = sta_info_get(sdata, peer);
                if (!sta)
                        return -ENOLINK;

                iee80211_tdls_recalc_chanctx(sdata, sta);
                iee80211_tdls_recalc_ht_protection(sdata, sta);

                set_sta_flag(sta, WLAN_STA_TDLS_PEER_AUTH);

                WARN_ON_ONCE(is_zero_ether_addr(sdata->u.mgd.tdls_peer) ||
                             !ether_addr_equal(sdata->u.mgd.tdls_peer, peer));
                break;
        case NL80211_TDLS_DISABLE_LINK:
                /*
                 * The teardown message in ieee80211_tdls_mgmt_teardown() was
                 * created while the queues were stopped, so it might still be
                 * pending. Before flushing the queues we need to be sure the
                 * message is handled by the tasklet handling pending messages,
                 * otherwise we might start destroying the station before
                 * sending the teardown packet.
                 * Note that this only forces the tasklet to flush pendings -
                 * not to stop the tasklet from rescheduling itself.
                 */
                tasklet_kill(&local->tx_pending_tasklet);
                /* flush a potentially queued teardown packet */
                ieee80211_flush_queues(local, sdata, false);

                ret = sta_info_destroy_addr(sdata, peer);

                iee80211_tdls_recalc_ht_protection(sdata, NULL);

                iee80211_tdls_recalc_chanctx(sdata, NULL);
                if (ret)
                        return ret;
                break;
        default:
                return -EOPNOTSUPP;
        }

        if (ether_addr_equal(sdata->u.mgd.tdls_peer, peer)) {
                wiphy_delayed_work_cancel(sdata->local->hw.wiphy,
                                          &sdata->u.mgd.tdls_peer_del_work);
                eth_zero_addr(sdata->u.mgd.tdls_peer);
        }

        wiphy_work_queue(sdata->local->hw.wiphy,
                         &sdata->deflink.u.mgd.request_smps_work);

        return 0;
}

void ieee80211_tdls_oper_request(struct ieee80211_vif *vif, const u8 *peer,
                                 enum nl80211_tdls_operation oper,
                                 u16 reason_code, gfp_t gfp)
{
        struct ieee80211_sub_if_data *sdata = vif_to_sdata(vif);

        if (vif->type != NL80211_IFTYPE_STATION || !vif->cfg.assoc) {
                sdata_err(sdata, "Discarding TDLS oper %d - not STA or disconnected\n",
                          oper);
                return;
        }

        cfg80211_tdls_oper_request(sdata->dev, peer, oper, reason_code, gfp);
}
EXPORT_SYMBOL(ieee80211_tdls_oper_request);

static void
iee80211_tdls_add_ch_switch_timing(u8 *buf, u16 switch_time, u16 switch_timeout)
{
        struct ieee80211_ch_switch_timing *ch_sw;

        *buf++ = WLAN_EID_CHAN_SWITCH_TIMING;
        *buf++ = sizeof(struct ieee80211_ch_switch_timing);

        ch_sw = (void *)buf;
        ch_sw->switch_time = cpu_to_le16(switch_time);
        ch_sw->switch_timeout = cpu_to_le16(switch_timeout);
}

/* find switch timing IE in SKB ready for Tx */
static const u8 *ieee80211_tdls_find_sw_timing_ie(struct sk_buff *skb)
{
        struct ieee80211_tdls_data *tf;
        const u8 *ie_start;

        /*
         * Get the offset for the new location of the switch timing IE.
         * The SKB network header will now point to the "payload_type"
         * element of the TDLS data frame struct.
         */
        tf = container_of(skb->data + skb_network_offset(skb),
                          struct ieee80211_tdls_data, payload_type);
        ie_start = tf->u.chan_switch_req.variable;
        return cfg80211_find_ie(WLAN_EID_CHAN_SWITCH_TIMING, ie_start,
                                skb->len - (ie_start - skb->data));
}

static struct sk_buff *
ieee80211_tdls_ch_sw_tmpl_get(struct sta_info *sta, u8 oper_class,
                              struct cfg80211_chan_def *chandef,
                              u32 *ch_sw_tm_ie_offset)
{
        struct ieee80211_sub_if_data *sdata = sta->sdata;
        u8 extra_ies[2 + sizeof(struct ieee80211_sec_chan_offs_ie) +
                     2 + sizeof(struct ieee80211_ch_switch_timing)];
        int extra_ies_len = 2 + sizeof(struct ieee80211_ch_switch_timing);
        u8 *pos = extra_ies;
        struct sk_buff *skb;
        int link_id = sta->sta.valid_links ? ffs(sta->sta.valid_links) - 1 : 0;

        /*
         * if chandef points to a wide channel add a Secondary-Channel
         * Offset information element
         */
        if (chandef->width == NL80211_CHAN_WIDTH_40) {
                struct ieee80211_sec_chan_offs_ie *sec_chan_ie;
                bool ht40plus;

                *pos++ = WLAN_EID_SECONDARY_CHANNEL_OFFSET;
                *pos++ = sizeof(*sec_chan_ie);
                sec_chan_ie = (void *)pos;

                ht40plus = cfg80211_get_chandef_type(chandef) ==
                                                        NL80211_CHAN_HT40PLUS;
                sec_chan_ie->sec_chan_offs = ht40plus ?
                                             IEEE80211_HT_PARAM_CHA_SEC_ABOVE :
                                             IEEE80211_HT_PARAM_CHA_SEC_BELOW;
                pos += sizeof(*sec_chan_ie);

                extra_ies_len += 2 + sizeof(struct ieee80211_sec_chan_offs_ie);
        }

        /* just set the values to 0, this is a template */
        iee80211_tdls_add_ch_switch_timing(pos, 0, 0);

        skb = ieee80211_tdls_build_mgmt_packet_data(sdata, sta->sta.addr,
                                              link_id,
                                              WLAN_TDLS_CHANNEL_SWITCH_REQUEST,
                                              0, 0, !sta->sta.tdls_initiator,
                                              extra_ies, extra_ies_len,
                                              oper_class, chandef);
        if (!skb)
                return NULL;

        skb = ieee80211_build_data_template(sdata, skb, 0);
        if (IS_ERR(skb)) {
                tdls_dbg(sdata, "Failed building TDLS channel switch frame\n");
                return NULL;
        }

        if (ch_sw_tm_ie_offset) {
                const u8 *tm_ie = ieee80211_tdls_find_sw_timing_ie(skb);

                if (!tm_ie) {
                        tdls_dbg(sdata, "No switch timing IE in TDLS switch\n");
                        dev_kfree_skb_any(skb);
                        return NULL;
                }

                *ch_sw_tm_ie_offset = tm_ie - skb->data;
        }

        tdls_dbg(sdata,
                 "TDLS channel switch request template for %pM ch %d width %d\n",
                 sta->sta.addr, chandef->chan->center_freq, chandef->width);
        return skb;
}

int
ieee80211_tdls_channel_switch(struct wiphy *wiphy, struct net_device *dev,
                              const u8 *addr, u8 oper_class,
                              struct cfg80211_chan_def *chandef)
{
        struct ieee80211_sub_if_data *sdata = IEEE80211_DEV_TO_SUB_IF(dev);
        struct ieee80211_local *local = sdata->local;
        struct sta_info *sta;
        struct sk_buff *skb = NULL;
        u32 ch_sw_tm_ie;
        int ret;

        lockdep_assert_wiphy(local->hw.wiphy);

        if (chandef->chan->freq_offset)
                /* this may work, but is untested */
                return -EOPNOTSUPP;

        sta = sta_info_get(sdata, addr);
        if (!sta) {
                tdls_dbg(sdata,
                         "Invalid TDLS peer %pM for channel switch request\n",
                         addr);
                ret = -ENOENT;
                goto out;
        }

        if (!test_sta_flag(sta, WLAN_STA_TDLS_CHAN_SWITCH)) {
                tdls_dbg(sdata, "TDLS channel switch unsupported by %pM\n",
                         addr);
                ret = -EOPNOTSUPP;
                goto out;
        }

        skb = ieee80211_tdls_ch_sw_tmpl_get(sta, oper_class, chandef,
                                            &ch_sw_tm_ie);
        if (!skb) {
                ret = -ENOENT;
                goto out;
        }

        ret = drv_tdls_channel_switch(local, sdata, &sta->sta, oper_class,
                                      chandef, skb, ch_sw_tm_ie);
        if (!ret)
                set_sta_flag(sta, WLAN_STA_TDLS_OFF_CHANNEL);

out:
        dev_kfree_skb_any(skb);
        return ret;
}

void
ieee80211_tdls_cancel_channel_switch(struct wiphy *wiphy,
                                     struct net_device *dev,
                                     const u8 *addr)
{
        struct ieee80211_sub_if_data *sdata = IEEE80211_DEV_TO_SUB_IF(dev);
        struct ieee80211_local *local = sdata->local;
        struct sta_info *sta;

        lockdep_assert_wiphy(local->hw.wiphy);

        sta = sta_info_get(sdata, addr);
        if (!sta) {
                tdls_dbg(sdata,
                         "Invalid TDLS peer %pM for channel switch cancel\n",
                         addr);
                return;
        }

        if (!test_sta_flag(sta, WLAN_STA_TDLS_OFF_CHANNEL)) {
                tdls_dbg(sdata, "TDLS channel switch not initiated by %pM\n",
                         addr);
                return;
        }

        drv_tdls_cancel_channel_switch(local, sdata, &sta->sta);
        clear_sta_flag(sta, WLAN_STA_TDLS_OFF_CHANNEL);
}

static struct sk_buff *
ieee80211_tdls_ch_sw_resp_tmpl_get(struct sta_info *sta,
                                   u32 *ch_sw_tm_ie_offset)
{
        struct ieee80211_sub_if_data *sdata = sta->sdata;
        struct sk_buff *skb;
        u8 extra_ies[2 + sizeof(struct ieee80211_ch_switch_timing)];
        int link_id = sta->sta.valid_links ? ffs(sta->sta.valid_links) - 1 : 0;

        /* initial timing are always zero in the template */
        iee80211_tdls_add_ch_switch_timing(extra_ies, 0, 0);

        skb = ieee80211_tdls_build_mgmt_packet_data(sdata, sta->sta.addr,
                                        link_id,
                                        WLAN_TDLS_CHANNEL_SWITCH_RESPONSE,
                                        0, 0, !sta->sta.tdls_initiator,
                                        extra_ies, sizeof(extra_ies), 0, NULL);
        if (!skb)
                return NULL;

        skb = ieee80211_build_data_template(sdata, skb, 0);
        if (IS_ERR(skb)) {
                tdls_dbg(sdata,
                         "Failed building TDLS channel switch resp frame\n");
                return NULL;
        }

        if (ch_sw_tm_ie_offset) {
                const u8 *tm_ie = ieee80211_tdls_find_sw_timing_ie(skb);

                if (!tm_ie) {
                        tdls_dbg(sdata,
                                 "No switch timing IE in TDLS switch resp\n");
                        dev_kfree_skb_any(skb);
                        return NULL;
                }

                *ch_sw_tm_ie_offset = tm_ie - skb->data;
        }

        tdls_dbg(sdata, "TDLS get channel switch response template for %pM\n",
                 sta->sta.addr);
        return skb;
}

static int
ieee80211_process_tdls_channel_switch_resp(struct ieee80211_sub_if_data *sdata,
                                           struct sk_buff *skb)
{
        struct ieee80211_local *local = sdata->local;
        struct ieee802_11_elems *elems = NULL;
        struct sta_info *sta;
        struct ieee80211_tdls_data *tf = (void *)skb->data;
        bool local_initiator;
        struct ieee80211_rx_status *rx_status = IEEE80211_SKB_RXCB(skb);
        int baselen = offsetof(typeof(*tf), u.chan_switch_resp.variable);
        struct ieee80211_tdls_ch_sw_params params = {};
        int ret;

        lockdep_assert_wiphy(local->hw.wiphy);

        params.action_code = WLAN_TDLS_CHANNEL_SWITCH_RESPONSE;
        params.timestamp = rx_status->device_timestamp;

        if (skb->len < baselen) {
                tdls_dbg(sdata, "TDLS channel switch resp too short: %d\n",
                         skb->len);
                return -EINVAL;
        }

        sta = sta_info_get(sdata, tf->sa);
        if (!sta || !test_sta_flag(sta, WLAN_STA_TDLS_PEER_AUTH)) {
                tdls_dbg(sdata, "TDLS chan switch from non-peer sta %pM\n",
                         tf->sa);
                ret = -EINVAL;
                goto out;
        }

        params.sta = &sta->sta;
        params.status = le16_to_cpu(tf->u.chan_switch_resp.status_code);
        if (params.status != 0) {
                ret = 0;
                goto call_drv;
        }

        elems = ieee802_11_parse_elems(tf->u.chan_switch_resp.variable,
                                       skb->len - baselen, false, NULL);
        if (!elems) {
                ret = -ENOMEM;
                goto out;
        }

        if (elems->parse_error) {
                tdls_dbg(sdata, "Invalid IEs in TDLS channel switch resp\n");
                ret = -EINVAL;
                goto out;
        }

        if (!elems->ch_sw_timing || !elems->lnk_id) {
                tdls_dbg(sdata, "TDLS channel switch resp - missing IEs\n");
                ret = -EINVAL;
                goto out;
        }

        /* validate the initiator is set correctly */
        local_initiator =
                !memcmp(elems->lnk_id->init_sta, sdata->vif.addr, ETH_ALEN);
        if (local_initiator == sta->sta.tdls_initiator) {
                tdls_dbg(sdata, "TDLS chan switch invalid lnk-id initiator\n");
                ret = -EINVAL;
                goto out;
        }

        params.switch_time = le16_to_cpu(elems->ch_sw_timing->switch_time);
        params.switch_timeout = le16_to_cpu(elems->ch_sw_timing->switch_timeout);

        params.tmpl_skb =
                ieee80211_tdls_ch_sw_resp_tmpl_get(sta, &params.ch_sw_tm_ie);
        if (!params.tmpl_skb) {
                ret = -ENOENT;
                goto out;
        }

        ret = 0;
call_drv:
        drv_tdls_recv_channel_switch(sdata->local, sdata, &params);

        tdls_dbg(sdata,
                 "TDLS channel switch response received from %pM status %d\n",
                 tf->sa, params.status);

out:
        dev_kfree_skb_any(params.tmpl_skb);
        kfree(elems);
        return ret;
}

static int
ieee80211_process_tdls_channel_switch_req(struct ieee80211_sub_if_data *sdata,
                                          struct sk_buff *skb)
{
        struct ieee80211_local *local = sdata->local;
        struct ieee802_11_elems *elems;
        struct cfg80211_chan_def chandef;
        struct ieee80211_channel *chan;
        enum nl80211_channel_type chan_type;
        int freq;
        u8 target_channel, oper_class;
        bool local_initiator;
        struct sta_info *sta;
        enum nl80211_band band;
        struct ieee80211_tdls_data *tf = (void *)skb->data;
        struct ieee80211_rx_status *rx_status = IEEE80211_SKB_RXCB(skb);
        int baselen = offsetof(typeof(*tf), u.chan_switch_req.variable);
        struct ieee80211_tdls_ch_sw_params params = {};
        int ret = 0;

        lockdep_assert_wiphy(local->hw.wiphy);

        params.action_code = WLAN_TDLS_CHANNEL_SWITCH_REQUEST;
        params.timestamp = rx_status->device_timestamp;

        if (skb->len < baselen) {
                tdls_dbg(sdata, "TDLS channel switch req too short: %d\n",
                         skb->len);
                return -EINVAL;
        }

        target_channel = tf->u.chan_switch_req.target_channel;
        oper_class = tf->u.chan_switch_req.oper_class;

        /*
         * We can't easily infer the channel band. The operating class is
         * ambiguous - there are multiple tables (US/Europe/JP/Global). The
         * solution here is to treat channels with number >14 as 5GHz ones,
         * and specifically check for the (oper_class, channel) combinations
         * where this doesn't hold. These are thankfully unique according to
         * IEEE802.11-2012.
         * We consider only the 2GHz and 5GHz bands and 20MHz+ channels as
         * valid here.
         */
        if ((oper_class == 112 || oper_class == 2 || oper_class == 3 ||
             oper_class == 4 || oper_class == 5 || oper_class == 6) &&
             target_channel < 14)
                band = NL80211_BAND_5GHZ;
        else
                band = target_channel < 14 ? NL80211_BAND_2GHZ :
                                             NL80211_BAND_5GHZ;

        freq = ieee80211_channel_to_frequency(target_channel, band);
        if (freq == 0) {
                tdls_dbg(sdata, "Invalid channel in TDLS chan switch: %d\n",
                         target_channel);
                return -EINVAL;
        }

        chan = ieee80211_get_channel(sdata->local->hw.wiphy, freq);
        if (!chan) {
                tdls_dbg(sdata,
                         "Unsupported channel for TDLS chan switch: %d\n",
                         target_channel);
                return -EINVAL;
        }

        elems = ieee802_11_parse_elems(tf->u.chan_switch_req.variable,
                                       skb->len - baselen, false, NULL);
        if (!elems)
                return -ENOMEM;

        if (elems->parse_error) {
                tdls_dbg(sdata, "Invalid IEs in TDLS channel switch req\n");
                ret = -EINVAL;
                goto free;
        }

        if (!elems->ch_sw_timing || !elems->lnk_id) {
                tdls_dbg(sdata, "TDLS channel switch req - missing IEs\n");
                ret = -EINVAL;
                goto free;
        }

        if (!elems->sec_chan_offs) {
                chan_type = NL80211_CHAN_HT20;
        } else {
                switch (elems->sec_chan_offs->sec_chan_offs) {
                case IEEE80211_HT_PARAM_CHA_SEC_ABOVE:
                        chan_type = NL80211_CHAN_HT40PLUS;
                        break;
                case IEEE80211_HT_PARAM_CHA_SEC_BELOW:
                        chan_type = NL80211_CHAN_HT40MINUS;
                        break;
                default:
                        chan_type = NL80211_CHAN_HT20;
                        break;
                }
        }

        cfg80211_chandef_create(&chandef, chan, chan_type);

        /* we will be active on the TDLS link */
        if (!cfg80211_reg_can_beacon_relax(sdata->local->hw.wiphy, &chandef,
                                           sdata->wdev.iftype)) {
                tdls_dbg(sdata, "TDLS chan switch to forbidden channel\n");
                ret = -EINVAL;
                goto free;
        }

        sta = sta_info_get(sdata, tf->sa);
        if (!sta || !test_sta_flag(sta, WLAN_STA_TDLS_PEER_AUTH)) {
                tdls_dbg(sdata, "TDLS chan switch from non-peer sta %pM\n",
                         tf->sa);
                ret = -EINVAL;
                goto out;
        }

        params.sta = &sta->sta;

        /* validate the initiator is set correctly */
        local_initiator =
                !memcmp(elems->lnk_id->init_sta, sdata->vif.addr, ETH_ALEN);
        if (local_initiator == sta->sta.tdls_initiator) {
                tdls_dbg(sdata, "TDLS chan switch invalid lnk-id initiator\n");
                ret = -EINVAL;
                goto out;
        }

        /* peer should have known better */
        if (!sta->sta.deflink.ht_cap.ht_supported && elems->sec_chan_offs &&
            elems->sec_chan_offs->sec_chan_offs) {
                tdls_dbg(sdata, "TDLS chan switch - wide chan unsupported\n");
                ret = -EOPNOTSUPP;
                goto out;
        }

        params.chandef = &chandef;
        params.switch_time = le16_to_cpu(elems->ch_sw_timing->switch_time);
        params.switch_timeout = le16_to_cpu(elems->ch_sw_timing->switch_timeout);

        params.tmpl_skb =
                ieee80211_tdls_ch_sw_resp_tmpl_get(sta,
                                                   &params.ch_sw_tm_ie);
        if (!params.tmpl_skb) {
                ret = -ENOENT;
                goto out;
        }

        drv_tdls_recv_channel_switch(sdata->local, sdata, &params);

        tdls_dbg(sdata,
                 "TDLS ch switch request received from %pM ch %d width %d\n",
                 tf->sa, params.chandef->chan->center_freq,
                 params.chandef->width);
out:
        dev_kfree_skb_any(params.tmpl_skb);
free:
        kfree(elems);
        return ret;
}

void
ieee80211_process_tdls_channel_switch(struct ieee80211_sub_if_data *sdata,
                                      struct sk_buff *skb)
{
        struct ieee80211_tdls_data *tf = (void *)skb->data;
        struct wiphy *wiphy = sdata->local->hw.wiphy;

        lockdep_assert_wiphy(wiphy);

        /* make sure the driver supports it */
        if (!(wiphy->features & NL80211_FEATURE_TDLS_CHANNEL_SWITCH))
                return;

        /* we want to access the entire packet */
        if (skb_linearize(skb))
                return;
        /*
         * The packet/size was already validated by mac80211 Rx path, only look
         * at the action type.
         */
        switch (tf->action_code) {
        case WLAN_TDLS_CHANNEL_SWITCH_REQUEST:
                ieee80211_process_tdls_channel_switch_req(sdata, skb);
                break;
        case WLAN_TDLS_CHANNEL_SWITCH_RESPONSE:
                ieee80211_process_tdls_channel_switch_resp(sdata, skb);
                break;
        default:
                WARN_ON_ONCE(1);
                return;
        }
}

void ieee80211_teardown_tdls_peers(struct ieee80211_link_data *link)
{
        struct ieee80211_sub_if_data *sdata = link->sdata;
        struct sta_info *sta;
        u16 reason = WLAN_REASON_TDLS_TEARDOWN_UNSPECIFIED;

        rcu_read_lock();
        list_for_each_entry_rcu(sta, &sdata->local->sta_list, list) {
                if (!sta->sta.tdls || sta->sdata != sdata || !sta->uploaded ||
                    !test_sta_flag(sta, WLAN_STA_AUTHORIZED))
                        continue;

                if (sta->deflink.link_id != link->link_id)
                        continue;

                ieee80211_tdls_oper_request(&sdata->vif, sta->sta.addr,
                                            NL80211_TDLS_TEARDOWN, reason,
                                            GFP_ATOMIC);
        }
        rcu_read_unlock();
}

void ieee80211_tdls_handle_disconnect(struct ieee80211_sub_if_data *sdata,
                                      const u8 *peer, u16 reason)
{
        struct ieee80211_sta *sta;

        rcu_read_lock();
        sta = ieee80211_find_sta(&sdata->vif, peer);
        if (!sta || !sta->tdls) {
                rcu_read_unlock();
                return;
        }
        rcu_read_unlock();

        tdls_dbg(sdata, "disconnected from TDLS peer %pM (Reason: %u=%s)\n",
                 peer, reason,
                 ieee80211_get_reason_code_string(reason));

        ieee80211_tdls_oper_request(&sdata->vif, peer,
                                    NL80211_TDLS_TEARDOWN,
                                    WLAN_REASON_TDLS_TEARDOWN_UNREACHABLE,
                                    GFP_ATOMIC);
}















































    4 









    2 



    1 


    1 















    4 




    4 






    3 


    2 
    1 


    4 






    3 

    3 










    1 












    4 

    3 


    1 






    2 

    1 
    1 





































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
// SPDX-License-Identifier: GPL-2.0-only
/*
 * GHASH: hash function for GCM (Galois/Counter Mode).
 *
 * Copyright (c) 2007 Nokia Siemens Networks - Mikko Herranen <mh1@iki.fi>
 * Copyright (c) 2009 Intel Corp.
 *   Author: Huang Ying <ying.huang@intel.com>
 */

/*
 * GHASH is a keyed hash function used in GCM authentication tag generation.
 *
 * The original GCM paper [1] presents GHASH as a function GHASH(H, A, C) which
 * takes a 16-byte hash key H, additional authenticated data A, and a ciphertext
 * C.  It formats A and C into a single byte string X, interprets X as a
 * polynomial over GF(2^128), and evaluates this polynomial at the point H.
 *
 * However, the NIST standard for GCM [2] presents GHASH as GHASH(H, X) where X
 * is the already-formatted byte string containing both A and C.
 *
 * "ghash" in the Linux crypto API uses the 'X' (pre-formatted) convention,
 * since the API supports only a single data stream per hash.  Thus, the
 * formatting of 'A' and 'C' is done in the "gcm" template, not in "ghash".
 *
 * The reason "ghash" is separate from "gcm" is to allow "gcm" to use an
 * accelerated "ghash" when a standalone accelerated "gcm(aes)" is unavailable.
 * It is generally inappropriate to use "ghash" for other purposes, since it is
 * an "ε-almost-XOR-universal hash function", not a cryptographic hash function.
 * It can only be used securely in crypto modes specially designed to use it.
 *
 * [1] The Galois/Counter Mode of Operation (GCM)
 *     (http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.694.695&rep=rep1&type=pdf)
 * [2] Recommendation for Block Cipher Modes of Operation: Galois/Counter Mode (GCM) and GMAC
 *     (https://csrc.nist.gov/publications/detail/sp/800-38d/final)
 */

#include <crypto/algapi.h>
#include <crypto/gf128mul.h>
#include <crypto/ghash.h>
#include <crypto/internal/hash.h>
#include <linux/crypto.h>
#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/module.h>

static int ghash_init(struct shash_desc *desc)
{
        struct ghash_desc_ctx *dctx = shash_desc_ctx(desc);

        memset(dctx, 0, sizeof(*dctx));

        return 0;
}

static int ghash_setkey(struct crypto_shash *tfm,
                        const u8 *key, unsigned int keylen)
{
        struct ghash_ctx *ctx = crypto_shash_ctx(tfm);
        be128 k;

        if (keylen != GHASH_BLOCK_SIZE)
                return -EINVAL;

        if (ctx->gf128)
                gf128mul_free_4k(ctx->gf128);

        BUILD_BUG_ON(sizeof(k) != GHASH_BLOCK_SIZE);
        memcpy(&k, key, GHASH_BLOCK_SIZE); /* avoid violating alignment rules */
        ctx->gf128 = gf128mul_init_4k_lle(&k);
        memzero_explicit(&k, GHASH_BLOCK_SIZE);

        if (!ctx->gf128)
                return -ENOMEM;

        return 0;
}

static int ghash_update(struct shash_desc *desc,
                         const u8 *src, unsigned int srclen)
{
        struct ghash_desc_ctx *dctx = shash_desc_ctx(desc);
        struct ghash_ctx *ctx = crypto_shash_ctx(desc->tfm);
        u8 *dst = dctx->buffer;

        if (dctx->bytes) {
                int n = min(srclen, dctx->bytes);
                u8 *pos = dst + (GHASH_BLOCK_SIZE - dctx->bytes);

                dctx->bytes -= n;
                srclen -= n;

                while (n--)
                        *pos++ ^= *src++;

                if (!dctx->bytes)
                        gf128mul_4k_lle((be128 *)dst, ctx->gf128);
        }

        while (srclen >= GHASH_BLOCK_SIZE) {
                crypto_xor(dst, src, GHASH_BLOCK_SIZE);
                gf128mul_4k_lle((be128 *)dst, ctx->gf128);
                src += GHASH_BLOCK_SIZE;
                srclen -= GHASH_BLOCK_SIZE;
        }

        if (srclen) {
                dctx->bytes = GHASH_BLOCK_SIZE - srclen;
                while (srclen--)
                        *dst++ ^= *src++;
        }

        return 0;
}

static void ghash_flush(struct ghash_ctx *ctx, struct ghash_desc_ctx *dctx)
{
        u8 *dst = dctx->buffer;

        if (dctx->bytes) {
                u8 *tmp = dst + (GHASH_BLOCK_SIZE - dctx->bytes);

                while (dctx->bytes--)
                        *tmp++ ^= 0;

                gf128mul_4k_lle((be128 *)dst, ctx->gf128);
        }

        dctx->bytes = 0;
}

static int ghash_final(struct shash_desc *desc, u8 *dst)
{
        struct ghash_desc_ctx *dctx = shash_desc_ctx(desc);
        struct ghash_ctx *ctx = crypto_shash_ctx(desc->tfm);
        u8 *buf = dctx->buffer;

        ghash_flush(ctx, dctx);
        memcpy(dst, buf, GHASH_BLOCK_SIZE);

        return 0;
}

static void ghash_exit_tfm(struct crypto_tfm *tfm)
{
        struct ghash_ctx *ctx = crypto_tfm_ctx(tfm);
        if (ctx->gf128)
                gf128mul_free_4k(ctx->gf128);
}

static struct shash_alg ghash_alg = {
        .digestsize        = GHASH_DIGEST_SIZE,
        .init                = ghash_init,
        .update                = ghash_update,
        .final                = ghash_final,
        .setkey                = ghash_setkey,
        .descsize        = sizeof(struct ghash_desc_ctx),
        .base                = {
                .cra_name                = "ghash",
                .cra_driver_name        = "ghash-generic",
                .cra_priority                = 100,
                .cra_blocksize                = GHASH_BLOCK_SIZE,
                .cra_ctxsize                = sizeof(struct ghash_ctx),
                .cra_module                = THIS_MODULE,
                .cra_exit                = ghash_exit_tfm,
        },
};

static int __init ghash_mod_init(void)
{
        return crypto_register_shash(&ghash_alg);
}

static void __exit ghash_mod_exit(void)
{
        crypto_unregister_shash(&ghash_alg);
}

subsys_initcall(ghash_mod_init);
module_exit(ghash_mod_exit);

MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("GHASH hash function");
MODULE_ALIAS_CRYPTO("ghash");
MODULE_ALIAS_CRYPTO("ghash-generic");
























































































































































































































































































































   18 












    3 
    3 
    3 


    2 












   37 
    8 

   17 




    7 




    5 







   21 










1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
// SPDX-License-Identifier: GPL-2.0-only
/* Copyright (C) 2005 Marc Kleine-Budde, Pengutronix
 * Copyright (C) 2006 Andrey Volkov, Varma Electronics
 * Copyright (C) 2008-2009 Wolfgang Grandegger <wg@grandegger.com>
 */

#include <linux/can/dev.h>
#include <linux/module.h>

#define MOD_DESC "CAN device driver interface"

MODULE_DESCRIPTION(MOD_DESC);
MODULE_LICENSE("GPL v2");
MODULE_AUTHOR("Wolfgang Grandegger <wg@grandegger.com>");

/* Local echo of CAN messages
 *
 * CAN network devices *should* support a local echo functionality
 * (see Documentation/networking/can.rst). To test the handling of CAN
 * interfaces that do not support the local echo both driver types are
 * implemented. In the case that the driver does not support the echo
 * the IFF_ECHO remains clear in dev->flags. This causes the PF_CAN core
 * to perform the echo as a fallback solution.
 */
void can_flush_echo_skb(struct net_device *dev)
{
        struct can_priv *priv = netdev_priv(dev);
        struct net_device_stats *stats = &dev->stats;
        int i;

        for (i = 0; i < priv->echo_skb_max; i++) {
                if (priv->echo_skb[i]) {
                        kfree_skb(priv->echo_skb[i]);
                        priv->echo_skb[i] = NULL;
                        stats->tx_dropped++;
                        stats->tx_aborted_errors++;
                }
        }
}

/* Put the skb on the stack to be looped backed locally lateron
 *
 * The function is typically called in the start_xmit function
 * of the device driver. The driver must protect access to
 * priv->echo_skb, if necessary.
 */
int can_put_echo_skb(struct sk_buff *skb, struct net_device *dev,
                     unsigned int idx, unsigned int frame_len)
{
        struct can_priv *priv = netdev_priv(dev);

        if (idx >= priv->echo_skb_max) {
                netdev_err(dev, "%s: BUG! Trying to access can_priv::echo_skb out of bounds (%u/max %u)\n",
                           __func__, idx, priv->echo_skb_max);
                return -EINVAL;
        }

        /* check flag whether this packet has to be looped back */
        if (!(dev->flags & IFF_ECHO) ||
            (skb->protocol != htons(ETH_P_CAN) &&
             skb->protocol != htons(ETH_P_CANFD) &&
             skb->protocol != htons(ETH_P_CANXL))) {
                kfree_skb(skb);
                return 0;
        }

        if (!priv->echo_skb[idx]) {
                skb = can_create_echo_skb(skb);
                if (!skb)
                        return -ENOMEM;

                /* make settings for echo to reduce code in irq context */
                skb->ip_summed = CHECKSUM_UNNECESSARY;
                skb->dev = dev;

                /* save frame_len to reuse it when transmission is completed */
                can_skb_prv(skb)->frame_len = frame_len;

                if (skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)
                        skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;

                skb_tx_timestamp(skb);

                /* save this skb for tx interrupt echo handling */
                priv->echo_skb[idx] = skb;
        } else {
                /* locking problem with netif_stop_queue() ?? */
                netdev_err(dev, "%s: BUG! echo_skb %d is occupied!\n", __func__, idx);
                kfree_skb(skb);
                return -EBUSY;
        }

        return 0;
}
EXPORT_SYMBOL_GPL(can_put_echo_skb);

struct sk_buff *
__can_get_echo_skb(struct net_device *dev, unsigned int idx,
                   unsigned int *len_ptr, unsigned int *frame_len_ptr)
{
        struct can_priv *priv = netdev_priv(dev);

        if (idx >= priv->echo_skb_max) {
                netdev_err(dev, "%s: BUG! Trying to access can_priv::echo_skb out of bounds (%u/max %u)\n",
                           __func__, idx, priv->echo_skb_max);
                return NULL;
        }

        if (priv->echo_skb[idx]) {
                /* Using "struct canfd_frame::len" for the frame
                 * length is supported on both CAN and CANFD frames.
                 */
                struct sk_buff *skb = priv->echo_skb[idx];
                struct can_skb_priv *can_skb_priv = can_skb_prv(skb);

                if (skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS)
                        skb_tstamp_tx(skb, skb_hwtstamps(skb));

                /* get the real payload length for netdev statistics */
                *len_ptr = can_skb_get_data_len(skb);

                if (frame_len_ptr)
                        *frame_len_ptr = can_skb_priv->frame_len;

                priv->echo_skb[idx] = NULL;

                if (skb->pkt_type == PACKET_LOOPBACK) {
                        skb->pkt_type = PACKET_BROADCAST;
                } else {
                        dev_consume_skb_any(skb);
                        return NULL;
                }

                return skb;
        }

        return NULL;
}

/* Get the skb from the stack and loop it back locally
 *
 * The function is typically called when the TX done interrupt
 * is handled in the device driver. The driver must protect
 * access to priv->echo_skb, if necessary.
 */
unsigned int can_get_echo_skb(struct net_device *dev, unsigned int idx,
                              unsigned int *frame_len_ptr)
{
        struct sk_buff *skb;
        unsigned int len;

        skb = __can_get_echo_skb(dev, idx, &len, frame_len_ptr);
        if (!skb)
                return 0;

        skb_get(skb);
        if (netif_rx(skb) == NET_RX_SUCCESS)
                dev_consume_skb_any(skb);
        else
                dev_kfree_skb_any(skb);

        return len;
}
EXPORT_SYMBOL_GPL(can_get_echo_skb);

/* Remove the skb from the stack and free it.
 *
 * The function is typically called when TX failed.
 */
void can_free_echo_skb(struct net_device *dev, unsigned int idx,
                       unsigned int *frame_len_ptr)
{
        struct can_priv *priv = netdev_priv(dev);

        if (idx >= priv->echo_skb_max) {
                netdev_err(dev, "%s: BUG! Trying to access can_priv::echo_skb out of bounds (%u/max %u)\n",
                           __func__, idx, priv->echo_skb_max);
                return;
        }

        if (priv->echo_skb[idx]) {
                struct sk_buff *skb = priv->echo_skb[idx];
                struct can_skb_priv *can_skb_priv = can_skb_prv(skb);

                if (frame_len_ptr)
                        *frame_len_ptr = can_skb_priv->frame_len;

                dev_kfree_skb_any(skb);
                priv->echo_skb[idx] = NULL;
        }
}
EXPORT_SYMBOL_GPL(can_free_echo_skb);

/* fill common values for CAN sk_buffs */
static void init_can_skb_reserve(struct sk_buff *skb)
{
        skb->pkt_type = PACKET_BROADCAST;
        skb->ip_summed = CHECKSUM_UNNECESSARY;

        skb_reset_mac_header(skb);
        skb_reset_network_header(skb);
        skb_reset_transport_header(skb);

        can_skb_reserve(skb);
        can_skb_prv(skb)->skbcnt = 0;
}

struct sk_buff *alloc_can_skb(struct net_device *dev, struct can_frame **cf)
{
        struct sk_buff *skb;

        skb = netdev_alloc_skb(dev, sizeof(struct can_skb_priv) +
                               sizeof(struct can_frame));
        if (unlikely(!skb)) {
                *cf = NULL;

                return NULL;
        }

        skb->protocol = htons(ETH_P_CAN);
        init_can_skb_reserve(skb);
        can_skb_prv(skb)->ifindex = dev->ifindex;

        *cf = skb_put_zero(skb, sizeof(struct can_frame));

        return skb;
}
EXPORT_SYMBOL_GPL(alloc_can_skb);

struct sk_buff *alloc_canfd_skb(struct net_device *dev,
                                struct canfd_frame **cfd)
{
        struct sk_buff *skb;

        skb = netdev_alloc_skb(dev, sizeof(struct can_skb_priv) +
                               sizeof(struct canfd_frame));
        if (unlikely(!skb)) {
                *cfd = NULL;

                return NULL;
        }

        skb->protocol = htons(ETH_P_CANFD);
        init_can_skb_reserve(skb);
        can_skb_prv(skb)->ifindex = dev->ifindex;

        *cfd = skb_put_zero(skb, sizeof(struct canfd_frame));

        /* set CAN FD flag by default */
        (*cfd)->flags = CANFD_FDF;

        return skb;
}
EXPORT_SYMBOL_GPL(alloc_canfd_skb);

struct sk_buff *alloc_canxl_skb(struct net_device *dev,
                                struct canxl_frame **cxl,
                                unsigned int data_len)
{
        struct sk_buff *skb;

        if (data_len < CANXL_MIN_DLEN || data_len > CANXL_MAX_DLEN)
                goto out_error;

        skb = netdev_alloc_skb(dev, sizeof(struct can_skb_priv) +
                               CANXL_HDR_SIZE + data_len);
        if (unlikely(!skb))
                goto out_error;

        skb->protocol = htons(ETH_P_CANXL);
        init_can_skb_reserve(skb);
        can_skb_prv(skb)->ifindex = dev->ifindex;

        *cxl = skb_put_zero(skb, CANXL_HDR_SIZE + data_len);

        /* set CAN XL flag and length information by default */
        (*cxl)->flags = CANXL_XLF;
        (*cxl)->len = data_len;

        return skb;

out_error:
        *cxl = NULL;

        return NULL;
}
EXPORT_SYMBOL_GPL(alloc_canxl_skb);

struct sk_buff *alloc_can_err_skb(struct net_device *dev, struct can_frame **cf)
{
        struct sk_buff *skb;

        skb = alloc_can_skb(dev, cf);
        if (unlikely(!skb))
                return NULL;

        (*cf)->can_id = CAN_ERR_FLAG;
        (*cf)->len = CAN_ERR_DLC;

        return skb;
}
EXPORT_SYMBOL_GPL(alloc_can_err_skb);

/* Check for outgoing skbs that have not been created by the CAN subsystem */
static bool can_skb_headroom_valid(struct net_device *dev, struct sk_buff *skb)
{
        /* af_packet creates a headroom of HH_DATA_MOD bytes which is fine */
        if (WARN_ON_ONCE(skb_headroom(skb) < sizeof(struct can_skb_priv)))
                return false;

        /* af_packet does not apply CAN skb specific settings */
        if (skb->ip_summed == CHECKSUM_NONE) {
                /* init headroom */
                can_skb_prv(skb)->ifindex = dev->ifindex;
                can_skb_prv(skb)->skbcnt = 0;

                skb->ip_summed = CHECKSUM_UNNECESSARY;

                /* perform proper loopback on capable devices */
                if (dev->flags & IFF_ECHO)
                        skb->pkt_type = PACKET_LOOPBACK;
                else
                        skb->pkt_type = PACKET_HOST;

                skb_reset_mac_header(skb);
                skb_reset_network_header(skb);
                skb_reset_transport_header(skb);

                /* set CANFD_FDF flag for CAN FD frames */
                if (can_is_canfd_skb(skb)) {
                        struct canfd_frame *cfd;

                        cfd = (struct canfd_frame *)skb->data;
                        cfd->flags |= CANFD_FDF;
                }
        }

        return true;
}

/* Drop a given socketbuffer if it does not contain a valid CAN frame. */
bool can_dropped_invalid_skb(struct net_device *dev, struct sk_buff *skb)
{
        switch (ntohs(skb->protocol)) {
        case ETH_P_CAN:
                if (!can_is_can_skb(skb))
                        goto inval_skb;
                break;

        case ETH_P_CANFD:
                if (!can_is_canfd_skb(skb))
                        goto inval_skb;
                break;

        case ETH_P_CANXL:
                if (!can_is_canxl_skb(skb))
                        goto inval_skb;
                break;

        default:
                goto inval_skb;
        }

        if (!can_skb_headroom_valid(dev, skb))
                goto inval_skb;

        return false;

inval_skb:
        kfree_skb(skb);
        dev->stats.tx_dropped++;
        return true;
}
EXPORT_SYMBOL_GPL(can_dropped_invalid_skb);



























































































































    3 


    3 


    2 
    2 










    3 


    3 


    2 
    2 











    1 






    1 
    1 

    1 















    3 


    3 


    2 
    2 











    1 


    1 
    1 







    2 


    2 
    2 







    2 


    2 










































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * Copyright (C)2003,2004 USAGI/WIDE Project
 *
 * Authors        Mitsuru KANDA  <mk@linux-ipv6.org>
 *                YOSHIFUJI Hideaki <yoshfuji@linux-ipv6.org>
 */

#define pr_fmt(fmt) "IPv6: " fmt

#include <linux/icmpv6.h>
#include <linux/init.h>
#include <linux/module.h>
#include <linux/mutex.h>
#include <linux/netdevice.h>
#include <linux/skbuff.h>
#include <linux/slab.h>
#include <net/ipv6.h>
#include <net/protocol.h>
#include <net/xfrm.h>

static struct xfrm6_tunnel __rcu *tunnel6_handlers __read_mostly;
static struct xfrm6_tunnel __rcu *tunnel46_handlers __read_mostly;
static struct xfrm6_tunnel __rcu *tunnelmpls6_handlers __read_mostly;
static DEFINE_MUTEX(tunnel6_mutex);

static inline int xfrm6_tunnel_mpls_supported(void)
{
        return IS_ENABLED(CONFIG_MPLS);
}

int xfrm6_tunnel_register(struct xfrm6_tunnel *handler, unsigned short family)
{
        struct xfrm6_tunnel __rcu **pprev;
        struct xfrm6_tunnel *t;
        int ret = -EEXIST;
        int priority = handler->priority;

        mutex_lock(&tunnel6_mutex);

        switch (family) {
        case AF_INET6:
                pprev = &tunnel6_handlers;
                break;
        case AF_INET:
                pprev = &tunnel46_handlers;
                break;
        case AF_MPLS:
                pprev = &tunnelmpls6_handlers;
                break;
        default:
                goto err;
        }

        for (; (t = rcu_dereference_protected(*pprev,
                        lockdep_is_held(&tunnel6_mutex))) != NULL;
             pprev = &t->next) {
                if (t->priority > priority)
                        break;
                if (t->priority == priority)
                        goto err;
        }

        handler->next = *pprev;
        rcu_assign_pointer(*pprev, handler);

        ret = 0;

err:
        mutex_unlock(&tunnel6_mutex);

        return ret;
}
EXPORT_SYMBOL(xfrm6_tunnel_register);

int xfrm6_tunnel_deregister(struct xfrm6_tunnel *handler, unsigned short family)
{
        struct xfrm6_tunnel __rcu **pprev;
        struct xfrm6_tunnel *t;
        int ret = -ENOENT;

        mutex_lock(&tunnel6_mutex);

        switch (family) {
        case AF_INET6:
                pprev = &tunnel6_handlers;
                break;
        case AF_INET:
                pprev = &tunnel46_handlers;
                break;
        case AF_MPLS:
                pprev = &tunnelmpls6_handlers;
                break;
        default:
                goto err;
        }

        for (; (t = rcu_dereference_protected(*pprev,
                        lockdep_is_held(&tunnel6_mutex))) != NULL;
             pprev = &t->next) {
                if (t == handler) {
                        *pprev = handler->next;
                        ret = 0;
                        break;
                }
        }

err:
        mutex_unlock(&tunnel6_mutex);

        synchronize_net();

        return ret;
}
EXPORT_SYMBOL(xfrm6_tunnel_deregister);

#define for_each_tunnel_rcu(head, handler)                \
        for (handler = rcu_dereference(head);                \
             handler != NULL;                                \
             handler = rcu_dereference(handler->next))        \

static int tunnelmpls6_rcv(struct sk_buff *skb)
{
        struct xfrm6_tunnel *handler;

        if (!pskb_may_pull(skb, sizeof(struct ipv6hdr)))
                goto drop;

        for_each_tunnel_rcu(tunnelmpls6_handlers, handler)
                if (!handler->handler(skb))
                        return 0;

        icmpv6_send(skb, ICMPV6_DEST_UNREACH, ICMPV6_PORT_UNREACH, 0);

drop:
        kfree_skb(skb);
        return 0;
}

static int tunnel6_rcv(struct sk_buff *skb)
{
        struct xfrm6_tunnel *handler;

        if (!pskb_may_pull(skb, sizeof(struct ipv6hdr)))
                goto drop;

        for_each_tunnel_rcu(tunnel6_handlers, handler)
                if (!handler->handler(skb))
                        return 0;

        icmpv6_send(skb, ICMPV6_DEST_UNREACH, ICMPV6_PORT_UNREACH, 0);

drop:
        kfree_skb(skb);
        return 0;
}

#if IS_ENABLED(CONFIG_INET6_XFRM_TUNNEL)
static int tunnel6_rcv_cb(struct sk_buff *skb, u8 proto, int err)
{
        struct xfrm6_tunnel __rcu *head;
        struct xfrm6_tunnel *handler;
        int ret;

        head = (proto == IPPROTO_IPV6) ? tunnel6_handlers : tunnel46_handlers;

        for_each_tunnel_rcu(head, handler) {
                if (handler->cb_handler) {
                        ret = handler->cb_handler(skb, err);
                        if (ret <= 0)
                                return ret;
                }
        }

        return 0;
}

static const struct xfrm_input_afinfo tunnel6_input_afinfo = {
        .family                =        AF_INET6,
        .is_ipip        =        true,
        .callback        =        tunnel6_rcv_cb,
};
#endif

static int tunnel46_rcv(struct sk_buff *skb)
{
        struct xfrm6_tunnel *handler;

        if (!pskb_may_pull(skb, sizeof(struct iphdr)))
                goto drop;

        for_each_tunnel_rcu(tunnel46_handlers, handler)
                if (!handler->handler(skb))
                        return 0;

        icmpv6_send(skb, ICMPV6_DEST_UNREACH, ICMPV6_PORT_UNREACH, 0);

drop:
        kfree_skb(skb);
        return 0;
}

static int tunnel6_err(struct sk_buff *skb, struct inet6_skb_parm *opt,
                        u8 type, u8 code, int offset, __be32 info)
{
        struct xfrm6_tunnel *handler;

        for_each_tunnel_rcu(tunnel6_handlers, handler)
                if (!handler->err_handler(skb, opt, type, code, offset, info))
                        return 0;

        return -ENOENT;
}

static int tunnel46_err(struct sk_buff *skb, struct inet6_skb_parm *opt,
                         u8 type, u8 code, int offset, __be32 info)
{
        struct xfrm6_tunnel *handler;

        for_each_tunnel_rcu(tunnel46_handlers, handler)
                if (!handler->err_handler(skb, opt, type, code, offset, info))
                        return 0;

        return -ENOENT;
}

static int tunnelmpls6_err(struct sk_buff *skb, struct inet6_skb_parm *opt,
                           u8 type, u8 code, int offset, __be32 info)
{
        struct xfrm6_tunnel *handler;

        for_each_tunnel_rcu(tunnelmpls6_handlers, handler)
                if (!handler->err_handler(skb, opt, type, code, offset, info))
                        return 0;

        return -ENOENT;
}

static const struct inet6_protocol tunnel6_protocol = {
        .handler        = tunnel6_rcv,
        .err_handler        = tunnel6_err,
        .flags          = INET6_PROTO_NOPOLICY|INET6_PROTO_FINAL,
};

static const struct inet6_protocol tunnel46_protocol = {
        .handler        = tunnel46_rcv,
        .err_handler        = tunnel46_err,
        .flags          = INET6_PROTO_NOPOLICY|INET6_PROTO_FINAL,
};

static const struct inet6_protocol tunnelmpls6_protocol = {
        .handler        = tunnelmpls6_rcv,
        .err_handler        = tunnelmpls6_err,
        .flags          = INET6_PROTO_NOPOLICY|INET6_PROTO_FINAL,
};

static int __init tunnel6_init(void)
{
        if (inet6_add_protocol(&tunnel6_protocol, IPPROTO_IPV6)) {
                pr_err("%s: can't add protocol\n", __func__);
                return -EAGAIN;
        }
        if (inet6_add_protocol(&tunnel46_protocol, IPPROTO_IPIP)) {
                pr_err("%s: can't add protocol\n", __func__);
                inet6_del_protocol(&tunnel6_protocol, IPPROTO_IPV6);
                return -EAGAIN;
        }
        if (xfrm6_tunnel_mpls_supported() &&
            inet6_add_protocol(&tunnelmpls6_protocol, IPPROTO_MPLS)) {
                pr_err("%s: can't add protocol\n", __func__);
                inet6_del_protocol(&tunnel6_protocol, IPPROTO_IPV6);
                inet6_del_protocol(&tunnel46_protocol, IPPROTO_IPIP);
                return -EAGAIN;
        }
#if IS_ENABLED(CONFIG_INET6_XFRM_TUNNEL)
        if (xfrm_input_register_afinfo(&tunnel6_input_afinfo)) {
                pr_err("%s: can't add input afinfo\n", __func__);
                inet6_del_protocol(&tunnel6_protocol, IPPROTO_IPV6);
                inet6_del_protocol(&tunnel46_protocol, IPPROTO_IPIP);
                if (xfrm6_tunnel_mpls_supported())
                        inet6_del_protocol(&tunnelmpls6_protocol, IPPROTO_MPLS);
                return -EAGAIN;
        }
#endif
        return 0;
}

static void __exit tunnel6_fini(void)
{
#if IS_ENABLED(CONFIG_INET6_XFRM_TUNNEL)
        if (xfrm_input_unregister_afinfo(&tunnel6_input_afinfo))
                pr_err("%s: can't remove input afinfo\n", __func__);
#endif
        if (inet6_del_protocol(&tunnel46_protocol, IPPROTO_IPIP))
                pr_err("%s: can't remove protocol\n", __func__);
        if (inet6_del_protocol(&tunnel6_protocol, IPPROTO_IPV6))
                pr_err("%s: can't remove protocol\n", __func__);
        if (xfrm6_tunnel_mpls_supported() &&
            inet6_del_protocol(&tunnelmpls6_protocol, IPPROTO_MPLS))
                pr_err("%s: can't remove protocol\n", __func__);
}

module_init(tunnel6_init);
module_exit(tunnel6_fini);
MODULE_DESCRIPTION("IP-in-IPv6 tunnel driver");
MODULE_LICENSE("GPL");














































































































































































































































































































































 2262 






 2259 


























 2261 




 2283 










































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
// SPDX-License-Identifier: GPL-2.0
/*
 * security/tomoyo/audit.c
 *
 * Copyright (C) 2005-2011  NTT DATA CORPORATION
 */

#include "common.h"
#include <linux/slab.h>

/**
 * tomoyo_print_bprm - Print "struct linux_binprm" for auditing.
 *
 * @bprm: Pointer to "struct linux_binprm".
 * @dump: Pointer to "struct tomoyo_page_dump".
 *
 * Returns the contents of @bprm on success, NULL otherwise.
 *
 * This function uses kzalloc(), so caller must kfree() if this function
 * didn't return NULL.
 */
static char *tomoyo_print_bprm(struct linux_binprm *bprm,
                               struct tomoyo_page_dump *dump)
{
        static const int tomoyo_buffer_len = 4096 * 2;
        char *buffer = kzalloc(tomoyo_buffer_len, GFP_NOFS);
        char *cp;
        char *last_start;
        int len;
        unsigned long pos = bprm->p;
        int offset = pos % PAGE_SIZE;
        int argv_count = bprm->argc;
        int envp_count = bprm->envc;
        bool truncated = false;

        if (!buffer)
                return NULL;
        len = snprintf(buffer, tomoyo_buffer_len - 1, "argv[]={ ");
        cp = buffer + len;
        if (!argv_count) {
                memmove(cp, "} envp[]={ ", 11);
                cp += 11;
        }
        last_start = cp;
        while (argv_count || envp_count) {
                if (!tomoyo_dump_page(bprm, pos, dump))
                        goto out;
                pos += PAGE_SIZE - offset;
                /* Read. */
                while (offset < PAGE_SIZE) {
                        const char *kaddr = dump->data;
                        const unsigned char c = kaddr[offset++];

                        if (cp == last_start)
                                *cp++ = '"';
                        if (cp >= buffer + tomoyo_buffer_len - 32) {
                                /* Reserve some room for "..." string. */
                                truncated = true;
                        } else if (c == '\\') {
                                *cp++ = '\\';
                                *cp++ = '\\';
                        } else if (c > ' ' && c < 127) {
                                *cp++ = c;
                        } else if (!c) {
                                *cp++ = '"';
                                *cp++ = ' ';
                                last_start = cp;
                        } else {
                                *cp++ = '\\';
                                *cp++ = (c >> 6) + '0';
                                *cp++ = ((c >> 3) & 7) + '0';
                                *cp++ = (c & 7) + '0';
                        }
                        if (c)
                                continue;
                        if (argv_count) {
                                if (--argv_count == 0) {
                                        if (truncated) {
                                                cp = last_start;
                                                memmove(cp, "... ", 4);
                                                cp += 4;
                                        }
                                        memmove(cp, "} envp[]={ ", 11);
                                        cp += 11;
                                        last_start = cp;
                                        truncated = false;
                                }
                        } else if (envp_count) {
                                if (--envp_count == 0) {
                                        if (truncated) {
                                                cp = last_start;
                                                memmove(cp, "... ", 4);
                                                cp += 4;
                                        }
                                }
                        }
                        if (!argv_count && !envp_count)
                                break;
                }
                offset = 0;
        }
        *cp++ = '}';
        *cp = '\0';
        return buffer;
out:
        snprintf(buffer, tomoyo_buffer_len - 1,
                 "argv[]={ ... } envp[]= { ... }");
        return buffer;
}

/**
 * tomoyo_filetype - Get string representation of file type.
 *
 * @mode: Mode value for stat().
 *
 * Returns file type string.
 */
static inline const char *tomoyo_filetype(const umode_t mode)
{
        switch (mode & S_IFMT) {
        case S_IFREG:
        case 0:
                return tomoyo_condition_keyword[TOMOYO_TYPE_IS_FILE];
        case S_IFDIR:
                return tomoyo_condition_keyword[TOMOYO_TYPE_IS_DIRECTORY];
        case S_IFLNK:
                return tomoyo_condition_keyword[TOMOYO_TYPE_IS_SYMLINK];
        case S_IFIFO:
                return tomoyo_condition_keyword[TOMOYO_TYPE_IS_FIFO];
        case S_IFSOCK:
                return tomoyo_condition_keyword[TOMOYO_TYPE_IS_SOCKET];
        case S_IFBLK:
                return tomoyo_condition_keyword[TOMOYO_TYPE_IS_BLOCK_DEV];
        case S_IFCHR:
                return tomoyo_condition_keyword[TOMOYO_TYPE_IS_CHAR_DEV];
        }
        return "unknown"; /* This should not happen. */
}

/**
 * tomoyo_print_header - Get header line of audit log.
 *
 * @r: Pointer to "struct tomoyo_request_info".
 *
 * Returns string representation.
 *
 * This function uses kmalloc(), so caller must kfree() if this function
 * didn't return NULL.
 */
static char *tomoyo_print_header(struct tomoyo_request_info *r)
{
        struct tomoyo_time stamp;
        const pid_t gpid = task_pid_nr(current);
        struct tomoyo_obj_info *obj = r->obj;
        static const int tomoyo_buffer_len = 4096;
        char *buffer = kmalloc(tomoyo_buffer_len, GFP_NOFS);
        int pos;
        u8 i;

        if (!buffer)
                return NULL;

        tomoyo_convert_time(ktime_get_real_seconds(), &stamp);

        pos = snprintf(buffer, tomoyo_buffer_len - 1,
                       "#%04u/%02u/%02u %02u:%02u:%02u# profile=%u mode=%s granted=%s (global-pid=%u) task={ pid=%u ppid=%u uid=%u gid=%u euid=%u egid=%u suid=%u sgid=%u fsuid=%u fsgid=%u }",
                       stamp.year, stamp.month, stamp.day, stamp.hour,
                       stamp.min, stamp.sec, r->profile, tomoyo_mode[r->mode],
                       str_yes_no(r->granted), gpid, tomoyo_sys_getpid(),
                       tomoyo_sys_getppid(),
                       from_kuid(&init_user_ns, current_uid()),
                       from_kgid(&init_user_ns, current_gid()),
                       from_kuid(&init_user_ns, current_euid()),
                       from_kgid(&init_user_ns, current_egid()),
                       from_kuid(&init_user_ns, current_suid()),
                       from_kgid(&init_user_ns, current_sgid()),
                       from_kuid(&init_user_ns, current_fsuid()),
                       from_kgid(&init_user_ns, current_fsgid()));
        if (!obj)
                goto no_obj_info;
        if (!obj->validate_done) {
                tomoyo_get_attributes(obj);
                obj->validate_done = true;
        }
        for (i = 0; i < TOMOYO_MAX_PATH_STAT; i++) {
                struct tomoyo_mini_stat *stat;
                unsigned int dev;
                umode_t mode;

                if (!obj->stat_valid[i])
                        continue;
                stat = &obj->stat[i];
                dev = stat->dev;
                mode = stat->mode;
                if (i & 1) {
                        pos += snprintf(buffer + pos,
                                        tomoyo_buffer_len - 1 - pos,
                                        " path%u.parent={ uid=%u gid=%u ino=%lu perm=0%o }",
                                        (i >> 1) + 1,
                                        from_kuid(&init_user_ns, stat->uid),
                                        from_kgid(&init_user_ns, stat->gid),
                                        (unsigned long)stat->ino,
                                        stat->mode & S_IALLUGO);
                        continue;
                }
                pos += snprintf(buffer + pos, tomoyo_buffer_len - 1 - pos,
                                " path%u={ uid=%u gid=%u ino=%lu major=%u minor=%u perm=0%o type=%s",
                                (i >> 1) + 1,
                                from_kuid(&init_user_ns, stat->uid),
                                from_kgid(&init_user_ns, stat->gid),
                                (unsigned long)stat->ino,
                                MAJOR(dev), MINOR(dev),
                                mode & S_IALLUGO, tomoyo_filetype(mode));
                if (S_ISCHR(mode) || S_ISBLK(mode)) {
                        dev = stat->rdev;
                        pos += snprintf(buffer + pos,
                                        tomoyo_buffer_len - 1 - pos,
                                        " dev_major=%u dev_minor=%u",
                                        MAJOR(dev), MINOR(dev));
                }
                pos += snprintf(buffer + pos, tomoyo_buffer_len - 1 - pos,
                                " }");
        }
no_obj_info:
        if (pos < tomoyo_buffer_len - 1)
                return buffer;
        kfree(buffer);
        return NULL;
}

/**
 * tomoyo_init_log - Allocate buffer for audit logs.
 *
 * @r:    Pointer to "struct tomoyo_request_info".
 * @len:  Buffer size needed for @fmt and @args.
 * @fmt:  The printf()'s format string.
 * @args: va_list structure for @fmt.
 *
 * Returns pointer to allocated memory.
 *
 * This function uses kzalloc(), so caller must kfree() if this function
 * didn't return NULL.
 */
char *tomoyo_init_log(struct tomoyo_request_info *r, int len, const char *fmt,
                      va_list args)
{
        char *buf = NULL;
        char *bprm_info = NULL;
        const char *header = NULL;
        char *realpath = NULL;
        const char *symlink = NULL;
        int pos;
        const char *domainname = r->domain->domainname->name;

        header = tomoyo_print_header(r);
        if (!header)
                return NULL;
        /* +10 is for '\n' etc. and '\0'. */
        len += strlen(domainname) + strlen(header) + 10;
        if (r->ee) {
                struct file *file = r->ee->bprm->file;

                realpath = tomoyo_realpath_from_path(&file->f_path);
                bprm_info = tomoyo_print_bprm(r->ee->bprm, &r->ee->dump);
                if (!realpath || !bprm_info)
                        goto out;
                /* +80 is for " exec={ realpath=\"%s\" argc=%d envc=%d %s }" */
                len += strlen(realpath) + 80 + strlen(bprm_info);
        } else if (r->obj && r->obj->symlink_target) {
                symlink = r->obj->symlink_target->name;
                /* +18 is for " symlink.target=\"%s\"" */
                len += 18 + strlen(symlink);
        }
        len = kmalloc_size_roundup(len);
        buf = kzalloc(len, GFP_NOFS);
        if (!buf)
                goto out;
        len--;
        pos = snprintf(buf, len, "%s", header);
        if (realpath) {
                struct linux_binprm *bprm = r->ee->bprm;

                pos += snprintf(buf + pos, len - pos,
                                " exec={ realpath=\"%s\" argc=%d envc=%d %s }",
                                realpath, bprm->argc, bprm->envc, bprm_info);
        } else if (symlink)
                pos += snprintf(buf + pos, len - pos, " symlink.target=\"%s\"",
                                symlink);
        pos += snprintf(buf + pos, len - pos, "\n%s\n", domainname);
        vsnprintf(buf + pos, len - pos, fmt, args);
out:
        kfree(realpath);
        kfree(bprm_info);
        kfree(header);
        return buf;
}

/* Wait queue for /sys/kernel/security/tomoyo/audit. */
static DECLARE_WAIT_QUEUE_HEAD(tomoyo_log_wait);

/* Structure for audit log. */
struct tomoyo_log {
        struct list_head list;
        char *log;
        int size;
};

/* The list for "struct tomoyo_log". */
static LIST_HEAD(tomoyo_log);

/* Lock for "struct list_head tomoyo_log". */
static DEFINE_SPINLOCK(tomoyo_log_lock);

/* Length of "struct list_head tomoyo_log". */
static unsigned int tomoyo_log_count;

/**
 * tomoyo_get_audit - Get audit mode.
 *
 * @ns:          Pointer to "struct tomoyo_policy_namespace".
 * @profile:     Profile number.
 * @index:       Index number of functionality.
 * @matched_acl: Pointer to "struct tomoyo_acl_info".
 * @is_granted:  True if granted log, false otherwise.
 *
 * Returns true if this request should be audited, false otherwise.
 */
static bool tomoyo_get_audit(const struct tomoyo_policy_namespace *ns,
                             const u8 profile, const u8 index,
                             const struct tomoyo_acl_info *matched_acl,
                             const bool is_granted)
{
        u8 mode;
        const u8 category = tomoyo_index2category[index] +
                TOMOYO_MAX_MAC_INDEX;
        struct tomoyo_profile *p;

        if (!tomoyo_policy_loaded)
                return false;
        p = tomoyo_profile(ns, profile);
        if (tomoyo_log_count >= p->pref[TOMOYO_PREF_MAX_AUDIT_LOG])
                return false;
        if (is_granted && matched_acl && matched_acl->cond &&
            matched_acl->cond->grant_log != TOMOYO_GRANTLOG_AUTO)
                return matched_acl->cond->grant_log == TOMOYO_GRANTLOG_YES;
        mode = p->config[index];
        if (mode == TOMOYO_CONFIG_USE_DEFAULT)
                mode = p->config[category];
        if (mode == TOMOYO_CONFIG_USE_DEFAULT)
                mode = p->default_config;
        if (is_granted)
                return mode & TOMOYO_CONFIG_WANT_GRANT_LOG;
        return mode & TOMOYO_CONFIG_WANT_REJECT_LOG;
}

/**
 * tomoyo_write_log2 - Write an audit log.
 *
 * @r:    Pointer to "struct tomoyo_request_info".
 * @len:  Buffer size needed for @fmt and @args.
 * @fmt:  The printf()'s format string.
 * @args: va_list structure for @fmt.
 *
 * Returns nothing.
 */
void tomoyo_write_log2(struct tomoyo_request_info *r, int len, const char *fmt,
                       va_list args)
{
        char *buf;
        struct tomoyo_log *entry;
        bool quota_exceeded = false;

        if (!tomoyo_get_audit(r->domain->ns, r->profile, r->type,
                              r->matched_acl, r->granted))
                goto out;
        buf = tomoyo_init_log(r, len, fmt, args);
        if (!buf)
                goto out;
        entry = kzalloc(sizeof(*entry), GFP_NOFS);
        if (!entry) {
                kfree(buf);
                goto out;
        }
        entry->log = buf;
        len = kmalloc_size_roundup(strlen(buf) + 1);
        /*
         * The entry->size is used for memory quota checks.
         * Don't go beyond strlen(entry->log).
         */
        entry->size = len + kmalloc_size_roundup(sizeof(*entry));
        spin_lock(&tomoyo_log_lock);
        if (tomoyo_memory_quota[TOMOYO_MEMORY_AUDIT] &&
            tomoyo_memory_used[TOMOYO_MEMORY_AUDIT] + entry->size >=
            tomoyo_memory_quota[TOMOYO_MEMORY_AUDIT]) {
                quota_exceeded = true;
        } else {
                tomoyo_memory_used[TOMOYO_MEMORY_AUDIT] += entry->size;
                list_add_tail(&entry->list, &tomoyo_log);
                tomoyo_log_count++;
        }
        spin_unlock(&tomoyo_log_lock);
        if (quota_exceeded) {
                kfree(buf);
                kfree(entry);
                goto out;
        }
        wake_up(&tomoyo_log_wait);
out:
        return;
}

/**
 * tomoyo_write_log - Write an audit log.
 *
 * @r:   Pointer to "struct tomoyo_request_info".
 * @fmt: The printf()'s format string, followed by parameters.
 *
 * Returns nothing.
 */
void tomoyo_write_log(struct tomoyo_request_info *r, const char *fmt, ...)
{
        va_list args;
        int len;

        va_start(args, fmt);
        len = vsnprintf(NULL, 0, fmt, args) + 1;
        va_end(args);
        va_start(args, fmt);
        tomoyo_write_log2(r, len, fmt, args);
        va_end(args);
}

/**
 * tomoyo_read_log - Read an audit log.
 *
 * @head: Pointer to "struct tomoyo_io_buffer".
 *
 * Returns nothing.
 */
void tomoyo_read_log(struct tomoyo_io_buffer *head)
{
        struct tomoyo_log *ptr = NULL;

        if (head->r.w_pos)
                return;
        kfree(head->read_buf);
        head->read_buf = NULL;
        spin_lock(&tomoyo_log_lock);
        if (!list_empty(&tomoyo_log)) {
                ptr = list_entry(tomoyo_log.next, typeof(*ptr), list);
                list_del(&ptr->list);
                tomoyo_log_count--;
                tomoyo_memory_used[TOMOYO_MEMORY_AUDIT] -= ptr->size;
        }
        spin_unlock(&tomoyo_log_lock);
        if (ptr) {
                head->read_buf = ptr->log;
                head->r.w[head->r.w_pos++] = head->read_buf;
                kfree(ptr);
        }
}

/**
 * tomoyo_poll_log - Wait for an audit log.
 *
 * @file: Pointer to "struct file".
 * @wait: Pointer to "poll_table". Maybe NULL.
 *
 * Returns EPOLLIN | EPOLLRDNORM when ready to read an audit log.
 */
__poll_t tomoyo_poll_log(struct file *file, poll_table *wait)
{
        if (tomoyo_log_count)
                return EPOLLIN | EPOLLRDNORM;
        poll_wait(file, &tomoyo_log_wait, wait);
        if (tomoyo_log_count)
                return EPOLLIN | EPOLLRDNORM;
        return 0;
}









































































    5 
















    5 































































    2 







    2 










    1 






    2 



    2 














    2 































    1 




    2 









    2 
























































    4 






    4 
    4 














































    4 






























































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright (C) ST-Ericsson AB 2010
 * Author:        Sjur Brendeland
 */

#define pr_fmt(fmt) KBUILD_MODNAME ":%s(): " fmt, __func__

#include <linux/kernel.h>
#include <linux/stddef.h>
#include <linux/slab.h>
#include <linux/netdevice.h>
#include <linux/module.h>
#include <net/caif/caif_layer.h>
#include <net/caif/cfpkt.h>
#include <net/caif/cfcnfg.h>
#include <net/caif/cfctrl.h>
#include <net/caif/cfmuxl.h>
#include <net/caif/cffrml.h>
#include <net/caif/cfserl.h>
#include <net/caif/cfsrvl.h>
#include <net/caif/caif_dev.h>

#define container_obj(layr) container_of(layr, struct cfcnfg, layer)

/* Information about CAIF physical interfaces held by Config Module in order
 * to manage physical interfaces
 */
struct cfcnfg_phyinfo {
        struct list_head node;
        bool up;

        /* Pointer to the layer below the MUX (framing layer) */
        struct cflayer *frm_layer;
        /* Pointer to the lowest actual physical layer */
        struct cflayer *phy_layer;
        /* Unique identifier of the physical interface */
        unsigned int id;
        /* Preference of the physical in interface */
        enum cfcnfg_phy_preference pref;

        /* Information about the physical device */
        struct dev_info dev_info;

        /* Interface index */
        int ifindex;

        /* Protocol head room added for CAIF link layer */
        int head_room;

        /* Use Start of frame checksum */
        bool use_fcs;
};

struct cfcnfg {
        struct cflayer layer;
        struct cflayer *ctrl;
        struct cflayer *mux;
        struct list_head phys;
        struct mutex lock;
};

static void cfcnfg_linkup_rsp(struct cflayer *layer, u8 channel_id,
                              enum cfctrl_srv serv, u8 phyid,
                              struct cflayer *adapt_layer);
static void cfcnfg_linkdestroy_rsp(struct cflayer *layer, u8 channel_id);
static void cfcnfg_reject_rsp(struct cflayer *layer, u8 channel_id,
                              struct cflayer *adapt_layer);
static void cfctrl_resp_func(void);
static void cfctrl_enum_resp(void);

struct cfcnfg *cfcnfg_create(void)
{
        struct cfcnfg *this;
        struct cfctrl_rsp *resp;

        might_sleep();

        /* Initiate this layer */
        this = kzalloc(sizeof(struct cfcnfg), GFP_ATOMIC);
        if (!this)
                return NULL;
        this->mux = cfmuxl_create();
        if (!this->mux)
                goto out_of_mem;
        this->ctrl = cfctrl_create();
        if (!this->ctrl)
                goto out_of_mem;
        /* Initiate response functions */
        resp = cfctrl_get_respfuncs(this->ctrl);
        resp->enum_rsp = cfctrl_enum_resp;
        resp->linkerror_ind = cfctrl_resp_func;
        resp->linkdestroy_rsp = cfcnfg_linkdestroy_rsp;
        resp->sleep_rsp = cfctrl_resp_func;
        resp->wake_rsp = cfctrl_resp_func;
        resp->restart_rsp = cfctrl_resp_func;
        resp->radioset_rsp = cfctrl_resp_func;
        resp->linksetup_rsp = cfcnfg_linkup_rsp;
        resp->reject_rsp = cfcnfg_reject_rsp;
        INIT_LIST_HEAD(&this->phys);

        cfmuxl_set_uplayer(this->mux, this->ctrl, 0);
        layer_set_dn(this->ctrl, this->mux);
        layer_set_up(this->ctrl, this);
        mutex_init(&this->lock);

        return this;
out_of_mem:
        synchronize_rcu();

        kfree(this->mux);
        kfree(this->ctrl);
        kfree(this);
        return NULL;
}

void cfcnfg_remove(struct cfcnfg *cfg)
{
        might_sleep();
        if (cfg) {
                synchronize_rcu();

                kfree(cfg->mux);
                cfctrl_remove(cfg->ctrl);
                kfree(cfg);
        }
}

static void cfctrl_resp_func(void)
{
}

static struct cfcnfg_phyinfo *cfcnfg_get_phyinfo_rcu(struct cfcnfg *cnfg,
                                                     u8 phyid)
{
        struct cfcnfg_phyinfo *phy;

        list_for_each_entry_rcu(phy, &cnfg->phys, node)
                if (phy->id == phyid)
                        return phy;
        return NULL;
}

static void cfctrl_enum_resp(void)
{
}

static struct dev_info *cfcnfg_get_phyid(struct cfcnfg *cnfg,
                                  enum cfcnfg_phy_preference phy_pref)
{
        /* Try to match with specified preference */
        struct cfcnfg_phyinfo *phy;

        list_for_each_entry_rcu(phy, &cnfg->phys, node) {
                if (phy->up && phy->pref == phy_pref &&
                                phy->frm_layer != NULL)

                        return &phy->dev_info;
        }

        /* Otherwise just return something */
        list_for_each_entry_rcu(phy, &cnfg->phys, node)
                if (phy->up)
                        return &phy->dev_info;

        return NULL;
}

static int cfcnfg_get_id_from_ifi(struct cfcnfg *cnfg, int ifi)
{
        struct cfcnfg_phyinfo *phy;

        list_for_each_entry_rcu(phy, &cnfg->phys, node)
                if (phy->ifindex == ifi && phy->up)
                        return phy->id;
        return -ENODEV;
}

int caif_disconnect_client(struct net *net, struct cflayer *adap_layer)
{
        u8 channel_id;
        struct cfcnfg *cfg = get_cfcnfg(net);

        caif_assert(adap_layer != NULL);
        cfctrl_cancel_req(cfg->ctrl, adap_layer);
        channel_id = adap_layer->id;
        if (channel_id != 0) {
                struct cflayer *servl;
                servl = cfmuxl_remove_uplayer(cfg->mux, channel_id);
                cfctrl_linkdown_req(cfg->ctrl, channel_id, adap_layer);
                if (servl != NULL)
                        layer_set_up(servl, NULL);
        } else
                pr_debug("nothing to disconnect\n");

        /* Do RCU sync before initiating cleanup */
        synchronize_rcu();
        if (adap_layer->ctrlcmd != NULL)
                adap_layer->ctrlcmd(adap_layer, CAIF_CTRLCMD_DEINIT_RSP, 0);
        return 0;

}
EXPORT_SYMBOL(caif_disconnect_client);

static void cfcnfg_linkdestroy_rsp(struct cflayer *layer, u8 channel_id)
{
}

static const int protohead[CFCTRL_SRV_MASK] = {
        [CFCTRL_SRV_VEI] = 4,
        [CFCTRL_SRV_DATAGRAM] = 7,
        [CFCTRL_SRV_UTIL] = 4,
        [CFCTRL_SRV_RFM] = 3,
        [CFCTRL_SRV_DBG] = 3,
};


static int caif_connect_req_to_link_param(struct cfcnfg *cnfg,
                                          struct caif_connect_request *s,
                                          struct cfctrl_link_param *l)
{
        struct dev_info *dev_info;
        enum cfcnfg_phy_preference pref;
        int res;

        memset(l, 0, sizeof(*l));
        /* In caif protocol low value is high priority */
        l->priority = CAIF_PRIO_MAX - s->priority + 1;

        if (s->ifindex != 0) {
                res = cfcnfg_get_id_from_ifi(cnfg, s->ifindex);
                if (res < 0)
                        return res;
                l->phyid = res;
        } else {
                switch (s->link_selector) {
                case CAIF_LINK_HIGH_BANDW:
                        pref = CFPHYPREF_HIGH_BW;
                        break;
                case CAIF_LINK_LOW_LATENCY:
                        pref = CFPHYPREF_LOW_LAT;
                        break;
                default:
                        return -EINVAL;
                }
                dev_info = cfcnfg_get_phyid(cnfg, pref);
                if (dev_info == NULL)
                        return -ENODEV;
                l->phyid = dev_info->id;
        }
        switch (s->protocol) {
        case CAIFPROTO_AT:
                l->linktype = CFCTRL_SRV_VEI;
                l->endpoint = (s->sockaddr.u.at.type >> 2) & 0x3;
                l->chtype = s->sockaddr.u.at.type & 0x3;
                break;
        case CAIFPROTO_DATAGRAM:
                l->linktype = CFCTRL_SRV_DATAGRAM;
                l->chtype = 0x00;
                l->u.datagram.connid = s->sockaddr.u.dgm.connection_id;
                break;
        case CAIFPROTO_DATAGRAM_LOOP:
                l->linktype = CFCTRL_SRV_DATAGRAM;
                l->chtype = 0x03;
                l->endpoint = 0x00;
                l->u.datagram.connid = s->sockaddr.u.dgm.connection_id;
                break;
        case CAIFPROTO_RFM:
                l->linktype = CFCTRL_SRV_RFM;
                l->u.datagram.connid = s->sockaddr.u.rfm.connection_id;
                strscpy(l->u.rfm.volume, s->sockaddr.u.rfm.volume,
                        sizeof(l->u.rfm.volume));
                break;
        case CAIFPROTO_UTIL:
                l->linktype = CFCTRL_SRV_UTIL;
                l->endpoint = 0x00;
                l->chtype = 0x00;
                strscpy(l->u.utility.name, s->sockaddr.u.util.service,
                        sizeof(l->u.utility.name));
                caif_assert(sizeof(l->u.utility.name) > 10);
                l->u.utility.paramlen = s->param.size;
                if (l->u.utility.paramlen > sizeof(l->u.utility.params))
                        l->u.utility.paramlen = sizeof(l->u.utility.params);

                memcpy(l->u.utility.params, s->param.data,
                       l->u.utility.paramlen);

                break;
        case CAIFPROTO_DEBUG:
                l->linktype = CFCTRL_SRV_DBG;
                l->endpoint = s->sockaddr.u.dbg.service;
                l->chtype = s->sockaddr.u.dbg.type;
                break;
        default:
                return -EINVAL;
        }
        return 0;
}

int caif_connect_client(struct net *net, struct caif_connect_request *conn_req,
                        struct cflayer *adap_layer, int *ifindex,
                        int *proto_head, int *proto_tail)
{
        struct cflayer *frml;
        struct cfcnfg_phyinfo *phy;
        int err;
        struct cfctrl_link_param param;
        struct cfcnfg *cfg = get_cfcnfg(net);

        rcu_read_lock();
        err = caif_connect_req_to_link_param(cfg, conn_req, &param);
        if (err)
                goto unlock;

        phy = cfcnfg_get_phyinfo_rcu(cfg, param.phyid);
        if (!phy) {
                err = -ENODEV;
                goto unlock;
        }
        err = -EINVAL;

        if (adap_layer == NULL) {
                pr_err("adap_layer is zero\n");
                goto unlock;
        }
        if (adap_layer->receive == NULL) {
                pr_err("adap_layer->receive is NULL\n");
                goto unlock;
        }
        if (adap_layer->ctrlcmd == NULL) {
                pr_err("adap_layer->ctrlcmd == NULL\n");
                goto unlock;
        }

        err = -ENODEV;
        frml = phy->frm_layer;
        if (frml == NULL) {
                pr_err("Specified PHY type does not exist!\n");
                goto unlock;
        }
        caif_assert(param.phyid == phy->id);
        caif_assert(phy->frm_layer->id ==
                     param.phyid);
        caif_assert(phy->phy_layer->id ==
                     param.phyid);

        *ifindex = phy->ifindex;
        *proto_tail = 2;
        *proto_head = protohead[param.linktype] + phy->head_room;

        rcu_read_unlock();

        /* FIXME: ENUMERATE INITIALLY WHEN ACTIVATING PHYSICAL INTERFACE */
        cfctrl_enum_req(cfg->ctrl, param.phyid);
        return cfctrl_linkup_request(cfg->ctrl, &param, adap_layer);

unlock:
        rcu_read_unlock();
        return err;
}
EXPORT_SYMBOL(caif_connect_client);

static void cfcnfg_reject_rsp(struct cflayer *layer, u8 channel_id,
                              struct cflayer *adapt_layer)
{
        if (adapt_layer != NULL && adapt_layer->ctrlcmd != NULL)
                adapt_layer->ctrlcmd(adapt_layer,
                                     CAIF_CTRLCMD_INIT_FAIL_RSP, 0);
}

static void
cfcnfg_linkup_rsp(struct cflayer *layer, u8 channel_id, enum cfctrl_srv serv,
                  u8 phyid, struct cflayer *adapt_layer)
{
        struct cfcnfg *cnfg = container_obj(layer);
        struct cflayer *servicel = NULL;
        struct cfcnfg_phyinfo *phyinfo;
        struct net_device *netdev;

        if (channel_id == 0) {
                pr_warn("received channel_id zero\n");
                if (adapt_layer != NULL && adapt_layer->ctrlcmd != NULL)
                        adapt_layer->ctrlcmd(adapt_layer,
                                                CAIF_CTRLCMD_INIT_FAIL_RSP, 0);
                return;
        }

        rcu_read_lock();

        if (adapt_layer == NULL) {
                pr_debug("link setup response but no client exist, send linkdown back\n");
                cfctrl_linkdown_req(cnfg->ctrl, channel_id, NULL);
                goto unlock;
        }

        caif_assert(cnfg != NULL);
        caif_assert(phyid != 0);

        phyinfo = cfcnfg_get_phyinfo_rcu(cnfg, phyid);
        if (phyinfo == NULL) {
                pr_err("ERROR: Link Layer Device disappeared while connecting\n");
                goto unlock;
        }

        caif_assert(phyinfo != NULL);
        caif_assert(phyinfo->id == phyid);
        caif_assert(phyinfo->phy_layer != NULL);
        caif_assert(phyinfo->phy_layer->id == phyid);

        adapt_layer->id = channel_id;

        switch (serv) {
        case CFCTRL_SRV_VEI:
                servicel = cfvei_create(channel_id, &phyinfo->dev_info);
                break;
        case CFCTRL_SRV_DATAGRAM:
                servicel = cfdgml_create(channel_id,
                                        &phyinfo->dev_info);
                break;
        case CFCTRL_SRV_RFM:
                netdev = phyinfo->dev_info.dev;
                servicel = cfrfml_create(channel_id, &phyinfo->dev_info,
                                                netdev->mtu);
                break;
        case CFCTRL_SRV_UTIL:
                servicel = cfutill_create(channel_id, &phyinfo->dev_info);
                break;
        case CFCTRL_SRV_VIDEO:
                servicel = cfvidl_create(channel_id, &phyinfo->dev_info);
                break;
        case CFCTRL_SRV_DBG:
                servicel = cfdbgl_create(channel_id, &phyinfo->dev_info);
                break;
        default:
                pr_err("Protocol error. Link setup response - unknown channel type\n");
                goto unlock;
        }
        if (!servicel)
                goto unlock;
        layer_set_dn(servicel, cnfg->mux);
        cfmuxl_set_uplayer(cnfg->mux, servicel, channel_id);
        layer_set_up(servicel, adapt_layer);
        layer_set_dn(adapt_layer, servicel);

        rcu_read_unlock();

        servicel->ctrlcmd(servicel, CAIF_CTRLCMD_INIT_RSP, 0);
        return;
unlock:
        rcu_read_unlock();
}

int
cfcnfg_add_phy_layer(struct cfcnfg *cnfg,
                     struct net_device *dev, struct cflayer *phy_layer,
                     enum cfcnfg_phy_preference pref,
                     struct cflayer *link_support,
                     bool fcs, int head_room)
{
        struct cflayer *frml;
        struct cfcnfg_phyinfo *phyinfo = NULL;
        int i, res = 0;
        u8 phyid;

        mutex_lock(&cnfg->lock);

        /* CAIF protocol allow maximum 6 link-layers */
        for (i = 0; i < 7; i++) {
                phyid = (dev->ifindex + i) & 0x7;
                if (phyid == 0)
                        continue;
                if (cfcnfg_get_phyinfo_rcu(cnfg, phyid) == NULL)
                        goto got_phyid;
        }
        pr_warn("Too many CAIF Link Layers (max 6)\n");
        res = -EEXIST;
        goto out;

got_phyid:
        phyinfo = kzalloc(sizeof(struct cfcnfg_phyinfo), GFP_ATOMIC);
        if (!phyinfo) {
                res = -ENOMEM;
                goto out;
        }

        phy_layer->id = phyid;
        phyinfo->pref = pref;
        phyinfo->id = phyid;
        phyinfo->dev_info.id = phyid;
        phyinfo->dev_info.dev = dev;
        phyinfo->phy_layer = phy_layer;
        phyinfo->ifindex = dev->ifindex;
        phyinfo->head_room = head_room;
        phyinfo->use_fcs = fcs;

        frml = cffrml_create(phyid, fcs);

        if (!frml) {
                res = -ENOMEM;
                goto out_err;
        }
        phyinfo->frm_layer = frml;
        layer_set_up(frml, cnfg->mux);

        if (link_support != NULL) {
                link_support->id = phyid;
                layer_set_dn(frml, link_support);
                layer_set_up(link_support, frml);
                layer_set_dn(link_support, phy_layer);
                layer_set_up(phy_layer, link_support);
        } else {
                layer_set_dn(frml, phy_layer);
                layer_set_up(phy_layer, frml);
        }

        list_add_rcu(&phyinfo->node, &cnfg->phys);
out:
        mutex_unlock(&cnfg->lock);
        return res;

out_err:
        kfree(phyinfo);
        mutex_unlock(&cnfg->lock);
        return res;
}
EXPORT_SYMBOL(cfcnfg_add_phy_layer);

int cfcnfg_set_phy_state(struct cfcnfg *cnfg, struct cflayer *phy_layer,
                         bool up)
{
        struct cfcnfg_phyinfo *phyinfo;

        rcu_read_lock();
        phyinfo = cfcnfg_get_phyinfo_rcu(cnfg, phy_layer->id);
        if (phyinfo == NULL) {
                rcu_read_unlock();
                return -ENODEV;
        }

        if (phyinfo->up == up) {
                rcu_read_unlock();
                return 0;
        }
        phyinfo->up = up;

        if (up) {
                cffrml_hold(phyinfo->frm_layer);
                cfmuxl_set_dnlayer(cnfg->mux, phyinfo->frm_layer,
                                        phy_layer->id);
        } else {
                cfmuxl_remove_dnlayer(cnfg->mux, phy_layer->id);
                cffrml_put(phyinfo->frm_layer);
        }

        rcu_read_unlock();
        return 0;
}
EXPORT_SYMBOL(cfcnfg_set_phy_state);

int cfcnfg_del_phy_layer(struct cfcnfg *cnfg, struct cflayer *phy_layer)
{
        struct cflayer *frml, *frml_dn;
        u16 phyid;
        struct cfcnfg_phyinfo *phyinfo;

        might_sleep();

        mutex_lock(&cnfg->lock);

        phyid = phy_layer->id;
        phyinfo = cfcnfg_get_phyinfo_rcu(cnfg, phyid);

        if (phyinfo == NULL) {
                mutex_unlock(&cnfg->lock);
                return 0;
        }
        caif_assert(phyid == phyinfo->id);
        caif_assert(phy_layer == phyinfo->phy_layer);
        caif_assert(phy_layer->id == phyid);
        caif_assert(phyinfo->frm_layer->id == phyid);

        list_del_rcu(&phyinfo->node);
        synchronize_rcu();

        /* Fail if reference count is not zero */
        if (cffrml_refcnt_read(phyinfo->frm_layer) != 0) {
                pr_info("Wait for device inuse\n");
                list_add_rcu(&phyinfo->node, &cnfg->phys);
                mutex_unlock(&cnfg->lock);
                return -EAGAIN;
        }

        frml = phyinfo->frm_layer;
        frml_dn = frml->dn;
        cffrml_set_uplayer(frml, NULL);
        cffrml_set_dnlayer(frml, NULL);
        if (phy_layer != frml_dn) {
                layer_set_up(frml_dn, NULL);
                layer_set_dn(frml_dn, NULL);
        }
        layer_set_up(phy_layer, NULL);

        if (phyinfo->phy_layer != frml_dn)
                kfree(frml_dn);

        cffrml_free(frml);
        kfree(phyinfo);
        mutex_unlock(&cnfg->lock);

        return 0;
}
EXPORT_SYMBOL(cfcnfg_del_phy_layer);

















































    4 




























    3 





    3 





    3 


















    1 










    1 

    1 















    1 





















    1 






    1 





    1 









   13 













   12 




   13 
   12 
   13 





   12 

















    6 
    1 

































































































































































































































































































































































































































































    6 



















    6 







































    6 






    6 

































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
// SPDX-License-Identifier: GPL-2.0
#include <linux/memcontrol.h>
#include <linux/rwsem.h>
#include <linux/shrinker.h>
#include <linux/rculist.h>
#include <trace/events/vmscan.h>

#include "internal.h"

LIST_HEAD(shrinker_list);
DEFINE_MUTEX(shrinker_mutex);

#ifdef CONFIG_MEMCG
static int shrinker_nr_max;

static inline int shrinker_unit_size(int nr_items)
{
        return (DIV_ROUND_UP(nr_items, SHRINKER_UNIT_BITS) * sizeof(struct shrinker_info_unit *));
}

static inline void shrinker_unit_free(struct shrinker_info *info, int start)
{
        struct shrinker_info_unit **unit;
        int nr, i;

        if (!info)
                return;

        unit = info->unit;
        nr = DIV_ROUND_UP(info->map_nr_max, SHRINKER_UNIT_BITS);

        for (i = start; i < nr; i++) {
                if (!unit[i])
                        break;

                kfree(unit[i]);
                unit[i] = NULL;
        }
}

static inline int shrinker_unit_alloc(struct shrinker_info *new,
                                       struct shrinker_info *old, int nid)
{
        struct shrinker_info_unit *unit;
        int nr = DIV_ROUND_UP(new->map_nr_max, SHRINKER_UNIT_BITS);
        int start = old ? DIV_ROUND_UP(old->map_nr_max, SHRINKER_UNIT_BITS) : 0;
        int i;

        for (i = start; i < nr; i++) {
                unit = kzalloc_node(sizeof(*unit), GFP_KERNEL, nid);
                if (!unit) {
                        shrinker_unit_free(new, start);
                        return -ENOMEM;
                }

                new->unit[i] = unit;
        }

        return 0;
}

void free_shrinker_info(struct mem_cgroup *memcg)
{
        struct mem_cgroup_per_node *pn;
        struct shrinker_info *info;
        int nid;

        for_each_node(nid) {
                pn = memcg->nodeinfo[nid];
                info = rcu_dereference_protected(pn->shrinker_info, true);
                shrinker_unit_free(info, 0);
                kvfree(info);
                rcu_assign_pointer(pn->shrinker_info, NULL);
        }
}

int alloc_shrinker_info(struct mem_cgroup *memcg)
{
        int nid, ret = 0;
        int array_size = 0;

        mutex_lock(&shrinker_mutex);
        array_size = shrinker_unit_size(shrinker_nr_max);
        for_each_node(nid) {
                struct shrinker_info *info = kvzalloc_node(sizeof(*info) + array_size,
                                                           GFP_KERNEL, nid);
                if (!info)
                        goto err;
                info->map_nr_max = shrinker_nr_max;
                if (shrinker_unit_alloc(info, NULL, nid)) {
                        kvfree(info);
                        goto err;
                }
                rcu_assign_pointer(memcg->nodeinfo[nid]->shrinker_info, info);
        }
        mutex_unlock(&shrinker_mutex);

        return ret;

err:
        mutex_unlock(&shrinker_mutex);
        free_shrinker_info(memcg);
        return -ENOMEM;
}

static struct shrinker_info *shrinker_info_protected(struct mem_cgroup *memcg,
                                                     int nid)
{
        return rcu_dereference_protected(memcg->nodeinfo[nid]->shrinker_info,
                                         lockdep_is_held(&shrinker_mutex));
}

static int expand_one_shrinker_info(struct mem_cgroup *memcg, int new_size,
                                    int old_size, int new_nr_max)
{
        struct shrinker_info *new, *old;
        struct mem_cgroup_per_node *pn;
        int nid;

        for_each_node(nid) {
                pn = memcg->nodeinfo[nid];
                old = shrinker_info_protected(memcg, nid);
                /* Not yet online memcg */
                if (!old)
                        return 0;

                /* Already expanded this shrinker_info */
                if (new_nr_max <= old->map_nr_max)
                        continue;

                new = kvzalloc_node(sizeof(*new) + new_size, GFP_KERNEL, nid);
                if (!new)
                        return -ENOMEM;

                new->map_nr_max = new_nr_max;

                memcpy(new->unit, old->unit, old_size);
                if (shrinker_unit_alloc(new, old, nid)) {
                        kvfree(new);
                        return -ENOMEM;
                }

                rcu_assign_pointer(pn->shrinker_info, new);
                kvfree_rcu(old, rcu);
        }

        return 0;
}

static int expand_shrinker_info(int new_id)
{
        int ret = 0;
        int new_nr_max = round_up(new_id + 1, SHRINKER_UNIT_BITS);
        int new_size, old_size = 0;
        struct mem_cgroup *memcg;

        if (!root_mem_cgroup)
                goto out;

        lockdep_assert_held(&shrinker_mutex);

        new_size = shrinker_unit_size(new_nr_max);
        old_size = shrinker_unit_size(shrinker_nr_max);

        memcg = mem_cgroup_iter(NULL, NULL, NULL);
        do {
                ret = expand_one_shrinker_info(memcg, new_size, old_size,
                                               new_nr_max);
                if (ret) {
                        mem_cgroup_iter_break(NULL, memcg);
                        goto out;
                }
        } while ((memcg = mem_cgroup_iter(NULL, memcg, NULL)) != NULL);
out:
        if (!ret)
                shrinker_nr_max = new_nr_max;

        return ret;
}

static inline int shrinker_id_to_index(int shrinker_id)
{
        return shrinker_id / SHRINKER_UNIT_BITS;
}

static inline int shrinker_id_to_offset(int shrinker_id)
{
        return shrinker_id % SHRINKER_UNIT_BITS;
}

static inline int calc_shrinker_id(int index, int offset)
{
        return index * SHRINKER_UNIT_BITS + offset;
}

void set_shrinker_bit(struct mem_cgroup *memcg, int nid, int shrinker_id)
{
        if (shrinker_id >= 0 && memcg && !mem_cgroup_is_root(memcg)) {
                struct shrinker_info *info;
                struct shrinker_info_unit *unit;

                rcu_read_lock();
                info = rcu_dereference(memcg->nodeinfo[nid]->shrinker_info);
                unit = info->unit[shrinker_id_to_index(shrinker_id)];
                if (!WARN_ON_ONCE(shrinker_id >= info->map_nr_max)) {
                        /* Pairs with smp mb in shrink_slab() */
                        smp_mb__before_atomic();
                        set_bit(shrinker_id_to_offset(shrinker_id), unit->map);
                }
                rcu_read_unlock();
        }
}

static DEFINE_IDR(shrinker_idr);

static int shrinker_memcg_alloc(struct shrinker *shrinker)
{
        int id, ret = -ENOMEM;

        if (mem_cgroup_disabled())
                return -ENOSYS;

        mutex_lock(&shrinker_mutex);
        id = idr_alloc(&shrinker_idr, shrinker, 0, 0, GFP_KERNEL);
        if (id < 0)
                goto unlock;

        if (id >= shrinker_nr_max) {
                if (expand_shrinker_info(id)) {
                        idr_remove(&shrinker_idr, id);
                        goto unlock;
                }
        }
        shrinker->id = id;
        ret = 0;
unlock:
        mutex_unlock(&shrinker_mutex);
        return ret;
}

static void shrinker_memcg_remove(struct shrinker *shrinker)
{
        int id = shrinker->id;

        BUG_ON(id < 0);

        lockdep_assert_held(&shrinker_mutex);

        idr_remove(&shrinker_idr, id);
}

static long xchg_nr_deferred_memcg(int nid, struct shrinker *shrinker,
                                   struct mem_cgroup *memcg)
{
        struct shrinker_info *info;
        struct shrinker_info_unit *unit;
        long nr_deferred;

        rcu_read_lock();
        info = rcu_dereference(memcg->nodeinfo[nid]->shrinker_info);
        unit = info->unit[shrinker_id_to_index(shrinker->id)];
        nr_deferred = atomic_long_xchg(&unit->nr_deferred[shrinker_id_to_offset(shrinker->id)], 0);
        rcu_read_unlock();

        return nr_deferred;
}

static long add_nr_deferred_memcg(long nr, int nid, struct shrinker *shrinker,
                                  struct mem_cgroup *memcg)
{
        struct shrinker_info *info;
        struct shrinker_info_unit *unit;
        long nr_deferred;

        rcu_read_lock();
        info = rcu_dereference(memcg->nodeinfo[nid]->shrinker_info);
        unit = info->unit[shrinker_id_to_index(shrinker->id)];
        nr_deferred =
                atomic_long_add_return(nr, &unit->nr_deferred[shrinker_id_to_offset(shrinker->id)]);
        rcu_read_unlock();

        return nr_deferred;
}

void reparent_shrinker_deferred(struct mem_cgroup *memcg)
{
        int nid, index, offset;
        long nr;
        struct mem_cgroup *parent;
        struct shrinker_info *child_info, *parent_info;
        struct shrinker_info_unit *child_unit, *parent_unit;

        parent = parent_mem_cgroup(memcg);
        if (!parent)
                parent = root_mem_cgroup;

        /* Prevent from concurrent shrinker_info expand */
        mutex_lock(&shrinker_mutex);
        for_each_node(nid) {
                child_info = shrinker_info_protected(memcg, nid);
                parent_info = shrinker_info_protected(parent, nid);
                for (index = 0; index < shrinker_id_to_index(child_info->map_nr_max); index++) {
                        child_unit = child_info->unit[index];
                        parent_unit = parent_info->unit[index];
                        for (offset = 0; offset < SHRINKER_UNIT_BITS; offset++) {
                                nr = atomic_long_read(&child_unit->nr_deferred[offset]);
                                atomic_long_add(nr, &parent_unit->nr_deferred[offset]);
                        }
                }
        }
        mutex_unlock(&shrinker_mutex);
}
#else
static int shrinker_memcg_alloc(struct shrinker *shrinker)
{
        return -ENOSYS;
}

static void shrinker_memcg_remove(struct shrinker *shrinker)
{
}

static long xchg_nr_deferred_memcg(int nid, struct shrinker *shrinker,
                                   struct mem_cgroup *memcg)
{
        return 0;
}

static long add_nr_deferred_memcg(long nr, int nid, struct shrinker *shrinker,
                                  struct mem_cgroup *memcg)
{
        return 0;
}
#endif /* CONFIG_MEMCG */

static long xchg_nr_deferred(struct shrinker *shrinker,
                             struct shrink_control *sc)
{
        int nid = sc->nid;

        if (!(shrinker->flags & SHRINKER_NUMA_AWARE))
                nid = 0;

        if (sc->memcg &&
            (shrinker->flags & SHRINKER_MEMCG_AWARE))
                return xchg_nr_deferred_memcg(nid, shrinker,
                                              sc->memcg);

        return atomic_long_xchg(&shrinker->nr_deferred[nid], 0);
}


static long add_nr_deferred(long nr, struct shrinker *shrinker,
                            struct shrink_control *sc)
{
        int nid = sc->nid;

        if (!(shrinker->flags & SHRINKER_NUMA_AWARE))
                nid = 0;

        if (sc->memcg &&
            (shrinker->flags & SHRINKER_MEMCG_AWARE))
                return add_nr_deferred_memcg(nr, nid, shrinker,
                                             sc->memcg);

        return atomic_long_add_return(nr, &shrinker->nr_deferred[nid]);
}

#define SHRINK_BATCH 128

static unsigned long do_shrink_slab(struct shrink_control *shrinkctl,
                                    struct shrinker *shrinker, int priority)
{
        unsigned long freed = 0;
        unsigned long long delta;
        long total_scan;
        long freeable;
        long nr;
        long new_nr;
        long batch_size = shrinker->batch ? shrinker->batch
                                          : SHRINK_BATCH;
        long scanned = 0, next_deferred;

        freeable = shrinker->count_objects(shrinker, shrinkctl);
        if (freeable == 0 || freeable == SHRINK_EMPTY)
                return freeable;

        /*
         * copy the current shrinker scan count into a local variable
         * and zero it so that other concurrent shrinker invocations
         * don't also do this scanning work.
         */
        nr = xchg_nr_deferred(shrinker, shrinkctl);

        if (shrinker->seeks) {
                delta = freeable >> priority;
                delta *= 4;
                do_div(delta, shrinker->seeks);
        } else {
                /*
                 * These objects don't require any IO to create. Trim
                 * them aggressively under memory pressure to keep
                 * them from causing refetches in the IO caches.
                 */
                delta = freeable / 2;
        }

        total_scan = nr >> priority;
        total_scan += delta;
        total_scan = min(total_scan, (2 * freeable));

        trace_mm_shrink_slab_start(shrinker, shrinkctl, nr,
                                   freeable, delta, total_scan, priority);

        /*
         * Normally, we should not scan less than batch_size objects in one
         * pass to avoid too frequent shrinker calls, but if the slab has less
         * than batch_size objects in total and we are really tight on memory,
         * we will try to reclaim all available objects, otherwise we can end
         * up failing allocations although there are plenty of reclaimable
         * objects spread over several slabs with usage less than the
         * batch_size.
         *
         * We detect the "tight on memory" situations by looking at the total
         * number of objects we want to scan (total_scan). If it is greater
         * than the total number of objects on slab (freeable), we must be
         * scanning at high prio and therefore should try to reclaim as much as
         * possible.
         */
        while (total_scan >= batch_size ||
               total_scan >= freeable) {
                unsigned long ret;
                unsigned long nr_to_scan = min(batch_size, total_scan);

                shrinkctl->nr_to_scan = nr_to_scan;
                shrinkctl->nr_scanned = nr_to_scan;
                ret = shrinker->scan_objects(shrinker, shrinkctl);
                if (ret == SHRINK_STOP)
                        break;
                freed += ret;

                count_vm_events(SLABS_SCANNED, shrinkctl->nr_scanned);
                total_scan -= shrinkctl->nr_scanned;
                scanned += shrinkctl->nr_scanned;

                cond_resched();
        }

        /*
         * The deferred work is increased by any new work (delta) that wasn't
         * done, decreased by old deferred work that was done now.
         *
         * And it is capped to two times of the freeable items.
         */
        next_deferred = max_t(long, (nr + delta - scanned), 0);
        next_deferred = min(next_deferred, (2 * freeable));

        /*
         * move the unused scan count back into the shrinker in a
         * manner that handles concurrent updates.
         */
        new_nr = add_nr_deferred(next_deferred, shrinker, shrinkctl);

        trace_mm_shrink_slab_end(shrinker, shrinkctl->nid, freed, nr, new_nr, total_scan);
        return freed;
}

#ifdef CONFIG_MEMCG
static unsigned long shrink_slab_memcg(gfp_t gfp_mask, int nid,
                        struct mem_cgroup *memcg, int priority)
{
        struct shrinker_info *info;
        unsigned long ret, freed = 0;
        int offset, index = 0;

        if (!mem_cgroup_online(memcg))
                return 0;

        /*
         * lockless algorithm of memcg shrink.
         *
         * The shrinker_info may be freed asynchronously via RCU in the
         * expand_one_shrinker_info(), so the rcu_read_lock() needs to be used
         * to ensure the existence of the shrinker_info.
         *
         * The shrinker_info_unit is never freed unless its corresponding memcg
         * is destroyed. Here we already hold the refcount of memcg, so the
         * memcg will not be destroyed, and of course shrinker_info_unit will
         * not be freed.
         *
         * So in the memcg shrink:
         *  step 1: use rcu_read_lock() to guarantee existence of the
         *          shrinker_info.
         *  step 2: after getting shrinker_info_unit we can safely release the
         *          RCU lock.
         *  step 3: traverse the bitmap and calculate shrinker_id
         *  step 4: use rcu_read_lock() to guarantee existence of the shrinker.
         *  step 5: use shrinker_id to find the shrinker, then use
         *          shrinker_try_get() to guarantee existence of the shrinker,
         *          then we can release the RCU lock to do do_shrink_slab() that
         *          may sleep.
         *  step 6: do shrinker_put() paired with step 5 to put the refcount,
         *          if the refcount reaches 0, then wake up the waiter in
         *          shrinker_free() by calling complete().
         *          Note: here is different from the global shrink, we don't
         *                need to acquire the RCU lock to guarantee existence of
         *                the shrinker, because we don't need to use this
         *                shrinker to traverse the next shrinker in the bitmap.
         *  step 7: we have already exited the read-side of rcu critical section
         *          before calling do_shrink_slab(), the shrinker_info may be
         *          released in expand_one_shrinker_info(), so go back to step 1
         *          to reacquire the shrinker_info.
         */
again:
        rcu_read_lock();
        info = rcu_dereference(memcg->nodeinfo[nid]->shrinker_info);
        if (unlikely(!info))
                goto unlock;

        if (index < shrinker_id_to_index(info->map_nr_max)) {
                struct shrinker_info_unit *unit;

                unit = info->unit[index];

                rcu_read_unlock();

                for_each_set_bit(offset, unit->map, SHRINKER_UNIT_BITS) {
                        struct shrink_control sc = {
                                .gfp_mask = gfp_mask,
                                .nid = nid,
                                .memcg = memcg,
                        };
                        struct shrinker *shrinker;
                        int shrinker_id = calc_shrinker_id(index, offset);

                        rcu_read_lock();
                        shrinker = idr_find(&shrinker_idr, shrinker_id);
                        if (unlikely(!shrinker || !shrinker_try_get(shrinker))) {
                                clear_bit(offset, unit->map);
                                rcu_read_unlock();
                                continue;
                        }
                        rcu_read_unlock();

                        /* Call non-slab shrinkers even though kmem is disabled */
                        if (!memcg_kmem_online() &&
                            !(shrinker->flags & SHRINKER_NONSLAB))
                                continue;

                        ret = do_shrink_slab(&sc, shrinker, priority);
                        if (ret == SHRINK_EMPTY) {
                                clear_bit(offset, unit->map);
                                /*
                                 * After the shrinker reported that it had no objects to
                                 * free, but before we cleared the corresponding bit in
                                 * the memcg shrinker map, a new object might have been
                                 * added. To make sure, we have the bit set in this
                                 * case, we invoke the shrinker one more time and reset
                                 * the bit if it reports that it is not empty anymore.
                                 * The memory barrier here pairs with the barrier in
                                 * set_shrinker_bit():
                                 *
                                 * list_lru_add()     shrink_slab_memcg()
                                 *   list_add_tail()    clear_bit()
                                 *   <MB>               <MB>
                                 *   set_bit()          do_shrink_slab()
                                 */
                                smp_mb__after_atomic();
                                ret = do_shrink_slab(&sc, shrinker, priority);
                                if (ret == SHRINK_EMPTY)
                                        ret = 0;
                                else
                                        set_shrinker_bit(memcg, nid, shrinker_id);
                        }
                        freed += ret;
                        shrinker_put(shrinker);
                }

                index++;
                goto again;
        }
unlock:
        rcu_read_unlock();
        return freed;
}
#else /* !CONFIG_MEMCG */
static unsigned long shrink_slab_memcg(gfp_t gfp_mask, int nid,
                        struct mem_cgroup *memcg, int priority)
{
        return 0;
}
#endif /* CONFIG_MEMCG */

/**
 * shrink_slab - shrink slab caches
 * @gfp_mask: allocation context
 * @nid: node whose slab caches to target
 * @memcg: memory cgroup whose slab caches to target
 * @priority: the reclaim priority
 *
 * Call the shrink functions to age shrinkable caches.
 *
 * @nid is passed along to shrinkers with SHRINKER_NUMA_AWARE set,
 * unaware shrinkers will receive a node id of 0 instead.
 *
 * @memcg specifies the memory cgroup to target. Unaware shrinkers
 * are called only if it is the root cgroup.
 *
 * @priority is sc->priority, we take the number of objects and >> by priority
 * in order to get the scan target.
 *
 * Returns the number of reclaimed slab objects.
 */
unsigned long shrink_slab(gfp_t gfp_mask, int nid, struct mem_cgroup *memcg,
                          int priority)
{
        unsigned long ret, freed = 0;
        struct shrinker *shrinker;

        /*
         * The root memcg might be allocated even though memcg is disabled
         * via "cgroup_disable=memory" boot parameter.  This could make
         * mem_cgroup_is_root() return false, then just run memcg slab
         * shrink, but skip global shrink.  This may result in premature
         * oom.
         */
        if (!mem_cgroup_disabled() && !mem_cgroup_is_root(memcg))
                return shrink_slab_memcg(gfp_mask, nid, memcg, priority);

        /*
         * lockless algorithm of global shrink.
         *
         * In the unregistration setp, the shrinker will be freed asynchronously
         * via RCU after its refcount reaches 0. So both rcu_read_lock() and
         * shrinker_try_get() can be used to ensure the existence of the shrinker.
         *
         * So in the global shrink:
         *  step 1: use rcu_read_lock() to guarantee existence of the shrinker
         *          and the validity of the shrinker_list walk.
         *  step 2: use shrinker_try_get() to try get the refcount, if successful,
         *          then the existence of the shrinker can also be guaranteed,
         *          so we can release the RCU lock to do do_shrink_slab() that
         *          may sleep.
         *  step 3: *MUST* to reacquire the RCU lock before calling shrinker_put(),
         *          which ensures that neither this shrinker nor the next shrinker
         *          will be freed in the next traversal operation.
         *  step 4: do shrinker_put() paired with step 2 to put the refcount,
         *          if the refcount reaches 0, then wake up the waiter in
         *          shrinker_free() by calling complete().
         */
        rcu_read_lock();
        list_for_each_entry_rcu(shrinker, &shrinker_list, list) {
                struct shrink_control sc = {
                        .gfp_mask = gfp_mask,
                        .nid = nid,
                        .memcg = memcg,
                };

                if (!shrinker_try_get(shrinker))
                        continue;

                rcu_read_unlock();

                ret = do_shrink_slab(&sc, shrinker, priority);
                if (ret == SHRINK_EMPTY)
                        ret = 0;
                freed += ret;

                rcu_read_lock();
                shrinker_put(shrinker);
        }

        rcu_read_unlock();
        cond_resched();
        return freed;
}

struct shrinker *shrinker_alloc(unsigned int flags, const char *fmt, ...)
{
        struct shrinker *shrinker;
        unsigned int size;
        va_list ap;
        int err;

        shrinker = kzalloc(sizeof(struct shrinker), GFP_KERNEL);
        if (!shrinker)
                return NULL;

        va_start(ap, fmt);
        err = shrinker_debugfs_name_alloc(shrinker, fmt, ap);
        va_end(ap);
        if (err)
                goto err_name;

        shrinker->flags = flags | SHRINKER_ALLOCATED;
        shrinker->seeks = DEFAULT_SEEKS;

        if (flags & SHRINKER_MEMCG_AWARE) {
                err = shrinker_memcg_alloc(shrinker);
                if (err == -ENOSYS) {
                        /* Memcg is not supported, fallback to non-memcg-aware shrinker. */
                        shrinker->flags &= ~SHRINKER_MEMCG_AWARE;
                        goto non_memcg;
                }

                if (err)
                        goto err_flags;

                return shrinker;
        }

non_memcg:
        /*
         * The nr_deferred is available on per memcg level for memcg aware
         * shrinkers, so only allocate nr_deferred in the following cases:
         *  - non-memcg-aware shrinkers
         *  - !CONFIG_MEMCG
         *  - memcg is disabled by kernel command line
         */
        size = sizeof(*shrinker->nr_deferred);
        if (flags & SHRINKER_NUMA_AWARE)
                size *= nr_node_ids;

        shrinker->nr_deferred = kzalloc(size, GFP_KERNEL);
        if (!shrinker->nr_deferred)
                goto err_flags;

        return shrinker;

err_flags:
        shrinker_debugfs_name_free(shrinker);
err_name:
        kfree(shrinker);
        return NULL;
}
EXPORT_SYMBOL_GPL(shrinker_alloc);

void shrinker_register(struct shrinker *shrinker)
{
        if (unlikely(!(shrinker->flags & SHRINKER_ALLOCATED))) {
                pr_warn("Must use shrinker_alloc() to dynamically allocate the shrinker");
                return;
        }

        mutex_lock(&shrinker_mutex);
        list_add_tail_rcu(&shrinker->list, &shrinker_list);
        shrinker->flags |= SHRINKER_REGISTERED;
        shrinker_debugfs_add(shrinker);
        mutex_unlock(&shrinker_mutex);

        init_completion(&shrinker->done);
        /*
         * Now the shrinker is fully set up, take the first reference to it to
         * indicate that lookup operations are now allowed to use it via
         * shrinker_try_get().
         */
        refcount_set(&shrinker->refcount, 1);
}
EXPORT_SYMBOL_GPL(shrinker_register);

static void shrinker_free_rcu_cb(struct rcu_head *head)
{
        struct shrinker *shrinker = container_of(head, struct shrinker, rcu);

        kfree(shrinker->nr_deferred);
        kfree(shrinker);
}

void shrinker_free(struct shrinker *shrinker)
{
        struct dentry *debugfs_entry = NULL;
        int debugfs_id;

        if (!shrinker)
                return;

        if (shrinker->flags & SHRINKER_REGISTERED) {
                /* drop the initial refcount */
                shrinker_put(shrinker);
                /*
                 * Wait for all lookups of the shrinker to complete, after that,
                 * no shrinker is running or will run again, then we can safely
                 * free it asynchronously via RCU and safely free the structure
                 * where the shrinker is located, such as super_block etc.
                 */
                wait_for_completion(&shrinker->done);
        }

        mutex_lock(&shrinker_mutex);
        if (shrinker->flags & SHRINKER_REGISTERED) {
                /*
                 * Now we can safely remove it from the shrinker_list and then
                 * free it.
                 */
                list_del_rcu(&shrinker->list);
                debugfs_entry = shrinker_debugfs_detach(shrinker, &debugfs_id);
                shrinker->flags &= ~SHRINKER_REGISTERED;
        }

        shrinker_debugfs_name_free(shrinker);

        if (shrinker->flags & SHRINKER_MEMCG_AWARE)
                shrinker_memcg_remove(shrinker);
        mutex_unlock(&shrinker_mutex);

        if (debugfs_entry)
                shrinker_debugfs_remove(debugfs_entry, debugfs_id);

        call_rcu(&shrinker->rcu, shrinker_free_rcu_cb);
}
EXPORT_SYMBOL_GPL(shrinker_free);
































































































































































































































































































































































































































































































 20777 















































    2 



1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __LINUX_PREEMPT_H
#define __LINUX_PREEMPT_H

/*
 * include/linux/preempt.h - macros for accessing and manipulating
 * preempt_count (used for kernel preemption, interrupt count, etc.)
 */

#include <linux/linkage.h>
#include <linux/cleanup.h>
#include <linux/types.h>

/*
 * We put the hardirq and softirq counter into the preemption
 * counter. The bitmask has the following meaning:
 *
 * - bits 0-7 are the preemption count (max preemption depth: 256)
 * - bits 8-15 are the softirq count (max # of softirqs: 256)
 *
 * The hardirq count could in theory be the same as the number of
 * interrupts in the system, but we run all interrupt handlers with
 * interrupts disabled, so we cannot have nesting interrupts. Though
 * there are a few palaeontologic drivers which reenable interrupts in
 * the handler, so we need more than one bit here.
 *
 *         PREEMPT_MASK:        0x000000ff
 *         SOFTIRQ_MASK:        0x0000ff00
 *         HARDIRQ_MASK:        0x000f0000
 *             NMI_MASK:        0x00f00000
 * PREEMPT_NEED_RESCHED:        0x80000000
 */
#define PREEMPT_BITS        8
#define SOFTIRQ_BITS        8
#define HARDIRQ_BITS        4
#define NMI_BITS        4

#define PREEMPT_SHIFT        0
#define SOFTIRQ_SHIFT        (PREEMPT_SHIFT + PREEMPT_BITS)
#define HARDIRQ_SHIFT        (SOFTIRQ_SHIFT + SOFTIRQ_BITS)
#define NMI_SHIFT        (HARDIRQ_SHIFT + HARDIRQ_BITS)

#define __IRQ_MASK(x)        ((1UL << (x))-1)

#define PREEMPT_MASK        (__IRQ_MASK(PREEMPT_BITS) << PREEMPT_SHIFT)
#define SOFTIRQ_MASK        (__IRQ_MASK(SOFTIRQ_BITS) << SOFTIRQ_SHIFT)
#define HARDIRQ_MASK        (__IRQ_MASK(HARDIRQ_BITS) << HARDIRQ_SHIFT)
#define NMI_MASK        (__IRQ_MASK(NMI_BITS)     << NMI_SHIFT)

#define PREEMPT_OFFSET        (1UL << PREEMPT_SHIFT)
#define SOFTIRQ_OFFSET        (1UL << SOFTIRQ_SHIFT)
#define HARDIRQ_OFFSET        (1UL << HARDIRQ_SHIFT)
#define NMI_OFFSET        (1UL << NMI_SHIFT)

#define SOFTIRQ_DISABLE_OFFSET        (2 * SOFTIRQ_OFFSET)

#define PREEMPT_DISABLED        (PREEMPT_DISABLE_OFFSET + PREEMPT_ENABLED)

/*
 * Disable preemption until the scheduler is running -- use an unconditional
 * value so that it also works on !PREEMPT_COUNT kernels.
 *
 * Reset by start_kernel()->sched_init()->init_idle()->init_idle_preempt_count().
 */
#define INIT_PREEMPT_COUNT        PREEMPT_OFFSET

/*
 * Initial preempt_count value; reflects the preempt_count schedule invariant
 * which states that during context switches:
 *
 *    preempt_count() == 2*PREEMPT_DISABLE_OFFSET
 *
 * Note: PREEMPT_DISABLE_OFFSET is 0 for !PREEMPT_COUNT kernels.
 * Note: See finish_task_switch().
 */
#define FORK_PREEMPT_COUNT        (2*PREEMPT_DISABLE_OFFSET + PREEMPT_ENABLED)

/* preempt_count() and related functions, depends on PREEMPT_NEED_RESCHED */
#include <asm/preempt.h>

/**
 * interrupt_context_level - return interrupt context level
 *
 * Returns the current interrupt context level.
 *  0 - normal context
 *  1 - softirq context
 *  2 - hardirq context
 *  3 - NMI context
 */
static __always_inline unsigned char interrupt_context_level(void)
{
        unsigned long pc = preempt_count();
        unsigned char level = 0;

        level += !!(pc & (NMI_MASK));
        level += !!(pc & (NMI_MASK | HARDIRQ_MASK));
        level += !!(pc & (NMI_MASK | HARDIRQ_MASK | SOFTIRQ_OFFSET));

        return level;
}

/*
 * These macro definitions avoid redundant invocations of preempt_count()
 * because such invocations would result in redundant loads given that
 * preempt_count() is commonly implemented with READ_ONCE().
 */

#define nmi_count()        (preempt_count() & NMI_MASK)
#define hardirq_count()        (preempt_count() & HARDIRQ_MASK)
#ifdef CONFIG_PREEMPT_RT
# define softirq_count()        (current->softirq_disable_cnt & SOFTIRQ_MASK)
# define irq_count()                ((preempt_count() & (NMI_MASK | HARDIRQ_MASK)) | softirq_count())
#else
# define softirq_count()        (preempt_count() & SOFTIRQ_MASK)
# define irq_count()                (preempt_count() & (NMI_MASK | HARDIRQ_MASK | SOFTIRQ_MASK))
#endif

/*
 * Macros to retrieve the current execution context:
 *
 * in_nmi()                - We're in NMI context
 * in_hardirq()                - We're in hard IRQ context
 * in_serving_softirq()        - We're in softirq context
 * in_task()                - We're in task context
 */
#define in_nmi()                (nmi_count())
#define in_hardirq()                (hardirq_count())
#define in_serving_softirq()        (softirq_count() & SOFTIRQ_OFFSET)
#ifdef CONFIG_PREEMPT_RT
# define in_task()                (!((preempt_count() & (NMI_MASK | HARDIRQ_MASK)) | in_serving_softirq()))
#else
# define in_task()                (!(preempt_count() & (NMI_MASK | HARDIRQ_MASK | SOFTIRQ_OFFSET)))
#endif

/*
 * The following macros are deprecated and should not be used in new code:
 * in_irq()       - Obsolete version of in_hardirq()
 * in_softirq()   - We have BH disabled, or are processing softirqs
 * in_interrupt() - We're in NMI,IRQ,SoftIRQ context or have BH disabled
 */
#define in_irq()                (hardirq_count())
#define in_softirq()                (softirq_count())
#define in_interrupt()                (irq_count())

/*
 * The preempt_count offset after preempt_disable();
 */
#if defined(CONFIG_PREEMPT_COUNT)
# define PREEMPT_DISABLE_OFFSET        PREEMPT_OFFSET
#else
# define PREEMPT_DISABLE_OFFSET        0
#endif

/*
 * The preempt_count offset after spin_lock()
 */
#if !defined(CONFIG_PREEMPT_RT)
#define PREEMPT_LOCK_OFFSET                PREEMPT_DISABLE_OFFSET
#else
/* Locks on RT do not disable preemption */
#define PREEMPT_LOCK_OFFSET                0
#endif

/*
 * The preempt_count offset needed for things like:
 *
 *  spin_lock_bh()
 *
 * Which need to disable both preemption (CONFIG_PREEMPT_COUNT) and
 * softirqs, such that unlock sequences of:
 *
 *  spin_unlock();
 *  local_bh_enable();
 *
 * Work as expected.
 */
#define SOFTIRQ_LOCK_OFFSET (SOFTIRQ_DISABLE_OFFSET + PREEMPT_LOCK_OFFSET)

/*
 * Are we running in atomic context?  WARNING: this macro cannot
 * always detect atomic context; in particular, it cannot know about
 * held spinlocks in non-preemptible kernels.  Thus it should not be
 * used in the general case to determine whether sleeping is possible.
 * Do not use in_atomic() in driver code.
 */
#define in_atomic()        (preempt_count() != 0)

/*
 * Check whether we were atomic before we did preempt_disable():
 * (used by the scheduler)
 */
#define in_atomic_preempt_off() (preempt_count() != PREEMPT_DISABLE_OFFSET)

#if defined(CONFIG_DEBUG_PREEMPT) || defined(CONFIG_TRACE_PREEMPT_TOGGLE)
extern void preempt_count_add(int val);
extern void preempt_count_sub(int val);
#define preempt_count_dec_and_test() \
        ({ preempt_count_sub(1); should_resched(0); })
#else
#define preempt_count_add(val)        __preempt_count_add(val)
#define preempt_count_sub(val)        __preempt_count_sub(val)
#define preempt_count_dec_and_test() __preempt_count_dec_and_test()
#endif

#define __preempt_count_inc() __preempt_count_add(1)
#define __preempt_count_dec() __preempt_count_sub(1)

#define preempt_count_inc() preempt_count_add(1)
#define preempt_count_dec() preempt_count_sub(1)

#ifdef CONFIG_PREEMPT_COUNT

#define preempt_disable() \
do { \
        preempt_count_inc(); \
        barrier(); \
} while (0)

#define sched_preempt_enable_no_resched() \
do { \
        barrier(); \
        preempt_count_dec(); \
} while (0)

#define preempt_enable_no_resched() sched_preempt_enable_no_resched()

#define preemptible()        (preempt_count() == 0 && !irqs_disabled())

#ifdef CONFIG_PREEMPTION
#define preempt_enable() \
do { \
        barrier(); \
        if (unlikely(preempt_count_dec_and_test())) \
                __preempt_schedule(); \
} while (0)

#define preempt_enable_notrace() \
do { \
        barrier(); \
        if (unlikely(__preempt_count_dec_and_test())) \
                __preempt_schedule_notrace(); \
} while (0)

#define preempt_check_resched() \
do { \
        if (should_resched(0)) \
                __preempt_schedule(); \
} while (0)

#else /* !CONFIG_PREEMPTION */
#define preempt_enable() \
do { \
        barrier(); \
        preempt_count_dec(); \
} while (0)

#define preempt_enable_notrace() \
do { \
        barrier(); \
        __preempt_count_dec(); \
} while (0)

#define preempt_check_resched() do { } while (0)
#endif /* CONFIG_PREEMPTION */

#define preempt_disable_notrace() \
do { \
        __preempt_count_inc(); \
        barrier(); \
} while (0)

#define preempt_enable_no_resched_notrace() \
do { \
        barrier(); \
        __preempt_count_dec(); \
} while (0)

#else /* !CONFIG_PREEMPT_COUNT */

/*
 * Even if we don't have any preemption, we need preempt disable/enable
 * to be barriers, so that we don't have things like get_user/put_user
 * that can cause faults and scheduling migrate into our preempt-protected
 * region.
 */
#define preempt_disable()                        barrier()
#define sched_preempt_enable_no_resched()        barrier()
#define preempt_enable_no_resched()                barrier()
#define preempt_enable()                        barrier()
#define preempt_check_resched()                        do { } while (0)

#define preempt_disable_notrace()                barrier()
#define preempt_enable_no_resched_notrace()        barrier()
#define preempt_enable_notrace()                barrier()
#define preemptible()                                0

#endif /* CONFIG_PREEMPT_COUNT */

#ifdef MODULE
/*
 * Modules have no business playing preemption tricks.
 */
#undef sched_preempt_enable_no_resched
#undef preempt_enable_no_resched
#undef preempt_enable_no_resched_notrace
#undef preempt_check_resched
#endif

#define preempt_set_need_resched() \
do { \
        set_preempt_need_resched(); \
} while (0)
#define preempt_fold_need_resched() \
do { \
        if (tif_need_resched()) \
                set_preempt_need_resched(); \
} while (0)

#ifdef CONFIG_PREEMPT_NOTIFIERS

struct preempt_notifier;

/**
 * preempt_ops - notifiers called when a task is preempted and rescheduled
 * @sched_in: we're about to be rescheduled:
 *    notifier: struct preempt_notifier for the task being scheduled
 *    cpu:  cpu we're scheduled on
 * @sched_out: we've just been preempted
 *    notifier: struct preempt_notifier for the task being preempted
 *    next: the task that's kicking us out
 *
 * Please note that sched_in and out are called under different
 * contexts.  sched_out is called with rq lock held and irq disabled
 * while sched_in is called without rq lock and irq enabled.  This
 * difference is intentional and depended upon by its users.
 */
struct preempt_ops {
        void (*sched_in)(struct preempt_notifier *notifier, int cpu);
        void (*sched_out)(struct preempt_notifier *notifier,
                          struct task_struct *next);
};

/**
 * preempt_notifier - key for installing preemption notifiers
 * @link: internal use
 * @ops: defines the notifier functions to be called
 *
 * Usually used in conjunction with container_of().
 */
struct preempt_notifier {
        struct hlist_node link;
        struct preempt_ops *ops;
};

void preempt_notifier_inc(void);
void preempt_notifier_dec(void);
void preempt_notifier_register(struct preempt_notifier *notifier);
void preempt_notifier_unregister(struct preempt_notifier *notifier);

static inline void preempt_notifier_init(struct preempt_notifier *notifier,
                                     struct preempt_ops *ops)
{
        /* INIT_HLIST_NODE() open coded, to avoid dependency on list.h */
        notifier->link.next = NULL;
        notifier->link.pprev = NULL;
        notifier->ops = ops;
}

#endif

#ifdef CONFIG_SMP

/*
 * Migrate-Disable and why it is undesired.
 *
 * When a preempted task becomes elegible to run under the ideal model (IOW it
 * becomes one of the M highest priority tasks), it might still have to wait
 * for the preemptee's migrate_disable() section to complete. Thereby suffering
 * a reduction in bandwidth in the exact duration of the migrate_disable()
 * section.
 *
 * Per this argument, the change from preempt_disable() to migrate_disable()
 * gets us:
 *
 * - a higher priority tasks gains reduced wake-up latency; with preempt_disable()
 *   it would have had to wait for the lower priority task.
 *
 * - a lower priority tasks; which under preempt_disable() could've instantly
 *   migrated away when another CPU becomes available, is now constrained
 *   by the ability to push the higher priority task away, which might itself be
 *   in a migrate_disable() section, reducing it's available bandwidth.
 *
 * IOW it trades latency / moves the interference term, but it stays in the
 * system, and as long as it remains unbounded, the system is not fully
 * deterministic.
 *
 *
 * The reason we have it anyway.
 *
 * PREEMPT_RT breaks a number of assumptions traditionally held. By forcing a
 * number of primitives into becoming preemptible, they would also allow
 * migration. This turns out to break a bunch of per-cpu usage. To this end,
 * all these primitives employ migirate_disable() to restore this implicit
 * assumption.
 *
 * This is a 'temporary' work-around at best. The correct solution is getting
 * rid of the above assumptions and reworking the code to employ explicit
 * per-cpu locking or short preempt-disable regions.
 *
 * The end goal must be to get rid of migrate_disable(), alternatively we need
 * a schedulability theory that does not depend on abritrary migration.
 *
 *
 * Notes on the implementation.
 *
 * The implementation is particularly tricky since existing code patterns
 * dictate neither migrate_disable() nor migrate_enable() is allowed to block.
 * This means that it cannot use cpus_read_lock() to serialize against hotplug,
 * nor can it easily migrate itself into a pending affinity mask change on
 * migrate_enable().
 *
 *
 * Note: even non-work-conserving schedulers like semi-partitioned depends on
 *       migration, so migrate_disable() is not only a problem for
 *       work-conserving schedulers.
 *
 */
extern void migrate_disable(void);
extern void migrate_enable(void);

#else

static inline void migrate_disable(void) { }
static inline void migrate_enable(void) { }

#endif /* CONFIG_SMP */

/**
 * preempt_disable_nested - Disable preemption inside a normally preempt disabled section
 *
 * Use for code which requires preemption protection inside a critical
 * section which has preemption disabled implicitly on non-PREEMPT_RT
 * enabled kernels, by e.g.:
 *  - holding a spinlock/rwlock
 *  - soft interrupt context
 *  - regular interrupt handlers
 *
 * On PREEMPT_RT enabled kernels spinlock/rwlock held sections, soft
 * interrupt context and regular interrupt handlers are preemptible and
 * only prevent migration. preempt_disable_nested() ensures that preemption
 * is disabled for cases which require CPU local serialization even on
 * PREEMPT_RT. For non-PREEMPT_RT kernels this is a NOP.
 *
 * The use cases are code sequences which are not serialized by a
 * particular lock instance, e.g.:
 *  - seqcount write side critical sections where the seqcount is not
 *    associated to a particular lock and therefore the automatic
 *    protection mechanism does not work. This prevents a live lock
 *    against a preempting high priority reader.
 *  - RMW per CPU variable updates like vmstat.
 */
/* Macro to avoid header recursion hell vs. lockdep */
#define preempt_disable_nested()                                \
do {                                                                \
        if (IS_ENABLED(CONFIG_PREEMPT_RT))                        \
                preempt_disable();                                \
        else                                                        \
                lockdep_assert_preemption_disabled();                \
} while (0)

/**
 * preempt_enable_nested - Undo the effect of preempt_disable_nested()
 */
static __always_inline void preempt_enable_nested(void)
{
        if (IS_ENABLED(CONFIG_PREEMPT_RT))
                preempt_enable();
}

DEFINE_LOCK_GUARD_0(preempt, preempt_disable(), preempt_enable())
DEFINE_LOCK_GUARD_0(preempt_notrace, preempt_disable_notrace(), preempt_enable_notrace())
DEFINE_LOCK_GUARD_0(migrate, migrate_disable(), migrate_enable())

#ifdef CONFIG_PREEMPT_DYNAMIC

extern bool preempt_model_none(void);
extern bool preempt_model_voluntary(void);
extern bool preempt_model_full(void);
extern bool preempt_model_lazy(void);

#else

static inline bool preempt_model_none(void)
{
        return IS_ENABLED(CONFIG_PREEMPT_NONE);
}
static inline bool preempt_model_voluntary(void)
{
        return IS_ENABLED(CONFIG_PREEMPT_VOLUNTARY);
}
static inline bool preempt_model_full(void)
{
        return IS_ENABLED(CONFIG_PREEMPT);
}

static inline bool preempt_model_lazy(void)
{
        return IS_ENABLED(CONFIG_PREEMPT_LAZY);
}

#endif

static inline bool preempt_model_rt(void)
{
        return IS_ENABLED(CONFIG_PREEMPT_RT);
}

/*
 * Does the preemption model allow non-cooperative preemption?
 *
 * For !CONFIG_PREEMPT_DYNAMIC kernels this is an exact match with
 * CONFIG_PREEMPTION; for CONFIG_PREEMPT_DYNAMIC this doesn't work as the
 * kernel is *built* with CONFIG_PREEMPTION=y but may run with e.g. the
 * PREEMPT_NONE model.
 */
static inline bool preempt_model_preemptible(void)
{
        return preempt_model_full() || preempt_model_lazy() || preempt_model_rt();
}

#endif /* __LINUX_PREEMPT_H */




















































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































    2 



    2 
    2 





























































    2 



























































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
// SPDX-License-Identifier: GPL-2.0-or-later
/* auditfilter.c -- filtering of audit events
 *
 * Copyright 2003-2004 Red Hat, Inc.
 * Copyright 2005 Hewlett-Packard Development Company, L.P.
 * Copyright 2005 IBM Corporation
 */

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#include <linux/kernel.h>
#include <linux/audit.h>
#include <linux/kthread.h>
#include <linux/mutex.h>
#include <linux/fs.h>
#include <linux/namei.h>
#include <linux/netlink.h>
#include <linux/sched.h>
#include <linux/slab.h>
#include <linux/security.h>
#include <net/net_namespace.h>
#include <net/sock.h>
#include "audit.h"

/*
 * Locking model:
 *
 * audit_filter_mutex:
 *                Synchronizes writes and blocking reads of audit's filterlist
 *                data.  Rcu is used to traverse the filterlist and access
 *                contents of structs audit_entry, audit_watch and opaque
 *                LSM rules during filtering.  If modified, these structures
 *                must be copied and replace their counterparts in the filterlist.
 *                An audit_parent struct is not accessed during filtering, so may
 *                be written directly provided audit_filter_mutex is held.
 */

/* Audit filter lists, defined in <linux/audit.h> */
struct list_head audit_filter_list[AUDIT_NR_FILTERS] = {
        LIST_HEAD_INIT(audit_filter_list[0]),
        LIST_HEAD_INIT(audit_filter_list[1]),
        LIST_HEAD_INIT(audit_filter_list[2]),
        LIST_HEAD_INIT(audit_filter_list[3]),
        LIST_HEAD_INIT(audit_filter_list[4]),
        LIST_HEAD_INIT(audit_filter_list[5]),
        LIST_HEAD_INIT(audit_filter_list[6]),
        LIST_HEAD_INIT(audit_filter_list[7]),
#if AUDIT_NR_FILTERS != 8
#error Fix audit_filter_list initialiser
#endif
};
static struct list_head audit_rules_list[AUDIT_NR_FILTERS] = {
        LIST_HEAD_INIT(audit_rules_list[0]),
        LIST_HEAD_INIT(audit_rules_list[1]),
        LIST_HEAD_INIT(audit_rules_list[2]),
        LIST_HEAD_INIT(audit_rules_list[3]),
        LIST_HEAD_INIT(audit_rules_list[4]),
        LIST_HEAD_INIT(audit_rules_list[5]),
        LIST_HEAD_INIT(audit_rules_list[6]),
        LIST_HEAD_INIT(audit_rules_list[7]),
};

DEFINE_MUTEX(audit_filter_mutex);

static void audit_free_lsm_field(struct audit_field *f)
{
        switch (f->type) {
        case AUDIT_SUBJ_USER:
        case AUDIT_SUBJ_ROLE:
        case AUDIT_SUBJ_TYPE:
        case AUDIT_SUBJ_SEN:
        case AUDIT_SUBJ_CLR:
        case AUDIT_OBJ_USER:
        case AUDIT_OBJ_ROLE:
        case AUDIT_OBJ_TYPE:
        case AUDIT_OBJ_LEV_LOW:
        case AUDIT_OBJ_LEV_HIGH:
                kfree(f->lsm_str);
                security_audit_rule_free(f->lsm_rule);
        }
}

static inline void audit_free_rule(struct audit_entry *e)
{
        int i;
        struct audit_krule *erule = &e->rule;

        /* some rules don't have associated watches */
        if (erule->watch)
                audit_put_watch(erule->watch);
        if (erule->fields)
                for (i = 0; i < erule->field_count; i++)
                        audit_free_lsm_field(&erule->fields[i]);
        kfree(erule->fields);
        kfree(erule->filterkey);
        kfree(e);
}

void audit_free_rule_rcu(struct rcu_head *head)
{
        struct audit_entry *e = container_of(head, struct audit_entry, rcu);
        audit_free_rule(e);
}

/* Initialize an audit filterlist entry. */
static inline struct audit_entry *audit_init_entry(u32 field_count)
{
        struct audit_entry *entry;
        struct audit_field *fields;

        entry = kzalloc(sizeof(*entry), GFP_KERNEL);
        if (unlikely(!entry))
                return NULL;

        fields = kcalloc(field_count, sizeof(*fields), GFP_KERNEL);
        if (unlikely(!fields)) {
                kfree(entry);
                return NULL;
        }
        entry->rule.fields = fields;

        return entry;
}

/* Unpack a filter field's string representation from user-space
 * buffer. */
char *audit_unpack_string(void **bufp, size_t *remain, size_t len)
{
        char *str;

        if (!*bufp || (len == 0) || (len > *remain))
                return ERR_PTR(-EINVAL);

        /* Of the currently implemented string fields, PATH_MAX
         * defines the longest valid length.
         */
        if (len > PATH_MAX)
                return ERR_PTR(-ENAMETOOLONG);

        str = kmalloc(len + 1, GFP_KERNEL);
        if (unlikely(!str))
                return ERR_PTR(-ENOMEM);

        memcpy(str, *bufp, len);
        str[len] = 0;
        *bufp += len;
        *remain -= len;

        return str;
}

/* Translate an inode field to kernel representation. */
static inline int audit_to_inode(struct audit_krule *krule,
                                 struct audit_field *f)
{
        if ((krule->listnr != AUDIT_FILTER_EXIT &&
             krule->listnr != AUDIT_FILTER_URING_EXIT) ||
            krule->inode_f || krule->watch || krule->tree ||
            (f->op != Audit_equal && f->op != Audit_not_equal))
                return -EINVAL;

        krule->inode_f = f;
        return 0;
}

static __u32 *classes[AUDIT_SYSCALL_CLASSES];

int __init audit_register_class(int class, unsigned *list)
{
        __u32 *p = kcalloc(AUDIT_BITMASK_SIZE, sizeof(__u32), GFP_KERNEL);
        if (!p)
                return -ENOMEM;
        while (*list != ~0U) {
                unsigned n = *list++;
                if (n >= AUDIT_BITMASK_SIZE * 32 - AUDIT_SYSCALL_CLASSES) {
                        kfree(p);
                        return -EINVAL;
                }
                p[AUDIT_WORD(n)] |= AUDIT_BIT(n);
        }
        if (class >= AUDIT_SYSCALL_CLASSES || classes[class]) {
                kfree(p);
                return -EINVAL;
        }
        classes[class] = p;
        return 0;
}

int audit_match_class(int class, unsigned syscall)
{
        if (unlikely(syscall >= AUDIT_BITMASK_SIZE * 32))
                return 0;
        if (unlikely(class >= AUDIT_SYSCALL_CLASSES || !classes[class]))
                return 0;
        return classes[class][AUDIT_WORD(syscall)] & AUDIT_BIT(syscall);
}

#ifdef CONFIG_AUDITSYSCALL
static inline int audit_match_class_bits(int class, u32 *mask)
{
        int i;

        if (classes[class]) {
                for (i = 0; i < AUDIT_BITMASK_SIZE; i++)
                        if (mask[i] & classes[class][i])
                                return 0;
        }
        return 1;
}

static int audit_match_signal(struct audit_entry *entry)
{
        struct audit_field *arch = entry->rule.arch_f;

        if (!arch) {
                /* When arch is unspecified, we must check both masks on biarch
                 * as syscall number alone is ambiguous. */
                return (audit_match_class_bits(AUDIT_CLASS_SIGNAL,
                                               entry->rule.mask) &&
                        audit_match_class_bits(AUDIT_CLASS_SIGNAL_32,
                                               entry->rule.mask));
        }

        switch (audit_classify_arch(arch->val)) {
        case 0: /* native */
                return (audit_match_class_bits(AUDIT_CLASS_SIGNAL,
                                               entry->rule.mask));
        case 1: /* 32bit on biarch */
                return (audit_match_class_bits(AUDIT_CLASS_SIGNAL_32,
                                               entry->rule.mask));
        default:
                return 1;
        }
}
#endif

/* Common user-space to kernel rule translation. */
static inline struct audit_entry *audit_to_entry_common(struct audit_rule_data *rule)
{
        unsigned listnr;
        struct audit_entry *entry;
        int i, err;

        err = -EINVAL;
        listnr = rule->flags & ~AUDIT_FILTER_PREPEND;
        switch (listnr) {
        default:
                goto exit_err;
#ifdef CONFIG_AUDITSYSCALL
        case AUDIT_FILTER_ENTRY:
                pr_err("AUDIT_FILTER_ENTRY is deprecated\n");
                goto exit_err;
        case AUDIT_FILTER_EXIT:
        case AUDIT_FILTER_URING_EXIT:
        case AUDIT_FILTER_TASK:
#endif
        case AUDIT_FILTER_USER:
        case AUDIT_FILTER_EXCLUDE:
        case AUDIT_FILTER_FS:
                ;
        }
        if (unlikely(rule->action == AUDIT_POSSIBLE)) {
                pr_err("AUDIT_POSSIBLE is deprecated\n");
                goto exit_err;
        }
        if (rule->action != AUDIT_NEVER && rule->action != AUDIT_ALWAYS)
                goto exit_err;
        if (rule->field_count > AUDIT_MAX_FIELDS)
                goto exit_err;

        err = -ENOMEM;
        entry = audit_init_entry(rule->field_count);
        if (!entry)
                goto exit_err;

        entry->rule.flags = rule->flags & AUDIT_FILTER_PREPEND;
        entry->rule.listnr = listnr;
        entry->rule.action = rule->action;
        entry->rule.field_count = rule->field_count;

        for (i = 0; i < AUDIT_BITMASK_SIZE; i++)
                entry->rule.mask[i] = rule->mask[i];

        for (i = 0; i < AUDIT_SYSCALL_CLASSES; i++) {
                int bit = AUDIT_BITMASK_SIZE * 32 - i - 1;
                __u32 *p = &entry->rule.mask[AUDIT_WORD(bit)];
                __u32 *class;

                if (!(*p & AUDIT_BIT(bit)))
                        continue;
                *p &= ~AUDIT_BIT(bit);
                class = classes[i];
                if (class) {
                        int j;
                        for (j = 0; j < AUDIT_BITMASK_SIZE; j++)
                                entry->rule.mask[j] |= class[j];
                }
        }

        return entry;

exit_err:
        return ERR_PTR(err);
}

static u32 audit_ops[] =
{
        [Audit_equal] = AUDIT_EQUAL,
        [Audit_not_equal] = AUDIT_NOT_EQUAL,
        [Audit_bitmask] = AUDIT_BIT_MASK,
        [Audit_bittest] = AUDIT_BIT_TEST,
        [Audit_lt] = AUDIT_LESS_THAN,
        [Audit_gt] = AUDIT_GREATER_THAN,
        [Audit_le] = AUDIT_LESS_THAN_OR_EQUAL,
        [Audit_ge] = AUDIT_GREATER_THAN_OR_EQUAL,
};

static u32 audit_to_op(u32 op)
{
        u32 n;
        for (n = Audit_equal; n < Audit_bad && audit_ops[n] != op; n++)
                ;
        return n;
}

/* check if an audit field is valid */
static int audit_field_valid(struct audit_entry *entry, struct audit_field *f)
{
        switch (f->type) {
        case AUDIT_MSGTYPE:
                if (entry->rule.listnr != AUDIT_FILTER_EXCLUDE &&
                    entry->rule.listnr != AUDIT_FILTER_USER)
                        return -EINVAL;
                break;
        case AUDIT_FSTYPE:
                if (entry->rule.listnr != AUDIT_FILTER_FS)
                        return -EINVAL;
                break;
        case AUDIT_PERM:
                if (entry->rule.listnr == AUDIT_FILTER_URING_EXIT)
                        return -EINVAL;
                break;
        }

        switch (entry->rule.listnr) {
        case AUDIT_FILTER_FS:
                switch (f->type) {
                case AUDIT_FSTYPE:
                case AUDIT_FILTERKEY:
                        break;
                default:
                        return -EINVAL;
                }
        }

        /* Check for valid field type and op */
        switch (f->type) {
        case AUDIT_ARG0:
        case AUDIT_ARG1:
        case AUDIT_ARG2:
        case AUDIT_ARG3:
        case AUDIT_PERS: /* <uapi/linux/personality.h> */
        case AUDIT_DEVMINOR:
                /* all ops are valid */
                break;
        case AUDIT_UID:
        case AUDIT_EUID:
        case AUDIT_SUID:
        case AUDIT_FSUID:
        case AUDIT_LOGINUID:
        case AUDIT_OBJ_UID:
        case AUDIT_GID:
        case AUDIT_EGID:
        case AUDIT_SGID:
        case AUDIT_FSGID:
        case AUDIT_OBJ_GID:
        case AUDIT_PID:
        case AUDIT_MSGTYPE:
        case AUDIT_PPID:
        case AUDIT_DEVMAJOR:
        case AUDIT_EXIT:
        case AUDIT_SUCCESS:
        case AUDIT_INODE:
        case AUDIT_SESSIONID:
        case AUDIT_SUBJ_SEN:
        case AUDIT_SUBJ_CLR:
        case AUDIT_OBJ_LEV_LOW:
        case AUDIT_OBJ_LEV_HIGH:
        case AUDIT_SADDR_FAM:
                /* bit ops are only useful on syscall args */
                if (f->op == Audit_bitmask || f->op == Audit_bittest)
                        return -EINVAL;
                break;
        case AUDIT_SUBJ_USER:
        case AUDIT_SUBJ_ROLE:
        case AUDIT_SUBJ_TYPE:
        case AUDIT_OBJ_USER:
        case AUDIT_OBJ_ROLE:
        case AUDIT_OBJ_TYPE:
        case AUDIT_WATCH:
        case AUDIT_DIR:
        case AUDIT_FILTERKEY:
        case AUDIT_LOGINUID_SET:
        case AUDIT_ARCH:
        case AUDIT_FSTYPE:
        case AUDIT_PERM:
        case AUDIT_FILETYPE:
        case AUDIT_FIELD_COMPARE:
        case AUDIT_EXE:
                /* only equal and not equal valid ops */
                if (f->op != Audit_not_equal && f->op != Audit_equal)
                        return -EINVAL;
                break;
        default:
                /* field not recognized */
                return -EINVAL;
        }

        /* Check for select valid field values */
        switch (f->type) {
        case AUDIT_LOGINUID_SET:
                if ((f->val != 0) && (f->val != 1))
                        return -EINVAL;
                break;
        case AUDIT_PERM:
                if (f->val & ~15)
                        return -EINVAL;
                break;
        case AUDIT_FILETYPE:
                if (f->val & ~S_IFMT)
                        return -EINVAL;
                break;
        case AUDIT_FIELD_COMPARE:
                if (f->val > AUDIT_MAX_FIELD_COMPARE)
                        return -EINVAL;
                break;
        case AUDIT_SADDR_FAM:
                if (f->val >= AF_MAX)
                        return -EINVAL;
                break;
        default:
                break;
        }

        return 0;
}

/* Translate struct audit_rule_data to kernel's rule representation. */
static struct audit_entry *audit_data_to_entry(struct audit_rule_data *data,
                                               size_t datasz)
{
        int err = 0;
        struct audit_entry *entry;
        void *bufp;
        size_t remain = datasz - sizeof(struct audit_rule_data);
        int i;
        char *str;
        struct audit_fsnotify_mark *audit_mark;

        entry = audit_to_entry_common(data);
        if (IS_ERR(entry))
                goto exit_nofree;

        bufp = data->buf;
        for (i = 0; i < data->field_count; i++) {
                struct audit_field *f = &entry->rule.fields[i];
                u32 f_val;

                err = -EINVAL;

                f->op = audit_to_op(data->fieldflags[i]);
                if (f->op == Audit_bad)
                        goto exit_free;

                f->type = data->fields[i];
                f_val = data->values[i];

                /* Support legacy tests for a valid loginuid */
                if ((f->type == AUDIT_LOGINUID) && (f_val == AUDIT_UID_UNSET)) {
                        f->type = AUDIT_LOGINUID_SET;
                        f_val = 0;
                        entry->rule.pflags |= AUDIT_LOGINUID_LEGACY;
                }

                err = audit_field_valid(entry, f);
                if (err)
                        goto exit_free;

                err = -EINVAL;
                switch (f->type) {
                case AUDIT_LOGINUID:
                case AUDIT_UID:
                case AUDIT_EUID:
                case AUDIT_SUID:
                case AUDIT_FSUID:
                case AUDIT_OBJ_UID:
                        f->uid = make_kuid(current_user_ns(), f_val);
                        if (!uid_valid(f->uid))
                                goto exit_free;
                        break;
                case AUDIT_GID:
                case AUDIT_EGID:
                case AUDIT_SGID:
                case AUDIT_FSGID:
                case AUDIT_OBJ_GID:
                        f->gid = make_kgid(current_user_ns(), f_val);
                        if (!gid_valid(f->gid))
                                goto exit_free;
                        break;
                case AUDIT_ARCH:
                        f->val = f_val;
                        entry->rule.arch_f = f;
                        break;
                case AUDIT_SUBJ_USER:
                case AUDIT_SUBJ_ROLE:
                case AUDIT_SUBJ_TYPE:
                case AUDIT_SUBJ_SEN:
                case AUDIT_SUBJ_CLR:
                case AUDIT_OBJ_USER:
                case AUDIT_OBJ_ROLE:
                case AUDIT_OBJ_TYPE:
                case AUDIT_OBJ_LEV_LOW:
                case AUDIT_OBJ_LEV_HIGH:
                        str = audit_unpack_string(&bufp, &remain, f_val);
                        if (IS_ERR(str)) {
                                err = PTR_ERR(str);
                                goto exit_free;
                        }
                        entry->rule.buflen += f_val;
                        f->lsm_str = str;
                        err = security_audit_rule_init(f->type, f->op, str,
                                                       (void **)&f->lsm_rule,
                                                       GFP_KERNEL);
                        /* Keep currently invalid fields around in case they
                         * become valid after a policy reload. */
                        if (err == -EINVAL) {
                                pr_warn("audit rule for LSM \'%s\' is invalid\n",
                                        str);
                                err = 0;
                        } else if (err)
                                goto exit_free;
                        break;
                case AUDIT_WATCH:
                        str = audit_unpack_string(&bufp, &remain, f_val);
                        if (IS_ERR(str)) {
                                err = PTR_ERR(str);
                                goto exit_free;
                        }
                        err = audit_to_watch(&entry->rule, str, f_val, f->op);
                        if (err) {
                                kfree(str);
                                goto exit_free;
                        }
                        entry->rule.buflen += f_val;
                        break;
                case AUDIT_DIR:
                        str = audit_unpack_string(&bufp, &remain, f_val);
                        if (IS_ERR(str)) {
                                err = PTR_ERR(str);
                                goto exit_free;
                        }
                        err = audit_make_tree(&entry->rule, str, f->op);
                        kfree(str);
                        if (err)
                                goto exit_free;
                        entry->rule.buflen += f_val;
                        break;
                case AUDIT_INODE:
                        f->val = f_val;
                        err = audit_to_inode(&entry->rule, f);
                        if (err)
                                goto exit_free;
                        break;
                case AUDIT_FILTERKEY:
                        if (entry->rule.filterkey || f_val > AUDIT_MAX_KEY_LEN)
                                goto exit_free;
                        str = audit_unpack_string(&bufp, &remain, f_val);
                        if (IS_ERR(str)) {
                                err = PTR_ERR(str);
                                goto exit_free;
                        }
                        entry->rule.buflen += f_val;
                        entry->rule.filterkey = str;
                        break;
                case AUDIT_EXE:
                        if (entry->rule.exe || f_val > PATH_MAX)
                                goto exit_free;
                        str = audit_unpack_string(&bufp, &remain, f_val);
                        if (IS_ERR(str)) {
                                err = PTR_ERR(str);
                                goto exit_free;
                        }
                        audit_mark = audit_alloc_mark(&entry->rule, str, f_val);
                        if (IS_ERR(audit_mark)) {
                                kfree(str);
                                err = PTR_ERR(audit_mark);
                                goto exit_free;
                        }
                        entry->rule.buflen += f_val;
                        entry->rule.exe = audit_mark;
                        break;
                default:
                        f->val = f_val;
                        break;
                }
        }

        if (entry->rule.inode_f && entry->rule.inode_f->op == Audit_not_equal)
                entry->rule.inode_f = NULL;

exit_nofree:
        return entry;

exit_free:
        if (entry->rule.tree)
                audit_put_tree(entry->rule.tree); /* that's the temporary one */
        if (entry->rule.exe)
                audit_remove_mark(entry->rule.exe); /* that's the template one */
        audit_free_rule(entry);
        return ERR_PTR(err);
}

/* Pack a filter field's string representation into data block. */
static inline size_t audit_pack_string(void **bufp, const char *str)
{
        size_t len = strlen(str);

        memcpy(*bufp, str, len);
        *bufp += len;

        return len;
}

/* Translate kernel rule representation to struct audit_rule_data. */
static struct audit_rule_data *audit_krule_to_data(struct audit_krule *krule)
{
        struct audit_rule_data *data;
        void *bufp;
        int i;

        data = kmalloc(struct_size(data, buf, krule->buflen), GFP_KERNEL);
        if (unlikely(!data))
                return NULL;
        memset(data, 0, sizeof(*data));

        data->flags = krule->flags | krule->listnr;
        data->action = krule->action;
        data->field_count = krule->field_count;
        bufp = data->buf;
        for (i = 0; i < data->field_count; i++) {
                struct audit_field *f = &krule->fields[i];

                data->fields[i] = f->type;
                data->fieldflags[i] = audit_ops[f->op];
                switch (f->type) {
                case AUDIT_SUBJ_USER:
                case AUDIT_SUBJ_ROLE:
                case AUDIT_SUBJ_TYPE:
                case AUDIT_SUBJ_SEN:
                case AUDIT_SUBJ_CLR:
                case AUDIT_OBJ_USER:
                case AUDIT_OBJ_ROLE:
                case AUDIT_OBJ_TYPE:
                case AUDIT_OBJ_LEV_LOW:
                case AUDIT_OBJ_LEV_HIGH:
                        data->buflen += data->values[i] =
                                audit_pack_string(&bufp, f->lsm_str);
                        break;
                case AUDIT_WATCH:
                        data->buflen += data->values[i] =
                                audit_pack_string(&bufp,
                                                  audit_watch_path(krule->watch));
                        break;
                case AUDIT_DIR:
                        data->buflen += data->values[i] =
                                audit_pack_string(&bufp,
                                                  audit_tree_path(krule->tree));
                        break;
                case AUDIT_FILTERKEY:
                        data->buflen += data->values[i] =
                                audit_pack_string(&bufp, krule->filterkey);
                        break;
                case AUDIT_EXE:
                        data->buflen += data->values[i] =
                                audit_pack_string(&bufp, audit_mark_path(krule->exe));
                        break;
                case AUDIT_LOGINUID_SET:
                        if (krule->pflags & AUDIT_LOGINUID_LEGACY && !f->val) {
                                data->fields[i] = AUDIT_LOGINUID;
                                data->values[i] = AUDIT_UID_UNSET;
                                break;
                        }
                        fallthrough;        /* if set */
                default:
                        data->values[i] = f->val;
                }
        }
        for (i = 0; i < AUDIT_BITMASK_SIZE; i++)
                data->mask[i] = krule->mask[i];

        return data;
}

/* Compare two rules in kernel format.  Considered success if rules
 * don't match. */
static int audit_compare_rule(struct audit_krule *a, struct audit_krule *b)
{
        int i;

        if (a->flags != b->flags ||
            a->pflags != b->pflags ||
            a->listnr != b->listnr ||
            a->action != b->action ||
            a->field_count != b->field_count)
                return 1;

        for (i = 0; i < a->field_count; i++) {
                if (a->fields[i].type != b->fields[i].type ||
                    a->fields[i].op != b->fields[i].op)
                        return 1;

                switch (a->fields[i].type) {
                case AUDIT_SUBJ_USER:
                case AUDIT_SUBJ_ROLE:
                case AUDIT_SUBJ_TYPE:
                case AUDIT_SUBJ_SEN:
                case AUDIT_SUBJ_CLR:
                case AUDIT_OBJ_USER:
                case AUDIT_OBJ_ROLE:
                case AUDIT_OBJ_TYPE:
                case AUDIT_OBJ_LEV_LOW:
                case AUDIT_OBJ_LEV_HIGH:
                        if (strcmp(a->fields[i].lsm_str, b->fields[i].lsm_str))
                                return 1;
                        break;
                case AUDIT_WATCH:
                        if (strcmp(audit_watch_path(a->watch),
                                   audit_watch_path(b->watch)))
                                return 1;
                        break;
                case AUDIT_DIR:
                        if (strcmp(audit_tree_path(a->tree),
                                   audit_tree_path(b->tree)))
                                return 1;
                        break;
                case AUDIT_FILTERKEY:
                        /* both filterkeys exist based on above type compare */
                        if (strcmp(a->filterkey, b->filterkey))
                                return 1;
                        break;
                case AUDIT_EXE:
                        /* both paths exist based on above type compare */
                        if (strcmp(audit_mark_path(a->exe),
                                   audit_mark_path(b->exe)))
                                return 1;
                        break;
                case AUDIT_UID:
                case AUDIT_EUID:
                case AUDIT_SUID:
                case AUDIT_FSUID:
                case AUDIT_LOGINUID:
                case AUDIT_OBJ_UID:
                        if (!uid_eq(a->fields[i].uid, b->fields[i].uid))
                                return 1;
                        break;
                case AUDIT_GID:
                case AUDIT_EGID:
                case AUDIT_SGID:
                case AUDIT_FSGID:
                case AUDIT_OBJ_GID:
                        if (!gid_eq(a->fields[i].gid, b->fields[i].gid))
                                return 1;
                        break;
                default:
                        if (a->fields[i].val != b->fields[i].val)
                                return 1;
                }
        }

        for (i = 0; i < AUDIT_BITMASK_SIZE; i++)
                if (a->mask[i] != b->mask[i])
                        return 1;

        return 0;
}

/* Duplicate LSM field information.  The lsm_rule is opaque, so must be
 * re-initialized. */
static inline int audit_dupe_lsm_field(struct audit_field *df,
                                           struct audit_field *sf)
{
        int ret;
        char *lsm_str;

        /* our own copy of lsm_str */
        lsm_str = kstrdup(sf->lsm_str, GFP_KERNEL);
        if (unlikely(!lsm_str))
                return -ENOMEM;
        df->lsm_str = lsm_str;

        /* our own (refreshed) copy of lsm_rule */
        ret = security_audit_rule_init(df->type, df->op, df->lsm_str,
                                       (void **)&df->lsm_rule, GFP_KERNEL);
        /* Keep currently invalid fields around in case they
         * become valid after a policy reload. */
        if (ret == -EINVAL) {
                pr_warn("audit rule for LSM \'%s\' is invalid\n",
                        df->lsm_str);
                ret = 0;
        }

        return ret;
}

/* Duplicate an audit rule.  This will be a deep copy with the exception
 * of the watch - that pointer is carried over.  The LSM specific fields
 * will be updated in the copy.  The point is to be able to replace the old
 * rule with the new rule in the filterlist, then free the old rule.
 * The rlist element is undefined; list manipulations are handled apart from
 * the initial copy. */
struct audit_entry *audit_dupe_rule(struct audit_krule *old)
{
        u32 fcount = old->field_count;
        struct audit_entry *entry;
        struct audit_krule *new;
        char *fk;
        int i, err = 0;

        entry = audit_init_entry(fcount);
        if (unlikely(!entry))
                return ERR_PTR(-ENOMEM);

        new = &entry->rule;
        new->flags = old->flags;
        new->pflags = old->pflags;
        new->listnr = old->listnr;
        new->action = old->action;
        for (i = 0; i < AUDIT_BITMASK_SIZE; i++)
                new->mask[i] = old->mask[i];
        new->prio = old->prio;
        new->buflen = old->buflen;
        new->inode_f = old->inode_f;
        new->field_count = old->field_count;

        /*
         * note that we are OK with not refcounting here; audit_match_tree()
         * never dereferences tree and we can't get false positives there
         * since we'd have to have rule gone from the list *and* removed
         * before the chunks found by lookup had been allocated, i.e. before
         * the beginning of list scan.
         */
        new->tree = old->tree;
        memcpy(new->fields, old->fields, sizeof(struct audit_field) * fcount);

        /* deep copy this information, updating the lsm_rule fields, because
         * the originals will all be freed when the old rule is freed. */
        for (i = 0; i < fcount; i++) {
                switch (new->fields[i].type) {
                case AUDIT_SUBJ_USER:
                case AUDIT_SUBJ_ROLE:
                case AUDIT_SUBJ_TYPE:
                case AUDIT_SUBJ_SEN:
                case AUDIT_SUBJ_CLR:
                case AUDIT_OBJ_USER:
                case AUDIT_OBJ_ROLE:
                case AUDIT_OBJ_TYPE:
                case AUDIT_OBJ_LEV_LOW:
                case AUDIT_OBJ_LEV_HIGH:
                        err = audit_dupe_lsm_field(&new->fields[i],
                                                       &old->fields[i]);
                        break;
                case AUDIT_FILTERKEY:
                        fk = kstrdup(old->filterkey, GFP_KERNEL);
                        if (unlikely(!fk))
                                err = -ENOMEM;
                        else
                                new->filterkey = fk;
                        break;
                case AUDIT_EXE:
                        err = audit_dupe_exe(new, old);
                        break;
                }
                if (err) {
                        if (new->exe)
                                audit_remove_mark(new->exe);
                        audit_free_rule(entry);
                        return ERR_PTR(err);
                }
        }

        if (old->watch) {
                audit_get_watch(old->watch);
                new->watch = old->watch;
        }

        return entry;
}

/* Find an existing audit rule.
 * Caller must hold audit_filter_mutex to prevent stale rule data. */
static struct audit_entry *audit_find_rule(struct audit_entry *entry,
                                           struct list_head **p)
{
        struct audit_entry *e, *found = NULL;
        struct list_head *list;
        int h;

        if (entry->rule.inode_f) {
                h = audit_hash_ino(entry->rule.inode_f->val);
                *p = list = &audit_inode_hash[h];
        } else if (entry->rule.watch) {
                /* we don't know the inode number, so must walk entire hash */
                for (h = 0; h < AUDIT_INODE_BUCKETS; h++) {
                        list = &audit_inode_hash[h];
                        list_for_each_entry(e, list, list)
                                if (!audit_compare_rule(&entry->rule, &e->rule)) {
                                        found = e;
                                        goto out;
                                }
                }
                goto out;
        } else {
                *p = list = &audit_filter_list[entry->rule.listnr];
        }

        list_for_each_entry(e, list, list)
                if (!audit_compare_rule(&entry->rule, &e->rule)) {
                        found = e;
                        goto out;
                }

out:
        return found;
}

static u64 prio_low = ~0ULL/2;
static u64 prio_high = ~0ULL/2 - 1;

/* Add rule to given filterlist if not a duplicate. */
static inline int audit_add_rule(struct audit_entry *entry)
{
        struct audit_entry *e;
        struct audit_watch *watch = entry->rule.watch;
        struct audit_tree *tree = entry->rule.tree;
        struct list_head *list;
        int err = 0;
#ifdef CONFIG_AUDITSYSCALL
        int dont_count = 0;

        /* If any of these, don't count towards total */
        switch (entry->rule.listnr) {
        case AUDIT_FILTER_USER:
        case AUDIT_FILTER_EXCLUDE:
        case AUDIT_FILTER_FS:
                dont_count = 1;
        }
#endif

        mutex_lock(&audit_filter_mutex);
        e = audit_find_rule(entry, &list);
        if (e) {
                mutex_unlock(&audit_filter_mutex);
                err = -EEXIST;
                /* normally audit_add_tree_rule() will free it on failure */
                if (tree)
                        audit_put_tree(tree);
                return err;
        }

        if (watch) {
                /* audit_filter_mutex is dropped and re-taken during this call */
                err = audit_add_watch(&entry->rule, &list);
                if (err) {
                        mutex_unlock(&audit_filter_mutex);
                        /*
                         * normally audit_add_tree_rule() will free it
                         * on failure
                         */
                        if (tree)
                                audit_put_tree(tree);
                        return err;
                }
        }
        if (tree) {
                err = audit_add_tree_rule(&entry->rule);
                if (err) {
                        mutex_unlock(&audit_filter_mutex);
                        return err;
                }
        }

        entry->rule.prio = ~0ULL;
        if (entry->rule.listnr == AUDIT_FILTER_EXIT ||
            entry->rule.listnr == AUDIT_FILTER_URING_EXIT) {
                if (entry->rule.flags & AUDIT_FILTER_PREPEND)
                        entry->rule.prio = ++prio_high;
                else
                        entry->rule.prio = --prio_low;
        }

        if (entry->rule.flags & AUDIT_FILTER_PREPEND) {
                list_add(&entry->rule.list,
                         &audit_rules_list[entry->rule.listnr]);
                list_add_rcu(&entry->list, list);
                entry->rule.flags &= ~AUDIT_FILTER_PREPEND;
        } else {
                list_add_tail(&entry->rule.list,
                              &audit_rules_list[entry->rule.listnr]);
                list_add_tail_rcu(&entry->list, list);
        }
#ifdef CONFIG_AUDITSYSCALL
        if (!dont_count)
                audit_n_rules++;

        if (!audit_match_signal(entry))
                audit_signals++;
#endif
        mutex_unlock(&audit_filter_mutex);

        return err;
}

/* Remove an existing rule from filterlist. */
int audit_del_rule(struct audit_entry *entry)
{
        struct audit_entry  *e;
        struct audit_tree *tree = entry->rule.tree;
        struct list_head *list;
        int ret = 0;
#ifdef CONFIG_AUDITSYSCALL
        int dont_count = 0;

        /* If any of these, don't count towards total */
        switch (entry->rule.listnr) {
        case AUDIT_FILTER_USER:
        case AUDIT_FILTER_EXCLUDE:
        case AUDIT_FILTER_FS:
                dont_count = 1;
        }
#endif

        mutex_lock(&audit_filter_mutex);
        e = audit_find_rule(entry, &list);
        if (!e) {
                ret = -ENOENT;
                goto out;
        }

        if (e->rule.watch)
                audit_remove_watch_rule(&e->rule);

        if (e->rule.tree)
                audit_remove_tree_rule(&e->rule);

        if (e->rule.exe)
                audit_remove_mark_rule(&e->rule);

#ifdef CONFIG_AUDITSYSCALL
        if (!dont_count)
                audit_n_rules--;

        if (!audit_match_signal(entry))
                audit_signals--;
#endif

        list_del_rcu(&e->list);
        list_del(&e->rule.list);
        call_rcu(&e->rcu, audit_free_rule_rcu);

out:
        mutex_unlock(&audit_filter_mutex);

        if (tree)
                audit_put_tree(tree);        /* that's the temporary one */

        return ret;
}

/* List rules using struct audit_rule_data. */
static void audit_list_rules(int seq, struct sk_buff_head *q)
{
        struct sk_buff *skb;
        struct audit_krule *r;
        int i;

        /* This is a blocking read, so use audit_filter_mutex instead of rcu
         * iterator to sync with list writers. */
        for (i = 0; i < AUDIT_NR_FILTERS; i++) {
                list_for_each_entry(r, &audit_rules_list[i], list) {
                        struct audit_rule_data *data;

                        data = audit_krule_to_data(r);
                        if (unlikely(!data))
                                break;
                        skb = audit_make_reply(seq, AUDIT_LIST_RULES, 0, 1,
                                               data,
                                               struct_size(data, buf, data->buflen));
                        if (skb)
                                skb_queue_tail(q, skb);
                        kfree(data);
                }
        }
        skb = audit_make_reply(seq, AUDIT_LIST_RULES, 1, 1, NULL, 0);
        if (skb)
                skb_queue_tail(q, skb);
}

/* Log rule additions and removals */
static void audit_log_rule_change(char *action, struct audit_krule *rule, int res)
{
        struct audit_buffer *ab;

        if (!audit_enabled)
                return;

        ab = audit_log_start(audit_context(), GFP_KERNEL, AUDIT_CONFIG_CHANGE);
        if (!ab)
                return;
        audit_log_session_info(ab);
        audit_log_task_context(ab);
        audit_log_format(ab, " op=%s", action);
        audit_log_key(ab, rule->filterkey);
        audit_log_format(ab, " list=%d res=%d", rule->listnr, res);
        audit_log_end(ab);
}

/**
 * audit_rule_change - apply all rules to the specified message type
 * @type: audit message type
 * @seq: netlink audit message sequence (serial) number
 * @data: payload data
 * @datasz: size of payload data
 */
int audit_rule_change(int type, int seq, void *data, size_t datasz)
{
        int err = 0;
        struct audit_entry *entry;

        switch (type) {
        case AUDIT_ADD_RULE:
                entry = audit_data_to_entry(data, datasz);
                if (IS_ERR(entry))
                        return PTR_ERR(entry);
                err = audit_add_rule(entry);
                audit_log_rule_change("add_rule", &entry->rule, !err);
                break;
        case AUDIT_DEL_RULE:
                entry = audit_data_to_entry(data, datasz);
                if (IS_ERR(entry))
                        return PTR_ERR(entry);
                err = audit_del_rule(entry);
                audit_log_rule_change("remove_rule", &entry->rule, !err);
                break;
        default:
                WARN_ON(1);
                return -EINVAL;
        }

        if (err || type == AUDIT_DEL_RULE) {
                if (entry->rule.exe)
                        audit_remove_mark(entry->rule.exe);
                audit_free_rule(entry);
        }

        return err;
}

/**
 * audit_list_rules_send - list the audit rules
 * @request_skb: skb of request we are replying to (used to target the reply)
 * @seq: netlink audit message sequence (serial) number
 */
int audit_list_rules_send(struct sk_buff *request_skb, int seq)
{
        struct task_struct *tsk;
        struct audit_netlink_list *dest;

        /* We can't just spew out the rules here because we might fill
         * the available socket buffer space and deadlock waiting for
         * auditctl to read from it... which isn't ever going to
         * happen if we're actually running in the context of auditctl
         * trying to _send_ the stuff */

        dest = kmalloc(sizeof(*dest), GFP_KERNEL);
        if (!dest)
                return -ENOMEM;
        dest->net = get_net(sock_net(NETLINK_CB(request_skb).sk));
        dest->portid = NETLINK_CB(request_skb).portid;
        skb_queue_head_init(&dest->q);

        mutex_lock(&audit_filter_mutex);
        audit_list_rules(seq, &dest->q);
        mutex_unlock(&audit_filter_mutex);

        tsk = kthread_run(audit_send_list_thread, dest, "audit_send_list");
        if (IS_ERR(tsk)) {
                skb_queue_purge(&dest->q);
                put_net(dest->net);
                kfree(dest);
                return PTR_ERR(tsk);
        }

        return 0;
}

int audit_comparator(u32 left, u32 op, u32 right)
{
        switch (op) {
        case Audit_equal:
                return (left == right);
        case Audit_not_equal:
                return (left != right);
        case Audit_lt:
                return (left < right);
        case Audit_le:
                return (left <= right);
        case Audit_gt:
                return (left > right);
        case Audit_ge:
                return (left >= right);
        case Audit_bitmask:
                return (left & right);
        case Audit_bittest:
                return ((left & right) == right);
        default:
                return 0;
        }
}

int audit_uid_comparator(kuid_t left, u32 op, kuid_t right)
{
        switch (op) {
        case Audit_equal:
                return uid_eq(left, right);
        case Audit_not_equal:
                return !uid_eq(left, right);
        case Audit_lt:
                return uid_lt(left, right);
        case Audit_le:
                return uid_lte(left, right);
        case Audit_gt:
                return uid_gt(left, right);
        case Audit_ge:
                return uid_gte(left, right);
        case Audit_bitmask:
        case Audit_bittest:
        default:
                return 0;
        }
}

int audit_gid_comparator(kgid_t left, u32 op, kgid_t right)
{
        switch (op) {
        case Audit_equal:
                return gid_eq(left, right);
        case Audit_not_equal:
                return !gid_eq(left, right);
        case Audit_lt:
                return gid_lt(left, right);
        case Audit_le:
                return gid_lte(left, right);
        case Audit_gt:
                return gid_gt(left, right);
        case Audit_ge:
                return gid_gte(left, right);
        case Audit_bitmask:
        case Audit_bittest:
        default:
                return 0;
        }
}

/**
 * parent_len - find the length of the parent portion of a pathname
 * @path: pathname of which to determine length
 */
int parent_len(const char *path)
{
        int plen;
        const char *p;

        plen = strlen(path);

        if (plen == 0)
                return plen;

        /* disregard trailing slashes */
        p = path + plen - 1;
        while ((*p == '/') && (p > path))
                p--;

        /* walk backward until we find the next slash or hit beginning */
        while ((*p != '/') && (p > path))
                p--;

        /* did we find a slash? Then increment to include it in path */
        if (*p == '/')
                p++;

        return p - path;
}

/**
 * audit_compare_dname_path - compare given dentry name with last component in
 *                               given path. Return of 0 indicates a match.
 * @dname:        dentry name that we're comparing
 * @path:        full pathname that we're comparing
 * @parentlen:        length of the parent if known. Passing in AUDIT_NAME_FULL
 *                 here indicates that we must compute this value.
 */
int audit_compare_dname_path(const struct qstr *dname, const char *path, int parentlen)
{
        int dlen, pathlen;
        const char *p;

        dlen = dname->len;
        pathlen = strlen(path);
        if (pathlen < dlen)
                return 1;

        parentlen = parentlen == AUDIT_NAME_FULL ? parent_len(path) : parentlen;
        if (pathlen - parentlen != dlen)
                return 1;

        p = path + parentlen;

        return strncmp(p, dname->name, dlen);
}

int audit_filter(int msgtype, unsigned int listtype)
{
        struct audit_entry *e;
        int ret = 1; /* Audit by default */

        rcu_read_lock();
        list_for_each_entry_rcu(e, &audit_filter_list[listtype], list) {
                int i, result = 0;

                for (i = 0; i < e->rule.field_count; i++) {
                        struct audit_field *f = &e->rule.fields[i];
                        struct lsm_prop prop = { };
                        pid_t pid;

                        switch (f->type) {
                        case AUDIT_PID:
                                pid = task_tgid_nr(current);
                                result = audit_comparator(pid, f->op, f->val);
                                break;
                        case AUDIT_UID:
                                result = audit_uid_comparator(current_uid(), f->op, f->uid);
                                break;
                        case AUDIT_GID:
                                result = audit_gid_comparator(current_gid(), f->op, f->gid);
                                break;
                        case AUDIT_LOGINUID:
                                result = audit_uid_comparator(audit_get_loginuid(current),
                                                              f->op, f->uid);
                                break;
                        case AUDIT_LOGINUID_SET:
                                result = audit_comparator(audit_loginuid_set(current),
                                                          f->op, f->val);
                                break;
                        case AUDIT_MSGTYPE:
                                result = audit_comparator(msgtype, f->op, f->val);
                                break;
                        case AUDIT_SUBJ_USER:
                        case AUDIT_SUBJ_ROLE:
                        case AUDIT_SUBJ_TYPE:
                        case AUDIT_SUBJ_SEN:
                        case AUDIT_SUBJ_CLR:
                                if (f->lsm_rule) {
                                        security_current_getlsmprop_subj(&prop);
                                        result = security_audit_rule_match(
                                                   &prop, f->type, f->op,
                                                   f->lsm_rule);
                                }
                                break;
                        case AUDIT_EXE:
                                result = audit_exe_compare(current, e->rule.exe);
                                if (f->op == Audit_not_equal)
                                        result = !result;
                                break;
                        default:
                                goto unlock_and_return;
                        }
                        if (result < 0) /* error */
                                goto unlock_and_return;
                        if (!result)
                                break;
                }
                if (result > 0) {
                        if (e->rule.action == AUDIT_NEVER || listtype == AUDIT_FILTER_EXCLUDE)
                                ret = 0;
                        break;
                }
        }
unlock_and_return:
        rcu_read_unlock();
        return ret;
}

static int update_lsm_rule(struct audit_krule *r)
{
        struct audit_entry *entry = container_of(r, struct audit_entry, rule);
        struct audit_entry *nentry;
        int err = 0;

        if (!security_audit_rule_known(r))
                return 0;

        nentry = audit_dupe_rule(r);
        if (entry->rule.exe)
                audit_remove_mark(entry->rule.exe);
        if (IS_ERR(nentry)) {
                /* save the first error encountered for the
                 * return value */
                err = PTR_ERR(nentry);
                audit_panic("error updating LSM filters");
                if (r->watch)
                        list_del(&r->rlist);
                list_del_rcu(&entry->list);
                list_del(&r->list);
        } else {
                if (r->watch || r->tree)
                        list_replace_init(&r->rlist, &nentry->rule.rlist);
                list_replace_rcu(&entry->list, &nentry->list);
                list_replace(&r->list, &nentry->rule.list);
        }
        call_rcu(&entry->rcu, audit_free_rule_rcu);

        return err;
}

/* This function will re-initialize the lsm_rule field of all applicable rules.
 * It will traverse the filter lists serarching for rules that contain LSM
 * specific filter fields.  When such a rule is found, it is copied, the
 * LSM field is re-initialized, and the old rule is replaced with the
 * updated rule. */
int audit_update_lsm_rules(void)
{
        struct audit_krule *r, *n;
        int i, err = 0;

        /* audit_filter_mutex synchronizes the writers */
        mutex_lock(&audit_filter_mutex);

        for (i = 0; i < AUDIT_NR_FILTERS; i++) {
                list_for_each_entry_safe(r, n, &audit_rules_list[i], list) {
                        int res = update_lsm_rule(r);
                        if (!err)
                                err = res;
                }
        }
        mutex_unlock(&audit_filter_mutex);

        return err;
}


























  134 









  133 



   99 


































  134 






1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
/* SPDX-License-Identifier: GPL-2.0-only */
/*
 * (C) 2008 Krzysztof Piotr Oledzki <ole@ans.pl>
 */

#ifndef _NF_CONNTRACK_ACCT_H
#define _NF_CONNTRACK_ACCT_H
#include <net/net_namespace.h>
#include <linux/netfilter/nf_conntrack_common.h>
#include <linux/netfilter/nf_conntrack_tuple_common.h>
#include <net/netfilter/nf_conntrack.h>
#include <net/netfilter/nf_conntrack_extend.h>

struct nf_conn_counter {
        atomic64_t packets;
        atomic64_t bytes;
};

struct nf_conn_acct {
        struct nf_conn_counter counter[IP_CT_DIR_MAX];
};

static inline
struct nf_conn_acct *nf_conn_acct_find(const struct nf_conn *ct)
{
        return nf_ct_ext_find(ct, NF_CT_EXT_ACCT);
}

static inline
struct nf_conn_acct *nf_ct_acct_ext_add(struct nf_conn *ct, gfp_t gfp)
{
#if IS_ENABLED(CONFIG_NF_CONNTRACK)
        struct net *net = nf_ct_net(ct);
        struct nf_conn_acct *acct;

        if (!net->ct.sysctl_acct)
                return NULL;

        acct = nf_ct_ext_add(ct, NF_CT_EXT_ACCT, gfp);
        if (!acct)
                pr_debug("failed to add accounting extension area");


        return acct;
#else
        return NULL;
#endif
}

/* Check if connection tracking accounting is enabled */
static inline bool nf_ct_acct_enabled(struct net *net)
{
#if IS_ENABLED(CONFIG_NF_CONNTRACK)
        return net->ct.sysctl_acct != 0;
#else
        return false;
#endif
}

/* Enable/disable connection tracking accounting */
static inline void nf_ct_set_acct(struct net *net, bool enable)
{
#if IS_ENABLED(CONFIG_NF_CONNTRACK)
        net->ct.sysctl_acct = enable;
#endif
}

void nf_ct_acct_add(struct nf_conn *ct, u32 dir, unsigned int packets,
                    unsigned int bytes);

static inline void nf_ct_acct_update(struct nf_conn *ct, u32 dir,
                                     unsigned int bytes)
{
#if IS_ENABLED(CONFIG_NF_CONNTRACK)
        nf_ct_acct_add(ct, dir, 1, bytes);
#endif
}

void nf_conntrack_acct_pernet_init(struct net *net);

#endif /* _NF_CONNTRACK_ACCT_H */























































































































    7 






    7 





    7 















































































































































































































































































































































































































































































































































































































  967 
  963 


  956 




















































  890 



  885 































    6 







    6 

    6 





  626 













  266 

    1 










  266 














  265 





  266 
























  267 






   81 


   81 




































































   55 
















   57 









































































































































   54 





   57 



















































   57 

























































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
// SPDX-License-Identifier: GPL-2.0
/*
 *  Copyright (C) 1995  Linus Torvalds
 *  Copyright (C) 2001, 2002 Andi Kleen, SuSE Labs.
 *  Copyright (C) 2008-2009, Red Hat Inc., Ingo Molnar
 */
#include <linux/sched.h>                /* test_thread_flag(), ...        */
#include <linux/sched/task_stack.h>        /* task_stack_*(), ...                */
#include <linux/kdebug.h>                /* oops_begin/end, ...                */
#include <linux/extable.h>                /* search_exception_tables        */
#include <linux/memblock.h>                /* max_low_pfn                        */
#include <linux/kfence.h>                /* kfence_handle_page_fault        */
#include <linux/kprobes.h>                /* NOKPROBE_SYMBOL, ...                */
#include <linux/mmiotrace.h>                /* kmmio_handler, ...                */
#include <linux/perf_event.h>                /* perf_sw_event                */
#include <linux/hugetlb.h>                /* hstate_index_to_shift        */
#include <linux/prefetch.h>                /* prefetchw                        */
#include <linux/context_tracking.h>        /* exception_enter(), ...        */
#include <linux/uaccess.h>                /* faulthandler_disabled()        */
#include <linux/efi.h>                        /* efi_crash_gracefully_on_page_fault()*/
#include <linux/mm_types.h>
#include <linux/mm.h>                        /* find_and_lock_vma() */
#include <linux/vmalloc.h>

#include <asm/cpufeature.h>                /* boot_cpu_has, ...                */
#include <asm/traps.h>                        /* dotraplinkage, ...                */
#include <asm/fixmap.h>                        /* VSYSCALL_ADDR                */
#include <asm/vsyscall.h>                /* emulate_vsyscall                */
#include <asm/vm86.h>                        /* struct vm86                        */
#include <asm/mmu_context.h>                /* vma_pkey()                        */
#include <asm/efi.h>                        /* efi_crash_gracefully_on_page_fault()*/
#include <asm/desc.h>                        /* store_idt(), ...                */
#include <asm/cpu_entry_area.h>                /* exception stack                */
#include <asm/pgtable_areas.h>                /* VMALLOC_START, ...                */
#include <asm/kvm_para.h>                /* kvm_handle_async_pf                */
#include <asm/vdso.h>                        /* fixup_vdso_exception()        */
#include <asm/irq_stack.h>
#include <asm/fred.h>
#include <asm/sev.h>                        /* snp_dump_hva_rmpentry()        */

#define CREATE_TRACE_POINTS
#include <asm/trace/exceptions.h>

/*
 * Returns 0 if mmiotrace is disabled, or if the fault is not
 * handled by mmiotrace:
 */
static nokprobe_inline int
kmmio_fault(struct pt_regs *regs, unsigned long addr)
{
        if (unlikely(is_kmmio_active()))
                if (kmmio_handler(regs, addr) == 1)
                        return -1;
        return 0;
}

/*
 * Prefetch quirks:
 *
 * 32-bit mode:
 *
 *   Sometimes AMD Athlon/Opteron CPUs report invalid exceptions on prefetch.
 *   Check that here and ignore it.  This is AMD erratum #91.
 *
 * 64-bit mode:
 *
 *   Sometimes the CPU reports invalid exceptions on prefetch.
 *   Check that here and ignore it.
 *
 * Opcode checker based on code by Richard Brunner.
 */
static inline int
check_prefetch_opcode(struct pt_regs *regs, unsigned char *instr,
                      unsigned char opcode, int *prefetch)
{
        unsigned char instr_hi = opcode & 0xf0;
        unsigned char instr_lo = opcode & 0x0f;

        switch (instr_hi) {
        case 0x20:
        case 0x30:
                /*
                 * Values 0x26,0x2E,0x36,0x3E are valid x86 prefixes.
                 * In X86_64 long mode, the CPU will signal invalid
                 * opcode if some of these prefixes are present so
                 * X86_64 will never get here anyway
                 */
                return ((instr_lo & 7) == 0x6);
#ifdef CONFIG_X86_64
        case 0x40:
                /*
                 * In 64-bit mode 0x40..0x4F are valid REX prefixes
                 */
                return (!user_mode(regs) || user_64bit_mode(regs));
#endif
        case 0x60:
                /* 0x64 thru 0x67 are valid prefixes in all modes. */
                return (instr_lo & 0xC) == 0x4;
        case 0xF0:
                /* 0xF0, 0xF2, 0xF3 are valid prefixes in all modes. */
                return !instr_lo || (instr_lo>>1) == 1;
        case 0x00:
                /* Prefetch instruction is 0x0F0D or 0x0F18 */
                if (get_kernel_nofault(opcode, instr))
                        return 0;

                *prefetch = (instr_lo == 0xF) &&
                        (opcode == 0x0D || opcode == 0x18);
                return 0;
        default:
                return 0;
        }
}

static bool is_amd_k8_pre_npt(void)
{
        struct cpuinfo_x86 *c = &boot_cpu_data;

        return unlikely(IS_ENABLED(CONFIG_CPU_SUP_AMD) &&
                        c->x86_vendor == X86_VENDOR_AMD &&
                        c->x86 == 0xf && c->x86_model < 0x40);
}

static int
is_prefetch(struct pt_regs *regs, unsigned long error_code, unsigned long addr)
{
        unsigned char *max_instr;
        unsigned char *instr;
        int prefetch = 0;

        /* Erratum #91 affects AMD K8, pre-NPT CPUs */
        if (!is_amd_k8_pre_npt())
                return 0;

        /*
         * If it was a exec (instruction fetch) fault on NX page, then
         * do not ignore the fault:
         */
        if (error_code & X86_PF_INSTR)
                return 0;

        instr = (void *)convert_ip_to_linear(current, regs);
        max_instr = instr + 15;

        /*
         * This code has historically always bailed out if IP points to a
         * not-present page (e.g. due to a race).  No one has ever
         * complained about this.
         */
        pagefault_disable();

        while (instr < max_instr) {
                unsigned char opcode;

                if (user_mode(regs)) {
                        if (get_user(opcode, (unsigned char __user *) instr))
                                break;
                } else {
                        if (get_kernel_nofault(opcode, instr))
                                break;
                }

                instr++;

                if (!check_prefetch_opcode(regs, instr, opcode, &prefetch))
                        break;
        }

        pagefault_enable();
        return prefetch;
}

DEFINE_SPINLOCK(pgd_lock);
LIST_HEAD(pgd_list);

#ifdef CONFIG_X86_32
static inline pmd_t *vmalloc_sync_one(pgd_t *pgd, unsigned long address)
{
        unsigned index = pgd_index(address);
        pgd_t *pgd_k;
        p4d_t *p4d, *p4d_k;
        pud_t *pud, *pud_k;
        pmd_t *pmd, *pmd_k;

        pgd += index;
        pgd_k = init_mm.pgd + index;

        if (!pgd_present(*pgd_k))
                return NULL;

        /*
         * set_pgd(pgd, *pgd_k); here would be useless on PAE
         * and redundant with the set_pmd() on non-PAE. As would
         * set_p4d/set_pud.
         */
        p4d = p4d_offset(pgd, address);
        p4d_k = p4d_offset(pgd_k, address);
        if (!p4d_present(*p4d_k))
                return NULL;

        pud = pud_offset(p4d, address);
        pud_k = pud_offset(p4d_k, address);
        if (!pud_present(*pud_k))
                return NULL;

        pmd = pmd_offset(pud, address);
        pmd_k = pmd_offset(pud_k, address);

        if (pmd_present(*pmd) != pmd_present(*pmd_k))
                set_pmd(pmd, *pmd_k);

        if (!pmd_present(*pmd_k))
                return NULL;
        else
                BUG_ON(pmd_pfn(*pmd) != pmd_pfn(*pmd_k));

        return pmd_k;
}

/*
 *   Handle a fault on the vmalloc or module mapping area
 *
 *   This is needed because there is a race condition between the time
 *   when the vmalloc mapping code updates the PMD to the point in time
 *   where it synchronizes this update with the other page-tables in the
 *   system.
 *
 *   In this race window another thread/CPU can map an area on the same
 *   PMD, finds it already present and does not synchronize it with the
 *   rest of the system yet. As a result v[mz]alloc might return areas
 *   which are not mapped in every page-table in the system, causing an
 *   unhandled page-fault when they are accessed.
 */
static noinline int vmalloc_fault(unsigned long address)
{
        unsigned long pgd_paddr;
        pmd_t *pmd_k;
        pte_t *pte_k;

        /* Make sure we are in vmalloc area: */
        if (!(address >= VMALLOC_START && address < VMALLOC_END))
                return -1;

        /*
         * Synchronize this task's top level page-table
         * with the 'reference' page table.
         *
         * Do _not_ use "current" here. We might be inside
         * an interrupt in the middle of a task switch..
         */
        pgd_paddr = read_cr3_pa();
        pmd_k = vmalloc_sync_one(__va(pgd_paddr), address);
        if (!pmd_k)
                return -1;

        if (pmd_leaf(*pmd_k))
                return 0;

        pte_k = pte_offset_kernel(pmd_k, address);
        if (!pte_present(*pte_k))
                return -1;

        return 0;
}
NOKPROBE_SYMBOL(vmalloc_fault);

void arch_sync_kernel_mappings(unsigned long start, unsigned long end)
{
        unsigned long addr;

        for (addr = start & PMD_MASK;
             addr >= TASK_SIZE_MAX && addr < VMALLOC_END;
             addr += PMD_SIZE) {
                struct page *page;

                spin_lock(&pgd_lock);
                list_for_each_entry(page, &pgd_list, lru) {
                        spinlock_t *pgt_lock;

                        /* the pgt_lock only for Xen */
                        pgt_lock = &pgd_page_get_mm(page)->page_table_lock;

                        spin_lock(pgt_lock);
                        vmalloc_sync_one(page_address(page), addr);
                        spin_unlock(pgt_lock);
                }
                spin_unlock(&pgd_lock);
        }
}

static bool low_pfn(unsigned long pfn)
{
        return pfn < max_low_pfn;
}

static void dump_pagetable(unsigned long address)
{
        pgd_t *base = __va(read_cr3_pa());
        pgd_t *pgd = &base[pgd_index(address)];
        p4d_t *p4d;
        pud_t *pud;
        pmd_t *pmd;
        pte_t *pte;

#ifdef CONFIG_X86_PAE
        pr_info("*pdpt = %016Lx ", pgd_val(*pgd));
        if (!low_pfn(pgd_val(*pgd) >> PAGE_SHIFT) || !pgd_present(*pgd))
                goto out;
#define pr_pde pr_cont
#else
#define pr_pde pr_info
#endif
        p4d = p4d_offset(pgd, address);
        pud = pud_offset(p4d, address);
        pmd = pmd_offset(pud, address);
        pr_pde("*pde = %0*Lx ", sizeof(*pmd) * 2, (u64)pmd_val(*pmd));
#undef pr_pde

        /*
         * We must not directly access the pte in the highpte
         * case if the page table is located in highmem.
         * And let's rather not kmap-atomic the pte, just in case
         * it's allocated already:
         */
        if (!low_pfn(pmd_pfn(*pmd)) || !pmd_present(*pmd) || pmd_leaf(*pmd))
                goto out;

        pte = pte_offset_kernel(pmd, address);
        pr_cont("*pte = %0*Lx ", sizeof(*pte) * 2, (u64)pte_val(*pte));
out:
        pr_cont("\n");
}

#else /* CONFIG_X86_64: */

#ifdef CONFIG_CPU_SUP_AMD
static const char errata93_warning[] =
KERN_ERR 
"******* Your BIOS seems to not contain a fix for K8 errata #93\n"
"******* Working around it, but it may cause SEGVs or burn power.\n"
"******* Please consider a BIOS update.\n"
"******* Disabling USB legacy in the BIOS may also help.\n";
#endif

static int bad_address(void *p)
{
        unsigned long dummy;

        return get_kernel_nofault(dummy, (unsigned long *)p);
}

static void dump_pagetable(unsigned long address)
{
        pgd_t *base = __va(read_cr3_pa());
        pgd_t *pgd = base + pgd_index(address);
        p4d_t *p4d;
        pud_t *pud;
        pmd_t *pmd;
        pte_t *pte;

        if (bad_address(pgd))
                goto bad;

        pr_info("PGD %lx ", pgd_val(*pgd));

        if (!pgd_present(*pgd))
                goto out;

        p4d = p4d_offset(pgd, address);
        if (bad_address(p4d))
                goto bad;

        pr_cont("P4D %lx ", p4d_val(*p4d));
        if (!p4d_present(*p4d) || p4d_leaf(*p4d))
                goto out;

        pud = pud_offset(p4d, address);
        if (bad_address(pud))
                goto bad;

        pr_cont("PUD %lx ", pud_val(*pud));
        if (!pud_present(*pud) || pud_leaf(*pud))
                goto out;

        pmd = pmd_offset(pud, address);
        if (bad_address(pmd))
                goto bad;

        pr_cont("PMD %lx ", pmd_val(*pmd));
        if (!pmd_present(*pmd) || pmd_leaf(*pmd))
                goto out;

        pte = pte_offset_kernel(pmd, address);
        if (bad_address(pte))
                goto bad;

        pr_cont("PTE %lx", pte_val(*pte));
out:
        pr_cont("\n");
        return;
bad:
        pr_info("BAD\n");
}

#endif /* CONFIG_X86_64 */

/*
 * Workaround for K8 erratum #93 & buggy BIOS.
 *
 * BIOS SMM functions are required to use a specific workaround
 * to avoid corruption of the 64bit RIP register on C stepping K8.
 *
 * A lot of BIOS that didn't get tested properly miss this.
 *
 * The OS sees this as a page fault with the upper 32bits of RIP cleared.
 * Try to work around it here.
 *
 * Note we only handle faults in kernel here.
 * Does nothing on 32-bit.
 */
static int is_errata93(struct pt_regs *regs, unsigned long address)
{
#if defined(CONFIG_X86_64) && defined(CONFIG_CPU_SUP_AMD)
        if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD
            || boot_cpu_data.x86 != 0xf)
                return 0;

        if (user_mode(regs))
                return 0;

        if (address != regs->ip)
                return 0;

        if ((address >> 32) != 0)
                return 0;

        address |= 0xffffffffUL << 32;
        if ((address >= (u64)_stext && address <= (u64)_etext) ||
            (address >= MODULES_VADDR && address <= MODULES_END)) {
                printk_once(errata93_warning);
                regs->ip = address;
                return 1;
        }
#endif
        return 0;
}

/*
 * Work around K8 erratum #100 K8 in compat mode occasionally jumps
 * to illegal addresses >4GB.
 *
 * We catch this in the page fault handler because these addresses
 * are not reachable. Just detect this case and return.  Any code
 * segment in LDT is compatibility mode.
 */
static int is_errata100(struct pt_regs *regs, unsigned long address)
{
#ifdef CONFIG_X86_64
        if ((regs->cs == __USER32_CS || (regs->cs & (1<<2))) && (address >> 32))
                return 1;
#endif
        return 0;
}

/* Pentium F0 0F C7 C8 bug workaround: */
static int is_f00f_bug(struct pt_regs *regs, unsigned long error_code,
                       unsigned long address)
{
#ifdef CONFIG_X86_F00F_BUG
        if (boot_cpu_has_bug(X86_BUG_F00F) && !(error_code & X86_PF_USER) &&
            idt_is_f00f_address(address)) {
                handle_invalid_op(regs);
                return 1;
        }
#endif
        return 0;
}

static void show_ldttss(const struct desc_ptr *gdt, const char *name, u16 index)
{
        u32 offset = (index >> 3) * sizeof(struct desc_struct);
        unsigned long addr;
        struct ldttss_desc desc;

        if (index == 0) {
                pr_alert("%s: NULL\n", name);
                return;
        }

        if (offset + sizeof(struct ldttss_desc) >= gdt->size) {
                pr_alert("%s: 0x%hx -- out of bounds\n", name, index);
                return;
        }

        if (copy_from_kernel_nofault(&desc, (void *)(gdt->address + offset),
                              sizeof(struct ldttss_desc))) {
                pr_alert("%s: 0x%hx -- GDT entry is not readable\n",
                         name, index);
                return;
        }

        addr = desc.base0 | (desc.base1 << 16) | ((unsigned long)desc.base2 << 24);
#ifdef CONFIG_X86_64
        addr |= ((u64)desc.base3 << 32);
#endif
        pr_alert("%s: 0x%hx -- base=0x%lx limit=0x%x\n",
                 name, index, addr, (desc.limit0 | (desc.limit1 << 16)));
}

static void
show_fault_oops(struct pt_regs *regs, unsigned long error_code, unsigned long address)
{
        if (!oops_may_print())
                return;

        if (error_code & X86_PF_INSTR) {
                unsigned int level;
                bool nx, rw;
                pgd_t *pgd;
                pte_t *pte;

                pgd = __va(read_cr3_pa());
                pgd += pgd_index(address);

                pte = lookup_address_in_pgd_attr(pgd, address, &level, &nx, &rw);

                if (pte && pte_present(*pte) && (!pte_exec(*pte) || nx))
                        pr_crit("kernel tried to execute NX-protected page - exploit attempt? (uid: %d)\n",
                                from_kuid(&init_user_ns, current_uid()));
                if (pte && pte_present(*pte) && pte_exec(*pte) && !nx &&
                                (pgd_flags(*pgd) & _PAGE_USER) &&
                                (__read_cr4() & X86_CR4_SMEP))
                        pr_crit("unable to execute userspace code (SMEP?) (uid: %d)\n",
                                from_kuid(&init_user_ns, current_uid()));
        }

        if (address < PAGE_SIZE && !user_mode(regs))
                pr_alert("BUG: kernel NULL pointer dereference, address: %px\n",
                        (void *)address);
        else
                pr_alert("BUG: unable to handle page fault for address: %px\n",
                        (void *)address);

        pr_alert("#PF: %s %s in %s mode\n",
                 (error_code & X86_PF_USER)  ? "user" : "supervisor",
                 (error_code & X86_PF_INSTR) ? "instruction fetch" :
                 (error_code & X86_PF_WRITE) ? "write access" :
                                               "read access",
                             user_mode(regs) ? "user" : "kernel");
        pr_alert("#PF: error_code(0x%04lx) - %s\n", error_code,
                 !(error_code & X86_PF_PROT) ? "not-present page" :
                 (error_code & X86_PF_RSVD)  ? "reserved bit violation" :
                 (error_code & X86_PF_PK)    ? "protection keys violation" :
                 (error_code & X86_PF_RMP)   ? "RMP violation" :
                                               "permissions violation");

        if (!(error_code & X86_PF_USER) && user_mode(regs)) {
                struct desc_ptr idt, gdt;
                u16 ldtr, tr;

                /*
                 * This can happen for quite a few reasons.  The more obvious
                 * ones are faults accessing the GDT, or LDT.  Perhaps
                 * surprisingly, if the CPU tries to deliver a benign or
                 * contributory exception from user code and gets a page fault
                 * during delivery, the page fault can be delivered as though
                 * it originated directly from user code.  This could happen
                 * due to wrong permissions on the IDT, GDT, LDT, TSS, or
                 * kernel or IST stack.
                 */
                store_idt(&idt);

                /* Usable even on Xen PV -- it's just slow. */
                native_store_gdt(&gdt);

                pr_alert("IDT: 0x%lx (limit=0x%hx) GDT: 0x%lx (limit=0x%hx)\n",
                         idt.address, idt.size, gdt.address, gdt.size);

                store_ldt(ldtr);
                show_ldttss(&gdt, "LDTR", ldtr);

                store_tr(tr);
                show_ldttss(&gdt, "TR", tr);
        }

        dump_pagetable(address);

        if (error_code & X86_PF_RMP)
                snp_dump_hva_rmpentry(address);
}

static noinline void
pgtable_bad(struct pt_regs *regs, unsigned long error_code,
            unsigned long address)
{
        struct task_struct *tsk;
        unsigned long flags;
        int sig;

        flags = oops_begin();
        tsk = current;
        sig = SIGKILL;

        printk(KERN_ALERT "%s: Corrupted page table at address %lx\n",
               tsk->comm, address);
        dump_pagetable(address);

        if (__die("Bad pagetable", regs, error_code))
                sig = 0;

        oops_end(flags, regs, sig);
}

static void sanitize_error_code(unsigned long address,
                                unsigned long *error_code)
{
        /*
         * To avoid leaking information about the kernel page
         * table layout, pretend that user-mode accesses to
         * kernel addresses are always protection faults.
         *
         * NB: This means that failed vsyscalls with vsyscall=none
         * will have the PROT bit.  This doesn't leak any
         * information and does not appear to cause any problems.
         */
        if (address >= TASK_SIZE_MAX)
                *error_code |= X86_PF_PROT;
}

static void set_signal_archinfo(unsigned long address,
                                unsigned long error_code)
{
        struct task_struct *tsk = current;

        tsk->thread.trap_nr = X86_TRAP_PF;
        tsk->thread.error_code = error_code | X86_PF_USER;
        tsk->thread.cr2 = address;
}

static noinline void
page_fault_oops(struct pt_regs *regs, unsigned long error_code,
                unsigned long address)
{
#ifdef CONFIG_VMAP_STACK
        struct stack_info info;
#endif
        unsigned long flags;
        int sig;

        if (user_mode(regs)) {
                /*
                 * Implicit kernel access from user mode?  Skip the stack
                 * overflow and EFI special cases.
                 */
                goto oops;
        }

#ifdef CONFIG_VMAP_STACK
        /*
         * Stack overflow?  During boot, we can fault near the initial
         * stack in the direct map, but that's not an overflow -- check
         * that we're in vmalloc space to avoid this.
         */
        if (is_vmalloc_addr((void *)address) &&
            get_stack_guard_info((void *)address, &info)) {
                /*
                 * We're likely to be running with very little stack space
                 * left.  It's plausible that we'd hit this condition but
                 * double-fault even before we get this far, in which case
                 * we're fine: the double-fault handler will deal with it.
                 *
                 * We don't want to make it all the way into the oops code
                 * and then double-fault, though, because we're likely to
                 * break the console driver and lose most of the stack dump.
                 */
                call_on_stack(__this_cpu_ist_top_va(DF) - sizeof(void*),
                              handle_stack_overflow,
                              ASM_CALL_ARG3,
                              , [arg1] "r" (regs), [arg2] "r" (address), [arg3] "r" (&info));

                unreachable();
        }
#endif

        /*
         * Buggy firmware could access regions which might page fault.  If
         * this happens, EFI has a special OOPS path that will try to
         * avoid hanging the system.
         */
        if (IS_ENABLED(CONFIG_EFI))
                efi_crash_gracefully_on_page_fault(address);

        /* Only not-present faults should be handled by KFENCE. */
        if (!(error_code & X86_PF_PROT) &&
            kfence_handle_page_fault(address, error_code & X86_PF_WRITE, regs))
                return;

oops:
        /*
         * Oops. The kernel tried to access some bad page. We'll have to
         * terminate things with extreme prejudice:
         */
        flags = oops_begin();

        show_fault_oops(regs, error_code, address);

        if (task_stack_end_corrupted(current))
                printk(KERN_EMERG "Thread overran stack, or stack corrupted\n");

        sig = SIGKILL;
        if (__die("Oops", regs, error_code))
                sig = 0;

        /* Executive summary in case the body of the oops scrolled away */
        printk(KERN_DEFAULT "CR2: %016lx\n", address);

        oops_end(flags, regs, sig);
}

static noinline void
kernelmode_fixup_or_oops(struct pt_regs *regs, unsigned long error_code,
                         unsigned long address, int signal, int si_code,
                         u32 pkey)
{
        WARN_ON_ONCE(user_mode(regs));

        /* Are we prepared to handle this kernel fault? */
        if (fixup_exception(regs, X86_TRAP_PF, error_code, address))
                return;

        /*
         * AMD erratum #91 manifests as a spurious page fault on a PREFETCH
         * instruction.
         */
        if (is_prefetch(regs, error_code, address))
                return;

        page_fault_oops(regs, error_code, address);
}

/*
 * Print out info about fatal segfaults, if the show_unhandled_signals
 * sysctl is set:
 */
static inline void
show_signal_msg(struct pt_regs *regs, unsigned long error_code,
                unsigned long address, struct task_struct *tsk)
{
        const char *loglvl = task_pid_nr(tsk) > 1 ? KERN_INFO : KERN_EMERG;
        /* This is a racy snapshot, but it's better than nothing. */
        int cpu = raw_smp_processor_id();

        if (!unhandled_signal(tsk, SIGSEGV))
                return;

        if (!printk_ratelimit())
                return;

        printk("%s%s[%d]: segfault at %lx ip %px sp %px error %lx",
                loglvl, tsk->comm, task_pid_nr(tsk), address,
                (void *)regs->ip, (void *)regs->sp, error_code);

        print_vma_addr(KERN_CONT " in ", regs->ip);

        /*
         * Dump the likely CPU where the fatal segfault happened.
         * This can help identify faulty hardware.
         */
        printk(KERN_CONT " likely on CPU %d (core %d, socket %d)", cpu,
               topology_core_id(cpu), topology_physical_package_id(cpu));


        printk(KERN_CONT "\n");

        show_opcodes(regs, loglvl);
}

static void
__bad_area_nosemaphore(struct pt_regs *regs, unsigned long error_code,
                       unsigned long address, u32 pkey, int si_code)
{
        struct task_struct *tsk = current;

        if (!user_mode(regs)) {
                kernelmode_fixup_or_oops(regs, error_code, address,
                                         SIGSEGV, si_code, pkey);
                return;
        }

        if (!(error_code & X86_PF_USER)) {
                /* Implicit user access to kernel memory -- just oops */
                page_fault_oops(regs, error_code, address);
                return;
        }

        /*
         * User mode accesses just cause a SIGSEGV.
         * It's possible to have interrupts off here:
         */
        local_irq_enable();

        /*
         * Valid to do another page fault here because this one came
         * from user space:
         */
        if (is_prefetch(regs, error_code, address))
                return;

        if (is_errata100(regs, address))
                return;

        sanitize_error_code(address, &error_code);

        if (fixup_vdso_exception(regs, X86_TRAP_PF, error_code, address))
                return;

        if (likely(show_unhandled_signals))
                show_signal_msg(regs, error_code, address, tsk);

        set_signal_archinfo(address, error_code);

        if (si_code == SEGV_PKUERR)
                force_sig_pkuerr((void __user *)address, pkey);
        else
                force_sig_fault(SIGSEGV, si_code, (void __user *)address);

        local_irq_disable();
}

static noinline void
bad_area_nosemaphore(struct pt_regs *regs, unsigned long error_code,
                     unsigned long address)
{
        __bad_area_nosemaphore(regs, error_code, address, 0, SEGV_MAPERR);
}

static void
__bad_area(struct pt_regs *regs, unsigned long error_code,
           unsigned long address, struct mm_struct *mm,
           struct vm_area_struct *vma, u32 pkey, int si_code)
{
        /*
         * Something tried to access memory that isn't in our memory map..
         * Fix it, but check if it's kernel or user first..
         */
        if (mm)
                mmap_read_unlock(mm);
        else
                vma_end_read(vma);

        __bad_area_nosemaphore(regs, error_code, address, pkey, si_code);
}

static inline bool bad_area_access_from_pkeys(unsigned long error_code,
                struct vm_area_struct *vma)
{
        /* This code is always called on the current mm */
        bool foreign = false;

        if (!cpu_feature_enabled(X86_FEATURE_OSPKE))
                return false;
        if (error_code & X86_PF_PK)
                return true;
        /* this checks permission keys on the VMA: */
        if (!arch_vma_access_permitted(vma, (error_code & X86_PF_WRITE),
                                       (error_code & X86_PF_INSTR), foreign))
                return true;
        return false;
}

static noinline void
bad_area_access_error(struct pt_regs *regs, unsigned long error_code,
                      unsigned long address, struct mm_struct *mm,
                      struct vm_area_struct *vma)
{
        /*
         * This OSPKE check is not strictly necessary at runtime.
         * But, doing it this way allows compiler optimizations
         * if pkeys are compiled out.
         */
        if (bad_area_access_from_pkeys(error_code, vma)) {
                /*
                 * A protection key fault means that the PKRU value did not allow
                 * access to some PTE.  Userspace can figure out what PKRU was
                 * from the XSAVE state.  This function captures the pkey from
                 * the vma and passes it to userspace so userspace can discover
                 * which protection key was set on the PTE.
                 *
                 * If we get here, we know that the hardware signaled a X86_PF_PK
                 * fault and that there was a VMA once we got in the fault
                 * handler.  It does *not* guarantee that the VMA we find here
                 * was the one that we faulted on.
                 *
                 * 1. T1   : mprotect_key(foo, PAGE_SIZE, pkey=4);
                 * 2. T1   : set PKRU to deny access to pkey=4, touches page
                 * 3. T1   : faults...
                 * 4.    T2: mprotect_key(foo, PAGE_SIZE, pkey=5);
                 * 5. T1   : enters fault handler, takes mmap_lock, etc...
                 * 6. T1   : reaches here, sees vma_pkey(vma)=5, when we really
                 *             faulted on a pte with its pkey=4.
                 */
                u32 pkey = vma_pkey(vma);

                __bad_area(regs, error_code, address, mm, vma, pkey, SEGV_PKUERR);
        } else {
                __bad_area(regs, error_code, address, mm, vma, 0, SEGV_ACCERR);
        }
}

static void
do_sigbus(struct pt_regs *regs, unsigned long error_code, unsigned long address,
          vm_fault_t fault)
{
        /* Kernel mode? Handle exceptions or die: */
        if (!user_mode(regs)) {
                kernelmode_fixup_or_oops(regs, error_code, address,
                                         SIGBUS, BUS_ADRERR, ARCH_DEFAULT_PKEY);
                return;
        }

        /* User-space => ok to do another page fault: */
        if (is_prefetch(regs, error_code, address))
                return;

        sanitize_error_code(address, &error_code);

        if (fixup_vdso_exception(regs, X86_TRAP_PF, error_code, address))
                return;

        set_signal_archinfo(address, error_code);

#ifdef CONFIG_MEMORY_FAILURE
        if (fault & (VM_FAULT_HWPOISON|VM_FAULT_HWPOISON_LARGE)) {
                struct task_struct *tsk = current;
                unsigned lsb = 0;

                pr_err(
        "MCE: Killing %s:%d due to hardware memory corruption fault at %lx\n",
                        tsk->comm, tsk->pid, address);
                if (fault & VM_FAULT_HWPOISON_LARGE)
                        lsb = hstate_index_to_shift(VM_FAULT_GET_HINDEX(fault));
                if (fault & VM_FAULT_HWPOISON)
                        lsb = PAGE_SHIFT;
                force_sig_mceerr(BUS_MCEERR_AR, (void __user *)address, lsb);
                return;
        }
#endif
        force_sig_fault(SIGBUS, BUS_ADRERR, (void __user *)address);
}

static int spurious_kernel_fault_check(unsigned long error_code, pte_t *pte)
{
        if ((error_code & X86_PF_WRITE) && !pte_write(*pte))
                return 0;

        if ((error_code & X86_PF_INSTR) && !pte_exec(*pte))
                return 0;

        return 1;
}

/*
 * Handle a spurious fault caused by a stale TLB entry.
 *
 * This allows us to lazily refresh the TLB when increasing the
 * permissions of a kernel page (RO -> RW or NX -> X).  Doing it
 * eagerly is very expensive since that implies doing a full
 * cross-processor TLB flush, even if no stale TLB entries exist
 * on other processors.
 *
 * Spurious faults may only occur if the TLB contains an entry with
 * fewer permission than the page table entry.  Non-present (P = 0)
 * and reserved bit (R = 1) faults are never spurious.
 *
 * There are no security implications to leaving a stale TLB when
 * increasing the permissions on a page.
 *
 * Returns non-zero if a spurious fault was handled, zero otherwise.
 *
 * See Intel Developer's Manual Vol 3 Section 4.10.4.3, bullet 3
 * (Optional Invalidation).
 */
static noinline int
spurious_kernel_fault(unsigned long error_code, unsigned long address)
{
        pgd_t *pgd;
        p4d_t *p4d;
        pud_t *pud;
        pmd_t *pmd;
        pte_t *pte;
        int ret;

        /*
         * Only writes to RO or instruction fetches from NX may cause
         * spurious faults.
         *
         * These could be from user or supervisor accesses but the TLB
         * is only lazily flushed after a kernel mapping protection
         * change, so user accesses are not expected to cause spurious
         * faults.
         */
        if (error_code != (X86_PF_WRITE | X86_PF_PROT) &&
            error_code != (X86_PF_INSTR | X86_PF_PROT))
                return 0;

        pgd = init_mm.pgd + pgd_index(address);
        if (!pgd_present(*pgd))
                return 0;

        p4d = p4d_offset(pgd, address);
        if (!p4d_present(*p4d))
                return 0;

        if (p4d_leaf(*p4d))
                return spurious_kernel_fault_check(error_code, (pte_t *) p4d);

        pud = pud_offset(p4d, address);
        if (!pud_present(*pud))
                return 0;

        if (pud_leaf(*pud))
                return spurious_kernel_fault_check(error_code, (pte_t *) pud);

        pmd = pmd_offset(pud, address);
        if (!pmd_present(*pmd))
                return 0;

        if (pmd_leaf(*pmd))
                return spurious_kernel_fault_check(error_code, (pte_t *) pmd);

        pte = pte_offset_kernel(pmd, address);
        if (!pte_present(*pte))
                return 0;

        ret = spurious_kernel_fault_check(error_code, pte);
        if (!ret)
                return 0;

        /*
         * Make sure we have permissions in PMD.
         * If not, then there's a bug in the page tables:
         */
        ret = spurious_kernel_fault_check(error_code, (pte_t *) pmd);
        WARN_ONCE(!ret, "PMD has incorrect permission bits\n");

        return ret;
}
NOKPROBE_SYMBOL(spurious_kernel_fault);

int show_unhandled_signals = 1;

static inline int
access_error(unsigned long error_code, struct vm_area_struct *vma)
{
        /* This is only called for the current mm, so: */
        bool foreign = false;

        /*
         * Read or write was blocked by protection keys.  This is
         * always an unconditional error and can never result in
         * a follow-up action to resolve the fault, like a COW.
         */
        if (error_code & X86_PF_PK)
                return 1;

        /*
         * SGX hardware blocked the access.  This usually happens
         * when the enclave memory contents have been destroyed, like
         * after a suspend/resume cycle. In any case, the kernel can't
         * fix the cause of the fault.  Handle the fault as an access
         * error even in cases where no actual access violation
         * occurred.  This allows userspace to rebuild the enclave in
         * response to the signal.
         */
        if (unlikely(error_code & X86_PF_SGX))
                return 1;

        /*
         * Make sure to check the VMA so that we do not perform
         * faults just to hit a X86_PF_PK as soon as we fill in a
         * page.
         */
        if (!arch_vma_access_permitted(vma, (error_code & X86_PF_WRITE),
                                       (error_code & X86_PF_INSTR), foreign))
                return 1;

        /*
         * Shadow stack accesses (PF_SHSTK=1) are only permitted to
         * shadow stack VMAs. All other accesses result in an error.
         */
        if (error_code & X86_PF_SHSTK) {
                if (unlikely(!(vma->vm_flags & VM_SHADOW_STACK)))
                        return 1;
                if (unlikely(!(vma->vm_flags & VM_WRITE)))
                        return 1;
                return 0;
        }

        if (error_code & X86_PF_WRITE) {
                /* write, present and write, not present: */
                if (unlikely(vma->vm_flags & VM_SHADOW_STACK))
                        return 1;
                if (unlikely(!(vma->vm_flags & VM_WRITE)))
                        return 1;
                return 0;
        }

        /* read, present: */
        if (unlikely(error_code & X86_PF_PROT))
                return 1;

        /* read, not present: */
        if (unlikely(!vma_is_accessible(vma)))
                return 1;

        return 0;
}

bool fault_in_kernel_space(unsigned long address)
{
        /*
         * On 64-bit systems, the vsyscall page is at an address above
         * TASK_SIZE_MAX, but is not considered part of the kernel
         * address space.
         */
        if (IS_ENABLED(CONFIG_X86_64) && is_vsyscall_vaddr(address))
                return false;

        return address >= TASK_SIZE_MAX;
}

/*
 * Called for all faults where 'address' is part of the kernel address
 * space.  Might get called for faults that originate from *code* that
 * ran in userspace or the kernel.
 */
static void
do_kern_addr_fault(struct pt_regs *regs, unsigned long hw_error_code,
                   unsigned long address)
{
        /*
         * Protection keys exceptions only happen on user pages.  We
         * have no user pages in the kernel portion of the address
         * space, so do not expect them here.
         */
        WARN_ON_ONCE(hw_error_code & X86_PF_PK);

#ifdef CONFIG_X86_32
        /*
         * We can fault-in kernel-space virtual memory on-demand. The
         * 'reference' page table is init_mm.pgd.
         *
         * NOTE! We MUST NOT take any locks for this case. We may
         * be in an interrupt or a critical region, and should
         * only copy the information from the master page table,
         * nothing more.
         *
         * Before doing this on-demand faulting, ensure that the
         * fault is not any of the following:
         * 1. A fault on a PTE with a reserved bit set.
         * 2. A fault caused by a user-mode access.  (Do not demand-
         *    fault kernel memory due to user-mode accesses).
         * 3. A fault caused by a page-level protection violation.
         *    (A demand fault would be on a non-present page which
         *     would have X86_PF_PROT==0).
         *
         * This is only needed to close a race condition on x86-32 in
         * the vmalloc mapping/unmapping code. See the comment above
         * vmalloc_fault() for details. On x86-64 the race does not
         * exist as the vmalloc mappings don't need to be synchronized
         * there.
         */
        if (!(hw_error_code & (X86_PF_RSVD | X86_PF_USER | X86_PF_PROT))) {
                if (vmalloc_fault(address) >= 0)
                        return;
        }
#endif

        if (is_f00f_bug(regs, hw_error_code, address))
                return;

        /* Was the fault spurious, caused by lazy TLB invalidation? */
        if (spurious_kernel_fault(hw_error_code, address))
                return;

        /* kprobes don't want to hook the spurious faults: */
        if (WARN_ON_ONCE(kprobe_page_fault(regs, X86_TRAP_PF)))
                return;

        /*
         * Note, despite being a "bad area", there are quite a few
         * acceptable reasons to get here, such as erratum fixups
         * and handling kernel code that can fault, like get_user().
         *
         * Don't take the mm semaphore here. If we fixup a prefetch
         * fault we could otherwise deadlock:
         */
        bad_area_nosemaphore(regs, hw_error_code, address);
}
NOKPROBE_SYMBOL(do_kern_addr_fault);

/*
 * Handle faults in the user portion of the address space.  Nothing in here
 * should check X86_PF_USER without a specific justification: for almost
 * all purposes, we should treat a normal kernel access to user memory
 * (e.g. get_user(), put_user(), etc.) the same as the WRUSS instruction.
 * The one exception is AC flag handling, which is, per the x86
 * architecture, special for WRUSS.
 */
static inline
void do_user_addr_fault(struct pt_regs *regs,
                        unsigned long error_code,
                        unsigned long address)
{
        struct vm_area_struct *vma;
        struct task_struct *tsk;
        struct mm_struct *mm;
        vm_fault_t fault;
        unsigned int flags = FAULT_FLAG_DEFAULT;

        tsk = current;
        mm = tsk->mm;

        if (unlikely((error_code & (X86_PF_USER | X86_PF_INSTR)) == X86_PF_INSTR)) {
                /*
                 * Whoops, this is kernel mode code trying to execute from
                 * user memory.  Unless this is AMD erratum #93, which
                 * corrupts RIP such that it looks like a user address,
                 * this is unrecoverable.  Don't even try to look up the
                 * VMA or look for extable entries.
                 */
                if (is_errata93(regs, address))
                        return;

                page_fault_oops(regs, error_code, address);
                return;
        }

        /* kprobes don't want to hook the spurious faults: */
        if (WARN_ON_ONCE(kprobe_page_fault(regs, X86_TRAP_PF)))
                return;

        /*
         * Reserved bits are never expected to be set on
         * entries in the user portion of the page tables.
         */
        if (unlikely(error_code & X86_PF_RSVD))
                pgtable_bad(regs, error_code, address);

        /*
         * If SMAP is on, check for invalid kernel (supervisor) access to user
         * pages in the user address space.  The odd case here is WRUSS,
         * which, according to the preliminary documentation, does not respect
         * SMAP and will have the USER bit set so, in all cases, SMAP
         * enforcement appears to be consistent with the USER bit.
         */
        if (unlikely(cpu_feature_enabled(X86_FEATURE_SMAP) &&
                     !(error_code & X86_PF_USER) &&
                     !(regs->flags & X86_EFLAGS_AC))) {
                /*
                 * No extable entry here.  This was a kernel access to an
                 * invalid pointer.  get_kernel_nofault() will not get here.
                 */
                page_fault_oops(regs, error_code, address);
                return;
        }

        /*
         * If we're in an interrupt, have no user context or are running
         * in a region with pagefaults disabled then we must not take the fault
         */
        if (unlikely(faulthandler_disabled() || !mm)) {
                bad_area_nosemaphore(regs, error_code, address);
                return;
        }

        /* Legacy check - remove this after verifying that it doesn't trigger */
        if (WARN_ON_ONCE(!(regs->flags & X86_EFLAGS_IF))) {
                bad_area_nosemaphore(regs, error_code, address);
                return;
        }

        local_irq_enable();

        perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address);

        /*
         * Read-only permissions can not be expressed in shadow stack PTEs.
         * Treat all shadow stack accesses as WRITE faults. This ensures
         * that the MM will prepare everything (e.g., break COW) such that
         * maybe_mkwrite() can create a proper shadow stack PTE.
         */
        if (error_code & X86_PF_SHSTK)
                flags |= FAULT_FLAG_WRITE;
        if (error_code & X86_PF_WRITE)
                flags |= FAULT_FLAG_WRITE;
        if (error_code & X86_PF_INSTR)
                flags |= FAULT_FLAG_INSTRUCTION;

        /*
         * We set FAULT_FLAG_USER based on the register state, not
         * based on X86_PF_USER. User space accesses that cause
         * system page faults are still user accesses.
         */
        if (user_mode(regs))
                flags |= FAULT_FLAG_USER;

#ifdef CONFIG_X86_64
        /*
         * Faults in the vsyscall page might need emulation.  The
         * vsyscall page is at a high address (>PAGE_OFFSET), but is
         * considered to be part of the user address space.
         *
         * The vsyscall page does not have a "real" VMA, so do this
         * emulation before we go searching for VMAs.
         *
         * PKRU never rejects instruction fetches, so we don't need
         * to consider the PF_PK bit.
         */
        if (is_vsyscall_vaddr(address)) {
                if (emulate_vsyscall(error_code, regs, address))
                        return;
        }
#endif

        if (!(flags & FAULT_FLAG_USER))
                goto lock_mmap;

        vma = lock_vma_under_rcu(mm, address);
        if (!vma)
                goto lock_mmap;

        if (unlikely(access_error(error_code, vma))) {
                bad_area_access_error(regs, error_code, address, NULL, vma);
                count_vm_vma_lock_event(VMA_LOCK_SUCCESS);
                return;
        }
        fault = handle_mm_fault(vma, address, flags | FAULT_FLAG_VMA_LOCK, regs);
        if (!(fault & (VM_FAULT_RETRY | VM_FAULT_COMPLETED)))
                vma_end_read(vma);

        if (!(fault & VM_FAULT_RETRY)) {
                count_vm_vma_lock_event(VMA_LOCK_SUCCESS);
                goto done;
        }
        count_vm_vma_lock_event(VMA_LOCK_RETRY);
        if (fault & VM_FAULT_MAJOR)
                flags |= FAULT_FLAG_TRIED;

        /* Quick path to respond to signals */
        if (fault_signal_pending(fault, regs)) {
                if (!user_mode(regs))
                        kernelmode_fixup_or_oops(regs, error_code, address,
                                                 SIGBUS, BUS_ADRERR,
                                                 ARCH_DEFAULT_PKEY);
                return;
        }
lock_mmap:

retry:
        vma = lock_mm_and_find_vma(mm, address, regs);
        if (unlikely(!vma)) {
                bad_area_nosemaphore(regs, error_code, address);
                return;
        }

        /*
         * Ok, we have a good vm_area for this memory access, so
         * we can handle it..
         */
        if (unlikely(access_error(error_code, vma))) {
                bad_area_access_error(regs, error_code, address, mm, vma);
                return;
        }

        /*
         * If for any reason at all we couldn't handle the fault,
         * make sure we exit gracefully rather than endlessly redo
         * the fault.  Since we never set FAULT_FLAG_RETRY_NOWAIT, if
         * we get VM_FAULT_RETRY back, the mmap_lock has been unlocked.
         *
         * Note that handle_userfault() may also release and reacquire mmap_lock
         * (and not return with VM_FAULT_RETRY), when returning to userland to
         * repeat the page fault later with a VM_FAULT_NOPAGE retval
         * (potentially after handling any pending signal during the return to
         * userland). The return to userland is identified whenever
         * FAULT_FLAG_USER|FAULT_FLAG_KILLABLE are both set in flags.
         */
        fault = handle_mm_fault(vma, address, flags, regs);

        if (fault_signal_pending(fault, regs)) {
                /*
                 * Quick path to respond to signals.  The core mm code
                 * has unlocked the mm for us if we get here.
                 */
                if (!user_mode(regs))
                        kernelmode_fixup_or_oops(regs, error_code, address,
                                                 SIGBUS, BUS_ADRERR,
                                                 ARCH_DEFAULT_PKEY);
                return;
        }

        /* The fault is fully completed (including releasing mmap lock) */
        if (fault & VM_FAULT_COMPLETED)
                return;

        /*
         * If we need to retry the mmap_lock has already been released,
         * and if there is a fatal signal pending there is no guarantee
         * that we made any progress. Handle this case first.
         */
        if (unlikely(fault & VM_FAULT_RETRY)) {
                flags |= FAULT_FLAG_TRIED;
                goto retry;
        }

        mmap_read_unlock(mm);
done:
        if (likely(!(fault & VM_FAULT_ERROR)))
                return;

        if (fatal_signal_pending(current) && !user_mode(regs)) {
                kernelmode_fixup_or_oops(regs, error_code, address,
                                         0, 0, ARCH_DEFAULT_PKEY);
                return;
        }

        if (fault & VM_FAULT_OOM) {
                /* Kernel mode? Handle exceptions or die: */
                if (!user_mode(regs)) {
                        kernelmode_fixup_or_oops(regs, error_code, address,
                                                 SIGSEGV, SEGV_MAPERR,
                                                 ARCH_DEFAULT_PKEY);
                        return;
                }

                /*
                 * We ran out of memory, call the OOM killer, and return the
                 * userspace (which will retry the fault, or kill us if we got
                 * oom-killed):
                 */
                pagefault_out_of_memory();
        } else {
                if (fault & (VM_FAULT_SIGBUS|VM_FAULT_HWPOISON|
                             VM_FAULT_HWPOISON_LARGE))
                        do_sigbus(regs, error_code, address, fault);
                else if (fault & VM_FAULT_SIGSEGV)
                        bad_area_nosemaphore(regs, error_code, address);
                else
                        BUG();
        }
}
NOKPROBE_SYMBOL(do_user_addr_fault);

static __always_inline void
trace_page_fault_entries(struct pt_regs *regs, unsigned long error_code,
                         unsigned long address)
{
        if (!trace_pagefault_enabled())
                return;

        if (user_mode(regs))
                trace_page_fault_user(address, regs, error_code);
        else
                trace_page_fault_kernel(address, regs, error_code);
}

static __always_inline void
handle_page_fault(struct pt_regs *regs, unsigned long error_code,
                              unsigned long address)
{
        trace_page_fault_entries(regs, error_code, address);

        if (unlikely(kmmio_fault(regs, address)))
                return;

        /* Was the fault on kernel-controlled part of the address space? */
        if (unlikely(fault_in_kernel_space(address))) {
                do_kern_addr_fault(regs, error_code, address);
        } else {
                do_user_addr_fault(regs, error_code, address);
                /*
                 * User address page fault handling might have reenabled
                 * interrupts. Fixing up all potential exit points of
                 * do_user_addr_fault() and its leaf functions is just not
                 * doable w/o creating an unholy mess or turning the code
                 * upside down.
                 */
                local_irq_disable();
        }
}

DEFINE_IDTENTRY_RAW_ERRORCODE(exc_page_fault)
{
        irqentry_state_t state;
        unsigned long address;

        address = cpu_feature_enabled(X86_FEATURE_FRED) ? fred_event_data(regs) : read_cr2();

        prefetchw(&current->mm->mmap_lock);

        /*
         * KVM uses #PF vector to deliver 'page not present' events to guests
         * (asynchronous page fault mechanism). The event happens when a
         * userspace task is trying to access some valid (from guest's point of
         * view) memory which is not currently mapped by the host (e.g. the
         * memory is swapped out). Note, the corresponding "page ready" event
         * which is injected when the memory becomes available, is delivered via
         * an interrupt mechanism and not a #PF exception
         * (see arch/x86/kernel/kvm.c: sysvec_kvm_asyncpf_interrupt()).
         *
         * We are relying on the interrupted context being sane (valid RSP,
         * relevant locks not held, etc.), which is fine as long as the
         * interrupted context had IF=1.  We are also relying on the KVM
         * async pf type field and CR2 being read consistently instead of
         * getting values from real and async page faults mixed up.
         *
         * Fingers crossed.
         *
         * The async #PF handling code takes care of idtentry handling
         * itself.
         */
        if (kvm_handle_async_pf(regs, (u32)address))
                return;

        /*
         * Entry handling for valid #PF from kernel mode is slightly
         * different: RCU is already watching and ct_irq_enter() must not
         * be invoked because a kernel fault on a user space address might
         * sleep.
         *
         * In case the fault hit a RCU idle region the conditional entry
         * code reenabled RCU to avoid subsequent wreckage which helps
         * debuggability.
         */
        state = irqentry_enter(regs);

        instrumentation_begin();
        handle_page_fault(regs, error_code, address);
        instrumentation_end();

        irqentry_exit(regs, state);
}
































































































   32 



    5 



    1 











1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
// SPDX-License-Identifier: GPL-2.0
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/device.h>
#include <linux/netdevice.h>

#include <net/bonding.h>
#include <net/bond_alb.h>

#if defined(CONFIG_DEBUG_FS) && !defined(CONFIG_NET_NS)

#include <linux/debugfs.h>
#include <linux/seq_file.h>

static struct dentry *bonding_debug_root;

/* Show RLB hash table */
static int bond_debug_rlb_hash_show(struct seq_file *m, void *v)
{
        struct bonding *bond = m->private;
        struct alb_bond_info *bond_info = &(BOND_ALB_INFO(bond));
        struct rlb_client_info *client_info;
        u32 hash_index;

        if (BOND_MODE(bond) != BOND_MODE_ALB)
                return 0;

        seq_printf(m, "SourceIP        DestinationIP   "
                        "Destination MAC   DEV\n");

        spin_lock_bh(&bond->mode_lock);

        hash_index = bond_info->rx_hashtbl_used_head;
        for (; hash_index != RLB_NULL_INDEX;
             hash_index = client_info->used_next) {
                client_info = &(bond_info->rx_hashtbl[hash_index]);
                seq_printf(m, "%-15pI4 %-15pI4 %-17pM %s\n",
                        &client_info->ip_src,
                        &client_info->ip_dst,
                        &client_info->mac_dst,
                        client_info->slave->dev->name);
        }

        spin_unlock_bh(&bond->mode_lock);

        return 0;
}
DEFINE_SHOW_ATTRIBUTE(bond_debug_rlb_hash);

void bond_debug_register(struct bonding *bond)
{
        bond->debug_dir =
                debugfs_create_dir(bond->dev->name, bonding_debug_root);

        debugfs_create_file("rlb_hash_table", 0400, bond->debug_dir,
                                bond, &bond_debug_rlb_hash_fops);
}

void bond_debug_unregister(struct bonding *bond)
{
        debugfs_remove_recursive(bond->debug_dir);
}

void bond_debug_reregister(struct bonding *bond)
{
        struct dentry *d;

        d = debugfs_rename(bonding_debug_root, bond->debug_dir,
                           bonding_debug_root, bond->dev->name);
        if (!IS_ERR(d)) {
                bond->debug_dir = d;
        } else {
                netdev_warn(bond->dev, "failed to reregister, so just unregister old one\n");
                bond_debug_unregister(bond);
        }
}

void __init bond_create_debugfs(void)
{
        bonding_debug_root = debugfs_create_dir("bonding", NULL);

        if (IS_ERR(bonding_debug_root))
                pr_warn("Warning: Cannot create bonding directory in debugfs\n");
}

void bond_destroy_debugfs(void)
{
        debugfs_remove_recursive(bonding_debug_root);
        bonding_debug_root = NULL;
}


#else /* !CONFIG_DEBUG_FS */

void bond_debug_register(struct bonding *bond)
{
}

void bond_debug_unregister(struct bonding *bond)
{
}

void bond_debug_reregister(struct bonding *bond)
{
}

void __init bond_create_debugfs(void)
{
}

void bond_destroy_debugfs(void)
{
}

#endif /* CONFIG_DEBUG_FS */



























  498 

















































  753 






  750 




  184 




  173 

  119 

    3 

  123 






   40 
   41 


   67 













  729 

















  263 


  261 


  260 


  249 
   83 

  119 




  256 
   14 












  195 



  187 
  263 


  171 
  263 
  144 










  272 
  275 



  264 

















   41 



   80 










   83 








   83 









   60 

    8 

   59 















   17 



   88 

   58 


   79 
    6 








   26 


   80 
    8 
   76 































































































































































































  436 
  432 
  430 





    4 















  797 
  796 

















  431 



  433 













  640 
























  496 






  498 

  498 














































































    4 






    2 
    4 














   23 






   23 














   34 





    9 
   25 



















   74 


    1 



   71 






















   60 






   60 

















   54 
    1 













   53 













  174 

  174 











  638 














    3 






    1 
    3 















  371 












  369 
  370 











    1 
    1 











    4 
  242 













  246 
  244 












    1 
    1 
















   74 


    1 



   73 






















   60 






   60 

















   55 
    1 





   54 













  173 

  172 











  639 



1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * net/core/dev_addr_lists.c - Functions for handling net device lists
 * Copyright (c) 2010 Jiri Pirko <jpirko@redhat.com>
 *
 * This file contains functions for working with unicast, multicast and device
 * addresses lists.
 */

#include <linux/netdevice.h>
#include <linux/rtnetlink.h>
#include <linux/export.h>
#include <linux/list.h>

#include "dev.h"

/*
 * General list handling functions
 */

static int __hw_addr_insert(struct netdev_hw_addr_list *list,
                            struct netdev_hw_addr *new, int addr_len)
{
        struct rb_node **ins_point = &list->tree.rb_node, *parent = NULL;
        struct netdev_hw_addr *ha;

        while (*ins_point) {
                int diff;

                ha = rb_entry(*ins_point, struct netdev_hw_addr, node);
                diff = memcmp(new->addr, ha->addr, addr_len);
                if (diff == 0)
                        diff = memcmp(&new->type, &ha->type, sizeof(new->type));

                parent = *ins_point;
                if (diff < 0)
                        ins_point = &parent->rb_left;
                else if (diff > 0)
                        ins_point = &parent->rb_right;
                else
                        return -EEXIST;
        }

        rb_link_node_rcu(&new->node, parent, ins_point);
        rb_insert_color(&new->node, &list->tree);

        return 0;
}

static struct netdev_hw_addr*
__hw_addr_create(const unsigned char *addr, int addr_len,
                 unsigned char addr_type, bool global, bool sync)
{
        struct netdev_hw_addr *ha;
        int alloc_size;

        alloc_size = sizeof(*ha);
        if (alloc_size < L1_CACHE_BYTES)
                alloc_size = L1_CACHE_BYTES;
        ha = kmalloc(alloc_size, GFP_ATOMIC);
        if (!ha)
                return NULL;
        memcpy(ha->addr, addr, addr_len);
        ha->type = addr_type;
        ha->refcount = 1;
        ha->global_use = global;
        ha->synced = sync ? 1 : 0;
        ha->sync_cnt = 0;

        return ha;
}

static int __hw_addr_add_ex(struct netdev_hw_addr_list *list,
                            const unsigned char *addr, int addr_len,
                            unsigned char addr_type, bool global, bool sync,
                            int sync_count, bool exclusive)
{
        struct rb_node **ins_point = &list->tree.rb_node, *parent = NULL;
        struct netdev_hw_addr *ha;

        if (addr_len > MAX_ADDR_LEN)
                return -EINVAL;

        while (*ins_point) {
                int diff;

                ha = rb_entry(*ins_point, struct netdev_hw_addr, node);
                diff = memcmp(addr, ha->addr, addr_len);
                if (diff == 0)
                        diff = memcmp(&addr_type, &ha->type, sizeof(addr_type));

                parent = *ins_point;
                if (diff < 0) {
                        ins_point = &parent->rb_left;
                } else if (diff > 0) {
                        ins_point = &parent->rb_right;
                } else {
                        if (exclusive)
                                return -EEXIST;
                        if (global) {
                                /* check if addr is already used as global */
                                if (ha->global_use)
                                        return 0;
                                else
                                        ha->global_use = true;
                        }
                        if (sync) {
                                if (ha->synced && sync_count)
                                        return -EEXIST;
                                else
                                        ha->synced++;
                        }
                        ha->refcount++;
                        return 0;
                }
        }

        ha = __hw_addr_create(addr, addr_len, addr_type, global, sync);
        if (!ha)
                return -ENOMEM;

        rb_link_node(&ha->node, parent, ins_point);
        rb_insert_color(&ha->node, &list->tree);

        list_add_tail_rcu(&ha->list, &list->list);
        list->count++;

        return 0;
}

static int __hw_addr_add(struct netdev_hw_addr_list *list,
                         const unsigned char *addr, int addr_len,
                         unsigned char addr_type)
{
        return __hw_addr_add_ex(list, addr, addr_len, addr_type, false, false,
                                0, false);
}

static int __hw_addr_del_entry(struct netdev_hw_addr_list *list,
                               struct netdev_hw_addr *ha, bool global,
                               bool sync)
{
        if (global && !ha->global_use)
                return -ENOENT;

        if (sync && !ha->synced)
                return -ENOENT;

        if (global)
                ha->global_use = false;

        if (sync)
                ha->synced--;

        if (--ha->refcount)
                return 0;

        rb_erase(&ha->node, &list->tree);

        list_del_rcu(&ha->list);
        kfree_rcu(ha, rcu_head);
        list->count--;
        return 0;
}

static struct netdev_hw_addr *__hw_addr_lookup(struct netdev_hw_addr_list *list,
                                               const unsigned char *addr, int addr_len,
                                               unsigned char addr_type)
{
        struct rb_node *node;

        node = list->tree.rb_node;

        while (node) {
                struct netdev_hw_addr *ha = rb_entry(node, struct netdev_hw_addr, node);
                int diff = memcmp(addr, ha->addr, addr_len);

                if (diff == 0 && addr_type)
                        diff = memcmp(&addr_type, &ha->type, sizeof(addr_type));

                if (diff < 0)
                        node = node->rb_left;
                else if (diff > 0)
                        node = node->rb_right;
                else
                        return ha;
        }

        return NULL;
}

static int __hw_addr_del_ex(struct netdev_hw_addr_list *list,
                            const unsigned char *addr, int addr_len,
                            unsigned char addr_type, bool global, bool sync)
{
        struct netdev_hw_addr *ha = __hw_addr_lookup(list, addr, addr_len, addr_type);

        if (!ha)
                return -ENOENT;
        return __hw_addr_del_entry(list, ha, global, sync);
}

static int __hw_addr_del(struct netdev_hw_addr_list *list,
                         const unsigned char *addr, int addr_len,
                         unsigned char addr_type)
{
        return __hw_addr_del_ex(list, addr, addr_len, addr_type, false, false);
}

static int __hw_addr_sync_one(struct netdev_hw_addr_list *to_list,
                               struct netdev_hw_addr *ha,
                               int addr_len)
{
        int err;

        err = __hw_addr_add_ex(to_list, ha->addr, addr_len, ha->type,
                               false, true, ha->sync_cnt, false);
        if (err && err != -EEXIST)
                return err;

        if (!err) {
                ha->sync_cnt++;
                ha->refcount++;
        }

        return 0;
}

static void __hw_addr_unsync_one(struct netdev_hw_addr_list *to_list,
                                 struct netdev_hw_addr_list *from_list,
                                 struct netdev_hw_addr *ha,
                                 int addr_len)
{
        int err;

        err = __hw_addr_del_ex(to_list, ha->addr, addr_len, ha->type,
                               false, true);
        if (err)
                return;
        ha->sync_cnt--;
        /* address on from list is not marked synced */
        __hw_addr_del_entry(from_list, ha, false, false);
}

static int __hw_addr_sync_multiple(struct netdev_hw_addr_list *to_list,
                                   struct netdev_hw_addr_list *from_list,
                                   int addr_len)
{
        int err = 0;
        struct netdev_hw_addr *ha, *tmp;

        list_for_each_entry_safe(ha, tmp, &from_list->list, list) {
                if (ha->sync_cnt == ha->refcount) {
                        __hw_addr_unsync_one(to_list, from_list, ha, addr_len);
                } else {
                        err = __hw_addr_sync_one(to_list, ha, addr_len);
                        if (err)
                                break;
                }
        }
        return err;
}

/* This function only works where there is a strict 1-1 relationship
 * between source and destination of they synch. If you ever need to
 * sync addresses to more then 1 destination, you need to use
 * __hw_addr_sync_multiple().
 */
int __hw_addr_sync(struct netdev_hw_addr_list *to_list,
                   struct netdev_hw_addr_list *from_list,
                   int addr_len)
{
        int err = 0;
        struct netdev_hw_addr *ha, *tmp;

        list_for_each_entry_safe(ha, tmp, &from_list->list, list) {
                if (!ha->sync_cnt) {
                        err = __hw_addr_sync_one(to_list, ha, addr_len);
                        if (err)
                                break;
                } else if (ha->refcount == 1)
                        __hw_addr_unsync_one(to_list, from_list, ha, addr_len);
        }
        return err;
}
EXPORT_SYMBOL(__hw_addr_sync);

void __hw_addr_unsync(struct netdev_hw_addr_list *to_list,
                      struct netdev_hw_addr_list *from_list,
                      int addr_len)
{
        struct netdev_hw_addr *ha, *tmp;

        list_for_each_entry_safe(ha, tmp, &from_list->list, list) {
                if (ha->sync_cnt)
                        __hw_addr_unsync_one(to_list, from_list, ha, addr_len);
        }
}
EXPORT_SYMBOL(__hw_addr_unsync);

/**
 *  __hw_addr_sync_dev - Synchronize device's multicast list
 *  @list: address list to synchronize
 *  @dev:  device to sync
 *  @sync: function to call if address should be added
 *  @unsync: function to call if address should be removed
 *
 *  This function is intended to be called from the ndo_set_rx_mode
 *  function of devices that require explicit address add/remove
 *  notifications.  The unsync function may be NULL in which case
 *  the addresses requiring removal will simply be removed without
 *  any notification to the device.
 **/
int __hw_addr_sync_dev(struct netdev_hw_addr_list *list,
                       struct net_device *dev,
                       int (*sync)(struct net_device *, const unsigned char *),
                       int (*unsync)(struct net_device *,
                                     const unsigned char *))
{
        struct netdev_hw_addr *ha, *tmp;
        int err;

        /* first go through and flush out any stale entries */
        list_for_each_entry_safe(ha, tmp, &list->list, list) {
                if (!ha->sync_cnt || ha->refcount != 1)
                        continue;

                /* if unsync is defined and fails defer unsyncing address */
                if (unsync && unsync(dev, ha->addr))
                        continue;

                ha->sync_cnt--;
                __hw_addr_del_entry(list, ha, false, false);
        }

        /* go through and sync new entries to the list */
        list_for_each_entry_safe(ha, tmp, &list->list, list) {
                if (ha->sync_cnt)
                        continue;

                err = sync(dev, ha->addr);
                if (err)
                        return err;

                ha->sync_cnt++;
                ha->refcount++;
        }

        return 0;
}
EXPORT_SYMBOL(__hw_addr_sync_dev);

/**
 *  __hw_addr_ref_sync_dev - Synchronize device's multicast address list taking
 *  into account references
 *  @list: address list to synchronize
 *  @dev:  device to sync
 *  @sync: function to call if address or reference on it should be added
 *  @unsync: function to call if address or some reference on it should removed
 *
 *  This function is intended to be called from the ndo_set_rx_mode
 *  function of devices that require explicit address or references on it
 *  add/remove notifications. The unsync function may be NULL in which case
 *  the addresses or references on it requiring removal will simply be
 *  removed without any notification to the device. That is responsibility of
 *  the driver to identify and distribute address or references on it between
 *  internal address tables.
 **/
int __hw_addr_ref_sync_dev(struct netdev_hw_addr_list *list,
                           struct net_device *dev,
                           int (*sync)(struct net_device *,
                                       const unsigned char *, int),
                           int (*unsync)(struct net_device *,
                                         const unsigned char *, int))
{
        struct netdev_hw_addr *ha, *tmp;
        int err, ref_cnt;

        /* first go through and flush out any unsynced/stale entries */
        list_for_each_entry_safe(ha, tmp, &list->list, list) {
                /* sync if address is not used */
                if ((ha->sync_cnt << 1) <= ha->refcount)
                        continue;

                /* if fails defer unsyncing address */
                ref_cnt = ha->refcount - ha->sync_cnt;
                if (unsync && unsync(dev, ha->addr, ref_cnt))
                        continue;

                ha->refcount = (ref_cnt << 1) + 1;
                ha->sync_cnt = ref_cnt;
                __hw_addr_del_entry(list, ha, false, false);
        }

        /* go through and sync updated/new entries to the list */
        list_for_each_entry_safe(ha, tmp, &list->list, list) {
                /* sync if address added or reused */
                if ((ha->sync_cnt << 1) >= ha->refcount)
                        continue;

                ref_cnt = ha->refcount - ha->sync_cnt;
                err = sync(dev, ha->addr, ref_cnt);
                if (err)
                        return err;

                ha->refcount = ref_cnt << 1;
                ha->sync_cnt = ref_cnt;
        }

        return 0;
}
EXPORT_SYMBOL(__hw_addr_ref_sync_dev);

/**
 *  __hw_addr_ref_unsync_dev - Remove synchronized addresses and references on
 *  it from device
 *  @list: address list to remove synchronized addresses (references on it) from
 *  @dev:  device to sync
 *  @unsync: function to call if address and references on it should be removed
 *
 *  Remove all addresses that were added to the device by
 *  __hw_addr_ref_sync_dev(). This function is intended to be called from the
 *  ndo_stop or ndo_open functions on devices that require explicit address (or
 *  references on it) add/remove notifications. If the unsync function pointer
 *  is NULL then this function can be used to just reset the sync_cnt for the
 *  addresses in the list.
 **/
void __hw_addr_ref_unsync_dev(struct netdev_hw_addr_list *list,
                              struct net_device *dev,
                              int (*unsync)(struct net_device *,
                                            const unsigned char *, int))
{
        struct netdev_hw_addr *ha, *tmp;

        list_for_each_entry_safe(ha, tmp, &list->list, list) {
                if (!ha->sync_cnt)
                        continue;

                /* if fails defer unsyncing address */
                if (unsync && unsync(dev, ha->addr, ha->sync_cnt))
                        continue;

                ha->refcount -= ha->sync_cnt - 1;
                ha->sync_cnt = 0;
                __hw_addr_del_entry(list, ha, false, false);
        }
}
EXPORT_SYMBOL(__hw_addr_ref_unsync_dev);

/**
 *  __hw_addr_unsync_dev - Remove synchronized addresses from device
 *  @list: address list to remove synchronized addresses from
 *  @dev:  device to sync
 *  @unsync: function to call if address should be removed
 *
 *  Remove all addresses that were added to the device by __hw_addr_sync_dev().
 *  This function is intended to be called from the ndo_stop or ndo_open
 *  functions on devices that require explicit address add/remove
 *  notifications.  If the unsync function pointer is NULL then this function
 *  can be used to just reset the sync_cnt for the addresses in the list.
 **/
void __hw_addr_unsync_dev(struct netdev_hw_addr_list *list,
                          struct net_device *dev,
                          int (*unsync)(struct net_device *,
                                        const unsigned char *))
{
        struct netdev_hw_addr *ha, *tmp;

        list_for_each_entry_safe(ha, tmp, &list->list, list) {
                if (!ha->sync_cnt)
                        continue;

                /* if unsync is defined and fails defer unsyncing address */
                if (unsync && unsync(dev, ha->addr))
                        continue;

                ha->sync_cnt--;
                __hw_addr_del_entry(list, ha, false, false);
        }
}
EXPORT_SYMBOL(__hw_addr_unsync_dev);

static void __hw_addr_flush(struct netdev_hw_addr_list *list)
{
        struct netdev_hw_addr *ha, *tmp;

        list->tree = RB_ROOT;
        list_for_each_entry_safe(ha, tmp, &list->list, list) {
                list_del_rcu(&ha->list);
                kfree_rcu(ha, rcu_head);
        }
        list->count = 0;
}

void __hw_addr_init(struct netdev_hw_addr_list *list)
{
        INIT_LIST_HEAD(&list->list);
        list->count = 0;
        list->tree = RB_ROOT;
}
EXPORT_SYMBOL(__hw_addr_init);

/*
 * Device addresses handling functions
 */

/* Check that netdev->dev_addr is not written to directly as this would
 * break the rbtree layout. All changes should go thru dev_addr_set() and co.
 * Remove this check in mid-2024.
 */
void dev_addr_check(struct net_device *dev)
{
        if (!memcmp(dev->dev_addr, dev->dev_addr_shadow, MAX_ADDR_LEN))
                return;

        netdev_warn(dev, "Current addr:  %*ph\n", MAX_ADDR_LEN, dev->dev_addr);
        netdev_warn(dev, "Expected addr: %*ph\n",
                    MAX_ADDR_LEN, dev->dev_addr_shadow);
        netdev_WARN(dev, "Incorrect netdev->dev_addr\n");
}

/**
 *        dev_addr_flush - Flush device address list
 *        @dev: device
 *
 *        Flush device address list and reset ->dev_addr.
 *
 *        The caller must hold the rtnl_mutex.
 */
void dev_addr_flush(struct net_device *dev)
{
        /* rtnl_mutex must be held here */
        dev_addr_check(dev);

        __hw_addr_flush(&dev->dev_addrs);
        dev->dev_addr = NULL;
}

/**
 *        dev_addr_init - Init device address list
 *        @dev: device
 *
 *        Init device address list and create the first element,
 *        used by ->dev_addr.
 *
 *        The caller must hold the rtnl_mutex.
 */
int dev_addr_init(struct net_device *dev)
{
        unsigned char addr[MAX_ADDR_LEN];
        struct netdev_hw_addr *ha;
        int err;

        /* rtnl_mutex must be held here */

        __hw_addr_init(&dev->dev_addrs);
        memset(addr, 0, sizeof(addr));
        err = __hw_addr_add(&dev->dev_addrs, addr, sizeof(addr),
                            NETDEV_HW_ADDR_T_LAN);
        if (!err) {
                /*
                 * Get the first (previously created) address from the list
                 * and set dev_addr pointer to this location.
                 */
                ha = list_first_entry(&dev->dev_addrs.list,
                                      struct netdev_hw_addr, list);
                dev->dev_addr = ha->addr;
        }
        return err;
}

void dev_addr_mod(struct net_device *dev, unsigned int offset,
                  const void *addr, size_t len)
{
        struct netdev_hw_addr *ha;

        dev_addr_check(dev);

        ha = container_of(dev->dev_addr, struct netdev_hw_addr, addr[0]);
        rb_erase(&ha->node, &dev->dev_addrs.tree);
        memcpy(&ha->addr[offset], addr, len);
        memcpy(&dev->dev_addr_shadow[offset], addr, len);
        WARN_ON(__hw_addr_insert(&dev->dev_addrs, ha, dev->addr_len));
}
EXPORT_SYMBOL(dev_addr_mod);

/**
 *        dev_addr_add - Add a device address
 *        @dev: device
 *        @addr: address to add
 *        @addr_type: address type
 *
 *        Add a device address to the device or increase the reference count if
 *        it already exists.
 *
 *        The caller must hold the rtnl_mutex.
 */
int dev_addr_add(struct net_device *dev, const unsigned char *addr,
                 unsigned char addr_type)
{
        int err;

        ASSERT_RTNL();

        err = dev_pre_changeaddr_notify(dev, addr, NULL);
        if (err)
                return err;
        err = __hw_addr_add(&dev->dev_addrs, addr, dev->addr_len, addr_type);
        if (!err)
                call_netdevice_notifiers(NETDEV_CHANGEADDR, dev);
        return err;
}
EXPORT_SYMBOL(dev_addr_add);

/**
 *        dev_addr_del - Release a device address.
 *        @dev: device
 *        @addr: address to delete
 *        @addr_type: address type
 *
 *        Release reference to a device address and remove it from the device
 *        if the reference count drops to zero.
 *
 *        The caller must hold the rtnl_mutex.
 */
int dev_addr_del(struct net_device *dev, const unsigned char *addr,
                 unsigned char addr_type)
{
        int err;
        struct netdev_hw_addr *ha;

        ASSERT_RTNL();

        /*
         * We can not remove the first address from the list because
         * dev->dev_addr points to that.
         */
        ha = list_first_entry(&dev->dev_addrs.list,
                              struct netdev_hw_addr, list);
        if (!memcmp(ha->addr, addr, dev->addr_len) &&
            ha->type == addr_type && ha->refcount == 1)
                return -ENOENT;

        err = __hw_addr_del(&dev->dev_addrs, addr, dev->addr_len,
                            addr_type);
        if (!err)
                call_netdevice_notifiers(NETDEV_CHANGEADDR, dev);
        return err;
}
EXPORT_SYMBOL(dev_addr_del);

/*
 * Unicast list handling functions
 */

/**
 *        dev_uc_add_excl - Add a global secondary unicast address
 *        @dev: device
 *        @addr: address to add
 */
int dev_uc_add_excl(struct net_device *dev, const unsigned char *addr)
{
        int err;

        netif_addr_lock_bh(dev);
        err = __hw_addr_add_ex(&dev->uc, addr, dev->addr_len,
                               NETDEV_HW_ADDR_T_UNICAST, true, false,
                               0, true);
        if (!err)
                __dev_set_rx_mode(dev);
        netif_addr_unlock_bh(dev);
        return err;
}
EXPORT_SYMBOL(dev_uc_add_excl);

/**
 *        dev_uc_add - Add a secondary unicast address
 *        @dev: device
 *        @addr: address to add
 *
 *        Add a secondary unicast address to the device or increase
 *        the reference count if it already exists.
 */
int dev_uc_add(struct net_device *dev, const unsigned char *addr)
{
        int err;

        netif_addr_lock_bh(dev);
        err = __hw_addr_add(&dev->uc, addr, dev->addr_len,
                            NETDEV_HW_ADDR_T_UNICAST);
        if (!err)
                __dev_set_rx_mode(dev);
        netif_addr_unlock_bh(dev);
        return err;
}
EXPORT_SYMBOL(dev_uc_add);

/**
 *        dev_uc_del - Release secondary unicast address.
 *        @dev: device
 *        @addr: address to delete
 *
 *        Release reference to a secondary unicast address and remove it
 *        from the device if the reference count drops to zero.
 */
int dev_uc_del(struct net_device *dev, const unsigned char *addr)
{
        int err;

        netif_addr_lock_bh(dev);
        err = __hw_addr_del(&dev->uc, addr, dev->addr_len,
                            NETDEV_HW_ADDR_T_UNICAST);
        if (!err)
                __dev_set_rx_mode(dev);
        netif_addr_unlock_bh(dev);
        return err;
}
EXPORT_SYMBOL(dev_uc_del);

/**
 *        dev_uc_sync - Synchronize device's unicast list to another device
 *        @to: destination device
 *        @from: source device
 *
 *        Add newly added addresses to the destination device and release
 *        addresses that have no users left. The source device must be
 *        locked by netif_addr_lock_bh.
 *
 *        This function is intended to be called from the dev->set_rx_mode
 *        function of layered software devices.  This function assumes that
 *        addresses will only ever be synced to the @to devices and no other.
 */
int dev_uc_sync(struct net_device *to, struct net_device *from)
{
        int err = 0;

        if (to->addr_len != from->addr_len)
                return -EINVAL;

        netif_addr_lock(to);
        err = __hw_addr_sync(&to->uc, &from->uc, to->addr_len);
        if (!err)
                __dev_set_rx_mode(to);
        netif_addr_unlock(to);
        return err;
}
EXPORT_SYMBOL(dev_uc_sync);

/**
 *        dev_uc_sync_multiple - Synchronize device's unicast list to another
 *        device, but allow for multiple calls to sync to multiple devices.
 *        @to: destination device
 *        @from: source device
 *
 *        Add newly added addresses to the destination device and release
 *        addresses that have been deleted from the source. The source device
 *        must be locked by netif_addr_lock_bh.
 *
 *        This function is intended to be called from the dev->set_rx_mode
 *        function of layered software devices.  It allows for a single source
 *        device to be synced to multiple destination devices.
 */
int dev_uc_sync_multiple(struct net_device *to, struct net_device *from)
{
        int err = 0;

        if (to->addr_len != from->addr_len)
                return -EINVAL;

        netif_addr_lock(to);
        err = __hw_addr_sync_multiple(&to->uc, &from->uc, to->addr_len);
        if (!err)
                __dev_set_rx_mode(to);
        netif_addr_unlock(to);
        return err;
}
EXPORT_SYMBOL(dev_uc_sync_multiple);

/**
 *        dev_uc_unsync - Remove synchronized addresses from the destination device
 *        @to: destination device
 *        @from: source device
 *
 *        Remove all addresses that were added to the destination device by
 *        dev_uc_sync(). This function is intended to be called from the
 *        dev->stop function of layered software devices.
 */
void dev_uc_unsync(struct net_device *to, struct net_device *from)
{
        if (to->addr_len != from->addr_len)
                return;

        /* netif_addr_lock_bh() uses lockdep subclass 0, this is okay for two
         * reasons:
         * 1) This is always called without any addr_list_lock, so as the
         *    outermost one here, it must be 0.
         * 2) This is called by some callers after unlinking the upper device,
         *    so the dev->lower_level becomes 1 again.
         * Therefore, the subclass for 'from' is 0, for 'to' is either 1 or
         * larger.
         */
        netif_addr_lock_bh(from);
        netif_addr_lock(to);
        __hw_addr_unsync(&to->uc, &from->uc, to->addr_len);
        __dev_set_rx_mode(to);
        netif_addr_unlock(to);
        netif_addr_unlock_bh(from);
}
EXPORT_SYMBOL(dev_uc_unsync);

/**
 *        dev_uc_flush - Flush unicast addresses
 *        @dev: device
 *
 *        Flush unicast addresses.
 */
void dev_uc_flush(struct net_device *dev)
{
        netif_addr_lock_bh(dev);
        __hw_addr_flush(&dev->uc);
        netif_addr_unlock_bh(dev);
}
EXPORT_SYMBOL(dev_uc_flush);

/**
 *        dev_uc_init - Init unicast address list
 *        @dev: device
 *
 *        Init unicast address list.
 */
void dev_uc_init(struct net_device *dev)
{
        __hw_addr_init(&dev->uc);
}
EXPORT_SYMBOL(dev_uc_init);

/*
 * Multicast list handling functions
 */

/**
 *        dev_mc_add_excl - Add a global secondary multicast address
 *        @dev: device
 *        @addr: address to add
 */
int dev_mc_add_excl(struct net_device *dev, const unsigned char *addr)
{
        int err;

        netif_addr_lock_bh(dev);
        err = __hw_addr_add_ex(&dev->mc, addr, dev->addr_len,
                               NETDEV_HW_ADDR_T_MULTICAST, true, false,
                               0, true);
        if (!err)
                __dev_set_rx_mode(dev);
        netif_addr_unlock_bh(dev);
        return err;
}
EXPORT_SYMBOL(dev_mc_add_excl);

static int __dev_mc_add(struct net_device *dev, const unsigned char *addr,
                        bool global)
{
        int err;

        netif_addr_lock_bh(dev);
        err = __hw_addr_add_ex(&dev->mc, addr, dev->addr_len,
                               NETDEV_HW_ADDR_T_MULTICAST, global, false,
                               0, false);
        if (!err)
                __dev_set_rx_mode(dev);
        netif_addr_unlock_bh(dev);
        return err;
}
/**
 *        dev_mc_add - Add a multicast address
 *        @dev: device
 *        @addr: address to add
 *
 *        Add a multicast address to the device or increase
 *        the reference count if it already exists.
 */
int dev_mc_add(struct net_device *dev, const unsigned char *addr)
{
        return __dev_mc_add(dev, addr, false);
}
EXPORT_SYMBOL(dev_mc_add);

/**
 *        dev_mc_add_global - Add a global multicast address
 *        @dev: device
 *        @addr: address to add
 *
 *        Add a global multicast address to the device.
 */
int dev_mc_add_global(struct net_device *dev, const unsigned char *addr)
{
        return __dev_mc_add(dev, addr, true);
}
EXPORT_SYMBOL(dev_mc_add_global);

static int __dev_mc_del(struct net_device *dev, const unsigned char *addr,
                        bool global)
{
        int err;

        netif_addr_lock_bh(dev);
        err = __hw_addr_del_ex(&dev->mc, addr, dev->addr_len,
                               NETDEV_HW_ADDR_T_MULTICAST, global, false);
        if (!err)
                __dev_set_rx_mode(dev);
        netif_addr_unlock_bh(dev);
        return err;
}

/**
 *        dev_mc_del - Delete a multicast address.
 *        @dev: device
 *        @addr: address to delete
 *
 *        Release reference to a multicast address and remove it
 *        from the device if the reference count drops to zero.
 */
int dev_mc_del(struct net_device *dev, const unsigned char *addr)
{
        return __dev_mc_del(dev, addr, false);
}
EXPORT_SYMBOL(dev_mc_del);

/**
 *        dev_mc_del_global - Delete a global multicast address.
 *        @dev: device
 *        @addr: address to delete
 *
 *        Release reference to a multicast address and remove it
 *        from the device if the reference count drops to zero.
 */
int dev_mc_del_global(struct net_device *dev, const unsigned char *addr)
{
        return __dev_mc_del(dev, addr, true);
}
EXPORT_SYMBOL(dev_mc_del_global);

/**
 *        dev_mc_sync - Synchronize device's multicast list to another device
 *        @to: destination device
 *        @from: source device
 *
 *        Add newly added addresses to the destination device and release
 *        addresses that have no users left. The source device must be
 *        locked by netif_addr_lock_bh.
 *
 *        This function is intended to be called from the ndo_set_rx_mode
 *        function of layered software devices.
 */
int dev_mc_sync(struct net_device *to, struct net_device *from)
{
        int err = 0;

        if (to->addr_len != from->addr_len)
                return -EINVAL;

        netif_addr_lock(to);
        err = __hw_addr_sync(&to->mc, &from->mc, to->addr_len);
        if (!err)
                __dev_set_rx_mode(to);
        netif_addr_unlock(to);
        return err;
}
EXPORT_SYMBOL(dev_mc_sync);

/**
 *        dev_mc_sync_multiple - Synchronize device's multicast list to another
 *        device, but allow for multiple calls to sync to multiple devices.
 *        @to: destination device
 *        @from: source device
 *
 *        Add newly added addresses to the destination device and release
 *        addresses that have no users left. The source device must be
 *        locked by netif_addr_lock_bh.
 *
 *        This function is intended to be called from the ndo_set_rx_mode
 *        function of layered software devices.  It allows for a single
 *        source device to be synced to multiple destination devices.
 */
int dev_mc_sync_multiple(struct net_device *to, struct net_device *from)
{
        int err = 0;

        if (to->addr_len != from->addr_len)
                return -EINVAL;

        netif_addr_lock(to);
        err = __hw_addr_sync_multiple(&to->mc, &from->mc, to->addr_len);
        if (!err)
                __dev_set_rx_mode(to);
        netif_addr_unlock(to);
        return err;
}
EXPORT_SYMBOL(dev_mc_sync_multiple);

/**
 *        dev_mc_unsync - Remove synchronized addresses from the destination device
 *        @to: destination device
 *        @from: source device
 *
 *        Remove all addresses that were added to the destination device by
 *        dev_mc_sync(). This function is intended to be called from the
 *        dev->stop function of layered software devices.
 */
void dev_mc_unsync(struct net_device *to, struct net_device *from)
{
        if (to->addr_len != from->addr_len)
                return;

        /* See the above comments inside dev_uc_unsync(). */
        netif_addr_lock_bh(from);
        netif_addr_lock(to);
        __hw_addr_unsync(&to->mc, &from->mc, to->addr_len);
        __dev_set_rx_mode(to);
        netif_addr_unlock(to);
        netif_addr_unlock_bh(from);
}
EXPORT_SYMBOL(dev_mc_unsync);

/**
 *        dev_mc_flush - Flush multicast addresses
 *        @dev: device
 *
 *        Flush multicast addresses.
 */
void dev_mc_flush(struct net_device *dev)
{
        netif_addr_lock_bh(dev);
        __hw_addr_flush(&dev->mc);
        netif_addr_unlock_bh(dev);
}
EXPORT_SYMBOL(dev_mc_flush);

/**
 *        dev_mc_init - Init multicast address list
 *        @dev: device
 *
 *        Init multicast address list.
 */
void dev_mc_init(struct net_device *dev)
{
        __hw_addr_init(&dev->mc);
}
EXPORT_SYMBOL(dev_mc_init);
























































































    6 












    1 









































































   13 





    9 







   13 
   13 
   13 







   41 





    5 




   41 






    1 






































   22 

   59 

   22 






































    6 



















































































































































































































    2 























































    1 































































































































































































   81 
    4 


























































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __NET_PKT_CLS_H
#define __NET_PKT_CLS_H

#include <linux/pkt_cls.h>
#include <linux/workqueue.h>
#include <net/sch_generic.h>
#include <net/act_api.h>
#include <net/net_namespace.h>

/* TC action not accessible from user space */
#define TC_ACT_CONSUMED                (TC_ACT_VALUE_MAX + 1)

/* Basic packet classifier frontend definitions. */

struct tcf_walker {
        int        stop;
        int        skip;
        int        count;
        bool        nonempty;
        unsigned long cookie;
        int        (*fn)(struct tcf_proto *, void *node, struct tcf_walker *);
};

int register_tcf_proto_ops(struct tcf_proto_ops *ops);
void unregister_tcf_proto_ops(struct tcf_proto_ops *ops);
#define NET_CLS_ALIAS_PREFIX "net-cls-"
#define MODULE_ALIAS_NET_CLS(kind)        MODULE_ALIAS(NET_CLS_ALIAS_PREFIX kind)

struct tcf_block_ext_info {
        enum flow_block_binder_type binder_type;
        tcf_chain_head_change_t *chain_head_change;
        void *chain_head_change_priv;
        u32 block_index;
};

struct tcf_qevent {
        struct tcf_block        *block;
        struct tcf_block_ext_info info;
        struct tcf_proto __rcu *filter_chain;
};

struct tcf_block_cb;
bool tcf_queue_work(struct rcu_work *rwork, work_func_t func);

#ifdef CONFIG_NET_CLS
struct tcf_chain *tcf_chain_get_by_act(struct tcf_block *block,
                                       u32 chain_index);
void tcf_chain_put_by_act(struct tcf_chain *chain);
struct tcf_chain *tcf_get_next_chain(struct tcf_block *block,
                                     struct tcf_chain *chain);
struct tcf_proto *tcf_get_next_proto(struct tcf_chain *chain,
                                     struct tcf_proto *tp);
void tcf_block_netif_keep_dst(struct tcf_block *block);
int tcf_block_get(struct tcf_block **p_block,
                  struct tcf_proto __rcu **p_filter_chain, struct Qdisc *q,
                  struct netlink_ext_ack *extack);
int tcf_block_get_ext(struct tcf_block **p_block, struct Qdisc *q,
                      struct tcf_block_ext_info *ei,
                      struct netlink_ext_ack *extack);
void tcf_block_put(struct tcf_block *block);
void tcf_block_put_ext(struct tcf_block *block, struct Qdisc *q,
                       struct tcf_block_ext_info *ei);
int tcf_exts_init_ex(struct tcf_exts *exts, struct net *net, int action,
                     int police, struct tcf_proto *tp, u32 handle, bool used_action_miss);

static inline bool tcf_block_shared(struct tcf_block *block)
{
        return block->index;
}

static inline bool tcf_block_non_null_shared(struct tcf_block *block)
{
        return block && block->index;
}

#ifdef CONFIG_NET_CLS_ACT
DECLARE_STATIC_KEY_FALSE(tcf_bypass_check_needed_key);

static inline bool tcf_block_bypass_sw(struct tcf_block *block)
{
        return block && block->bypass_wanted;
}
#endif

static inline struct Qdisc *tcf_block_q(struct tcf_block *block)
{
        WARN_ON(tcf_block_shared(block));
        return block->q;
}

int tcf_classify(struct sk_buff *skb,
                 const struct tcf_block *block,
                 const struct tcf_proto *tp, struct tcf_result *res,
                 bool compat_mode);

static inline bool tc_cls_stats_dump(struct tcf_proto *tp,
                                     struct tcf_walker *arg,
                                     void *filter)
{
        if (arg->count >= arg->skip && arg->fn(tp, filter, arg) < 0) {
                arg->stop = 1;
                return false;
        }

        arg->count++;
        return true;
}

#else
static inline bool tcf_block_shared(struct tcf_block *block)
{
        return false;
}

static inline bool tcf_block_non_null_shared(struct tcf_block *block)
{
        return false;
}

static inline
int tcf_block_get(struct tcf_block **p_block,
                  struct tcf_proto __rcu **p_filter_chain, struct Qdisc *q,
                  struct netlink_ext_ack *extack)
{
        return 0;
}

static inline
int tcf_block_get_ext(struct tcf_block **p_block, struct Qdisc *q,
                      struct tcf_block_ext_info *ei,
                      struct netlink_ext_ack *extack)
{
        return 0;
}

static inline void tcf_block_put(struct tcf_block *block)
{
}

static inline
void tcf_block_put_ext(struct tcf_block *block, struct Qdisc *q,
                       struct tcf_block_ext_info *ei)
{
}

static inline struct Qdisc *tcf_block_q(struct tcf_block *block)
{
        return NULL;
}

static inline int tcf_classify(struct sk_buff *skb,
                               const struct tcf_block *block,
                               const struct tcf_proto *tp,
                               struct tcf_result *res, bool compat_mode)
{
        return TC_ACT_UNSPEC;
}

#endif

static inline unsigned long
__cls_set_class(unsigned long *clp, unsigned long cl)
{
        return xchg(clp, cl);
}

static inline void
__tcf_bind_filter(struct Qdisc *q, struct tcf_result *r, unsigned long base)
{
        unsigned long cl;

        cl = q->ops->cl_ops->bind_tcf(q, base, r->classid);
        cl = __cls_set_class(&r->class, cl);
        if (cl)
                q->ops->cl_ops->unbind_tcf(q, cl);
}

static inline void
tcf_bind_filter(struct tcf_proto *tp, struct tcf_result *r, unsigned long base)
{
        struct Qdisc *q = tp->chain->block->q;

        /* Check q as it is not set for shared blocks. In that case,
         * setting class is not supported.
         */
        if (!q)
                return;
        sch_tree_lock(q);
        __tcf_bind_filter(q, r, base);
        sch_tree_unlock(q);
}

static inline void
__tcf_unbind_filter(struct Qdisc *q, struct tcf_result *r)
{
        unsigned long cl;

        if ((cl = __cls_set_class(&r->class, 0)) != 0)
                q->ops->cl_ops->unbind_tcf(q, cl);
}

static inline void
tcf_unbind_filter(struct tcf_proto *tp, struct tcf_result *r)
{
        struct Qdisc *q = tp->chain->block->q;

        if (!q)
                return;
        __tcf_unbind_filter(q, r);
}

static inline void tc_cls_bind_class(u32 classid, unsigned long cl,
                                     void *q, struct tcf_result *res,
                                     unsigned long base)
{
        if (res->classid == classid) {
                if (cl)
                        __tcf_bind_filter(q, res, base);
                else
                        __tcf_unbind_filter(q, res);
        }
}

struct tcf_exts {
#ifdef CONFIG_NET_CLS_ACT
        __u32        type; /* for backward compat(TCA_OLD_COMPAT) */
        int nr_actions;
        struct tc_action **actions;
        struct net        *net;
        netns_tracker        ns_tracker;
        struct tcf_exts_miss_cookie_node *miss_cookie_node;
#endif
        /* Map to export classifier specific extension TLV types to the
         * generic extensions API. Unsupported extensions must be set to 0.
         */
        int action;
        int police;
};

static inline int tcf_exts_init(struct tcf_exts *exts, struct net *net,
                                int action, int police)
{
#ifdef CONFIG_NET_CLS
        return tcf_exts_init_ex(exts, net, action, police, NULL, 0, false);
#else
        return -EOPNOTSUPP;
#endif
}

/* Return false if the netns is being destroyed in cleanup_net(). Callers
 * need to do cleanup synchronously in this case, otherwise may race with
 * tc_action_net_exit(). Return true for other cases.
 */
static inline bool tcf_exts_get_net(struct tcf_exts *exts)
{
#ifdef CONFIG_NET_CLS_ACT
        exts->net = maybe_get_net(exts->net);
        if (exts->net)
                netns_tracker_alloc(exts->net, &exts->ns_tracker, GFP_KERNEL);
        return exts->net != NULL;
#else
        return true;
#endif
}

static inline void tcf_exts_put_net(struct tcf_exts *exts)
{
#ifdef CONFIG_NET_CLS_ACT
        if (exts->net)
                put_net_track(exts->net, &exts->ns_tracker);
#endif
}

#ifdef CONFIG_NET_CLS_ACT
#define tcf_exts_for_each_action(i, a, exts) \
        for (i = 0; i < TCA_ACT_MAX_PRIO && ((a) = (exts)->actions[i]); i++)
#else
#define tcf_exts_for_each_action(i, a, exts) \
        for (; 0; (void)(i), (void)(a), (void)(exts))
#endif

#define tcf_act_for_each_action(i, a, actions) \
        for (i = 0; i < TCA_ACT_MAX_PRIO && ((a) = actions[i]); i++)

static inline bool tc_act_in_hw(struct tc_action *act)
{
        return !!act->in_hw_count;
}

static inline void
tcf_exts_hw_stats_update(const struct tcf_exts *exts,
                         struct flow_stats *stats,
                         bool use_act_stats)
{
#ifdef CONFIG_NET_CLS_ACT
        int i;

        for (i = 0; i < exts->nr_actions; i++) {
                struct tc_action *a = exts->actions[i];

                if (use_act_stats || tc_act_in_hw(a)) {
                        if (!tcf_action_update_hw_stats(a))
                                continue;
                }

                preempt_disable();
                tcf_action_stats_update(a, stats->bytes, stats->pkts, stats->drops,
                                        stats->lastused, true);
                preempt_enable();

                a->used_hw_stats = stats->used_hw_stats;
                a->used_hw_stats_valid = stats->used_hw_stats_valid;
        }
#endif
}

/**
 * tcf_exts_has_actions - check if at least one action is present
 * @exts: tc filter extensions handle
 *
 * Returns true if at least one action is present.
 */
static inline bool tcf_exts_has_actions(struct tcf_exts *exts)
{
#ifdef CONFIG_NET_CLS_ACT
        return exts->nr_actions;
#else
        return false;
#endif
}

/**
 * tcf_exts_exec - execute tc filter extensions
 * @skb: socket buffer
 * @exts: tc filter extensions handle
 * @res: desired result
 *
 * Executes all configured extensions. Returns TC_ACT_OK on a normal execution,
 * a negative number if the filter must be considered unmatched or
 * a positive action code (TC_ACT_*) which must be returned to the
 * underlying layer.
 */
static inline int
tcf_exts_exec(struct sk_buff *skb, struct tcf_exts *exts,
              struct tcf_result *res)
{
#ifdef CONFIG_NET_CLS_ACT
        return tcf_action_exec(skb, exts->actions, exts->nr_actions, res);
#endif
        return TC_ACT_OK;
}

static inline int
tcf_exts_exec_ex(struct sk_buff *skb, struct tcf_exts *exts, int act_index,
                 struct tcf_result *res)
{
#ifdef CONFIG_NET_CLS_ACT
        return tcf_action_exec(skb, exts->actions + act_index,
                               exts->nr_actions - act_index, res);
#else
        return TC_ACT_OK;
#endif
}

int tcf_exts_validate(struct net *net, struct tcf_proto *tp,
                      struct nlattr **tb, struct nlattr *rate_tlv,
                      struct tcf_exts *exts, u32 flags,
                      struct netlink_ext_ack *extack);
int tcf_exts_validate_ex(struct net *net, struct tcf_proto *tp, struct nlattr **tb,
                         struct nlattr *rate_tlv, struct tcf_exts *exts,
                         u32 flags, u32 fl_flags, struct netlink_ext_ack *extack);
void tcf_exts_destroy(struct tcf_exts *exts);
void tcf_exts_change(struct tcf_exts *dst, struct tcf_exts *src);
int tcf_exts_dump(struct sk_buff *skb, struct tcf_exts *exts);
int tcf_exts_terse_dump(struct sk_buff *skb, struct tcf_exts *exts);
int tcf_exts_dump_stats(struct sk_buff *skb, struct tcf_exts *exts);

/**
 * struct tcf_pkt_info - packet information
 *
 * @ptr: start of the pkt data
 * @nexthdr: offset of the next header
 */
struct tcf_pkt_info {
        unsigned char *                ptr;
        int                        nexthdr;
};

#ifdef CONFIG_NET_EMATCH

struct tcf_ematch_ops;

/**
 * struct tcf_ematch - extended match (ematch)
 * 
 * @matchid: identifier to allow userspace to reidentify a match
 * @flags: flags specifying attributes and the relation to other matches
 * @ops: the operations lookup table of the corresponding ematch module
 * @datalen: length of the ematch specific configuration data
 * @data: ematch specific data
 * @net: the network namespace
 */
struct tcf_ematch {
        struct tcf_ematch_ops * ops;
        unsigned long                data;
        unsigned int                datalen;
        u16                        matchid;
        u16                        flags;
        struct net                *net;
};

static inline int tcf_em_is_container(struct tcf_ematch *em)
{
        return !em->ops;
}

static inline int tcf_em_is_simple(struct tcf_ematch *em)
{
        return em->flags & TCF_EM_SIMPLE;
}

static inline int tcf_em_is_inverted(struct tcf_ematch *em)
{
        return em->flags & TCF_EM_INVERT;
}

static inline int tcf_em_last_match(struct tcf_ematch *em)
{
        return (em->flags & TCF_EM_REL_MASK) == TCF_EM_REL_END;
}

static inline int tcf_em_early_end(struct tcf_ematch *em, int result)
{
        if (tcf_em_last_match(em))
                return 1;

        if (result == 0 && em->flags & TCF_EM_REL_AND)
                return 1;

        if (result != 0 && em->flags & TCF_EM_REL_OR)
                return 1;

        return 0;
}
        
/**
 * struct tcf_ematch_tree - ematch tree handle
 *
 * @hdr: ematch tree header supplied by userspace
 * @matches: array of ematches
 */
struct tcf_ematch_tree {
        struct tcf_ematch_tree_hdr hdr;
        struct tcf_ematch *        matches;
        
};

/**
 * struct tcf_ematch_ops - ematch module operations
 * 
 * @kind: identifier (kind) of this ematch module
 * @datalen: length of expected configuration data (optional)
 * @change: called during validation (optional)
 * @match: called during ematch tree evaluation, must return 1/0
 * @destroy: called during destroyage (optional)
 * @dump: called during dumping process (optional)
 * @owner: owner, must be set to THIS_MODULE
 * @link: link to previous/next ematch module (internal use)
 */
struct tcf_ematch_ops {
        int                        kind;
        int                        datalen;
        int                        (*change)(struct net *net, void *,
                                          int, struct tcf_ematch *);
        int                        (*match)(struct sk_buff *, struct tcf_ematch *,
                                         struct tcf_pkt_info *);
        void                        (*destroy)(struct tcf_ematch *);
        int                        (*dump)(struct sk_buff *, struct tcf_ematch *);
        struct module                *owner;
        struct list_head        link;
};

int tcf_em_register(struct tcf_ematch_ops *);
void tcf_em_unregister(struct tcf_ematch_ops *);
int tcf_em_tree_validate(struct tcf_proto *, struct nlattr *,
                         struct tcf_ematch_tree *);
void tcf_em_tree_destroy(struct tcf_ematch_tree *);
int tcf_em_tree_dump(struct sk_buff *, struct tcf_ematch_tree *, int);
int __tcf_em_tree_match(struct sk_buff *, struct tcf_ematch_tree *,
                        struct tcf_pkt_info *);

/**
 * tcf_em_tree_match - evaluate an ematch tree
 *
 * @skb: socket buffer of the packet in question
 * @tree: ematch tree to be used for evaluation
 * @info: packet information examined by classifier
 *
 * This function matches @skb against the ematch tree in @tree by going
 * through all ematches respecting their logic relations returning
 * as soon as the result is obvious.
 *
 * Returns 1 if the ematch tree as-one matches, no ematches are configured
 * or ematch is not enabled in the kernel, otherwise 0 is returned.
 */
static inline int tcf_em_tree_match(struct sk_buff *skb,
                                    struct tcf_ematch_tree *tree,
                                    struct tcf_pkt_info *info)
{
        if (tree->hdr.nmatches)
                return __tcf_em_tree_match(skb, tree, info);
        else
                return 1;
}

#define MODULE_ALIAS_TCF_EMATCH(kind)        MODULE_ALIAS("ematch-kind-" __stringify(kind))

#else /* CONFIG_NET_EMATCH */

struct tcf_ematch_tree {
};

#define tcf_em_tree_validate(tp, tb, t) ((void)(t), 0)
#define tcf_em_tree_destroy(t) do { (void)(t); } while(0)
#define tcf_em_tree_dump(skb, t, tlv) (0)
#define tcf_em_tree_match(skb, t, info) ((void)(info), 1)

#endif /* CONFIG_NET_EMATCH */

static inline unsigned char * tcf_get_base_ptr(struct sk_buff *skb, int layer)
{
        switch (layer) {
                case TCF_LAYER_LINK:
                        return skb_mac_header(skb);
                case TCF_LAYER_NETWORK:
                        return skb_network_header(skb);
                case TCF_LAYER_TRANSPORT:
                        return skb_transport_header(skb);
        }

        return NULL;
}

static inline int tcf_valid_offset(const struct sk_buff *skb,
                                   const unsigned char *ptr, const int len)
{
        return likely((ptr + len) <= skb_tail_pointer(skb) &&
                      ptr >= skb->head &&
                      (ptr <= (ptr + len)));
}

static inline int
tcf_change_indev(struct net *net, struct nlattr *indev_tlv,
                 struct netlink_ext_ack *extack)
{
        char indev[IFNAMSIZ];
        struct net_device *dev;

        if (nla_strscpy(indev, indev_tlv, IFNAMSIZ) < 0) {
                NL_SET_ERR_MSG_ATTR(extack, indev_tlv,
                                    "Interface name too long");
                return -EINVAL;
        }
        dev = __dev_get_by_name(net, indev);
        if (!dev) {
                NL_SET_ERR_MSG_ATTR(extack, indev_tlv,
                                    "Network device not found");
                return -ENODEV;
        }
        return dev->ifindex;
}

static inline bool
tcf_match_indev(struct sk_buff *skb, int ifindex)
{
        if (!ifindex)
                return true;
        if  (!skb->skb_iif)
                return false;
        return ifindex == skb->skb_iif;
}

int tc_setup_offload_action(struct flow_action *flow_action,
                            const struct tcf_exts *exts,
                            struct netlink_ext_ack *extack);
void tc_cleanup_offload_action(struct flow_action *flow_action);
int tc_setup_action(struct flow_action *flow_action,
                    struct tc_action *actions[],
                    u32 miss_cookie_base,
                    struct netlink_ext_ack *extack);

int tc_setup_cb_call(struct tcf_block *block, enum tc_setup_type type,
                     void *type_data, bool err_stop, bool rtnl_held);
int tc_setup_cb_add(struct tcf_block *block, struct tcf_proto *tp,
                    enum tc_setup_type type, void *type_data, bool err_stop,
                    u32 *flags, unsigned int *in_hw_count, bool rtnl_held);
int tc_setup_cb_replace(struct tcf_block *block, struct tcf_proto *tp,
                        enum tc_setup_type type, void *type_data, bool err_stop,
                        u32 *old_flags, unsigned int *old_in_hw_count,
                        u32 *new_flags, unsigned int *new_in_hw_count,
                        bool rtnl_held);
int tc_setup_cb_destroy(struct tcf_block *block, struct tcf_proto *tp,
                        enum tc_setup_type type, void *type_data, bool err_stop,
                        u32 *flags, unsigned int *in_hw_count, bool rtnl_held);
int tc_setup_cb_reoffload(struct tcf_block *block, struct tcf_proto *tp,
                          bool add, flow_setup_cb_t *cb,
                          enum tc_setup_type type, void *type_data,
                          void *cb_priv, u32 *flags, unsigned int *in_hw_count);
unsigned int tcf_exts_num_actions(struct tcf_exts *exts);

#ifdef CONFIG_NET_CLS_ACT
int tcf_qevent_init(struct tcf_qevent *qe, struct Qdisc *sch,
                    enum flow_block_binder_type binder_type,
                    struct nlattr *block_index_attr,
                    struct netlink_ext_ack *extack);
void tcf_qevent_destroy(struct tcf_qevent *qe, struct Qdisc *sch);
int tcf_qevent_validate_change(struct tcf_qevent *qe, struct nlattr *block_index_attr,
                               struct netlink_ext_ack *extack);
struct sk_buff *tcf_qevent_handle(struct tcf_qevent *qe, struct Qdisc *sch, struct sk_buff *skb,
                                  struct sk_buff **to_free, int *ret);
int tcf_qevent_dump(struct sk_buff *skb, int attr_name, struct tcf_qevent *qe);
#else
static inline int tcf_qevent_init(struct tcf_qevent *qe, struct Qdisc *sch,
                                  enum flow_block_binder_type binder_type,
                                  struct nlattr *block_index_attr,
                                  struct netlink_ext_ack *extack)
{
        return 0;
}

static inline void tcf_qevent_destroy(struct tcf_qevent *qe, struct Qdisc *sch)
{
}

static inline int tcf_qevent_validate_change(struct tcf_qevent *qe, struct nlattr *block_index_attr,
                                             struct netlink_ext_ack *extack)
{
        return 0;
}

static inline struct sk_buff *
tcf_qevent_handle(struct tcf_qevent *qe, struct Qdisc *sch, struct sk_buff *skb,
                  struct sk_buff **to_free, int *ret)
{
        return skb;
}

static inline int tcf_qevent_dump(struct sk_buff *skb, int attr_name, struct tcf_qevent *qe)
{
        return 0;
}
#endif

struct tc_cls_u32_knode {
        struct tcf_exts *exts;
        struct tcf_result *res;
        struct tc_u32_sel *sel;
        u32 handle;
        u32 val;
        u32 mask;
        u32 link_handle;
        u8 fshift;
};

struct tc_cls_u32_hnode {
        u32 handle;
        u32 prio;
        unsigned int divisor;
};

enum tc_clsu32_command {
        TC_CLSU32_NEW_KNODE,
        TC_CLSU32_REPLACE_KNODE,
        TC_CLSU32_DELETE_KNODE,
        TC_CLSU32_NEW_HNODE,
        TC_CLSU32_REPLACE_HNODE,
        TC_CLSU32_DELETE_HNODE,
};

struct tc_cls_u32_offload {
        struct flow_cls_common_offload common;
        /* knode values */
        enum tc_clsu32_command command;
        union {
                struct tc_cls_u32_knode knode;
                struct tc_cls_u32_hnode hnode;
        };
};

static inline bool tc_can_offload(const struct net_device *dev)
{
        return dev->features & NETIF_F_HW_TC;
}

static inline bool tc_can_offload_extack(const struct net_device *dev,
                                         struct netlink_ext_ack *extack)
{
        bool can = tc_can_offload(dev);

        if (!can)
                NL_SET_ERR_MSG(extack, "TC offload is disabled on net device");

        return can;
}

static inline bool
tc_cls_can_offload_and_chain0(const struct net_device *dev,
                              struct flow_cls_common_offload *common)
{
        if (!tc_can_offload_extack(dev, common->extack))
                return false;
        if (common->chain_index) {
                NL_SET_ERR_MSG(common->extack,
                               "Driver supports only offload of chain 0");
                return false;
        }
        return true;
}

static inline bool tc_skip_hw(u32 flags)
{
        return (flags & TCA_CLS_FLAGS_SKIP_HW) ? true : false;
}

static inline bool tc_skip_sw(u32 flags)
{
        return (flags & TCA_CLS_FLAGS_SKIP_SW) ? true : false;
}

/* SKIP_HW and SKIP_SW are mutually exclusive flags. */
static inline bool tc_flags_valid(u32 flags)
{
        if (flags & ~(TCA_CLS_FLAGS_SKIP_HW | TCA_CLS_FLAGS_SKIP_SW |
                      TCA_CLS_FLAGS_VERBOSE))
                return false;

        flags &= TCA_CLS_FLAGS_SKIP_HW | TCA_CLS_FLAGS_SKIP_SW;
        if (!(flags ^ (TCA_CLS_FLAGS_SKIP_HW | TCA_CLS_FLAGS_SKIP_SW)))
                return false;

        return true;
}

static inline bool tc_in_hw(u32 flags)
{
        return (flags & TCA_CLS_FLAGS_IN_HW) ? true : false;
}

static inline void
tc_cls_common_offload_init(struct flow_cls_common_offload *cls_common,
                           const struct tcf_proto *tp, u32 flags,
                           struct netlink_ext_ack *extack)
{
        cls_common->chain_index = tp->chain->index;
        cls_common->protocol = tp->protocol;
        cls_common->prio = tp->prio >> 16;
        cls_common->skip_sw = tc_skip_sw(flags);
        if (tc_skip_sw(flags) || flags & TCA_CLS_FLAGS_VERBOSE)
                cls_common->extack = extack;
}

#if IS_ENABLED(CONFIG_NET_TC_SKB_EXT)
static inline struct tc_skb_ext *tc_skb_ext_alloc(struct sk_buff *skb)
{
        struct tc_skb_ext *tc_skb_ext = skb_ext_add(skb, TC_SKB_EXT);

        if (tc_skb_ext)
                memset(tc_skb_ext, 0, sizeof(*tc_skb_ext));
        return tc_skb_ext;
}
#endif

enum tc_matchall_command {
        TC_CLSMATCHALL_REPLACE,
        TC_CLSMATCHALL_DESTROY,
        TC_CLSMATCHALL_STATS,
};

struct tc_cls_matchall_offload {
        struct flow_cls_common_offload common;
        enum