Total coverage: 213210 (12%)of 1884075 




































    1 




  112 














  247 

   73 











   76 



















  345 

   27 






1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __SOCK_DIAG_H__
#define __SOCK_DIAG_H__

#include <linux/netlink.h>
#include <linux/user_namespace.h>
#include <net/net_namespace.h>
#include <net/sock.h>
#include <uapi/linux/sock_diag.h>

struct sk_buff;
struct nlmsghdr;
struct sock;

struct sock_diag_handler {
        struct module *owner;
        __u8 family;
        int (*dump)(struct sk_buff *skb, struct nlmsghdr *nlh);
        int (*get_info)(struct sk_buff *skb, struct sock *sk);
        int (*destroy)(struct sk_buff *skb, struct nlmsghdr *nlh);
};

int sock_diag_register(const struct sock_diag_handler *h);
void sock_diag_unregister(const struct sock_diag_handler *h);

struct sock_diag_inet_compat {
        struct module *owner;
        int (*fn)(struct sk_buff *skb, struct nlmsghdr *nlh);
};

void sock_diag_register_inet_compat(const struct sock_diag_inet_compat *ptr);
void sock_diag_unregister_inet_compat(const struct sock_diag_inet_compat *ptr);

u64 __sock_gen_cookie(struct sock *sk);

static inline u64 sock_gen_cookie(struct sock *sk)
{
        u64 cookie;

        preempt_disable();
        cookie = __sock_gen_cookie(sk);
        preempt_enable();

        return cookie;
}

int sock_diag_check_cookie(struct sock *sk, const __u32 *cookie);
void sock_diag_save_cookie(struct sock *sk, __u32 *cookie);

int sock_diag_put_meminfo(struct sock *sk, struct sk_buff *skb, int attr);
int sock_diag_put_filterinfo(bool may_report_filterinfo, struct sock *sk,
                             struct sk_buff *skb, int attrtype);

static inline
enum sknetlink_groups sock_diag_destroy_group(const struct sock *sk)
{
        switch (sk->sk_family) {
        case AF_INET:
                if (sk->sk_type == SOCK_RAW)
                        return SKNLGRP_NONE;

                switch (sk->sk_protocol) {
                case IPPROTO_TCP:
                        return SKNLGRP_INET_TCP_DESTROY;
                case IPPROTO_UDP:
                        return SKNLGRP_INET_UDP_DESTROY;
                default:
                        return SKNLGRP_NONE;
                }
        case AF_INET6:
                if (sk->sk_type == SOCK_RAW)
                        return SKNLGRP_NONE;

                switch (sk->sk_protocol) {
                case IPPROTO_TCP:
                        return SKNLGRP_INET6_TCP_DESTROY;
                case IPPROTO_UDP:
                        return SKNLGRP_INET6_UDP_DESTROY;
                default:
                        return SKNLGRP_NONE;
                }
        default:
                return SKNLGRP_NONE;
        }
}

static inline
bool sock_diag_has_destroy_listeners(const struct sock *sk)
{
        const struct net *n = sock_net(sk);
        const enum sknetlink_groups group = sock_diag_destroy_group(sk);

        return group != SKNLGRP_NONE && n->diag_nlsk &&
                netlink_has_listeners(n->diag_nlsk, group);
}
void sock_diag_broadcast_destroy(struct sock *sk);

int sock_diag_destroy(struct sock *sk, int err);
#endif



























































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
// SPDX-License-Identifier: GPL-2.0-only
/*
 * 32-bit compatibility support for ELF format executables and core dumps.
 *
 * Copyright (C) 2007 Red Hat, Inc.  All rights reserved.
 *
 * Red Hat Author: Roland McGrath.
 *
 * This file is used in a 64-bit kernel that wants to support 32-bit ELF.
 * asm/elf.h is responsible for defining the compat_* and COMPAT_* macros
 * used below, with definitions appropriate for 32-bit ABI compatibility.
 *
 * We use macros to rename the ABI types and machine-dependent
 * functions used in binfmt_elf.c to compat versions.
 */

#include <linux/elfcore-compat.h>
#include <linux/time.h>

#define ELF_COMPAT        1

/*
 * Rename the basic ELF layout types to refer to the 32-bit class of files.
 */
#undef        ELF_CLASS
#define ELF_CLASS        ELFCLASS32

#undef        elfhdr
#undef        elf_phdr
#undef        elf_shdr
#undef        elf_note
#undef        elf_addr_t
#undef        ELF_GNU_PROPERTY_ALIGN
#define elfhdr                elf32_hdr
#define elf_phdr        elf32_phdr
#define elf_shdr        elf32_shdr
#define elf_note        elf32_note
#define elf_addr_t        Elf32_Addr
#define ELF_GNU_PROPERTY_ALIGN        ELF32_GNU_PROPERTY_ALIGN

/*
 * Some data types as stored in coredump.
 */
#define user_long_t                compat_long_t
#define user_siginfo_t                compat_siginfo_t
#define copy_siginfo_to_external        copy_siginfo_to_external32

/*
 * The machine-dependent core note format types are defined in elfcore-compat.h,
 * which requires asm/elf.h to define compat_elf_gregset_t et al.
 */
#define elf_prstatus        compat_elf_prstatus
#define elf_prstatus_common        compat_elf_prstatus_common
#define elf_prpsinfo        compat_elf_prpsinfo

#undef ns_to_kernel_old_timeval
#define ns_to_kernel_old_timeval ns_to_old_timeval32

/*
 * To use this file, asm/elf.h must define compat_elf_check_arch.
 * The other following macros can be defined if the compat versions
 * differ from the native ones, or omitted when they match.
 */

#undef        elf_check_arch
#define        elf_check_arch        compat_elf_check_arch

#ifdef        COMPAT_ELF_PLATFORM
#undef        ELF_PLATFORM
#define        ELF_PLATFORM                COMPAT_ELF_PLATFORM
#endif

#ifdef        COMPAT_ELF_HWCAP
#undef        ELF_HWCAP
#define        ELF_HWCAP                COMPAT_ELF_HWCAP
#endif

#ifdef        COMPAT_ELF_HWCAP2
#undef        ELF_HWCAP2
#define        ELF_HWCAP2                COMPAT_ELF_HWCAP2
#endif

#ifdef        COMPAT_ELF_HWCAP3
#undef        ELF_HWCAP3
#define        ELF_HWCAP3                COMPAT_ELF_HWCAP3
#endif

#ifdef        COMPAT_ELF_HWCAP4
#undef        ELF_HWCAP4
#define        ELF_HWCAP4                COMPAT_ELF_HWCAP4
#endif

#ifdef        COMPAT_ARCH_DLINFO
#undef        ARCH_DLINFO
#define        ARCH_DLINFO                COMPAT_ARCH_DLINFO
#endif

#ifdef        COMPAT_ELF_ET_DYN_BASE
#undef        ELF_ET_DYN_BASE
#define        ELF_ET_DYN_BASE                COMPAT_ELF_ET_DYN_BASE
#endif

#ifdef        COMPAT_ELF_PLAT_INIT
#undef        ELF_PLAT_INIT
#define        ELF_PLAT_INIT                COMPAT_ELF_PLAT_INIT
#endif

#ifdef        COMPAT_SET_PERSONALITY
#undef        SET_PERSONALITY
#define        SET_PERSONALITY                COMPAT_SET_PERSONALITY
#endif

#ifdef        compat_start_thread
#define COMPAT_START_THREAD(ex, regs, new_ip, new_sp)        \
        compat_start_thread(regs, new_ip, new_sp)
#endif

#ifdef        COMPAT_START_THREAD
#undef        START_THREAD
#define START_THREAD                COMPAT_START_THREAD
#endif

#ifdef compat_arch_setup_additional_pages
#define COMPAT_ARCH_SETUP_ADDITIONAL_PAGES(bprm, ex, interpreter) \
        compat_arch_setup_additional_pages(bprm, interpreter)
#endif

#ifdef        COMPAT_ARCH_SETUP_ADDITIONAL_PAGES
#undef        ARCH_HAS_SETUP_ADDITIONAL_PAGES
#define ARCH_HAS_SETUP_ADDITIONAL_PAGES 1
#undef        ARCH_SETUP_ADDITIONAL_PAGES
#define        ARCH_SETUP_ADDITIONAL_PAGES COMPAT_ARCH_SETUP_ADDITIONAL_PAGES
#endif

#ifdef        compat_elf_read_implies_exec
#undef        elf_read_implies_exec
#define        elf_read_implies_exec compat_elf_read_implies_exec
#endif

/*
 * Rename a few of the symbols that binfmt_elf.c will define.
 * These are all local so the names don't really matter, but it
 * might make some debugging less confusing not to duplicate them.
 */
#define elf_format                compat_elf_format
#define init_elf_binfmt                init_compat_elf_binfmt
#define exit_elf_binfmt                exit_compat_elf_binfmt
#define binfmt_elf_test_cases        compat_binfmt_elf_test_cases
#define binfmt_elf_test_suite        compat_binfmt_elf_test_suite

/*
 * We share all the actual code with the native (64-bit) version.
 */
#include "binfmt_elf.c"










































































































    3 

    3 































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
// SPDX-License-Identifier: GPL-2.0-only
/*
 *  Copyright (c) 2013
 *  Minchan Kim <minchan@kernel.org>
 */
#include <linux/types.h>
#include <linux/mutex.h>
#include <linux/slab.h>
#include <linux/bio.h>
#include <linux/sched.h>
#include <linux/wait.h>
#include <linux/cpumask.h>

#include "squashfs_fs.h"
#include "squashfs_fs_sb.h"
#include "decompressor.h"
#include "squashfs.h"

/*
 * This file implements multi-threaded decompression in the
 * decompressor framework
 */


/*
 * The reason that multiply two is that a CPU can request new I/O
 * while it is waiting previous request.
 */
#define MAX_DECOMPRESSOR        (num_online_cpus() * 2)


static int squashfs_max_decompressors(void)
{
        return MAX_DECOMPRESSOR;
}

struct squashfs_stream {
        void                        *comp_opts;
        struct list_head        strm_list;
        struct mutex                mutex;
        int                        avail_decomp;
        wait_queue_head_t        wait;
};


struct decomp_stream {
        void *stream;
        struct list_head list;
};


static void put_decomp_stream(struct decomp_stream *decomp_strm,
                                struct squashfs_stream *stream)
{
        mutex_lock(&stream->mutex);
        list_add(&decomp_strm->list, &stream->strm_list);
        mutex_unlock(&stream->mutex);
        wake_up(&stream->wait);
}

static void *squashfs_decompressor_create(struct squashfs_sb_info *msblk,
                                void *comp_opts)
{
        struct squashfs_stream *stream;
        struct decomp_stream *decomp_strm = NULL;
        int err = -ENOMEM;

        stream = kzalloc(sizeof(*stream), GFP_KERNEL);
        if (!stream)
                goto out;

        stream->comp_opts = comp_opts;
        mutex_init(&stream->mutex);
        INIT_LIST_HEAD(&stream->strm_list);
        init_waitqueue_head(&stream->wait);

        /*
         * We should have a decompressor at least as default
         * so if we fail to allocate new decompressor dynamically,
         * we could always fall back to default decompressor and
         * file system works.
         */
        decomp_strm = kmalloc(sizeof(*decomp_strm), GFP_KERNEL);
        if (!decomp_strm)
                goto out;

        decomp_strm->stream = msblk->decompressor->init(msblk,
                                                stream->comp_opts);
        if (IS_ERR(decomp_strm->stream)) {
                err = PTR_ERR(decomp_strm->stream);
                goto out;
        }

        list_add(&decomp_strm->list, &stream->strm_list);
        stream->avail_decomp = 1;
        return stream;

out:
        kfree(decomp_strm);
        kfree(stream);
        return ERR_PTR(err);
}


static void squashfs_decompressor_destroy(struct squashfs_sb_info *msblk)
{
        struct squashfs_stream *stream = msblk->stream;
        if (stream) {
                struct decomp_stream *decomp_strm;

                while (!list_empty(&stream->strm_list)) {
                        decomp_strm = list_entry(stream->strm_list.prev,
                                                struct decomp_stream, list);
                        list_del(&decomp_strm->list);
                        msblk->decompressor->free(decomp_strm->stream);
                        kfree(decomp_strm);
                        stream->avail_decomp--;
                }
                WARN_ON(stream->avail_decomp);
                kfree(stream->comp_opts);
                kfree(stream);
        }
}


static struct decomp_stream *get_decomp_stream(struct squashfs_sb_info *msblk,
                                        struct squashfs_stream *stream)
{
        struct decomp_stream *decomp_strm;

        while (1) {
                mutex_lock(&stream->mutex);

                /* There is available decomp_stream */
                if (!list_empty(&stream->strm_list)) {
                        decomp_strm = list_entry(stream->strm_list.prev,
                                struct decomp_stream, list);
                        list_del(&decomp_strm->list);
                        mutex_unlock(&stream->mutex);
                        break;
                }

                /*
                 * If there is no available decomp and already full,
                 * let's wait for releasing decomp from other users.
                 */
                if (stream->avail_decomp >= msblk->max_thread_num)
                        goto wait;

                /* Let's allocate new decomp */
                decomp_strm = kmalloc(sizeof(*decomp_strm), GFP_KERNEL);
                if (!decomp_strm)
                        goto wait;

                decomp_strm->stream = msblk->decompressor->init(msblk,
                                                stream->comp_opts);
                if (IS_ERR(decomp_strm->stream)) {
                        kfree(decomp_strm);
                        goto wait;
                }

                stream->avail_decomp++;
                WARN_ON(stream->avail_decomp > msblk->max_thread_num);

                mutex_unlock(&stream->mutex);
                break;
wait:
                /*
                 * If system memory is tough, let's for other's
                 * releasing instead of hurting VM because it could
                 * make page cache thrashing.
                 */
                mutex_unlock(&stream->mutex);
                wait_event(stream->wait,
                        !list_empty(&stream->strm_list));
        }

        return decomp_strm;
}


static int squashfs_decompress(struct squashfs_sb_info *msblk, struct bio *bio,
                        int offset, int length,
                        struct squashfs_page_actor *output)
{
        int res;
        struct squashfs_stream *stream = msblk->stream;
        struct decomp_stream *decomp_stream = get_decomp_stream(msblk, stream);
        res = msblk->decompressor->decompress(msblk, decomp_stream->stream,
                bio, offset, length, output);
        put_decomp_stream(decomp_stream, stream);
        if (res < 0)
                ERROR("%s decompression failed, data probably corrupt\n",
                        msblk->decompressor->name);
        return res;
}

const struct squashfs_decompressor_thread_ops squashfs_decompressor_multi = {
        .create = squashfs_decompressor_create,
        .destroy = squashfs_decompressor_destroy,
        .decompress = squashfs_decompress,
        .max_decompressors = squashfs_max_decompressors,
};















































































    1 


    1 





































































































































































































































































    1 















    1 







    1 




































































































































































































































































































































































































































































































































































































































































































































   13 























   13 




   13 






    2 



   13 





    3 


    3 









  287 




    1 










    1 


  290 








    1 




















  285 















  286 























    3 






    1 



    8 






    7 










   12 








   12 
    4 



    2 




    1 


    1 


















    1 






    4 









    2 








    1 
    1 




    3 
    3 
    3 








    3 


    3 









































  214 

























  215 











  125 
  146 


















  192 









  194 












  192 


















  212 




  216 
  216 





































































  116 





  114 
  116 


























  502 






  194 
  191 





  505 
  261 


  271 



  126 





   69 

   15 




   67 


    8 









  126 





   21 

   21 

   19 




































    1 










    1 










    4 


















    1 




    2 




    1 






    1 
    1 




    1 
    1 







































    2 


















    1 



    1 

















































































































    1 








    1 












    1 








    1 






































































































































































































   13 










   13 

    3 









    9 
    6 








    7 





    7 



































































































    2 











































    2 
















    2 
    2 




    1 


    1 












    1 

















































































































































































































































































   19 





















   19 

























































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
// SPDX-License-Identifier: GPL-2.0-only
#include <linux/types.h>
#include <linux/skbuff.h>
#include <linux/socket.h>
#include <linux/sysctl.h>
#include <linux/net.h>
#include <linux/module.h>
#include <linux/if_arp.h>
#include <linux/ipv6.h>
#include <linux/mpls.h>
#include <linux/netconf.h>
#include <linux/nospec.h>
#include <linux/vmalloc.h>
#include <linux/percpu.h>
#include <net/gso.h>
#include <net/ip.h>
#include <net/dst.h>
#include <net/sock.h>
#include <net/arp.h>
#include <net/ip_fib.h>
#include <net/netevent.h>
#include <net/ip_tunnels.h>
#include <net/netns/generic.h>
#if IS_ENABLED(CONFIG_IPV6)
#include <net/ipv6.h>
#endif
#include <net/ipv6_stubs.h>
#include <net/rtnh.h>
#include "internal.h"

/* max memory we will use for mpls_route */
#define MAX_MPLS_ROUTE_MEM        4096

/* Maximum number of labels to look ahead at when selecting a path of
 * a multipath route
 */
#define MAX_MP_SELECT_LABELS 4

#define MPLS_NEIGH_TABLE_UNSPEC (NEIGH_LINK_TABLE + 1)

static int label_limit = (1 << 20) - 1;
static int ttl_max = 255;

#if IS_ENABLED(CONFIG_NET_IP_TUNNEL)
static size_t ipgre_mpls_encap_hlen(struct ip_tunnel_encap *e)
{
        return sizeof(struct mpls_shim_hdr);
}

static const struct ip_tunnel_encap_ops mpls_iptun_ops = {
        .encap_hlen        = ipgre_mpls_encap_hlen,
};

static int ipgre_tunnel_encap_add_mpls_ops(void)
{
        return ip_tunnel_encap_add_ops(&mpls_iptun_ops, TUNNEL_ENCAP_MPLS);
}

static void ipgre_tunnel_encap_del_mpls_ops(void)
{
        ip_tunnel_encap_del_ops(&mpls_iptun_ops, TUNNEL_ENCAP_MPLS);
}
#else
static int ipgre_tunnel_encap_add_mpls_ops(void)
{
        return 0;
}

static void ipgre_tunnel_encap_del_mpls_ops(void)
{
}
#endif

static void rtmsg_lfib(int event, u32 label, struct mpls_route *rt,
                       struct nlmsghdr *nlh, struct net *net, u32 portid,
                       unsigned int nlm_flags);

static struct mpls_route *mpls_route_input_rcu(struct net *net, unsigned index)
{
        struct mpls_route *rt = NULL;

        if (index < net->mpls.platform_labels) {
                struct mpls_route __rcu **platform_label =
                        rcu_dereference_rtnl(net->mpls.platform_label);
                rt = rcu_dereference_rtnl(platform_label[index]);
        }
        return rt;
}

bool mpls_output_possible(const struct net_device *dev)
{
        return dev && (dev->flags & IFF_UP) && netif_carrier_ok(dev);
}
EXPORT_SYMBOL_GPL(mpls_output_possible);

static u8 *__mpls_nh_via(struct mpls_route *rt, struct mpls_nh *nh)
{
        return (u8 *)nh + rt->rt_via_offset;
}

static const u8 *mpls_nh_via(const struct mpls_route *rt,
                             const struct mpls_nh *nh)
{
        return __mpls_nh_via((struct mpls_route *)rt, (struct mpls_nh *)nh);
}

static unsigned int mpls_nh_header_size(const struct mpls_nh *nh)
{
        /* The size of the layer 2.5 labels to be added for this route */
        return nh->nh_labels * sizeof(struct mpls_shim_hdr);
}

unsigned int mpls_dev_mtu(const struct net_device *dev)
{
        /* The amount of data the layer 2 frame can hold */
        return dev->mtu;
}
EXPORT_SYMBOL_GPL(mpls_dev_mtu);

bool mpls_pkt_too_big(const struct sk_buff *skb, unsigned int mtu)
{
        if (skb->len <= mtu)
                return false;

        if (skb_is_gso(skb) && skb_gso_validate_network_len(skb, mtu))
                return false;

        return true;
}
EXPORT_SYMBOL_GPL(mpls_pkt_too_big);

void mpls_stats_inc_outucastpkts(struct net_device *dev,
                                 const struct sk_buff *skb)
{
        struct mpls_dev *mdev;

        if (skb->protocol == htons(ETH_P_MPLS_UC)) {
                mdev = mpls_dev_get(dev);
                if (mdev)
                        MPLS_INC_STATS_LEN(mdev, skb->len,
                                           tx_packets,
                                           tx_bytes);
        } else if (skb->protocol == htons(ETH_P_IP)) {
                IP_UPD_PO_STATS(dev_net(dev), IPSTATS_MIB_OUT, skb->len);
#if IS_ENABLED(CONFIG_IPV6)
        } else if (skb->protocol == htons(ETH_P_IPV6)) {
                struct inet6_dev *in6dev = __in6_dev_get(dev);

                if (in6dev)
                        IP6_UPD_PO_STATS(dev_net(dev), in6dev,
                                         IPSTATS_MIB_OUT, skb->len);
#endif
        }
}
EXPORT_SYMBOL_GPL(mpls_stats_inc_outucastpkts);

static u32 mpls_multipath_hash(struct mpls_route *rt, struct sk_buff *skb)
{
        struct mpls_entry_decoded dec;
        unsigned int mpls_hdr_len = 0;
        struct mpls_shim_hdr *hdr;
        bool eli_seen = false;
        int label_index;
        u32 hash = 0;

        for (label_index = 0; label_index < MAX_MP_SELECT_LABELS;
             label_index++) {
                mpls_hdr_len += sizeof(*hdr);
                if (!pskb_may_pull(skb, mpls_hdr_len))
                        break;

                /* Read and decode the current label */
                hdr = mpls_hdr(skb) + label_index;
                dec = mpls_entry_decode(hdr);

                /* RFC6790 - reserved labels MUST NOT be used as keys
                 * for the load-balancing function
                 */
                if (likely(dec.label >= MPLS_LABEL_FIRST_UNRESERVED)) {
                        hash = jhash_1word(dec.label, hash);

                        /* The entropy label follows the entropy label
                         * indicator, so this means that the entropy
                         * label was just added to the hash - no need to
                         * go any deeper either in the label stack or in the
                         * payload
                         */
                        if (eli_seen)
                                break;
                } else if (dec.label == MPLS_LABEL_ENTROPY) {
                        eli_seen = true;
                }

                if (!dec.bos)
                        continue;

                /* found bottom label; does skb have room for a header? */
                if (pskb_may_pull(skb, mpls_hdr_len + sizeof(struct iphdr))) {
                        const struct iphdr *v4hdr;

                        v4hdr = (const struct iphdr *)(hdr + 1);
                        if (v4hdr->version == 4) {
                                hash = jhash_3words(ntohl(v4hdr->saddr),
                                                    ntohl(v4hdr->daddr),
                                                    v4hdr->protocol, hash);
                        } else if (v4hdr->version == 6 &&
                                   pskb_may_pull(skb, mpls_hdr_len +
                                                 sizeof(struct ipv6hdr))) {
                                const struct ipv6hdr *v6hdr;

                                v6hdr = (const struct ipv6hdr *)(hdr + 1);
                                hash = __ipv6_addr_jhash(&v6hdr->saddr, hash);
                                hash = __ipv6_addr_jhash(&v6hdr->daddr, hash);
                                hash = jhash_1word(v6hdr->nexthdr, hash);
                        }
                }

                break;
        }

        return hash;
}

static struct mpls_nh *mpls_get_nexthop(struct mpls_route *rt, u8 index)
{
        return (struct mpls_nh *)((u8 *)rt->rt_nh + index * rt->rt_nh_size);
}

/* number of alive nexthops (rt->rt_nhn_alive) and the flags for
 * a next hop (nh->nh_flags) are modified by netdev event handlers.
 * Since those fields can change at any moment, use READ_ONCE to
 * access both.
 */
static const struct mpls_nh *mpls_select_multipath(struct mpls_route *rt,
                                                   struct sk_buff *skb)
{
        u32 hash = 0;
        int nh_index = 0;
        int n = 0;
        u8 alive;

        /* No need to look further into packet if there's only
         * one path
         */
        if (rt->rt_nhn == 1)
                return rt->rt_nh;

        alive = READ_ONCE(rt->rt_nhn_alive);
        if (alive == 0)
                return NULL;

        hash = mpls_multipath_hash(rt, skb);
        nh_index = hash % alive;
        if (alive == rt->rt_nhn)
                goto out;
        for_nexthops(rt) {
                unsigned int nh_flags = READ_ONCE(nh->nh_flags);

                if (nh_flags & (RTNH_F_DEAD | RTNH_F_LINKDOWN))
                        continue;
                if (n == nh_index)
                        return nh;
                n++;
        } endfor_nexthops(rt);

out:
        return mpls_get_nexthop(rt, nh_index);
}

static bool mpls_egress(struct net *net, struct mpls_route *rt,
                        struct sk_buff *skb, struct mpls_entry_decoded dec)
{
        enum mpls_payload_type payload_type;
        bool success = false;

        /* The IPv4 code below accesses through the IPv4 header
         * checksum, which is 12 bytes into the packet.
         * The IPv6 code below accesses through the IPv6 hop limit
         * which is 8 bytes into the packet.
         *
         * For all supported cases there should always be at least 12
         * bytes of packet data present.  The IPv4 header is 20 bytes
         * without options and the IPv6 header is always 40 bytes
         * long.
         */
        if (!pskb_may_pull(skb, 12))
                return false;

        payload_type = rt->rt_payload_type;
        if (payload_type == MPT_UNSPEC)
                payload_type = ip_hdr(skb)->version;

        switch (payload_type) {
        case MPT_IPV4: {
                struct iphdr *hdr4 = ip_hdr(skb);
                u8 new_ttl;
                skb->protocol = htons(ETH_P_IP);

                /* If propagating TTL, take the decremented TTL from
                 * the incoming MPLS header, otherwise decrement the
                 * TTL, but only if not 0 to avoid underflow.
                 */
                if (rt->rt_ttl_propagate == MPLS_TTL_PROP_ENABLED ||
                    (rt->rt_ttl_propagate == MPLS_TTL_PROP_DEFAULT &&
                     net->mpls.ip_ttl_propagate))
                        new_ttl = dec.ttl;
                else
                        new_ttl = hdr4->ttl ? hdr4->ttl - 1 : 0;

                csum_replace2(&hdr4->check,
                              htons(hdr4->ttl << 8),
                              htons(new_ttl << 8));
                hdr4->ttl = new_ttl;
                success = true;
                break;
        }
        case MPT_IPV6: {
                struct ipv6hdr *hdr6 = ipv6_hdr(skb);
                skb->protocol = htons(ETH_P_IPV6);

                /* If propagating TTL, take the decremented TTL from
                 * the incoming MPLS header, otherwise decrement the
                 * hop limit, but only if not 0 to avoid underflow.
                 */
                if (rt->rt_ttl_propagate == MPLS_TTL_PROP_ENABLED ||
                    (rt->rt_ttl_propagate == MPLS_TTL_PROP_DEFAULT &&
                     net->mpls.ip_ttl_propagate))
                        hdr6->hop_limit = dec.ttl;
                else if (hdr6->hop_limit)
                        hdr6->hop_limit = hdr6->hop_limit - 1;
                success = true;
                break;
        }
        case MPT_UNSPEC:
                /* Should have decided which protocol it is by now */
                break;
        }

        return success;
}

static int mpls_forward(struct sk_buff *skb, struct net_device *dev,
                        struct packet_type *pt, struct net_device *orig_dev)
{
        struct net *net = dev_net(dev);
        struct mpls_shim_hdr *hdr;
        const struct mpls_nh *nh;
        struct mpls_route *rt;
        struct mpls_entry_decoded dec;
        struct net_device *out_dev;
        struct mpls_dev *out_mdev;
        struct mpls_dev *mdev;
        unsigned int hh_len;
        unsigned int new_header_size;
        unsigned int mtu;
        int err;

        /* Careful this entire function runs inside of an rcu critical section */

        mdev = mpls_dev_get(dev);
        if (!mdev)
                goto drop;

        MPLS_INC_STATS_LEN(mdev, skb->len, rx_packets,
                           rx_bytes);

        if (!mdev->input_enabled) {
                MPLS_INC_STATS(mdev, rx_dropped);
                goto drop;
        }

        if (skb->pkt_type != PACKET_HOST)
                goto err;

        if ((skb = skb_share_check(skb, GFP_ATOMIC)) == NULL)
                goto err;

        if (!pskb_may_pull(skb, sizeof(*hdr)))
                goto err;

        skb_dst_drop(skb);

        /* Read and decode the label */
        hdr = mpls_hdr(skb);
        dec = mpls_entry_decode(hdr);

        rt = mpls_route_input_rcu(net, dec.label);
        if (!rt) {
                MPLS_INC_STATS(mdev, rx_noroute);
                goto drop;
        }

        nh = mpls_select_multipath(rt, skb);
        if (!nh)
                goto err;

        /* Pop the label */
        skb_pull(skb, sizeof(*hdr));
        skb_reset_network_header(skb);

        skb_orphan(skb);

        if (skb_warn_if_lro(skb))
                goto err;

        skb_forward_csum(skb);

        /* Verify ttl is valid */
        if (dec.ttl <= 1)
                goto err;

        /* Find the output device */
        out_dev = nh->nh_dev;
        if (!mpls_output_possible(out_dev))
                goto tx_err;

        /* Verify the destination can hold the packet */
        new_header_size = mpls_nh_header_size(nh);
        mtu = mpls_dev_mtu(out_dev);
        if (mpls_pkt_too_big(skb, mtu - new_header_size))
                goto tx_err;

        hh_len = LL_RESERVED_SPACE(out_dev);
        if (!out_dev->header_ops)
                hh_len = 0;

        /* Ensure there is enough space for the headers in the skb */
        if (skb_cow(skb, hh_len + new_header_size))
                goto tx_err;

        skb->dev = out_dev;
        skb->protocol = htons(ETH_P_MPLS_UC);

        dec.ttl -= 1;
        if (unlikely(!new_header_size && dec.bos)) {
                /* Penultimate hop popping */
                if (!mpls_egress(dev_net(out_dev), rt, skb, dec))
                        goto err;
        } else {
                bool bos;
                int i;
                skb_push(skb, new_header_size);
                skb_reset_network_header(skb);
                /* Push the new labels */
                hdr = mpls_hdr(skb);
                bos = dec.bos;
                for (i = nh->nh_labels - 1; i >= 0; i--) {
                        hdr[i] = mpls_entry_encode(nh->nh_label[i],
                                                   dec.ttl, 0, bos);
                        bos = false;
                }
        }

        mpls_stats_inc_outucastpkts(out_dev, skb);

        /* If via wasn't specified then send out using device address */
        if (nh->nh_via_table == MPLS_NEIGH_TABLE_UNSPEC)
                err = neigh_xmit(NEIGH_LINK_TABLE, out_dev,
                                 out_dev->dev_addr, skb);
        else
                err = neigh_xmit(nh->nh_via_table, out_dev,
                                 mpls_nh_via(rt, nh), skb);
        if (err)
                net_dbg_ratelimited("%s: packet transmission failed: %d\n",
                                    __func__, err);
        return 0;

tx_err:
        out_mdev = out_dev ? mpls_dev_get(out_dev) : NULL;
        if (out_mdev)
                MPLS_INC_STATS(out_mdev, tx_errors);
        goto drop;
err:
        MPLS_INC_STATS(mdev, rx_errors);
drop:
        kfree_skb(skb);
        return NET_RX_DROP;
}

static struct packet_type mpls_packet_type __read_mostly = {
        .type = cpu_to_be16(ETH_P_MPLS_UC),
        .func = mpls_forward,
};

static const struct nla_policy rtm_mpls_policy[RTA_MAX+1] = {
        [RTA_DST]                = { .type = NLA_U32 },
        [RTA_OIF]                = { .type = NLA_U32 },
        [RTA_TTL_PROPAGATE]        = { .type = NLA_U8 },
};

struct mpls_route_config {
        u32                        rc_protocol;
        u32                        rc_ifindex;
        u8                        rc_via_table;
        u8                        rc_via_alen;
        u8                        rc_via[MAX_VIA_ALEN];
        u32                        rc_label;
        u8                        rc_ttl_propagate;
        u8                        rc_output_labels;
        u32                        rc_output_label[MAX_NEW_LABELS];
        u32                        rc_nlflags;
        enum mpls_payload_type        rc_payload_type;
        struct nl_info                rc_nlinfo;
        struct rtnexthop        *rc_mp;
        int                        rc_mp_len;
};

/* all nexthops within a route have the same size based on max
 * number of labels and max via length for a hop
 */
static struct mpls_route *mpls_rt_alloc(u8 num_nh, u8 max_alen, u8 max_labels)
{
        u8 nh_size = MPLS_NH_SIZE(max_labels, max_alen);
        struct mpls_route *rt;
        size_t size;

        size = sizeof(*rt) + num_nh * nh_size;
        if (size > MAX_MPLS_ROUTE_MEM)
                return ERR_PTR(-EINVAL);

        rt = kzalloc(size, GFP_KERNEL);
        if (!rt)
                return ERR_PTR(-ENOMEM);

        rt->rt_nhn = num_nh;
        rt->rt_nhn_alive = num_nh;
        rt->rt_nh_size = nh_size;
        rt->rt_via_offset = MPLS_NH_VIA_OFF(max_labels);

        return rt;
}

static void mpls_rt_free(struct mpls_route *rt)
{
        if (rt)
                kfree_rcu(rt, rt_rcu);
}

static void mpls_notify_route(struct net *net, unsigned index,
                              struct mpls_route *old, struct mpls_route *new,
                              const struct nl_info *info)
{
        struct nlmsghdr *nlh = info ? info->nlh : NULL;
        unsigned portid = info ? info->portid : 0;
        int event = new ? RTM_NEWROUTE : RTM_DELROUTE;
        struct mpls_route *rt = new ? new : old;
        unsigned nlm_flags = (old && new) ? NLM_F_REPLACE : 0;
        /* Ignore reserved labels for now */
        if (rt && (index >= MPLS_LABEL_FIRST_UNRESERVED))
                rtmsg_lfib(event, index, rt, nlh, net, portid, nlm_flags);
}

static void mpls_route_update(struct net *net, unsigned index,
                              struct mpls_route *new,
                              const struct nl_info *info)
{
        struct mpls_route __rcu **platform_label;
        struct mpls_route *rt;

        ASSERT_RTNL();

        platform_label = rtnl_dereference(net->mpls.platform_label);
        rt = rtnl_dereference(platform_label[index]);
        rcu_assign_pointer(platform_label[index], new);

        mpls_notify_route(net, index, rt, new, info);

        /* If we removed a route free it now */
        mpls_rt_free(rt);
}

static unsigned find_free_label(struct net *net)
{
        struct mpls_route __rcu **platform_label;
        size_t platform_labels;
        unsigned index;

        platform_label = rtnl_dereference(net->mpls.platform_label);
        platform_labels = net->mpls.platform_labels;
        for (index = MPLS_LABEL_FIRST_UNRESERVED; index < platform_labels;
             index++) {
                if (!rtnl_dereference(platform_label[index]))
                        return index;
        }
        return LABEL_NOT_SPECIFIED;
}

#if IS_ENABLED(CONFIG_INET)
static struct net_device *inet_fib_lookup_dev(struct net *net,
                                              const void *addr)
{
        struct net_device *dev;
        struct rtable *rt;
        struct in_addr daddr;

        memcpy(&daddr, addr, sizeof(struct in_addr));
        rt = ip_route_output(net, daddr.s_addr, 0, 0, 0, RT_SCOPE_UNIVERSE);
        if (IS_ERR(rt))
                return ERR_CAST(rt);

        dev = rt->dst.dev;
        dev_hold(dev);

        ip_rt_put(rt);

        return dev;
}
#else
static struct net_device *inet_fib_lookup_dev(struct net *net,
                                              const void *addr)
{
        return ERR_PTR(-EAFNOSUPPORT);
}
#endif

#if IS_ENABLED(CONFIG_IPV6)
static struct net_device *inet6_fib_lookup_dev(struct net *net,
                                               const void *addr)
{
        struct net_device *dev;
        struct dst_entry *dst;
        struct flowi6 fl6;

        if (!ipv6_stub)
                return ERR_PTR(-EAFNOSUPPORT);

        memset(&fl6, 0, sizeof(fl6));
        memcpy(&fl6.daddr, addr, sizeof(struct in6_addr));
        dst = ipv6_stub->ipv6_dst_lookup_flow(net, NULL, &fl6, NULL);
        if (IS_ERR(dst))
                return ERR_CAST(dst);

        dev = dst->dev;
        dev_hold(dev);
        dst_release(dst);

        return dev;
}
#else
static struct net_device *inet6_fib_lookup_dev(struct net *net,
                                               const void *addr)
{
        return ERR_PTR(-EAFNOSUPPORT);
}
#endif

static struct net_device *find_outdev(struct net *net,
                                      struct mpls_route *rt,
                                      struct mpls_nh *nh, int oif)
{
        struct net_device *dev = NULL;

        if (!oif) {
                switch (nh->nh_via_table) {
                case NEIGH_ARP_TABLE:
                        dev = inet_fib_lookup_dev(net, mpls_nh_via(rt, nh));
                        break;
                case NEIGH_ND_TABLE:
                        dev = inet6_fib_lookup_dev(net, mpls_nh_via(rt, nh));
                        break;
                case NEIGH_LINK_TABLE:
                        break;
                }
        } else {
                dev = dev_get_by_index(net, oif);
        }

        if (!dev)
                return ERR_PTR(-ENODEV);

        if (IS_ERR(dev))
                return dev;

        /* The caller is holding rtnl anyways, so release the dev reference */
        dev_put(dev);

        return dev;
}

static int mpls_nh_assign_dev(struct net *net, struct mpls_route *rt,
                              struct mpls_nh *nh, int oif)
{
        struct net_device *dev = NULL;
        int err = -ENODEV;

        dev = find_outdev(net, rt, nh, oif);
        if (IS_ERR(dev)) {
                err = PTR_ERR(dev);
                dev = NULL;
                goto errout;
        }

        /* Ensure this is a supported device */
        err = -EINVAL;
        if (!mpls_dev_get(dev))
                goto errout;

        if ((nh->nh_via_table == NEIGH_LINK_TABLE) &&
            (dev->addr_len != nh->nh_via_alen))
                goto errout;

        nh->nh_dev = dev;

        if (!(dev->flags & IFF_UP)) {
                nh->nh_flags |= RTNH_F_DEAD;
        } else {
                unsigned int flags;

                flags = dev_get_flags(dev);
                if (!(flags & (IFF_RUNNING | IFF_LOWER_UP)))
                        nh->nh_flags |= RTNH_F_LINKDOWN;
        }

        return 0;

errout:
        return err;
}

static int nla_get_via(const struct nlattr *nla, u8 *via_alen, u8 *via_table,
                       u8 via_addr[], struct netlink_ext_ack *extack)
{
        struct rtvia *via = nla_data(nla);
        int err = -EINVAL;
        int alen;

        if (nla_len(nla) < offsetof(struct rtvia, rtvia_addr)) {
                NL_SET_ERR_MSG_ATTR(extack, nla,
                                    "Invalid attribute length for RTA_VIA");
                goto errout;
        }
        alen = nla_len(nla) -
                        offsetof(struct rtvia, rtvia_addr);
        if (alen > MAX_VIA_ALEN) {
                NL_SET_ERR_MSG_ATTR(extack, nla,
                                    "Invalid address length for RTA_VIA");
                goto errout;
        }

        /* Validate the address family */
        switch (via->rtvia_family) {
        case AF_PACKET:
                *via_table = NEIGH_LINK_TABLE;
                break;
        case AF_INET:
                *via_table = NEIGH_ARP_TABLE;
                if (alen != 4)
                        goto errout;
                break;
        case AF_INET6:
                *via_table = NEIGH_ND_TABLE;
                if (alen != 16)
                        goto errout;
                break;
        default:
                /* Unsupported address family */
                goto errout;
        }

        memcpy(via_addr, via->rtvia_addr, alen);
        *via_alen = alen;
        err = 0;

errout:
        return err;
}

static int mpls_nh_build_from_cfg(struct mpls_route_config *cfg,
                                  struct mpls_route *rt)
{
        struct net *net = cfg->rc_nlinfo.nl_net;
        struct mpls_nh *nh = rt->rt_nh;
        int err;
        int i;

        if (!nh)
                return -ENOMEM;

        nh->nh_labels = cfg->rc_output_labels;
        for (i = 0; i < nh->nh_labels; i++)
                nh->nh_label[i] = cfg->rc_output_label[i];

        nh->nh_via_table = cfg->rc_via_table;
        memcpy(__mpls_nh_via(rt, nh), cfg->rc_via, cfg->rc_via_alen);
        nh->nh_via_alen = cfg->rc_via_alen;

        err = mpls_nh_assign_dev(net, rt, nh, cfg->rc_ifindex);
        if (err)
                goto errout;

        if (nh->nh_flags & (RTNH_F_DEAD | RTNH_F_LINKDOWN))
                rt->rt_nhn_alive--;

        return 0;

errout:
        return err;
}

static int mpls_nh_build(struct net *net, struct mpls_route *rt,
                         struct mpls_nh *nh, int oif, struct nlattr *via,
                         struct nlattr *newdst, u8 max_labels,
                         struct netlink_ext_ack *extack)
{
        int err = -ENOMEM;

        if (!nh)
                goto errout;

        if (newdst) {
                err = nla_get_labels(newdst, max_labels, &nh->nh_labels,
                                     nh->nh_label, extack);
                if (err)
                        goto errout;
        }

        if (via) {
                err = nla_get_via(via, &nh->nh_via_alen, &nh->nh_via_table,
                                  __mpls_nh_via(rt, nh), extack);
                if (err)
                        goto errout;
        } else {
                nh->nh_via_table = MPLS_NEIGH_TABLE_UNSPEC;
        }

        err = mpls_nh_assign_dev(net, rt, nh, oif);
        if (err)
                goto errout;

        return 0;

errout:
        return err;
}

static u8 mpls_count_nexthops(struct rtnexthop *rtnh, int len,
                              u8 cfg_via_alen, u8 *max_via_alen,
                              u8 *max_labels)
{
        int remaining = len;
        u8 nhs = 0;

        *max_via_alen = 0;
        *max_labels = 0;

        while (rtnh_ok(rtnh, remaining)) {
                struct nlattr *nla, *attrs = rtnh_attrs(rtnh);
                int attrlen;
                u8 n_labels = 0;

                attrlen = rtnh_attrlen(rtnh);
                nla = nla_find(attrs, attrlen, RTA_VIA);
                if (nla && nla_len(nla) >=
                    offsetof(struct rtvia, rtvia_addr)) {
                        int via_alen = nla_len(nla) -
                                offsetof(struct rtvia, rtvia_addr);

                        if (via_alen <= MAX_VIA_ALEN)
                                *max_via_alen = max_t(u16, *max_via_alen,
                                                      via_alen);
                }

                nla = nla_find(attrs, attrlen, RTA_NEWDST);
                if (nla &&
                    nla_get_labels(nla, MAX_NEW_LABELS, &n_labels,
                                   NULL, NULL) != 0)
                        return 0;

                *max_labels = max_t(u8, *max_labels, n_labels);

                /* number of nexthops is tracked by a u8.
                 * Check for overflow.
                 */
                if (nhs == 255)
                        return 0;
                nhs++;

                rtnh = rtnh_next(rtnh, &remaining);
        }

        /* leftover implies invalid nexthop configuration, discard it */
        return remaining > 0 ? 0 : nhs;
}

static int mpls_nh_build_multi(struct mpls_route_config *cfg,
                               struct mpls_route *rt, u8 max_labels,
                               struct netlink_ext_ack *extack)
{
        struct rtnexthop *rtnh = cfg->rc_mp;
        struct nlattr *nla_via, *nla_newdst;
        int remaining = cfg->rc_mp_len;
        int err = 0;
        u8 nhs = 0;

        change_nexthops(rt) {
                int attrlen;

                nla_via = NULL;
                nla_newdst = NULL;

                err = -EINVAL;
                if (!rtnh_ok(rtnh, remaining))
                        goto errout;

                /* neither weighted multipath nor any flags
                 * are supported
                 */
                if (rtnh->rtnh_hops || rtnh->rtnh_flags)
                        goto errout;

                attrlen = rtnh_attrlen(rtnh);
                if (attrlen > 0) {
                        struct nlattr *attrs = rtnh_attrs(rtnh);

                        nla_via = nla_find(attrs, attrlen, RTA_VIA);
                        nla_newdst = nla_find(attrs, attrlen, RTA_NEWDST);
                }

                err = mpls_nh_build(cfg->rc_nlinfo.nl_net, rt, nh,
                                    rtnh->rtnh_ifindex, nla_via, nla_newdst,
                                    max_labels, extack);
                if (err)
                        goto errout;

                if (nh->nh_flags & (RTNH_F_DEAD | RTNH_F_LINKDOWN))
                        rt->rt_nhn_alive--;

                rtnh = rtnh_next(rtnh, &remaining);
                nhs++;
        } endfor_nexthops(rt);

        rt->rt_nhn = nhs;

        return 0;

errout:
        return err;
}

static bool mpls_label_ok(struct net *net, unsigned int *index,
                          struct netlink_ext_ack *extack)
{
        bool is_ok = true;

        /* Reserved labels may not be set */
        if (*index < MPLS_LABEL_FIRST_UNRESERVED) {
                NL_SET_ERR_MSG(extack,
                               "Invalid label - must be MPLS_LABEL_FIRST_UNRESERVED or higher");
                is_ok = false;
        }

        /* The full 20 bit range may not be supported. */
        if (is_ok && *index >= net->mpls.platform_labels) {
                NL_SET_ERR_MSG(extack,
                               "Label >= configured maximum in platform_labels");
                is_ok = false;
        }

        *index = array_index_nospec(*index, net->mpls.platform_labels);
        return is_ok;
}

static int mpls_route_add(struct mpls_route_config *cfg,
                          struct netlink_ext_ack *extack)
{
        struct mpls_route __rcu **platform_label;
        struct net *net = cfg->rc_nlinfo.nl_net;
        struct mpls_route *rt, *old;
        int err = -EINVAL;
        u8 max_via_alen;
        unsigned index;
        u8 max_labels;
        u8 nhs;

        index = cfg->rc_label;

        /* If a label was not specified during insert pick one */
        if ((index == LABEL_NOT_SPECIFIED) &&
            (cfg->rc_nlflags & NLM_F_CREATE)) {
                index = find_free_label(net);
        }

        if (!mpls_label_ok(net, &index, extack))
                goto errout;

        /* Append makes no sense with mpls */
        err = -EOPNOTSUPP;
        if (cfg->rc_nlflags & NLM_F_APPEND) {
                NL_SET_ERR_MSG(extack, "MPLS does not support route append");
                goto errout;
        }

        err = -EEXIST;
        platform_label = rtnl_dereference(net->mpls.platform_label);
        old = rtnl_dereference(platform_label[index]);
        if ((cfg->rc_nlflags & NLM_F_EXCL) && old)
                goto errout;

        err = -EEXIST;
        if (!(cfg->rc_nlflags & NLM_F_REPLACE) && old)
                goto errout;

        err = -ENOENT;
        if (!(cfg->rc_nlflags & NLM_F_CREATE) && !old)
                goto errout;

        err = -EINVAL;
        if (cfg->rc_mp) {
                nhs = mpls_count_nexthops(cfg->rc_mp, cfg->rc_mp_len,
                                          cfg->rc_via_alen, &max_via_alen,
                                          &max_labels);
        } else {
                max_via_alen = cfg->rc_via_alen;
                max_labels = cfg->rc_output_labels;
                nhs = 1;
        }

        if (nhs == 0) {
                NL_SET_ERR_MSG(extack, "Route does not contain a nexthop");
                goto errout;
        }

        rt = mpls_rt_alloc(nhs, max_via_alen, max_labels);
        if (IS_ERR(rt)) {
                err = PTR_ERR(rt);
                goto errout;
        }

        rt->rt_protocol = cfg->rc_protocol;
        rt->rt_payload_type = cfg->rc_payload_type;
        rt->rt_ttl_propagate = cfg->rc_ttl_propagate;

        if (cfg->rc_mp)
                err = mpls_nh_build_multi(cfg, rt, max_labels, extack);
        else
                err = mpls_nh_build_from_cfg(cfg, rt);
        if (err)
                goto freert;

        mpls_route_update(net, index, rt, &cfg->rc_nlinfo);

        return 0;

freert:
        mpls_rt_free(rt);
errout:
        return err;
}

static int mpls_route_del(struct mpls_route_config *cfg,
                          struct netlink_ext_ack *extack)
{
        struct net *net = cfg->rc_nlinfo.nl_net;
        unsigned index;
        int err = -EINVAL;

        index = cfg->rc_label;

        if (!mpls_label_ok(net, &index, extack))
                goto errout;

        mpls_route_update(net, index, NULL, &cfg->rc_nlinfo);

        err = 0;
errout:
        return err;
}

static void mpls_get_stats(struct mpls_dev *mdev,
                           struct mpls_link_stats *stats)
{
        struct mpls_pcpu_stats *p;
        int i;

        memset(stats, 0, sizeof(*stats));

        for_each_possible_cpu(i) {
                struct mpls_link_stats local;
                unsigned int start;

                p = per_cpu_ptr(mdev->stats, i);
                do {
                        start = u64_stats_fetch_begin(&p->syncp);
                        local = p->stats;
                } while (u64_stats_fetch_retry(&p->syncp, start));

                stats->rx_packets        += local.rx_packets;
                stats->rx_bytes                += local.rx_bytes;
                stats->tx_packets        += local.tx_packets;
                stats->tx_bytes                += local.tx_bytes;
                stats->rx_errors        += local.rx_errors;
                stats->tx_errors        += local.tx_errors;
                stats->rx_dropped        += local.rx_dropped;
                stats->tx_dropped        += local.tx_dropped;
                stats->rx_noroute        += local.rx_noroute;
        }
}

static int mpls_fill_stats_af(struct sk_buff *skb,
                              const struct net_device *dev)
{
        struct mpls_link_stats *stats;
        struct mpls_dev *mdev;
        struct nlattr *nla;

        mdev = mpls_dev_get(dev);
        if (!mdev)
                return -ENODATA;

        nla = nla_reserve_64bit(skb, MPLS_STATS_LINK,
                                sizeof(struct mpls_link_stats),
                                MPLS_STATS_UNSPEC);
        if (!nla)
                return -EMSGSIZE;

        stats = nla_data(nla);
        mpls_get_stats(mdev, stats);

        return 0;
}

static size_t mpls_get_stats_af_size(const struct net_device *dev)
{
        struct mpls_dev *mdev;

        mdev = mpls_dev_get(dev);
        if (!mdev)
                return 0;

        return nla_total_size_64bit(sizeof(struct mpls_link_stats));
}

static int mpls_netconf_fill_devconf(struct sk_buff *skb, struct mpls_dev *mdev,
                                     u32 portid, u32 seq, int event,
                                     unsigned int flags, int type)
{
        struct nlmsghdr  *nlh;
        struct netconfmsg *ncm;
        bool all = false;

        nlh = nlmsg_put(skb, portid, seq, event, sizeof(struct netconfmsg),
                        flags);
        if (!nlh)
                return -EMSGSIZE;

        if (type == NETCONFA_ALL)
                all = true;

        ncm = nlmsg_data(nlh);
        ncm->ncm_family = AF_MPLS;

        if (nla_put_s32(skb, NETCONFA_IFINDEX, mdev->dev->ifindex) < 0)
                goto nla_put_failure;

        if ((all || type == NETCONFA_INPUT) &&
            nla_put_s32(skb, NETCONFA_INPUT,
                        READ_ONCE(mdev->input_enabled)) < 0)
                goto nla_put_failure;

        nlmsg_end(skb, nlh);
        return 0;

nla_put_failure:
        nlmsg_cancel(skb, nlh);
        return -EMSGSIZE;
}

static int mpls_netconf_msgsize_devconf(int type)
{
        int size = NLMSG_ALIGN(sizeof(struct netconfmsg))
                        + nla_total_size(4); /* NETCONFA_IFINDEX */
        bool all = false;

        if (type == NETCONFA_ALL)
                all = true;

        if (all || type == NETCONFA_INPUT)
                size += nla_total_size(4);

        return size;
}

static void mpls_netconf_notify_devconf(struct net *net, int event,
                                        int type, struct mpls_dev *mdev)
{
        struct sk_buff *skb;
        int err = -ENOBUFS;

        skb = nlmsg_new(mpls_netconf_msgsize_devconf(type), GFP_KERNEL);
        if (!skb)
                goto errout;

        err = mpls_netconf_fill_devconf(skb, mdev, 0, 0, event, 0, type);
        if (err < 0) {
                /* -EMSGSIZE implies BUG in mpls_netconf_msgsize_devconf() */
                WARN_ON(err == -EMSGSIZE);
                kfree_skb(skb);
                goto errout;
        }

        rtnl_notify(skb, net, 0, RTNLGRP_MPLS_NETCONF, NULL, GFP_KERNEL);
        return;
errout:
        rtnl_set_sk_err(net, RTNLGRP_MPLS_NETCONF, err);
}

static const struct nla_policy devconf_mpls_policy[NETCONFA_MAX + 1] = {
        [NETCONFA_IFINDEX]        = { .len = sizeof(int) },
};

static int mpls_netconf_valid_get_req(struct sk_buff *skb,
                                      const struct nlmsghdr *nlh,
                                      struct nlattr **tb,
                                      struct netlink_ext_ack *extack)
{
        int i, err;

        if (nlh->nlmsg_len < nlmsg_msg_size(sizeof(struct netconfmsg))) {
                NL_SET_ERR_MSG_MOD(extack,
                                   "Invalid header for netconf get request");
                return -EINVAL;
        }

        if (!netlink_strict_get_check(skb))
                return nlmsg_parse_deprecated(nlh, sizeof(struct netconfmsg),
                                              tb, NETCONFA_MAX,
                                              devconf_mpls_policy, extack);

        err = nlmsg_parse_deprecated_strict(nlh, sizeof(struct netconfmsg),
                                            tb, NETCONFA_MAX,
                                            devconf_mpls_policy, extack);
        if (err)
                return err;

        for (i = 0; i <= NETCONFA_MAX; i++) {
                if (!tb[i])
                        continue;

                switch (i) {
                case NETCONFA_IFINDEX:
                        break;
                default:
                        NL_SET_ERR_MSG_MOD(extack, "Unsupported attribute in netconf get request");
                        return -EINVAL;
                }
        }

        return 0;
}

static int mpls_netconf_get_devconf(struct sk_buff *in_skb,
                                    struct nlmsghdr *nlh,
                                    struct netlink_ext_ack *extack)
{
        struct net *net = sock_net(in_skb->sk);
        struct nlattr *tb[NETCONFA_MAX + 1];
        struct net_device *dev;
        struct mpls_dev *mdev;
        struct sk_buff *skb;
        int ifindex;
        int err;

        err = mpls_netconf_valid_get_req(in_skb, nlh, tb, extack);
        if (err < 0)
                goto errout;

        err = -EINVAL;
        if (!tb[NETCONFA_IFINDEX])
                goto errout;

        ifindex = nla_get_s32(tb[NETCONFA_IFINDEX]);
        dev = __dev_get_by_index(net, ifindex);
        if (!dev)
                goto errout;

        mdev = mpls_dev_get(dev);
        if (!mdev)
                goto errout;

        err = -ENOBUFS;
        skb = nlmsg_new(mpls_netconf_msgsize_devconf(NETCONFA_ALL), GFP_KERNEL);
        if (!skb)
                goto errout;

        err = mpls_netconf_fill_devconf(skb, mdev,
                                        NETLINK_CB(in_skb).portid,
                                        nlh->nlmsg_seq, RTM_NEWNETCONF, 0,
                                        NETCONFA_ALL);
        if (err < 0) {
                /* -EMSGSIZE implies BUG in mpls_netconf_msgsize_devconf() */
                WARN_ON(err == -EMSGSIZE);
                kfree_skb(skb);
                goto errout;
        }
        err = rtnl_unicast(skb, net, NETLINK_CB(in_skb).portid);
errout:
        return err;
}

static int mpls_netconf_dump_devconf(struct sk_buff *skb,
                                     struct netlink_callback *cb)
{
        const struct nlmsghdr *nlh = cb->nlh;
        struct net *net = sock_net(skb->sk);
        struct {
                unsigned long ifindex;
        } *ctx = (void *)cb->ctx;
        struct net_device *dev;
        struct mpls_dev *mdev;
        int err = 0;

        if (cb->strict_check) {
                struct netlink_ext_ack *extack = cb->extack;
                struct netconfmsg *ncm;

                if (nlh->nlmsg_len < nlmsg_msg_size(sizeof(*ncm))) {
                        NL_SET_ERR_MSG_MOD(extack, "Invalid header for netconf dump request");
                        return -EINVAL;
                }

                if (nlmsg_attrlen(nlh, sizeof(*ncm))) {
                        NL_SET_ERR_MSG_MOD(extack, "Invalid data after header in netconf dump request");
                        return -EINVAL;
                }
        }

        rcu_read_lock();
        for_each_netdev_dump(net, dev, ctx->ifindex) {
                mdev = mpls_dev_get(dev);
                if (!mdev)
                        continue;
                err = mpls_netconf_fill_devconf(skb, mdev,
                                                NETLINK_CB(cb->skb).portid,
                                                nlh->nlmsg_seq,
                                                RTM_NEWNETCONF,
                                                NLM_F_MULTI,
                                                NETCONFA_ALL);
                if (err < 0)
                        break;
        }
        rcu_read_unlock();

        return err;
}

#define MPLS_PERDEV_SYSCTL_OFFSET(field)        \
        (&((struct mpls_dev *)0)->field)

static int mpls_conf_proc(const struct ctl_table *ctl, int write,
                          void *buffer, size_t *lenp, loff_t *ppos)
{
        int oval = *(int *)ctl->data;
        int ret = proc_dointvec(ctl, write, buffer, lenp, ppos);

        if (write) {
                struct mpls_dev *mdev = ctl->extra1;
                int i = (int *)ctl->data - (int *)mdev;
                struct net *net = ctl->extra2;
                int val = *(int *)ctl->data;

                if (i == offsetof(struct mpls_dev, input_enabled) &&
                    val != oval) {
                        mpls_netconf_notify_devconf(net, RTM_NEWNETCONF,
                                                    NETCONFA_INPUT, mdev);
                }
        }

        return ret;
}

static const struct ctl_table mpls_dev_table[] = {
        {
                .procname        = "input",
                .maxlen                = sizeof(int),
                .mode                = 0644,
                .proc_handler        = mpls_conf_proc,
                .data                = MPLS_PERDEV_SYSCTL_OFFSET(input_enabled),
        },
};

static int mpls_dev_sysctl_register(struct net_device *dev,
                                    struct mpls_dev *mdev)
{
        char path[sizeof("net/mpls/conf/") + IFNAMSIZ];
        size_t table_size = ARRAY_SIZE(mpls_dev_table);
        struct net *net = dev_net(dev);
        struct ctl_table *table;
        int i;

        table = kmemdup(&mpls_dev_table, sizeof(mpls_dev_table), GFP_KERNEL);
        if (!table)
                goto out;

        /* Table data contains only offsets relative to the base of
         * the mdev at this point, so make them absolute.
         */
        for (i = 0; i < table_size; i++) {
                table[i].data = (char *)mdev + (uintptr_t)table[i].data;
                table[i].extra1 = mdev;
                table[i].extra2 = net;
        }

        snprintf(path, sizeof(path), "net/mpls/conf/%s", dev->name);

        mdev->sysctl = register_net_sysctl_sz(net, path, table, table_size);
        if (!mdev->sysctl)
                goto free;

        mpls_netconf_notify_devconf(net, RTM_NEWNETCONF, NETCONFA_ALL, mdev);
        return 0;

free:
        kfree(table);
out:
        mdev->sysctl = NULL;
        return -ENOBUFS;
}

static void mpls_dev_sysctl_unregister(struct net_device *dev,
                                       struct mpls_dev *mdev)
{
        struct net *net = dev_net(dev);
        const struct ctl_table *table;

        if (!mdev->sysctl)
                return;

        table = mdev->sysctl->ctl_table_arg;
        unregister_net_sysctl_table(mdev->sysctl);
        kfree(table);

        mpls_netconf_notify_devconf(net, RTM_DELNETCONF, 0, mdev);
}

static struct mpls_dev *mpls_add_dev(struct net_device *dev)
{
        struct mpls_dev *mdev;
        int err = -ENOMEM;
        int i;

        ASSERT_RTNL();

        mdev = kzalloc(sizeof(*mdev), GFP_KERNEL);
        if (!mdev)
                return ERR_PTR(err);

        mdev->stats = alloc_percpu(struct mpls_pcpu_stats);
        if (!mdev->stats)
                goto free;

        for_each_possible_cpu(i) {
                struct mpls_pcpu_stats *mpls_stats;

                mpls_stats = per_cpu_ptr(mdev->stats, i);
                u64_stats_init(&mpls_stats->syncp);
        }

        mdev->dev = dev;

        err = mpls_dev_sysctl_register(dev, mdev);
        if (err)
                goto free;

        rcu_assign_pointer(dev->mpls_ptr, mdev);

        return mdev;

free:
        free_percpu(mdev->stats);
        kfree(mdev);
        return ERR_PTR(err);
}

static void mpls_dev_destroy_rcu(struct rcu_head *head)
{
        struct mpls_dev *mdev = container_of(head, struct mpls_dev, rcu);

        free_percpu(mdev->stats);
        kfree(mdev);
}

static int mpls_ifdown(struct net_device *dev, int event)
{
        struct mpls_route __rcu **platform_label;
        struct net *net = dev_net(dev);
        unsigned index;

        platform_label = rtnl_dereference(net->mpls.platform_label);
        for (index = 0; index < net->mpls.platform_labels; index++) {
                struct mpls_route *rt = rtnl_dereference(platform_label[index]);
                bool nh_del = false;
                u8 alive = 0;

                if (!rt)
                        continue;

                if (event == NETDEV_UNREGISTER) {
                        u8 deleted = 0;

                        for_nexthops(rt) {
                                if (!nh->nh_dev || nh->nh_dev == dev)
                                        deleted++;
                                if (nh->nh_dev == dev)
                                        nh_del = true;
                        } endfor_nexthops(rt);

                        /* if there are no more nexthops, delete the route */
                        if (deleted == rt->rt_nhn) {
                                mpls_route_update(net, index, NULL, NULL);
                                continue;
                        }

                        if (nh_del) {
                                size_t size = sizeof(*rt) + rt->rt_nhn *
                                        rt->rt_nh_size;
                                struct mpls_route *orig = rt;

                                rt = kmemdup(orig, size, GFP_KERNEL);
                                if (!rt)
                                        return -ENOMEM;
                        }
                }

                change_nexthops(rt) {
                        unsigned int nh_flags = nh->nh_flags;

                        if (nh->nh_dev != dev)
                                goto next;

                        switch (event) {
                        case NETDEV_DOWN:
                        case NETDEV_UNREGISTER:
                                nh_flags |= RTNH_F_DEAD;
                                fallthrough;
                        case NETDEV_CHANGE:
                                nh_flags |= RTNH_F_LINKDOWN;
                                break;
                        }
                        if (event == NETDEV_UNREGISTER)
                                nh->nh_dev = NULL;

                        if (nh->nh_flags != nh_flags)
                                WRITE_ONCE(nh->nh_flags, nh_flags);
next:
                        if (!(nh_flags & (RTNH_F_DEAD | RTNH_F_LINKDOWN)))
                                alive++;
                } endfor_nexthops(rt);

                WRITE_ONCE(rt->rt_nhn_alive, alive);

                if (nh_del)
                        mpls_route_update(net, index, rt, NULL);
        }

        return 0;
}

static void mpls_ifup(struct net_device *dev, unsigned int flags)
{
        struct mpls_route __rcu **platform_label;
        struct net *net = dev_net(dev);
        unsigned index;
        u8 alive;

        platform_label = rtnl_dereference(net->mpls.platform_label);
        for (index = 0; index < net->mpls.platform_labels; index++) {
                struct mpls_route *rt = rtnl_dereference(platform_label[index]);

                if (!rt)
                        continue;

                alive = 0;
                change_nexthops(rt) {
                        unsigned int nh_flags = nh->nh_flags;

                        if (!(nh_flags & flags)) {
                                alive++;
                                continue;
                        }
                        if (nh->nh_dev != dev)
                                continue;
                        alive++;
                        nh_flags &= ~flags;
                        WRITE_ONCE(nh->nh_flags, nh_flags);
                } endfor_nexthops(rt);

                WRITE_ONCE(rt->rt_nhn_alive, alive);
        }
}

static int mpls_dev_notify(struct notifier_block *this, unsigned long event,
                           void *ptr)
{
        struct net_device *dev = netdev_notifier_info_to_dev(ptr);
        struct mpls_dev *mdev;
        unsigned int flags;
        int err;

        if (event == NETDEV_REGISTER) {
                mdev = mpls_add_dev(dev);
                if (IS_ERR(mdev))
                        return notifier_from_errno(PTR_ERR(mdev));

                return NOTIFY_OK;
        }

        mdev = mpls_dev_get(dev);
        if (!mdev)
                return NOTIFY_OK;

        switch (event) {

        case NETDEV_DOWN:
                err = mpls_ifdown(dev, event);
                if (err)
                        return notifier_from_errno(err);
                break;
        case NETDEV_UP:
                flags = dev_get_flags(dev);
                if (flags & (IFF_RUNNING | IFF_LOWER_UP))
                        mpls_ifup(dev, RTNH_F_DEAD | RTNH_F_LINKDOWN);
                else
                        mpls_ifup(dev, RTNH_F_DEAD);
                break;
        case NETDEV_CHANGE:
                flags = dev_get_flags(dev);
                if (flags & (IFF_RUNNING | IFF_LOWER_UP)) {
                        mpls_ifup(dev, RTNH_F_DEAD | RTNH_F_LINKDOWN);
                } else {
                        err = mpls_ifdown(dev, event);
                        if (err)
                                return notifier_from_errno(err);
                }
                break;
        case NETDEV_UNREGISTER:
                err = mpls_ifdown(dev, event);
                if (err)
                        return notifier_from_errno(err);
                mdev = mpls_dev_get(dev);
                if (mdev) {
                        mpls_dev_sysctl_unregister(dev, mdev);
                        RCU_INIT_POINTER(dev->mpls_ptr, NULL);
                        call_rcu(&mdev->rcu, mpls_dev_destroy_rcu);
                }
                break;
        case NETDEV_CHANGENAME:
                mdev = mpls_dev_get(dev);
                if (mdev) {
                        mpls_dev_sysctl_unregister(dev, mdev);
                        err = mpls_dev_sysctl_register(dev, mdev);
                        if (err)
                                return notifier_from_errno(err);
                }
                break;
        }
        return NOTIFY_OK;
}

static struct notifier_block mpls_dev_notifier = {
        .notifier_call = mpls_dev_notify,
};

static int nla_put_via(struct sk_buff *skb,
                       u8 table, const void *addr, int alen)
{
        static const int table_to_family[NEIGH_NR_TABLES + 1] = {
                AF_INET, AF_INET6, AF_PACKET,
        };
        struct nlattr *nla;
        struct rtvia *via;
        int family = AF_UNSPEC;

        nla = nla_reserve(skb, RTA_VIA, alen + 2);
        if (!nla)
                return -EMSGSIZE;

        if (table <= NEIGH_NR_TABLES)
                family = table_to_family[table];

        via = nla_data(nla);
        via->rtvia_family = family;
        memcpy(via->rtvia_addr, addr, alen);
        return 0;
}

int nla_put_labels(struct sk_buff *skb, int attrtype,
                   u8 labels, const u32 label[])
{
        struct nlattr *nla;
        struct mpls_shim_hdr *nla_label;
        bool bos;
        int i;
        nla = nla_reserve(skb, attrtype, labels*4);
        if (!nla)
                return -EMSGSIZE;

        nla_label = nla_data(nla);
        bos = true;
        for (i = labels - 1; i >= 0; i--) {
                nla_label[i] = mpls_entry_encode(label[i], 0, 0, bos);
                bos = false;
        }

        return 0;
}
EXPORT_SYMBOL_GPL(nla_put_labels);

int nla_get_labels(const struct nlattr *nla, u8 max_labels, u8 *labels,
                   u32 label[], struct netlink_ext_ack *extack)
{
        unsigned len = nla_len(nla);
        struct mpls_shim_hdr *nla_label;
        u8 nla_labels;
        bool bos;
        int i;

        /* len needs to be an even multiple of 4 (the label size). Number
         * of labels is a u8 so check for overflow.
         */
        if (len & 3 || len / 4 > 255) {
                NL_SET_ERR_MSG_ATTR(extack, nla,
                                    "Invalid length for labels attribute");
                return -EINVAL;
        }

        /* Limit the number of new labels allowed */
        nla_labels = len/4;
        if (nla_labels > max_labels) {
                NL_SET_ERR_MSG(extack, "Too many labels");
                return -EINVAL;
        }

        /* when label == NULL, caller wants number of labels */
        if (!label)
                goto out;

        nla_label = nla_data(nla);
        bos = true;
        for (i = nla_labels - 1; i >= 0; i--, bos = false) {
                struct mpls_entry_decoded dec;
                dec = mpls_entry_decode(nla_label + i);

                /* Ensure the bottom of stack flag is properly set
                 * and ttl and tc are both clear.
                 */
                if (dec.ttl) {
                        NL_SET_ERR_MSG_ATTR(extack, nla,
                                            "TTL in label must be 0");
                        return -EINVAL;
                }

                if (dec.tc) {
                        NL_SET_ERR_MSG_ATTR(extack, nla,
                                            "Traffic class in label must be 0");
                        return -EINVAL;
                }

                if (dec.bos != bos) {
                        NL_SET_BAD_ATTR(extack, nla);
                        if (bos) {
                                NL_SET_ERR_MSG(extack,
                                               "BOS bit must be set in first label");
                        } else {
                                NL_SET_ERR_MSG(extack,
                                               "BOS bit can only be set in first label");
                        }
                        return -EINVAL;
                }

                switch (dec.label) {
                case MPLS_LABEL_IMPLNULL:
                        /* RFC3032: This is a label that an LSR may
                         * assign and distribute, but which never
                         * actually appears in the encapsulation.
                         */
                        NL_SET_ERR_MSG_ATTR(extack, nla,
                                            "Implicit NULL Label (3) can not be used in encapsulation");
                        return -EINVAL;
                }

                label[i] = dec.label;
        }
out:
        *labels = nla_labels;
        return 0;
}
EXPORT_SYMBOL_GPL(nla_get_labels);

static int rtm_to_route_config(struct sk_buff *skb,
                               struct nlmsghdr *nlh,
                               struct mpls_route_config *cfg,
                               struct netlink_ext_ack *extack)
{
        struct rtmsg *rtm;
        struct nlattr *tb[RTA_MAX+1];
        int index;
        int err;

        err = nlmsg_parse_deprecated(nlh, sizeof(*rtm), tb, RTA_MAX,
                                     rtm_mpls_policy, extack);
        if (err < 0)
                goto errout;

        err = -EINVAL;
        rtm = nlmsg_data(nlh);

        if (rtm->rtm_family != AF_MPLS) {
                NL_SET_ERR_MSG(extack, "Invalid address family in rtmsg");
                goto errout;
        }
        if (rtm->rtm_dst_len != 20) {
                NL_SET_ERR_MSG(extack, "rtm_dst_len must be 20 for MPLS");
                goto errout;
        }
        if (rtm->rtm_src_len != 0) {
                NL_SET_ERR_MSG(extack, "rtm_src_len must be 0 for MPLS");
                goto errout;
        }
        if (rtm->rtm_tos != 0) {
                NL_SET_ERR_MSG(extack, "rtm_tos must be 0 for MPLS");
                goto errout;
        }
        if (rtm->rtm_table != RT_TABLE_MAIN) {
                NL_SET_ERR_MSG(extack,
                               "MPLS only supports the main route table");
                goto errout;
        }
        /* Any value is acceptable for rtm_protocol */

        /* As mpls uses destination specific addresses
         * (or source specific address in the case of multicast)
         * all addresses have universal scope.
         */
        if (rtm->rtm_scope != RT_SCOPE_UNIVERSE) {
                NL_SET_ERR_MSG(extack,
                               "Invalid route scope  - MPLS only supports UNIVERSE");
                goto errout;
        }
        if (rtm->rtm_type != RTN_UNICAST) {
                NL_SET_ERR_MSG(extack,
                               "Invalid route type - MPLS only supports UNICAST");
                goto errout;
        }
        if (rtm->rtm_flags != 0) {
                NL_SET_ERR_MSG(extack, "rtm_flags must be 0 for MPLS");
                goto errout;
        }

        cfg->rc_label                = LABEL_NOT_SPECIFIED;
        cfg->rc_protocol        = rtm->rtm_protocol;
        cfg->rc_via_table        = MPLS_NEIGH_TABLE_UNSPEC;
        cfg->rc_ttl_propagate        = MPLS_TTL_PROP_DEFAULT;
        cfg->rc_nlflags                = nlh->nlmsg_flags;
        cfg->rc_nlinfo.portid        = NETLINK_CB(skb).portid;
        cfg->rc_nlinfo.nlh        = nlh;
        cfg->rc_nlinfo.nl_net        = sock_net(skb->sk);

        for (index = 0; index <= RTA_MAX; index++) {
                struct nlattr *nla = tb[index];
                if (!nla)
                        continue;

                switch (index) {
                case RTA_OIF:
                        cfg->rc_ifindex = nla_get_u32(nla);
                        break;
                case RTA_NEWDST:
                        if (nla_get_labels(nla, MAX_NEW_LABELS,
                                           &cfg->rc_output_labels,
                                           cfg->rc_output_label, extack))
                                goto errout;
                        break;
                case RTA_DST:
                {
                        u8 label_count;
                        if (nla_get_labels(nla, 1, &label_count,
                                           &cfg->rc_label, extack))
                                goto errout;

                        if (!mpls_label_ok(cfg->rc_nlinfo.nl_net,
                                           &cfg->rc_label, extack))
                                goto errout;
                        break;
                }
                case RTA_GATEWAY:
                        NL_SET_ERR_MSG(extack, "MPLS does not support RTA_GATEWAY attribute");
                        goto errout;
                case RTA_VIA:
                {
                        if (nla_get_via(nla, &cfg->rc_via_alen,
                                        &cfg->rc_via_table, cfg->rc_via,
                                        extack))
                                goto errout;
                        break;
                }
                case RTA_MULTIPATH:
                {
                        cfg->rc_mp = nla_data(nla);
                        cfg->rc_mp_len = nla_len(nla);
                        break;
                }
                case RTA_TTL_PROPAGATE:
                {
                        u8 ttl_propagate = nla_get_u8(nla);

                        if (ttl_propagate > 1) {
                                NL_SET_ERR_MSG_ATTR(extack, nla,
                                                    "RTA_TTL_PROPAGATE can only be 0 or 1");
                                goto errout;
                        }
                        cfg->rc_ttl_propagate = ttl_propagate ?
                                MPLS_TTL_PROP_ENABLED :
                                MPLS_TTL_PROP_DISABLED;
                        break;
                }
                default:
                        NL_SET_ERR_MSG_ATTR(extack, nla, "Unknown attribute");
                        /* Unsupported attribute */
                        goto errout;
                }
        }

        err = 0;
errout:
        return err;
}

static int mpls_rtm_delroute(struct sk_buff *skb, struct nlmsghdr *nlh,
                             struct netlink_ext_ack *extack)
{
        struct mpls_route_config *cfg;
        int err;

        cfg = kzalloc(sizeof(*cfg), GFP_KERNEL);
        if (!cfg)
                return -ENOMEM;

        err = rtm_to_route_config(skb, nlh, cfg, extack);
        if (err < 0)
                goto out;

        err = mpls_route_del(cfg, extack);
out:
        kfree(cfg);

        return err;
}


static int mpls_rtm_newroute(struct sk_buff *skb, struct nlmsghdr *nlh,
                             struct netlink_ext_ack *extack)
{
        struct mpls_route_config *cfg;
        int err;

        cfg = kzalloc(sizeof(*cfg), GFP_KERNEL);
        if (!cfg)
                return -ENOMEM;

        err = rtm_to_route_config(skb, nlh, cfg, extack);
        if (err < 0)
                goto out;

        err = mpls_route_add(cfg, extack);
out:
        kfree(cfg);

        return err;
}

static int mpls_dump_route(struct sk_buff *skb, u32 portid, u32 seq, int event,
                           u32 label, struct mpls_route *rt, int flags)
{
        struct net_device *dev;
        struct nlmsghdr *nlh;
        struct rtmsg *rtm;

        nlh = nlmsg_put(skb, portid, seq, event, sizeof(*rtm), flags);
        if (nlh == NULL)
                return -EMSGSIZE;

        rtm = nlmsg_data(nlh);
        rtm->rtm_family = AF_MPLS;
        rtm->rtm_dst_len = 20;
        rtm->rtm_src_len = 0;
        rtm->rtm_tos = 0;
        rtm->rtm_table = RT_TABLE_MAIN;
        rtm->rtm_protocol = rt->rt_protocol;
        rtm->rtm_scope = RT_SCOPE_UNIVERSE;
        rtm->rtm_type = RTN_UNICAST;
        rtm->rtm_flags = 0;

        if (nla_put_labels(skb, RTA_DST, 1, &label))
                goto nla_put_failure;

        if (rt->rt_ttl_propagate != MPLS_TTL_PROP_DEFAULT) {
                bool ttl_propagate =
                        rt->rt_ttl_propagate == MPLS_TTL_PROP_ENABLED;

                if (nla_put_u8(skb, RTA_TTL_PROPAGATE,
                               ttl_propagate))
                        goto nla_put_failure;
        }
        if (rt->rt_nhn == 1) {
                const struct mpls_nh *nh = rt->rt_nh;

                if (nh->nh_labels &&
                    nla_put_labels(skb, RTA_NEWDST, nh->nh_labels,
                                   nh->nh_label))
                        goto nla_put_failure;
                if (nh->nh_via_table != MPLS_NEIGH_TABLE_UNSPEC &&
                    nla_put_via(skb, nh->nh_via_table, mpls_nh_via(rt, nh),
                                nh->nh_via_alen))
                        goto nla_put_failure;
                dev = nh->nh_dev;
                if (dev && nla_put_u32(skb, RTA_OIF, dev->ifindex))
                        goto nla_put_failure;
                if (nh->nh_flags & RTNH_F_LINKDOWN)
                        rtm->rtm_flags |= RTNH_F_LINKDOWN;
                if (nh->nh_flags & RTNH_F_DEAD)
                        rtm->rtm_flags |= RTNH_F_DEAD;
        } else {
                struct rtnexthop *rtnh;
                struct nlattr *mp;
                u8 linkdown = 0;
                u8 dead = 0;

                mp = nla_nest_start_noflag(skb, RTA_MULTIPATH);
                if (!mp)
                        goto nla_put_failure;

                for_nexthops(rt) {
                        dev = nh->nh_dev;
                        if (!dev)
                                continue;

                        rtnh = nla_reserve_nohdr(skb, sizeof(*rtnh));
                        if (!rtnh)
                                goto nla_put_failure;

                        rtnh->rtnh_ifindex = dev->ifindex;
                        if (nh->nh_flags & RTNH_F_LINKDOWN) {
                                rtnh->rtnh_flags |= RTNH_F_LINKDOWN;
                                linkdown++;
                        }
                        if (nh->nh_flags & RTNH_F_DEAD) {
                                rtnh->rtnh_flags |= RTNH_F_DEAD;
                                dead++;
                        }

                        if (nh->nh_labels && nla_put_labels(skb, RTA_NEWDST,
                                                            nh->nh_labels,
                                                            nh->nh_label))
                                goto nla_put_failure;
                        if (nh->nh_via_table != MPLS_NEIGH_TABLE_UNSPEC &&
                            nla_put_via(skb, nh->nh_via_table,
                                        mpls_nh_via(rt, nh),
                                        nh->nh_via_alen))
                                goto nla_put_failure;

                        /* length of rtnetlink header + attributes */
                        rtnh->rtnh_len = nlmsg_get_pos(skb) - (void *)rtnh;
                } endfor_nexthops(rt);

                if (linkdown == rt->rt_nhn)
                        rtm->rtm_flags |= RTNH_F_LINKDOWN;
                if (dead == rt->rt_nhn)
                        rtm->rtm_flags |= RTNH_F_DEAD;

                nla_nest_end(skb, mp);
        }

        nlmsg_end(skb, nlh);
        return 0;

nla_put_failure:
        nlmsg_cancel(skb, nlh);
        return -EMSGSIZE;
}

#if IS_ENABLED(CONFIG_INET)
static int mpls_valid_fib_dump_req(struct net *net, const struct nlmsghdr *nlh,
                                   struct fib_dump_filter *filter,
                                   struct netlink_callback *cb)
{
        return ip_valid_fib_dump_req(net, nlh, filter, cb);
}
#else
static int mpls_valid_fib_dump_req(struct net *net, const struct nlmsghdr *nlh,
                                   struct fib_dump_filter *filter,
                                   struct netlink_callback *cb)
{
        struct netlink_ext_ack *extack = cb->extack;
        struct nlattr *tb[RTA_MAX + 1];
        struct rtmsg *rtm;
        int err, i;

        rtm = nlmsg_payload(nlh, sizeof(*rtm));
        if (!rtm) {
                NL_SET_ERR_MSG_MOD(extack, "Invalid header for FIB dump request");
                return -EINVAL;
        }

        if (rtm->rtm_dst_len || rtm->rtm_src_len  || rtm->rtm_tos   ||
            rtm->rtm_table   || rtm->rtm_scope    || rtm->rtm_type  ||
            rtm->rtm_flags) {
                NL_SET_ERR_MSG_MOD(extack, "Invalid values in header for FIB dump request");
                return -EINVAL;
        }

        if (rtm->rtm_protocol) {
                filter->protocol = rtm->rtm_protocol;
                filter->filter_set = 1;
                cb->answer_flags = NLM_F_DUMP_FILTERED;
        }

        err = nlmsg_parse_deprecated_strict(nlh, sizeof(*rtm), tb, RTA_MAX,
                                            rtm_mpls_policy, extack);
        if (err < 0)
                return err;

        for (i = 0; i <= RTA_MAX; ++i) {
                int ifindex;

                if (i == RTA_OIF) {
                        ifindex = nla_get_u32(tb[i]);
                        filter->dev = __dev_get_by_index(net, ifindex);
                        if (!filter->dev)
                                return -ENODEV;
                        filter->filter_set = 1;
                } else if (tb[i]) {
                        NL_SET_ERR_MSG_MOD(extack, "Unsupported attribute in dump request");
                        return -EINVAL;
                }
        }

        return 0;
}
#endif

static bool mpls_rt_uses_dev(struct mpls_route *rt,
                             const struct net_device *dev)
{
        if (rt->rt_nhn == 1) {
                struct mpls_nh *nh = rt->rt_nh;

                if (nh->nh_dev == dev)
                        return true;
        } else {
                for_nexthops(rt) {
                        if (nh->nh_dev == dev)
                                return true;
                } endfor_nexthops(rt);
        }

        return false;
}

static int mpls_dump_routes(struct sk_buff *skb, struct netlink_callback *cb)
{
        const struct nlmsghdr *nlh = cb->nlh;
        struct net *net = sock_net(skb->sk);
        struct mpls_route __rcu **platform_label;
        struct fib_dump_filter filter = {
                .rtnl_held = true,
        };
        unsigned int flags = NLM_F_MULTI;
        size_t platform_labels;
        unsigned int index;

        ASSERT_RTNL();

        if (cb->strict_check) {
                int err;

                err = mpls_valid_fib_dump_req(net, nlh, &filter, cb);
                if (err < 0)
                        return err;

                /* for MPLS, there is only 1 table with fixed type and flags.
                 * If either are set in the filter then return nothing.
                 */
                if ((filter.table_id && filter.table_id != RT_TABLE_MAIN) ||
                    (filter.rt_type && filter.rt_type != RTN_UNICAST) ||
                     filter.flags)
                        return skb->len;
        }

        index = cb->args[0];
        if (index < MPLS_LABEL_FIRST_UNRESERVED)
                index = MPLS_LABEL_FIRST_UNRESERVED;

        platform_label = rtnl_dereference(net->mpls.platform_label);
        platform_labels = net->mpls.platform_labels;

        if (filter.filter_set)
                flags |= NLM_F_DUMP_FILTERED;

        for (; index < platform_labels; index++) {
                struct mpls_route *rt;

                rt = rtnl_dereference(platform_label[index]);
                if (!rt)
                        continue;

                if ((filter.dev && !mpls_rt_uses_dev(rt, filter.dev)) ||
                    (filter.protocol && rt->rt_protocol != filter.protocol))
                        continue;

                if (mpls_dump_route(skb, NETLINK_CB(cb->skb).portid,
                                    cb->nlh->nlmsg_seq, RTM_NEWROUTE,
                                    index, rt, flags) < 0)
                        break;
        }
        cb->args[0] = index;

        return skb->len;
}

static inline size_t lfib_nlmsg_size(struct mpls_route *rt)
{
        size_t payload =
                NLMSG_ALIGN(sizeof(struct rtmsg))
                + nla_total_size(4)                        /* RTA_DST */
                + nla_total_size(1);                        /* RTA_TTL_PROPAGATE */

        if (rt->rt_nhn == 1) {
                struct mpls_nh *nh = rt->rt_nh;

                if (nh->nh_dev)
                        payload += nla_total_size(4); /* RTA_OIF */
                if (nh->nh_via_table != MPLS_NEIGH_TABLE_UNSPEC) /* RTA_VIA */
                        payload += nla_total_size(2 + nh->nh_via_alen);
                if (nh->nh_labels) /* RTA_NEWDST */
                        payload += nla_total_size(nh->nh_labels * 4);
        } else {
                /* each nexthop is packed in an attribute */
                size_t nhsize = 0;

                for_nexthops(rt) {
                        if (!nh->nh_dev)
                                continue;
                        nhsize += nla_total_size(sizeof(struct rtnexthop));
                        /* RTA_VIA */
                        if (nh->nh_via_table != MPLS_NEIGH_TABLE_UNSPEC)
                                nhsize += nla_total_size(2 + nh->nh_via_alen);
                        if (nh->nh_labels)
                                nhsize += nla_total_size(nh->nh_labels * 4);
                } endfor_nexthops(rt);
                /* nested attribute */
                payload += nla_total_size(nhsize);
        }

        return payload;
}

static void rtmsg_lfib(int event, u32 label, struct mpls_route *rt,
                       struct nlmsghdr *nlh, struct net *net, u32 portid,
                       unsigned int nlm_flags)
{
        struct sk_buff *skb;
        u32 seq = nlh ? nlh->nlmsg_seq : 0;
        int err = -ENOBUFS;

        skb = nlmsg_new(lfib_nlmsg_size(rt), GFP_KERNEL);
        if (skb == NULL)
                goto errout;

        err = mpls_dump_route(skb, portid, seq, event, label, rt, nlm_flags);
        if (err < 0) {
                /* -EMSGSIZE implies BUG in lfib_nlmsg_size */
                WARN_ON(err == -EMSGSIZE);
                kfree_skb(skb);
                goto errout;
        }
        rtnl_notify(skb, net, portid, RTNLGRP_MPLS_ROUTE, nlh, GFP_KERNEL);

        return;
errout:
        rtnl_set_sk_err(net, RTNLGRP_MPLS_ROUTE, err);
}

static int mpls_valid_getroute_req(struct sk_buff *skb,
                                   const struct nlmsghdr *nlh,
                                   struct nlattr **tb,
                                   struct netlink_ext_ack *extack)
{
        struct rtmsg *rtm;
        int i, err;

        rtm = nlmsg_payload(nlh, sizeof(*rtm));
        if (!rtm) {
                NL_SET_ERR_MSG_MOD(extack,
                                   "Invalid header for get route request");
                return -EINVAL;
        }

        if (!netlink_strict_get_check(skb))
                return nlmsg_parse_deprecated(nlh, sizeof(*rtm), tb, RTA_MAX,
                                              rtm_mpls_policy, extack);

        if ((rtm->rtm_dst_len && rtm->rtm_dst_len != 20) ||
            rtm->rtm_src_len || rtm->rtm_tos || rtm->rtm_table ||
            rtm->rtm_protocol || rtm->rtm_scope || rtm->rtm_type) {
                NL_SET_ERR_MSG_MOD(extack, "Invalid values in header for get route request");
                return -EINVAL;
        }
        if (rtm->rtm_flags & ~RTM_F_FIB_MATCH) {
                NL_SET_ERR_MSG_MOD(extack,
                                   "Invalid flags for get route request");
                return -EINVAL;
        }

        err = nlmsg_parse_deprecated_strict(nlh, sizeof(*rtm), tb, RTA_MAX,
                                            rtm_mpls_policy, extack);
        if (err)
                return err;

        if ((tb[RTA_DST] || tb[RTA_NEWDST]) && !rtm->rtm_dst_len) {
                NL_SET_ERR_MSG_MOD(extack, "rtm_dst_len must be 20 for MPLS");
                return -EINVAL;
        }

        for (i = 0; i <= RTA_MAX; i++) {
                if (!tb[i])
                        continue;

                switch (i) {
                case RTA_DST:
                case RTA_NEWDST:
                        break;
                default:
                        NL_SET_ERR_MSG_MOD(extack, "Unsupported attribute in get route request");
                        return -EINVAL;
                }
        }

        return 0;
}

static int mpls_getroute(struct sk_buff *in_skb, struct nlmsghdr *in_nlh,
                         struct netlink_ext_ack *extack)
{
        struct net *net = sock_net(in_skb->sk);
        u32 portid = NETLINK_CB(in_skb).portid;
        u32 in_label = LABEL_NOT_SPECIFIED;
        struct nlattr *tb[RTA_MAX + 1];
        u32 labels[MAX_NEW_LABELS];
        struct mpls_shim_hdr *hdr;
        unsigned int hdr_size = 0;
        const struct mpls_nh *nh;
        struct net_device *dev;
        struct mpls_route *rt;
        struct rtmsg *rtm, *r;
        struct nlmsghdr *nlh;
        struct sk_buff *skb;
        u8 n_labels;
        int err;

        err = mpls_valid_getroute_req(in_skb, in_nlh, tb, extack);
        if (err < 0)
                goto errout;

        rtm = nlmsg_data(in_nlh);

        if (tb[RTA_DST]) {
                u8 label_count;

                if (nla_get_labels(tb[RTA_DST], 1, &label_count,
                                   &in_label, extack)) {
                        err = -EINVAL;
                        goto errout;
                }

                if (!mpls_label_ok(net, &in_label, extack)) {
                        err = -EINVAL;
                        goto errout;
                }
        }

        rt = mpls_route_input_rcu(net, in_label);
        if (!rt) {
                err = -ENETUNREACH;
                goto errout;
        }

        if (rtm->rtm_flags & RTM_F_FIB_MATCH) {
                skb = nlmsg_new(lfib_nlmsg_size(rt), GFP_KERNEL);
                if (!skb) {
                        err = -ENOBUFS;
                        goto errout;
                }

                err = mpls_dump_route(skb, portid, in_nlh->nlmsg_seq,
                                      RTM_NEWROUTE, in_label, rt, 0);
                if (err < 0) {
                        /* -EMSGSIZE implies BUG in lfib_nlmsg_size */
                        WARN_ON(err == -EMSGSIZE);
                        goto errout_free;
                }

                return rtnl_unicast(skb, net, portid);
        }

        if (tb[RTA_NEWDST]) {
                if (nla_get_labels(tb[RTA_NEWDST], MAX_NEW_LABELS, &n_labels,
                                   labels, extack) != 0) {
                        err = -EINVAL;
                        goto errout;
                }

                hdr_size = n_labels * sizeof(struct mpls_shim_hdr);
        }

        skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
        if (!skb) {
                err = -ENOBUFS;
                goto errout;
        }

        skb->protocol = htons(ETH_P_MPLS_UC);

        if (hdr_size) {
                bool bos;
                int i;

                if (skb_cow(skb, hdr_size)) {
                        err = -ENOBUFS;
                        goto errout_free;
                }

                skb_reserve(skb, hdr_size);
                skb_push(skb, hdr_size);
                skb_reset_network_header(skb);

                /* Push new labels */
                hdr = mpls_hdr(skb);
                bos = true;
                for (i = n_labels - 1; i >= 0; i--) {
                        hdr[i] = mpls_entry_encode(labels[i],
                                                   1, 0, bos);
                        bos = false;
                }
        }

        nh = mpls_select_multipath(rt, skb);
        if (!nh) {
                err = -ENETUNREACH;
                goto errout_free;
        }

        if (hdr_size) {
                skb_pull(skb, hdr_size);
                skb_reset_network_header(skb);
        }

        nlh = nlmsg_put(skb, portid, in_nlh->nlmsg_seq,
                        RTM_NEWROUTE, sizeof(*r), 0);
        if (!nlh) {
                err = -EMSGSIZE;
                goto errout_free;
        }

        r = nlmsg_data(nlh);
        r->rtm_family         = AF_MPLS;
        r->rtm_dst_len        = 20;
        r->rtm_src_len        = 0;
        r->rtm_table        = RT_TABLE_MAIN;
        r->rtm_type        = RTN_UNICAST;
        r->rtm_scope        = RT_SCOPE_UNIVERSE;
        r->rtm_protocol = rt->rt_protocol;
        r->rtm_flags        = 0;

        if (nla_put_labels(skb, RTA_DST, 1, &in_label))
                goto nla_put_failure;

        if (nh->nh_labels &&
            nla_put_labels(skb, RTA_NEWDST, nh->nh_labels,
                           nh->nh_label))
                goto nla_put_failure;

        if (nh->nh_via_table != MPLS_NEIGH_TABLE_UNSPEC &&
            nla_put_via(skb, nh->nh_via_table, mpls_nh_via(rt, nh),
                        nh->nh_via_alen))
                goto nla_put_failure;
        dev = nh->nh_dev;
        if (dev && nla_put_u32(skb, RTA_OIF, dev->ifindex))
                goto nla_put_failure;

        nlmsg_end(skb, nlh);

        err = rtnl_unicast(skb, net, portid);
errout:
        return err;

nla_put_failure:
        nlmsg_cancel(skb, nlh);
        err = -EMSGSIZE;
errout_free:
        kfree_skb(skb);
        return err;
}

static int resize_platform_label_table(struct net *net, size_t limit)
{
        size_t size = sizeof(struct mpls_route *) * limit;
        size_t old_limit;
        size_t cp_size;
        struct mpls_route __rcu **labels = NULL, **old;
        struct mpls_route *rt0 = NULL, *rt2 = NULL;
        unsigned index;

        if (size) {
                labels = kvzalloc(size, GFP_KERNEL);
                if (!labels)
                        goto nolabels;
        }

        /* In case the predefined labels need to be populated */
        if (limit > MPLS_LABEL_IPV4NULL) {
                struct net_device *lo = net->loopback_dev;
                rt0 = mpls_rt_alloc(1, lo->addr_len, 0);
                if (IS_ERR(rt0))
                        goto nort0;
                rt0->rt_nh->nh_dev = lo;
                rt0->rt_protocol = RTPROT_KERNEL;
                rt0->rt_payload_type = MPT_IPV4;
                rt0->rt_ttl_propagate = MPLS_TTL_PROP_DEFAULT;
                rt0->rt_nh->nh_via_table = NEIGH_LINK_TABLE;
                rt0->rt_nh->nh_via_alen = lo->addr_len;
                memcpy(__mpls_nh_via(rt0, rt0->rt_nh), lo->dev_addr,
                       lo->addr_len);
        }
        if (limit > MPLS_LABEL_IPV6NULL) {
                struct net_device *lo = net->loopback_dev;
                rt2 = mpls_rt_alloc(1, lo->addr_len, 0);
                if (IS_ERR(rt2))
                        goto nort2;
                rt2->rt_nh->nh_dev = lo;
                rt2->rt_protocol = RTPROT_KERNEL;
                rt2->rt_payload_type = MPT_IPV6;
                rt2->rt_ttl_propagate = MPLS_TTL_PROP_DEFAULT;
                rt2->rt_nh->nh_via_table = NEIGH_LINK_TABLE;
                rt2->rt_nh->nh_via_alen = lo->addr_len;
                memcpy(__mpls_nh_via(rt2, rt2->rt_nh), lo->dev_addr,
                       lo->addr_len);
        }

        rtnl_lock();
        /* Remember the original table */
        old = rtnl_dereference(net->mpls.platform_label);
        old_limit = net->mpls.platform_labels;

        /* Free any labels beyond the new table */
        for (index = limit; index < old_limit; index++)
                mpls_route_update(net, index, NULL, NULL);

        /* Copy over the old labels */
        cp_size = size;
        if (old_limit < limit)
                cp_size = old_limit * sizeof(struct mpls_route *);

        memcpy(labels, old, cp_size);

        /* If needed set the predefined labels */
        if ((old_limit <= MPLS_LABEL_IPV6NULL) &&
            (limit > MPLS_LABEL_IPV6NULL)) {
                RCU_INIT_POINTER(labels[MPLS_LABEL_IPV6NULL], rt2);
                rt2 = NULL;
        }

        if ((old_limit <= MPLS_LABEL_IPV4NULL) &&
            (limit > MPLS_LABEL_IPV4NULL)) {
                RCU_INIT_POINTER(labels[MPLS_LABEL_IPV4NULL], rt0);
                rt0 = NULL;
        }

        /* Update the global pointers */
        net->mpls.platform_labels = limit;
        rcu_assign_pointer(net->mpls.platform_label, labels);

        rtnl_unlock();

        mpls_rt_free(rt2);
        mpls_rt_free(rt0);

        if (old) {
                synchronize_rcu();
                kvfree(old);
        }
        return 0;

nort2:
        mpls_rt_free(rt0);
nort0:
        kvfree(labels);
nolabels:
        return -ENOMEM;
}

static int mpls_platform_labels(const struct ctl_table *table, int write,
                                void *buffer, size_t *lenp, loff_t *ppos)
{
        struct net *net = table->data;
        int platform_labels = net->mpls.platform_labels;
        int ret;
        struct ctl_table tmp = {
                .procname        = table->procname,
                .data                = &platform_labels,
                .maxlen                = sizeof(int),
                .mode                = table->mode,
                .extra1                = SYSCTL_ZERO,
                .extra2                = &label_limit,
        };

        ret = proc_dointvec_minmax(&tmp, write, buffer, lenp, ppos);

        if (write && ret == 0)
                ret = resize_platform_label_table(net, platform_labels);

        return ret;
}

#define MPLS_NS_SYSCTL_OFFSET(field)                \
        (&((struct net *)0)->field)

static const struct ctl_table mpls_table[] = {
        {
                .procname        = "platform_labels",
                .data                = NULL,
                .maxlen                = sizeof(int),
                .mode                = 0644,
                .proc_handler        = mpls_platform_labels,
        },
        {
                .procname        = "ip_ttl_propagate",
                .data                = MPLS_NS_SYSCTL_OFFSET(mpls.ip_ttl_propagate),
                .maxlen                = sizeof(int),
                .mode                = 0644,
                .proc_handler        = proc_dointvec_minmax,
                .extra1                = SYSCTL_ZERO,
                .extra2                = SYSCTL_ONE,
        },
        {
                .procname        = "default_ttl",
                .data                = MPLS_NS_SYSCTL_OFFSET(mpls.default_ttl),
                .maxlen                = sizeof(int),
                .mode                = 0644,
                .proc_handler        = proc_dointvec_minmax,
                .extra1                = SYSCTL_ONE,
                .extra2                = &ttl_max,
        },
};

static int mpls_net_init(struct net *net)
{
        size_t table_size = ARRAY_SIZE(mpls_table);
        struct ctl_table *table;
        int i;

        net->mpls.platform_labels = 0;
        net->mpls.platform_label = NULL;
        net->mpls.ip_ttl_propagate = 1;
        net->mpls.default_ttl = 255;

        table = kmemdup(mpls_table, sizeof(mpls_table), GFP_KERNEL);
        if (table == NULL)
                return -ENOMEM;

        /* Table data contains only offsets relative to the base of
         * the mdev at this point, so make them absolute.
         */
        for (i = 0; i < table_size; i++)
                table[i].data = (char *)net + (uintptr_t)table[i].data;

        net->mpls.ctl = register_net_sysctl_sz(net, "net/mpls", table,
                                               table_size);
        if (net->mpls.ctl == NULL) {
                kfree(table);
                return -ENOMEM;
        }

        return 0;
}

static void mpls_net_exit(struct net *net)
{
        struct mpls_route __rcu **platform_label;
        size_t platform_labels;
        const struct ctl_table *table;
        unsigned int index;

        table = net->mpls.ctl->ctl_table_arg;
        unregister_net_sysctl_table(net->mpls.ctl);
        kfree(table);

        /* An rcu grace period has passed since there was a device in
         * the network namespace (and thus the last in flight packet)
         * left this network namespace.  This is because
         * unregister_netdevice_many and netdev_run_todo has completed
         * for each network device that was in this network namespace.
         *
         * As such no additional rcu synchronization is necessary when
         * freeing the platform_label table.
         */
        rtnl_lock();
        platform_label = rtnl_dereference(net->mpls.platform_label);
        platform_labels = net->mpls.platform_labels;
        for (index = 0; index < platform_labels; index++) {
                struct mpls_route *rt = rtnl_dereference(platform_label[index]);
                RCU_INIT_POINTER(platform_label[index], NULL);
                mpls_notify_route(net, index, rt, NULL, NULL);
                mpls_rt_free(rt);
        }
        rtnl_unlock();

        kvfree(platform_label);
}

static struct pernet_operations mpls_net_ops = {
        .init = mpls_net_init,
        .exit = mpls_net_exit,
};

static struct rtnl_af_ops mpls_af_ops __read_mostly = {
        .family                   = AF_MPLS,
        .fill_stats_af           = mpls_fill_stats_af,
        .get_stats_af_size = mpls_get_stats_af_size,
};

static const struct rtnl_msg_handler mpls_rtnl_msg_handlers[] __initdata_or_module = {
        {THIS_MODULE, PF_MPLS, RTM_NEWROUTE, mpls_rtm_newroute, NULL, 0},
        {THIS_MODULE, PF_MPLS, RTM_DELROUTE, mpls_rtm_delroute, NULL, 0},
        {THIS_MODULE, PF_MPLS, RTM_GETROUTE, mpls_getroute, mpls_dump_routes, 0},
        {THIS_MODULE, PF_MPLS, RTM_GETNETCONF,
         mpls_netconf_get_devconf, mpls_netconf_dump_devconf,
         RTNL_FLAG_DUMP_UNLOCKED},
};

static int __init mpls_init(void)
{
        int err;

        BUILD_BUG_ON(sizeof(struct mpls_shim_hdr) != 4);

        err = register_pernet_subsys(&mpls_net_ops);
        if (err)
                goto out;

        err = register_netdevice_notifier(&mpls_dev_notifier);
        if (err)
                goto out_unregister_pernet;

        dev_add_pack(&mpls_packet_type);

        err = rtnl_af_register(&mpls_af_ops);
        if (err)
                goto out_unregister_dev_type;

        err = rtnl_register_many(mpls_rtnl_msg_handlers);
        if (err)
                goto out_unregister_rtnl_af;

        err = ipgre_tunnel_encap_add_mpls_ops();
        if (err) {
                pr_err("Can't add mpls over gre tunnel ops\n");
                goto out_unregister_rtnl;
        }

        err = 0;
out:
        return err;

out_unregister_rtnl:
        rtnl_unregister_many(mpls_rtnl_msg_handlers);
out_unregister_rtnl_af:
        rtnl_af_unregister(&mpls_af_ops);
out_unregister_dev_type:
        dev_remove_pack(&mpls_packet_type);
out_unregister_pernet:
        unregister_pernet_subsys(&mpls_net_ops);
        goto out;
}
module_init(mpls_init);

static void __exit mpls_exit(void)
{
        rtnl_unregister_all(PF_MPLS);
        rtnl_af_unregister(&mpls_af_ops);
        dev_remove_pack(&mpls_packet_type);
        unregister_netdevice_notifier(&mpls_dev_notifier);
        unregister_pernet_subsys(&mpls_net_ops);
        ipgre_tunnel_encap_del_mpls_ops();
}
module_exit(mpls_exit);

MODULE_DESCRIPTION("MultiProtocol Label Switching");
MODULE_LICENSE("GPL v2");
MODULE_ALIAS_NETPROTO(PF_MPLS);




























































    3 

    3 

    3 

    2 
    1 
    3 

    3 








   45 


    1 


    1 


    1 



    1 



    1 

   19 


   18 






    1 

   20 










   17 
   20 








   25 









    5 





    1 




    4 







   13 





    1 






    1 


    1 



    1 

   11 




























   17 









   15 








   11 

    2 





   12 




    1 
    1 



































































































































































































































   11 













    4 







    1 







    1 


    1 

    1 




























   48 






    1 


   31 

   16 







   45 






   28 






    7 







    1 



   36 




   39 

    3 








   44 















   11 




   11 











   22 










    1 
    1 





    1 

    1 







    1 

    5 







    1 

    3 




    1 


    1 




    1 


    1 

    1 




    1 



    1 




















































    5 


    1 


    1 


    1 

    1 

    1 














    2 




    2 





















   16 








   16 


    1 






























   14 










    3 






    1 

    1 





    2 





















    5 

    5 











































    4 




    1 







    4 
    1 








    5 









    5 









    6 




    1 




    5 
























































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
// SPDX-License-Identifier: GPL-2.0-or-later
/* AF_RXRPC implementation
 *
 * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved.
 * Written by David Howells (dhowells@redhat.com)
 */

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#include <linux/module.h>
#include <linux/kernel.h>
#include <linux/net.h>
#include <linux/slab.h>
#include <linux/skbuff.h>
#include <linux/random.h>
#include <linux/poll.h>
#include <linux/proc_fs.h>
#include <linux/key-type.h>
#include <net/net_namespace.h>
#include <net/sock.h>
#include <net/af_rxrpc.h>
#define CREATE_TRACE_POINTS
#include "ar-internal.h"

MODULE_DESCRIPTION("RxRPC network protocol");
MODULE_AUTHOR("Red Hat, Inc.");
MODULE_LICENSE("GPL");
MODULE_ALIAS_NETPROTO(PF_RXRPC);

unsigned int rxrpc_debug; // = RXRPC_DEBUG_KPROTO;
module_param_named(debug, rxrpc_debug, uint, 0644);
MODULE_PARM_DESC(debug, "RxRPC debugging mask");

static struct proto rxrpc_proto;
static const struct proto_ops rxrpc_rpc_ops;

/* current debugging ID */
atomic_t rxrpc_debug_id;
EXPORT_SYMBOL(rxrpc_debug_id);

/* count of skbs currently in use */
atomic_t rxrpc_n_rx_skbs;

struct workqueue_struct *rxrpc_workqueue;

static void rxrpc_sock_destructor(struct sock *);

/*
 * see if an RxRPC socket is currently writable
 */
static inline int rxrpc_writable(struct sock *sk)
{
        return refcount_read(&sk->sk_wmem_alloc) < (size_t) sk->sk_sndbuf;
}

/*
 * wait for write bufferage to become available
 */
static void rxrpc_write_space(struct sock *sk)
{
        _enter("%p", sk);
        rcu_read_lock();
        if (rxrpc_writable(sk)) {
                struct socket_wq *wq = rcu_dereference(sk->sk_wq);

                if (skwq_has_sleeper(wq))
                        wake_up_interruptible(&wq->wait);
                sk_wake_async_rcu(sk, SOCK_WAKE_SPACE, POLL_OUT);
        }
        rcu_read_unlock();
}

/*
 * validate an RxRPC address
 */
static int rxrpc_validate_address(struct rxrpc_sock *rx,
                                  struct sockaddr_rxrpc *srx,
                                  int len)
{
        unsigned int tail;

        if (len < sizeof(struct sockaddr_rxrpc))
                return -EINVAL;

        if (srx->srx_family != AF_RXRPC)
                return -EAFNOSUPPORT;

        if (srx->transport_type != SOCK_DGRAM)
                return -ESOCKTNOSUPPORT;

        len -= offsetof(struct sockaddr_rxrpc, transport);
        if (srx->transport_len < sizeof(sa_family_t) ||
            srx->transport_len > len)
                return -EINVAL;

        switch (srx->transport.family) {
        case AF_INET:
                if (rx->family != AF_INET &&
                    rx->family != AF_INET6)
                        return -EAFNOSUPPORT;
                if (srx->transport_len < sizeof(struct sockaddr_in))
                        return -EINVAL;
                tail = offsetof(struct sockaddr_rxrpc, transport.sin.__pad);
                break;

#ifdef CONFIG_AF_RXRPC_IPV6
        case AF_INET6:
                if (rx->family != AF_INET6)
                        return -EAFNOSUPPORT;
                if (srx->transport_len < sizeof(struct sockaddr_in6))
                        return -EINVAL;
                tail = offsetof(struct sockaddr_rxrpc, transport) +
                        sizeof(struct sockaddr_in6);
                break;
#endif

        default:
                return -EAFNOSUPPORT;
        }

        if (tail < len)
                memset((void *)srx + tail, 0, len - tail);
        _debug("INET: %pISp", &srx->transport);
        return 0;
}

/*
 * bind a local address to an RxRPC socket
 */
static int rxrpc_bind(struct socket *sock, struct sockaddr *saddr, int len)
{
        struct sockaddr_rxrpc *srx = (struct sockaddr_rxrpc *)saddr;
        struct rxrpc_local *local;
        struct rxrpc_sock *rx = rxrpc_sk(sock->sk);
        u16 service_id;
        int ret;

        _enter("%p,%p,%d", rx, saddr, len);

        ret = rxrpc_validate_address(rx, srx, len);
        if (ret < 0)
                goto error;
        service_id = srx->srx_service;

        lock_sock(&rx->sk);

        switch (rx->sk.sk_state) {
        case RXRPC_UNBOUND:
                rx->srx = *srx;
                local = rxrpc_lookup_local(sock_net(&rx->sk), &rx->srx);
                if (IS_ERR(local)) {
                        ret = PTR_ERR(local);
                        goto error_unlock;
                }

                if (service_id) {
                        write_lock(&local->services_lock);
                        if (local->service)
                                goto service_in_use;
                        rx->local = local;
                        local->service = rx;
                        write_unlock(&local->services_lock);

                        rx->sk.sk_state = RXRPC_SERVER_BOUND;
                } else {
                        rx->local = local;
                        rx->sk.sk_state = RXRPC_CLIENT_BOUND;
                }
                break;

        case RXRPC_SERVER_BOUND:
                ret = -EINVAL;
                if (service_id == 0)
                        goto error_unlock;
                ret = -EADDRINUSE;
                if (service_id == rx->srx.srx_service)
                        goto error_unlock;
                ret = -EINVAL;
                srx->srx_service = rx->srx.srx_service;
                if (memcmp(srx, &rx->srx, sizeof(*srx)) != 0)
                        goto error_unlock;
                rx->second_service = service_id;
                rx->sk.sk_state = RXRPC_SERVER_BOUND2;
                break;

        default:
                ret = -EINVAL;
                goto error_unlock;
        }

        release_sock(&rx->sk);
        _leave(" = 0");
        return 0;

service_in_use:
        write_unlock(&local->services_lock);
        rxrpc_unuse_local(local, rxrpc_local_unuse_bind);
        rxrpc_put_local(local, rxrpc_local_put_bind);
        ret = -EADDRINUSE;
error_unlock:
        release_sock(&rx->sk);
error:
        _leave(" = %d", ret);
        return ret;
}

/*
 * set the number of pending calls permitted on a listening socket
 */
static int rxrpc_listen(struct socket *sock, int backlog)
{
        struct sock *sk = sock->sk;
        struct rxrpc_sock *rx = rxrpc_sk(sk);
        unsigned int max, old;
        int ret;

        _enter("%p,%d", rx, backlog);

        lock_sock(&rx->sk);

        switch (rx->sk.sk_state) {
        case RXRPC_UNBOUND:
                ret = -EADDRNOTAVAIL;
                break;
        case RXRPC_SERVER_BOUND:
        case RXRPC_SERVER_BOUND2:
                ASSERT(rx->local != NULL);
                max = READ_ONCE(rxrpc_max_backlog);
                ret = -EINVAL;
                if (backlog == INT_MAX)
                        backlog = max;
                else if (backlog < 0 || backlog > max)
                        break;
                old = sk->sk_max_ack_backlog;
                sk->sk_max_ack_backlog = backlog;
                ret = rxrpc_service_prealloc(rx, GFP_KERNEL);
                if (ret == 0)
                        rx->sk.sk_state = RXRPC_SERVER_LISTENING;
                else
                        sk->sk_max_ack_backlog = old;
                break;
        case RXRPC_SERVER_LISTENING:
                if (backlog == 0) {
                        rx->sk.sk_state = RXRPC_SERVER_LISTEN_DISABLED;
                        sk->sk_max_ack_backlog = 0;
                        rxrpc_discard_prealloc(rx);
                        ret = 0;
                        break;
                }
                fallthrough;
        default:
                ret = -EBUSY;
                break;
        }

        release_sock(&rx->sk);
        _leave(" = %d", ret);
        return ret;
}

/**
 * rxrpc_kernel_lookup_peer - Obtain remote transport endpoint for an address
 * @sock: The socket through which it will be accessed
 * @srx: The network address
 * @gfp: Allocation flags
 *
 * Lookup or create a remote transport endpoint record for the specified
 * address.
 *
 * Return: The peer record found with a reference, %NULL if no record is found
 * or a negative error code if the address is invalid or unsupported.
 */
struct rxrpc_peer *rxrpc_kernel_lookup_peer(struct socket *sock,
                                            struct sockaddr_rxrpc *srx, gfp_t gfp)
{
        struct rxrpc_sock *rx = rxrpc_sk(sock->sk);
        int ret;

        ret = rxrpc_validate_address(rx, srx, sizeof(*srx));
        if (ret < 0)
                return ERR_PTR(ret);

        return rxrpc_lookup_peer(rx->local, srx, gfp);
}
EXPORT_SYMBOL(rxrpc_kernel_lookup_peer);

/**
 * rxrpc_kernel_get_peer - Get a reference on a peer
 * @peer: The peer to get a reference on (may be NULL).
 *
 * Get a reference for a remote peer record (if not NULL).
 *
 * Return: The @peer argument.
 */
struct rxrpc_peer *rxrpc_kernel_get_peer(struct rxrpc_peer *peer)
{
        return peer ? rxrpc_get_peer(peer, rxrpc_peer_get_application) : NULL;
}
EXPORT_SYMBOL(rxrpc_kernel_get_peer);

/**
 * rxrpc_kernel_put_peer - Allow a kernel app to drop a peer reference
 * @peer: The peer to drop a ref on
 *
 * Drop a reference on a peer record.
 */
void rxrpc_kernel_put_peer(struct rxrpc_peer *peer)
{
        rxrpc_put_peer(peer, rxrpc_peer_put_application);
}
EXPORT_SYMBOL(rxrpc_kernel_put_peer);

/**
 * rxrpc_kernel_begin_call - Allow a kernel service to begin a call
 * @sock: The socket on which to make the call
 * @peer: The peer to contact
 * @key: The security context to use (defaults to socket setting)
 * @user_call_ID: The ID to use
 * @tx_total_len: Total length of data to transmit during the call (or -1)
 * @hard_timeout: The maximum lifespan of the call in sec
 * @gfp: The allocation constraints
 * @notify_rx: Where to send notifications instead of socket queue
 * @service_id: The ID of the service to contact
 * @upgrade: Request service upgrade for call
 * @interruptibility: The call is interruptible, or can be canceled.
 * @debug_id: The debug ID for tracing to be assigned to the call
 *
 * Allow a kernel service to begin a call on the nominated socket.  This just
 * sets up all the internal tracking structures and allocates connection and
 * call IDs as appropriate.
 *
 * The default socket destination address and security may be overridden by
 * supplying @srx and @key.
 *
 * Return: The new call or an error code.
 */
struct rxrpc_call *rxrpc_kernel_begin_call(struct socket *sock,
                                           struct rxrpc_peer *peer,
                                           struct key *key,
                                           unsigned long user_call_ID,
                                           s64 tx_total_len,
                                           u32 hard_timeout,
                                           gfp_t gfp,
                                           rxrpc_notify_rx_t notify_rx,
                                           u16 service_id,
                                           bool upgrade,
                                           enum rxrpc_interruptibility interruptibility,
                                           unsigned int debug_id)
{
        struct rxrpc_conn_parameters cp;
        struct rxrpc_call_params p;
        struct rxrpc_call *call;
        struct rxrpc_sock *rx = rxrpc_sk(sock->sk);

        _enter(",,%x,%lx", key_serial(key), user_call_ID);

        if (WARN_ON_ONCE(peer->local != rx->local))
                return ERR_PTR(-EIO);

        lock_sock(&rx->sk);

        if (!key)
                key = rx->key;
        if (key && !key->payload.data[0])
                key = NULL; /* a no-security key */

        memset(&p, 0, sizeof(p));
        p.user_call_ID                = user_call_ID;
        p.tx_total_len                = tx_total_len;
        p.interruptibility        = interruptibility;
        p.kernel                = true;
        p.timeouts.hard                = hard_timeout;

        memset(&cp, 0, sizeof(cp));
        cp.local                = rx->local;
        cp.peer                        = peer;
        cp.key                        = key;
        cp.security_level        = rx->min_sec_level;
        cp.exclusive                = false;
        cp.upgrade                = upgrade;
        cp.service_id                = service_id;
        call = rxrpc_new_client_call(rx, &cp, &p, gfp, debug_id);
        /* The socket has been unlocked. */
        if (!IS_ERR(call)) {
                call->notify_rx = notify_rx;
                mutex_unlock(&call->user_mutex);
        }

        _leave(" = %p", call);
        return call;
}
EXPORT_SYMBOL(rxrpc_kernel_begin_call);

/*
 * Dummy function used to stop the notifier talking to recvmsg().
 */
static void rxrpc_dummy_notify_rx(struct sock *sk, struct rxrpc_call *rxcall,
                                  unsigned long call_user_ID)
{
}

/**
 * rxrpc_kernel_shutdown_call - Allow a kernel service to shut down a call it was using
 * @sock: The socket the call is on
 * @call: The call to end
 *
 * Allow a kernel service to shut down a call it was using.  The call must be
 * complete before this is called (the call should be aborted if necessary).
 */
void rxrpc_kernel_shutdown_call(struct socket *sock, struct rxrpc_call *call)
{
        _enter("%d{%d}", call->debug_id, refcount_read(&call->ref));

        mutex_lock(&call->user_mutex);
        if (!test_bit(RXRPC_CALL_RELEASED, &call->flags)) {
                rxrpc_release_call(rxrpc_sk(sock->sk), call);

                /* Make sure we're not going to call back into a kernel service */
                if (call->notify_rx) {
                        spin_lock_irq(&call->notify_lock);
                        call->notify_rx = rxrpc_dummy_notify_rx;
                        spin_unlock_irq(&call->notify_lock);
                }
        }
        mutex_unlock(&call->user_mutex);
}
EXPORT_SYMBOL(rxrpc_kernel_shutdown_call);

/**
 * rxrpc_kernel_put_call - Release a reference to a call
 * @sock: The socket the call is on
 * @call: The call to put
 *
 * Drop the application's ref on an rxrpc call.
 */
void rxrpc_kernel_put_call(struct socket *sock, struct rxrpc_call *call)
{
        rxrpc_put_call(call, rxrpc_call_put_kernel);
}
EXPORT_SYMBOL(rxrpc_kernel_put_call);

/**
 * rxrpc_kernel_check_life - Check to see whether a call is still alive
 * @sock: The socket the call is on
 * @call: The call to check
 *
 * Allow a kernel service to find out whether a call is still alive - whether
 * it has completed successfully and all received data has been consumed.
 *
 * Return: %true if the call is still ongoing and %false if it has completed.
 */
bool rxrpc_kernel_check_life(const struct socket *sock,
                             const struct rxrpc_call *call)
{
        if (!rxrpc_call_is_complete(call))
                return true;
        if (call->completion != RXRPC_CALL_SUCCEEDED)
                return false;
        return !skb_queue_empty(&call->recvmsg_queue);
}
EXPORT_SYMBOL(rxrpc_kernel_check_life);

/**
 * rxrpc_kernel_set_notifications - Set table of callback operations
 * @sock: The socket to install table upon
 * @app_ops: Callback operation table to set
 *
 * Allow a kernel service to set a table of event notifications on a socket.
 */
void rxrpc_kernel_set_notifications(struct socket *sock,
                                    const struct rxrpc_kernel_ops *app_ops)
{
        struct rxrpc_sock *rx = rxrpc_sk(sock->sk);

        rx->app_ops = app_ops;
}
EXPORT_SYMBOL(rxrpc_kernel_set_notifications);

/*
 * connect an RxRPC socket
 * - this just targets it at a specific destination; no actual connection
 *   negotiation takes place
 */
static int rxrpc_connect(struct socket *sock, struct sockaddr *addr,
                         int addr_len, int flags)
{
        struct sockaddr_rxrpc *srx = (struct sockaddr_rxrpc *)addr;
        struct rxrpc_sock *rx = rxrpc_sk(sock->sk);
        int ret;

        _enter("%p,%p,%d,%d", rx, addr, addr_len, flags);

        ret = rxrpc_validate_address(rx, srx, addr_len);
        if (ret < 0) {
                _leave(" = %d [bad addr]", ret);
                return ret;
        }

        lock_sock(&rx->sk);

        ret = -EISCONN;
        if (test_bit(RXRPC_SOCK_CONNECTED, &rx->flags))
                goto error;

        switch (rx->sk.sk_state) {
        case RXRPC_UNBOUND:
                rx->sk.sk_state = RXRPC_CLIENT_UNBOUND;
                break;
        case RXRPC_CLIENT_UNBOUND:
        case RXRPC_CLIENT_BOUND:
                break;
        default:
                ret = -EBUSY;
                goto error;
        }

        rx->connect_srx = *srx;
        set_bit(RXRPC_SOCK_CONNECTED, &rx->flags);
        ret = 0;

error:
        release_sock(&rx->sk);
        return ret;
}

/*
 * send a message through an RxRPC socket
 * - in a client this does a number of things:
 *   - finds/sets up a connection for the security specified (if any)
 *   - initiates a call (ID in control data)
 *   - ends the request phase of a call (if MSG_MORE is not set)
 *   - sends a call data packet
 *   - may send an abort (abort code in control data)
 */
static int rxrpc_sendmsg(struct socket *sock, struct msghdr *m, size_t len)
{
        struct rxrpc_local *local;
        struct rxrpc_sock *rx = rxrpc_sk(sock->sk);
        int ret;

        _enter(",{%d},,%zu", rx->sk.sk_state, len);

        if (m->msg_flags & MSG_OOB)
                return -EOPNOTSUPP;

        if (m->msg_name) {
                ret = rxrpc_validate_address(rx, m->msg_name, m->msg_namelen);
                if (ret < 0) {
                        _leave(" = %d [bad addr]", ret);
                        return ret;
                }
        }

        lock_sock(&rx->sk);

        switch (rx->sk.sk_state) {
        case RXRPC_UNBOUND:
        case RXRPC_CLIENT_UNBOUND:
                rx->srx.srx_family = AF_RXRPC;
                rx->srx.srx_service = 0;
                rx->srx.transport_type = SOCK_DGRAM;
                rx->srx.transport.family = rx->family;
                switch (rx->family) {
                case AF_INET:
                        rx->srx.transport_len = sizeof(struct sockaddr_in);
                        break;
#ifdef CONFIG_AF_RXRPC_IPV6
                case AF_INET6:
                        rx->srx.transport_len = sizeof(struct sockaddr_in6);
                        break;
#endif
                default:
                        ret = -EAFNOSUPPORT;
                        goto error_unlock;
                }
                local = rxrpc_lookup_local(sock_net(sock->sk), &rx->srx);
                if (IS_ERR(local)) {
                        ret = PTR_ERR(local);
                        goto error_unlock;
                }

                rx->local = local;
                rx->sk.sk_state = RXRPC_CLIENT_BOUND;
                fallthrough;

        case RXRPC_CLIENT_BOUND:
                if (!m->msg_name &&
                    test_bit(RXRPC_SOCK_CONNECTED, &rx->flags)) {
                        m->msg_name = &rx->connect_srx;
                        m->msg_namelen = sizeof(rx->connect_srx);
                }
                fallthrough;
        case RXRPC_SERVER_BOUND:
        case RXRPC_SERVER_LISTENING:
                if (m->msg_flags & MSG_OOB)
                        ret = rxrpc_sendmsg_oob(rx, m, len);
                else
                        ret = rxrpc_do_sendmsg(rx, m, len);
                /* The socket has been unlocked */
                goto out;
        default:
                ret = -EINVAL;
                goto error_unlock;
        }

error_unlock:
        release_sock(&rx->sk);
out:
        _leave(" = %d", ret);
        return ret;
}

int rxrpc_sock_set_min_security_level(struct sock *sk, unsigned int val)
{
        if (sk->sk_state != RXRPC_UNBOUND)
                return -EISCONN;
        if (val > RXRPC_SECURITY_MAX)
                return -EINVAL;
        lock_sock(sk);
        rxrpc_sk(sk)->min_sec_level = val;
        release_sock(sk);
        return 0;
}
EXPORT_SYMBOL(rxrpc_sock_set_min_security_level);

/*
 * set RxRPC socket options
 */
static int rxrpc_setsockopt(struct socket *sock, int level, int optname,
                            sockptr_t optval, unsigned int optlen)
{
        struct rxrpc_sock *rx = rxrpc_sk(sock->sk);
        unsigned int min_sec_level, val;
        u16 service_upgrade[2];
        int ret;

        _enter(",%d,%d,,%d", level, optname, optlen);

        lock_sock(&rx->sk);
        ret = -EOPNOTSUPP;

        if (level == SOL_RXRPC) {
                switch (optname) {
                case RXRPC_EXCLUSIVE_CONNECTION:
                        ret = -EINVAL;
                        if (optlen != 0)
                                goto error;
                        ret = -EISCONN;
                        if (rx->sk.sk_state != RXRPC_UNBOUND)
                                goto error;
                        rx->exclusive = true;
                        goto success;

                case RXRPC_SECURITY_KEY:
                        ret = -EINVAL;
                        if (rx->key)
                                goto error;
                        ret = -EISCONN;
                        if (rx->sk.sk_state != RXRPC_UNBOUND)
                                goto error;
                        ret = rxrpc_request_key(rx, optval, optlen);
                        goto error;

                case RXRPC_SECURITY_KEYRING:
                        ret = -EINVAL;
                        if (rx->key)
                                goto error;
                        ret = -EISCONN;
                        if (rx->sk.sk_state != RXRPC_UNBOUND)
                                goto error;
                        ret = rxrpc_server_keyring(rx, optval, optlen);
                        goto error;

                case RXRPC_MIN_SECURITY_LEVEL:
                        ret = -EINVAL;
                        if (optlen != sizeof(unsigned int))
                                goto error;
                        ret = -EISCONN;
                        if (rx->sk.sk_state != RXRPC_UNBOUND)
                                goto error;
                        ret = copy_safe_from_sockptr(&min_sec_level,
                                                     sizeof(min_sec_level),
                                                     optval, optlen);
                        if (ret)
                                goto error;
                        ret = -EINVAL;
                        if (min_sec_level > RXRPC_SECURITY_MAX)
                                goto error;
                        rx->min_sec_level = min_sec_level;
                        goto success;

                case RXRPC_UPGRADEABLE_SERVICE:
                        ret = -EINVAL;
                        if (optlen != sizeof(service_upgrade) ||
                            rx->service_upgrade.from != 0)
                                goto error;
                        ret = -EISCONN;
                        if (rx->sk.sk_state != RXRPC_SERVER_BOUND2)
                                goto error;
                        ret = -EFAULT;
                        if (copy_from_sockptr(service_upgrade, optval,
                                           sizeof(service_upgrade)) != 0)
                                goto error;
                        ret = -EINVAL;
                        if ((service_upgrade[0] != rx->srx.srx_service ||
                             service_upgrade[1] != rx->second_service) &&
                            (service_upgrade[0] != rx->second_service ||
                             service_upgrade[1] != rx->srx.srx_service))
                                goto error;
                        rx->service_upgrade.from = service_upgrade[0];
                        rx->service_upgrade.to = service_upgrade[1];
                        goto success;

                case RXRPC_MANAGE_RESPONSE:
                        ret = -EINVAL;
                        if (optlen != sizeof(unsigned int))
                                goto error;
                        ret = -EISCONN;
                        if (rx->sk.sk_state != RXRPC_UNBOUND)
                                goto error;
                        ret = copy_safe_from_sockptr(&val, sizeof(val),
                                                     optval, optlen);
                        if (ret)
                                goto error;
                        ret = -EINVAL;
                        if (val > 1)
                                goto error;
                        if (val)
                                set_bit(RXRPC_SOCK_MANAGE_RESPONSE, &rx->flags);
                        else
                                clear_bit(RXRPC_SOCK_MANAGE_RESPONSE, &rx->flags);
                        goto success;

                default:
                        break;
                }
        }

success:
        ret = 0;
error:
        release_sock(&rx->sk);
        return ret;
}

/*
 * Get socket options.
 */
static int rxrpc_getsockopt(struct socket *sock, int level, int optname,
                            char __user *optval, int __user *_optlen)
{
        int optlen;

        if (level != SOL_RXRPC)
                return -EOPNOTSUPP;

        if (get_user(optlen, _optlen))
                return -EFAULT;

        switch (optname) {
        case RXRPC_SUPPORTED_CMSG:
                if (optlen < sizeof(int))
                        return -ETOOSMALL;
                if (put_user(RXRPC__SUPPORTED - 1, (int __user *)optval) ||
                    put_user(sizeof(int), _optlen))
                        return -EFAULT;
                return 0;

        default:
                return -EOPNOTSUPP;
        }
}

/*
 * permit an RxRPC socket to be polled
 */
static __poll_t rxrpc_poll(struct file *file, struct socket *sock,
                               poll_table *wait)
{
        struct sock *sk = sock->sk;
        struct rxrpc_sock *rx = rxrpc_sk(sk);
        __poll_t mask;

        sock_poll_wait(file, sock, wait);
        mask = 0;

        /* the socket is readable if there are any messages waiting on the Rx
         * queue */
        if (!list_empty(&rx->recvmsg_q))
                mask |= EPOLLIN | EPOLLRDNORM;

        /* the socket is writable if there is space to add new data to the
         * socket; there is no guarantee that any particular call in progress
         * on the socket may have space in the Tx ACK window */
        if (rxrpc_writable(sk))
                mask |= EPOLLOUT | EPOLLWRNORM;

        return mask;
}

/*
 * create an RxRPC socket
 */
static int rxrpc_create(struct net *net, struct socket *sock, int protocol,
                        int kern)
{
        struct rxrpc_net *rxnet;
        struct rxrpc_sock *rx;
        struct sock *sk;

        _enter("%p,%d", sock, protocol);

        /* we support transport protocol UDP/UDP6 only */
        if (protocol != PF_INET &&
            IS_ENABLED(CONFIG_AF_RXRPC_IPV6) && protocol != PF_INET6)
                return -EPROTONOSUPPORT;

        if (sock->type != SOCK_DGRAM)
                return -ESOCKTNOSUPPORT;

        sock->ops = &rxrpc_rpc_ops;
        sock->state = SS_UNCONNECTED;

        sk = sk_alloc(net, PF_RXRPC, GFP_KERNEL, &rxrpc_proto, kern);
        if (!sk)
                return -ENOMEM;

        sock_init_data(sock, sk);
        sock_set_flag(sk, SOCK_RCU_FREE);
        sk->sk_state                = RXRPC_UNBOUND;
        sk->sk_write_space        = rxrpc_write_space;
        sk->sk_max_ack_backlog        = 0;
        sk->sk_destruct                = rxrpc_sock_destructor;

        rx = rxrpc_sk(sk);
        rx->family = protocol;
        rx->calls = RB_ROOT;

        spin_lock_init(&rx->incoming_lock);
        skb_queue_head_init(&rx->recvmsg_oobq);
        rx->pending_oobq = RB_ROOT;
        INIT_LIST_HEAD(&rx->sock_calls);
        INIT_LIST_HEAD(&rx->to_be_accepted);
        INIT_LIST_HEAD(&rx->recvmsg_q);
        spin_lock_init(&rx->recvmsg_lock);
        rwlock_init(&rx->call_lock);
        memset(&rx->srx, 0, sizeof(rx->srx));

        rxnet = rxrpc_net(sock_net(&rx->sk));
        timer_reduce(&rxnet->peer_keepalive_timer, jiffies + 1);

        _leave(" = 0 [%p]", rx);
        return 0;
}

/*
 * Kill all the calls on a socket and shut it down.
 */
static int rxrpc_shutdown(struct socket *sock, int flags)
{
        struct sock *sk = sock->sk;
        struct rxrpc_sock *rx = rxrpc_sk(sk);
        int ret = 0;

        _enter("%p,%d", sk, flags);

        if (flags != SHUT_RDWR)
                return -EOPNOTSUPP;
        if (sk->sk_state == RXRPC_CLOSE)
                return -ESHUTDOWN;

        lock_sock(sk);

        if (sk->sk_state < RXRPC_CLOSE) {
                spin_lock_irq(&rx->recvmsg_lock);
                sk->sk_state = RXRPC_CLOSE;
                sk->sk_shutdown = SHUTDOWN_MASK;
                spin_unlock_irq(&rx->recvmsg_lock);
        } else {
                ret = -ESHUTDOWN;
        }

        rxrpc_discard_prealloc(rx);

        release_sock(sk);
        return ret;
}

/*
 * Purge the out-of-band queue.
 */
static void rxrpc_purge_oob_queue(struct sock *sk)
{
        struct rxrpc_sock *rx = rxrpc_sk(sk);
        struct sk_buff *skb;

        while ((skb = skb_dequeue(&rx->recvmsg_oobq)))
                rxrpc_kernel_free_oob(skb);
        while (!RB_EMPTY_ROOT(&rx->pending_oobq)) {
                skb = rb_entry(rx->pending_oobq.rb_node, struct sk_buff, rbnode);
                rb_erase(&skb->rbnode, &rx->pending_oobq);
                rxrpc_kernel_free_oob(skb);
        }
}

/*
 * RxRPC socket destructor
 */
static void rxrpc_sock_destructor(struct sock *sk)
{
        _enter("%p", sk);

        rxrpc_purge_oob_queue(sk);
        rxrpc_purge_queue(&sk->sk_receive_queue);

        WARN_ON(refcount_read(&sk->sk_wmem_alloc));
        WARN_ON(!sk_unhashed(sk));
        WARN_ON(sk->sk_socket);

        if (!sock_flag(sk, SOCK_DEAD)) {
                printk("Attempt to release alive rxrpc socket: %p\n", sk);
                return;
        }
}

/*
 * release an RxRPC socket
 */
static int rxrpc_release_sock(struct sock *sk)
{
        struct rxrpc_sock *rx = rxrpc_sk(sk);

        _enter("%p{%d,%d}", sk, sk->sk_state, refcount_read(&sk->sk_refcnt));

        /* declare the socket closed for business */
        sock_orphan(sk);
        sk->sk_shutdown = SHUTDOWN_MASK;

        /* We want to kill off all connections from a service socket
         * as fast as possible because we can't share these; client
         * sockets, on the other hand, can share an endpoint.
         */
        switch (sk->sk_state) {
        case RXRPC_SERVER_BOUND:
        case RXRPC_SERVER_BOUND2:
        case RXRPC_SERVER_LISTENING:
        case RXRPC_SERVER_LISTEN_DISABLED:
                rx->local->service_closed = true;
                break;
        }

        spin_lock_irq(&rx->recvmsg_lock);
        sk->sk_state = RXRPC_CLOSE;
        spin_unlock_irq(&rx->recvmsg_lock);

        if (rx->local && rx->local->service == rx) {
                write_lock(&rx->local->services_lock);
                rx->local->service = NULL;
                write_unlock(&rx->local->services_lock);
        }

        /* try to flush out this socket */
        rxrpc_discard_prealloc(rx);
        rxrpc_release_calls_on_socket(rx);
        flush_workqueue(rxrpc_workqueue);
        rxrpc_purge_oob_queue(sk);
        rxrpc_purge_queue(&sk->sk_receive_queue);

        rxrpc_unuse_local(rx->local, rxrpc_local_unuse_release_sock);
        rxrpc_put_local(rx->local, rxrpc_local_put_release_sock);
        rx->local = NULL;
        key_put(rx->key);
        rx->key = NULL;
        key_put(rx->securities);
        rx->securities = NULL;
        sock_put(sk);

        _leave(" = 0");
        return 0;
}

/*
 * release an RxRPC BSD socket on close() or equivalent
 */
static int rxrpc_release(struct socket *sock)
{
        struct sock *sk = sock->sk;

        _enter("%p{%p}", sock, sk);

        if (!sk)
                return 0;

        sock->sk = NULL;

        return rxrpc_release_sock(sk);
}

/*
 * RxRPC network protocol
 */
static const struct proto_ops rxrpc_rpc_ops = {
        .family                = PF_RXRPC,
        .owner                = THIS_MODULE,
        .release        = rxrpc_release,
        .bind                = rxrpc_bind,
        .connect        = rxrpc_connect,
        .socketpair        = sock_no_socketpair,
        .accept                = sock_no_accept,
        .getname        = sock_no_getname,
        .poll                = rxrpc_poll,
        .ioctl                = sock_no_ioctl,
        .listen                = rxrpc_listen,
        .shutdown        = rxrpc_shutdown,
        .setsockopt        = rxrpc_setsockopt,
        .getsockopt        = rxrpc_getsockopt,
        .sendmsg        = rxrpc_sendmsg,
        .recvmsg        = rxrpc_recvmsg,
        .mmap                = sock_no_mmap,
};

static struct proto rxrpc_proto = {
        .name                = "RXRPC",
        .owner                = THIS_MODULE,
        .obj_size        = sizeof(struct rxrpc_sock),
        .max_header        = sizeof(struct rxrpc_wire_header),
};

static const struct net_proto_family rxrpc_family_ops = {
        .family        = PF_RXRPC,
        .create = rxrpc_create,
        .owner        = THIS_MODULE,
};

/*
 * initialise and register the RxRPC protocol
 */
static int __init af_rxrpc_init(void)
{
        int ret = -1;

        BUILD_BUG_ON(sizeof(struct rxrpc_skb_priv) > sizeof_field(struct sk_buff, cb));

        ret = -ENOMEM;
        rxrpc_gen_version_string();
        rxrpc_call_jar = kmem_cache_create(
                "rxrpc_call_jar", sizeof(struct rxrpc_call), 0,
                SLAB_HWCACHE_ALIGN, NULL);
        if (!rxrpc_call_jar) {
                pr_notice("Failed to allocate call jar\n");
                goto error_call_jar;
        }

        rxrpc_workqueue = alloc_ordered_workqueue("krxrpcd", WQ_HIGHPRI | WQ_MEM_RECLAIM);
        if (!rxrpc_workqueue) {
                pr_notice("Failed to allocate work queue\n");
                goto error_work_queue;
        }

        ret = rxrpc_init_security();
        if (ret < 0) {
                pr_crit("Cannot initialise security\n");
                goto error_security;
        }

        ret = register_pernet_device(&rxrpc_net_ops);
        if (ret)
                goto error_pernet;

        ret = proto_register(&rxrpc_proto, 1);
        if (ret < 0) {
                pr_crit("Cannot register protocol\n");
                goto error_proto;
        }

        ret = sock_register(&rxrpc_family_ops);
        if (ret < 0) {
                pr_crit("Cannot register socket family\n");
                goto error_sock;
        }

        ret = register_key_type(&key_type_rxrpc);
        if (ret < 0) {
                pr_crit("Cannot register client key type\n");
                goto error_key_type;
        }

        ret = register_key_type(&key_type_rxrpc_s);
        if (ret < 0) {
                pr_crit("Cannot register server key type\n");
                goto error_key_type_s;
        }

        ret = rxrpc_sysctl_init();
        if (ret < 0) {
                pr_crit("Cannot register sysctls\n");
                goto error_sysctls;
        }

        return 0;

error_sysctls:
        unregister_key_type(&key_type_rxrpc_s);
error_key_type_s:
        unregister_key_type(&key_type_rxrpc);
error_key_type:
        sock_unregister(PF_RXRPC);
error_sock:
        proto_unregister(&rxrpc_proto);
error_proto:
        unregister_pernet_device(&rxrpc_net_ops);
error_pernet:
        rxrpc_exit_security();
error_security:
        destroy_workqueue(rxrpc_workqueue);
error_work_queue:
        kmem_cache_destroy(rxrpc_call_jar);
error_call_jar:
        return ret;
}

/*
 * unregister the RxRPC protocol
 */
static void __exit af_rxrpc_exit(void)
{
        _enter("");
        rxrpc_sysctl_exit();
        unregister_key_type(&key_type_rxrpc_s);
        unregister_key_type(&key_type_rxrpc);
        sock_unregister(PF_RXRPC);
        proto_unregister(&rxrpc_proto);
        unregister_pernet_device(&rxrpc_net_ops);
        ASSERTCMP(atomic_read(&rxrpc_n_rx_skbs), ==, 0);

        /* Make sure the local and peer records pinned by any dying connections
         * are released.
         */
        rcu_barrier();

        destroy_workqueue(rxrpc_workqueue);
        rxrpc_exit_security();
        kmem_cache_destroy(rxrpc_call_jar);
        _leave("");
}

module_init(af_rxrpc_init);
module_exit(af_rxrpc_exit);











































































































































































































































































































































































































































































    3 




    3 
    3 

    3 








    3 

























   33 




   30 



    6 




    6 
    6 





































































    3 



    3 






    3 






    3 





    3 















    3 





















    3 




    3 


    3 



















    3 































    2 





























































    3 






































    3 





    3 







































    3 
    2 


    3 





















    3 


    3 











    3 












    3 
    3 
    3 
    2 




    3 


    3 































    3 





    3 


    3 




    3 




    3 





    3 































    3 
    3 
    3 






    3 


    3 
    3 






    3 

    3 


    3 

    3 






















    3 








    3 










    3 

























    3 




    3 









    3 
    3 






    3 




    3 


    3 










    3 




    3 



    3 
    3 





    3 
    3 





    3 







    3 
    3 



    3 













































































































































































































    3 


    3 

    3 
















































    3 






    3 

    3 

    3 











    3 













    3 
    3 


























    3 









































    3 












    3 



    3 

    3 


    3 












    2 




    2 





    2 















    2 







    2 



    2 


    2 







    2 
























    3 



    3 
    3 
    3 



    3 
    3 














    3 
    3 
    3 
    3 










    3 















    3 















    3 









    3 














    3 




    3 




    3 


    3 




    3 











    3 


    3 




    3 
























    3 


    3 
    3 










    3 














    1 


















   26 









   17 



   17 


   17 

















   17 












   17 



   17 






   17 

























   17 
   17 
   17 







   17 










    2 




   30 





   30 

























































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright (C) 2010-2013 Felix Fietkau <nbd@openwrt.org>
 * Copyright (C) 2019-2022 Intel Corporation
 */
#include <linux/netdevice.h>
#include <linux/types.h>
#include <linux/skbuff.h>
#include <linux/debugfs.h>
#include <linux/random.h>
#include <linux/moduleparam.h>
#include <linux/ieee80211.h>
#include <linux/minmax.h>
#include <net/mac80211.h>
#include "rate.h"
#include "sta_info.h"
#include "rc80211_minstrel_ht.h"

#define AVG_AMPDU_SIZE        16
#define AVG_PKT_SIZE        1200

/* Number of bits for an average sized packet */
#define MCS_NBITS ((AVG_PKT_SIZE * AVG_AMPDU_SIZE) << 3)

/* Number of symbols for a packet with (bps) bits per symbol */
#define MCS_NSYMS(bps) DIV_ROUND_UP(MCS_NBITS, (bps))

/* Transmission time (nanoseconds) for a packet containing (syms) symbols */
#define MCS_SYMBOL_TIME(sgi, syms)                                        \
        (sgi ?                                                                \
          ((syms) * 18000 + 4000) / 5 :        /* syms * 3.6 us */                \
          ((syms) * 1000) << 2                /* syms * 4 us */                \
        )

/* Transmit duration for the raw data part of an average sized packet */
#define MCS_DURATION(streams, sgi, bps) \
        (MCS_SYMBOL_TIME(sgi, MCS_NSYMS((streams) * (bps))) / AVG_AMPDU_SIZE)

#define BW_20                        0
#define BW_40                        1
#define BW_80                        2

/*
 * Define group sort order: HT40 -> SGI -> #streams
 */
#define GROUP_IDX(_streams, _sgi, _ht40)        \
        MINSTREL_HT_GROUP_0 +                        \
        MINSTREL_MAX_STREAMS * 2 * _ht40 +        \
        MINSTREL_MAX_STREAMS * _sgi +        \
        _streams - 1

#define _MAX(a, b) (((a)>(b))?(a):(b))

#define GROUP_SHIFT(duration)                                                \
        _MAX(0, 16 - __builtin_clz(duration))

/* MCS rate information for an MCS group */
#define __MCS_GROUP(_streams, _sgi, _ht40, _s)                                \
        [GROUP_IDX(_streams, _sgi, _ht40)] = {                                \
        .streams = _streams,                                                \
        .shift = _s,                                                        \
        .bw = _ht40,                                                        \
        .flags =                                                        \
                IEEE80211_TX_RC_MCS |                                        \
                (_sgi ? IEEE80211_TX_RC_SHORT_GI : 0) |                        \
                (_ht40 ? IEEE80211_TX_RC_40_MHZ_WIDTH : 0),                \
        .duration = {                                                        \
                MCS_DURATION(_streams, _sgi, _ht40 ? 54 : 26) >> _s,        \
                MCS_DURATION(_streams, _sgi, _ht40 ? 108 : 52) >> _s,        \
                MCS_DURATION(_streams, _sgi, _ht40 ? 162 : 78) >> _s,        \
                MCS_DURATION(_streams, _sgi, _ht40 ? 216 : 104) >> _s,        \
                MCS_DURATION(_streams, _sgi, _ht40 ? 324 : 156) >> _s,        \
                MCS_DURATION(_streams, _sgi, _ht40 ? 432 : 208) >> _s,        \
                MCS_DURATION(_streams, _sgi, _ht40 ? 486 : 234) >> _s,        \
                MCS_DURATION(_streams, _sgi, _ht40 ? 540 : 260) >> _s        \
        }                                                                \
}

#define MCS_GROUP_SHIFT(_streams, _sgi, _ht40)                                \
        GROUP_SHIFT(MCS_DURATION(_streams, _sgi, _ht40 ? 54 : 26))

#define MCS_GROUP(_streams, _sgi, _ht40)                                \
        __MCS_GROUP(_streams, _sgi, _ht40,                                \
                    MCS_GROUP_SHIFT(_streams, _sgi, _ht40))

#define VHT_GROUP_IDX(_streams, _sgi, _bw)                                \
        (MINSTREL_VHT_GROUP_0 +                                                \
         MINSTREL_MAX_STREAMS * 2 * (_bw) +                                \
         MINSTREL_MAX_STREAMS * (_sgi) +                                \
         (_streams) - 1)

#define BW2VBPS(_bw, r3, r2, r1)                                        \
        (_bw == BW_80 ? r3 : _bw == BW_40 ? r2 : r1)

#define __VHT_GROUP(_streams, _sgi, _bw, _s)                                \
        [VHT_GROUP_IDX(_streams, _sgi, _bw)] = {                        \
        .streams = _streams,                                                \
        .shift = _s,                                                        \
        .bw = _bw,                                                        \
        .flags =                                                        \
                IEEE80211_TX_RC_VHT_MCS |                                \
                (_sgi ? IEEE80211_TX_RC_SHORT_GI : 0) |                        \
                (_bw == BW_80 ? IEEE80211_TX_RC_80_MHZ_WIDTH :                \
                 _bw == BW_40 ? IEEE80211_TX_RC_40_MHZ_WIDTH : 0),        \
        .duration = {                                                        \
                MCS_DURATION(_streams, _sgi,                                \
                             BW2VBPS(_bw,  117,  54,  26)) >> _s,        \
                MCS_DURATION(_streams, _sgi,                                \
                             BW2VBPS(_bw,  234, 108,  52)) >> _s,        \
                MCS_DURATION(_streams, _sgi,                                \
                             BW2VBPS(_bw,  351, 162,  78)) >> _s,        \
                MCS_DURATION(_streams, _sgi,                                \
                             BW2VBPS(_bw,  468, 216, 104)) >> _s,        \
                MCS_DURATION(_streams, _sgi,                                \
                             BW2VBPS(_bw,  702, 324, 156)) >> _s,        \
                MCS_DURATION(_streams, _sgi,                                \
                             BW2VBPS(_bw,  936, 432, 208)) >> _s,        \
                MCS_DURATION(_streams, _sgi,                                \
                             BW2VBPS(_bw, 1053, 486, 234)) >> _s,        \
                MCS_DURATION(_streams, _sgi,                                \
                             BW2VBPS(_bw, 1170, 540, 260)) >> _s,        \
                MCS_DURATION(_streams, _sgi,                                \
                             BW2VBPS(_bw, 1404, 648, 312)) >> _s,        \
                MCS_DURATION(_streams, _sgi,                                \
                             BW2VBPS(_bw, 1560, 720, 346)) >> _s        \
        }                                                                \
}

#define VHT_GROUP_SHIFT(_streams, _sgi, _bw)                                \
        GROUP_SHIFT(MCS_DURATION(_streams, _sgi,                        \
                                 BW2VBPS(_bw,  117,  54,  26)))

#define VHT_GROUP(_streams, _sgi, _bw)                                        \
        __VHT_GROUP(_streams, _sgi, _bw,                                \
                    VHT_GROUP_SHIFT(_streams, _sgi, _bw))

#define CCK_DURATION(_bitrate, _short)                        \
        (1000 * (10 /* SIFS */ +                        \
         (_short ? 72 + 24 : 144 + 48) +                \
         (8 * (AVG_PKT_SIZE + 4) * 10) / (_bitrate)))

#define CCK_DURATION_LIST(_short, _s)                        \
        CCK_DURATION(10, _short) >> _s,                        \
        CCK_DURATION(20, _short) >> _s,                        \
        CCK_DURATION(55, _short) >> _s,                        \
        CCK_DURATION(110, _short) >> _s

#define __CCK_GROUP(_s)                                        \
        [MINSTREL_CCK_GROUP] = {                        \
                .streams = 1,                                \
                .flags = 0,                                \
                .shift = _s,                                \
                .duration = {                                \
                        CCK_DURATION_LIST(false, _s),        \
                        CCK_DURATION_LIST(true, _s)        \
                }                                        \
        }

#define CCK_GROUP_SHIFT                                        \
        GROUP_SHIFT(CCK_DURATION(10, false))

#define CCK_GROUP __CCK_GROUP(CCK_GROUP_SHIFT)

#define OFDM_DURATION(_bitrate)                                \
        (1000 * (16 /* SIFS + signal ext */ +                \
         16 /* T_PREAMBLE */ +                                \
         4 /* T_SIGNAL */ +                                \
         4 * (((16 + 80 * (AVG_PKT_SIZE + 4) + 6) /        \
              ((_bitrate) * 4)))))

#define OFDM_DURATION_LIST(_s)                                \
        OFDM_DURATION(60) >> _s,                        \
        OFDM_DURATION(90) >> _s,                        \
        OFDM_DURATION(120) >> _s,                        \
        OFDM_DURATION(180) >> _s,                        \
        OFDM_DURATION(240) >> _s,                        \
        OFDM_DURATION(360) >> _s,                        \
        OFDM_DURATION(480) >> _s,                        \
        OFDM_DURATION(540) >> _s

#define __OFDM_GROUP(_s)                                \
        [MINSTREL_OFDM_GROUP] = {                        \
                .streams = 1,                                \
                .flags = 0,                                \
                .shift = _s,                                \
                .duration = {                                \
                        OFDM_DURATION_LIST(_s),                \
                }                                        \
        }

#define OFDM_GROUP_SHIFT                                \
        GROUP_SHIFT(OFDM_DURATION(60))

#define OFDM_GROUP __OFDM_GROUP(OFDM_GROUP_SHIFT)


static bool minstrel_vht_only = true;
module_param(minstrel_vht_only, bool, 0644);
MODULE_PARM_DESC(minstrel_vht_only,
                 "Use only VHT rates when VHT is supported by sta.");

/*
 * To enable sufficiently targeted rate sampling, MCS rates are divided into
 * groups, based on the number of streams and flags (HT40, SGI) that they
 * use.
 *
 * Sortorder has to be fixed for GROUP_IDX macro to be applicable:
 * BW -> SGI -> #streams
 */
const struct mcs_group minstrel_mcs_groups[] = {
        MCS_GROUP(1, 0, BW_20),
        MCS_GROUP(2, 0, BW_20),
        MCS_GROUP(3, 0, BW_20),
        MCS_GROUP(4, 0, BW_20),

        MCS_GROUP(1, 1, BW_20),
        MCS_GROUP(2, 1, BW_20),
        MCS_GROUP(3, 1, BW_20),
        MCS_GROUP(4, 1, BW_20),

        MCS_GROUP(1, 0, BW_40),
        MCS_GROUP(2, 0, BW_40),
        MCS_GROUP(3, 0, BW_40),
        MCS_GROUP(4, 0, BW_40),

        MCS_GROUP(1, 1, BW_40),
        MCS_GROUP(2, 1, BW_40),
        MCS_GROUP(3, 1, BW_40),
        MCS_GROUP(4, 1, BW_40),

        CCK_GROUP,
        OFDM_GROUP,

        VHT_GROUP(1, 0, BW_20),
        VHT_GROUP(2, 0, BW_20),
        VHT_GROUP(3, 0, BW_20),
        VHT_GROUP(4, 0, BW_20),

        VHT_GROUP(1, 1, BW_20),
        VHT_GROUP(2, 1, BW_20),
        VHT_GROUP(3, 1, BW_20),
        VHT_GROUP(4, 1, BW_20),

        VHT_GROUP(1, 0, BW_40),
        VHT_GROUP(2, 0, BW_40),
        VHT_GROUP(3, 0, BW_40),
        VHT_GROUP(4, 0, BW_40),

        VHT_GROUP(1, 1, BW_40),
        VHT_GROUP(2, 1, BW_40),
        VHT_GROUP(3, 1, BW_40),
        VHT_GROUP(4, 1, BW_40),

        VHT_GROUP(1, 0, BW_80),
        VHT_GROUP(2, 0, BW_80),
        VHT_GROUP(3, 0, BW_80),
        VHT_GROUP(4, 0, BW_80),

        VHT_GROUP(1, 1, BW_80),
        VHT_GROUP(2, 1, BW_80),
        VHT_GROUP(3, 1, BW_80),
        VHT_GROUP(4, 1, BW_80),
};

const s16 minstrel_cck_bitrates[4] = { 10, 20, 55, 110 };
const s16 minstrel_ofdm_bitrates[8] = { 60, 90, 120, 180, 240, 360, 480, 540 };
static u8 sample_table[SAMPLE_COLUMNS][MCS_GROUP_RATES] __read_mostly;
static const u8 minstrel_sample_seq[] = {
        MINSTREL_SAMPLE_TYPE_INC,
        MINSTREL_SAMPLE_TYPE_JUMP,
        MINSTREL_SAMPLE_TYPE_INC,
        MINSTREL_SAMPLE_TYPE_JUMP,
        MINSTREL_SAMPLE_TYPE_INC,
        MINSTREL_SAMPLE_TYPE_SLOW,
};

static void
minstrel_ht_update_rates(struct minstrel_priv *mp, struct minstrel_ht_sta *mi);

/*
 * Some VHT MCSes are invalid (when Ndbps / Nes is not an integer)
 * e.g for MCS9@20MHzx1Nss: Ndbps=8x52*(5/6) Nes=1
 *
 * Returns the valid mcs map for struct minstrel_mcs_group_data.supported
 */
static u16
minstrel_get_valid_vht_rates(int bw, int nss, __le16 mcs_map)
{
        u16 mask = 0;

        if (bw == BW_20) {
                if (nss != 3 && nss != 6)
                        mask = BIT(9);
        } else if (bw == BW_80) {
                if (nss == 3 || nss == 7)
                        mask = BIT(6);
                else if (nss == 6)
                        mask = BIT(9);
        } else {
                WARN_ON(bw != BW_40);
        }

        switch ((le16_to_cpu(mcs_map) >> (2 * (nss - 1))) & 3) {
        case IEEE80211_VHT_MCS_SUPPORT_0_7:
                mask |= 0x300;
                break;
        case IEEE80211_VHT_MCS_SUPPORT_0_8:
                mask |= 0x200;
                break;
        case IEEE80211_VHT_MCS_SUPPORT_0_9:
                break;
        default:
                mask = 0x3ff;
        }

        return 0x3ff & ~mask;
}

static bool
minstrel_ht_is_legacy_group(int group)
{
        return group == MINSTREL_CCK_GROUP ||
               group == MINSTREL_OFDM_GROUP;
}

/*
 * Look up an MCS group index based on mac80211 rate information
 */
static int
minstrel_ht_get_group_idx(struct ieee80211_tx_rate *rate)
{
        return GROUP_IDX((rate->idx / 8) + 1,
                         !!(rate->flags & IEEE80211_TX_RC_SHORT_GI),
                         !!(rate->flags & IEEE80211_TX_RC_40_MHZ_WIDTH));
}

/*
 * Look up an MCS group index based on new cfg80211 rate_info.
 */
static int
minstrel_ht_ri_get_group_idx(struct rate_info *rate)
{
        return GROUP_IDX((rate->mcs / 8) + 1,
                         !!(rate->flags & RATE_INFO_FLAGS_SHORT_GI),
                         !!(rate->bw & RATE_INFO_BW_40));
}

static int
minstrel_vht_get_group_idx(struct ieee80211_tx_rate *rate)
{
        return VHT_GROUP_IDX(ieee80211_rate_get_vht_nss(rate),
                             !!(rate->flags & IEEE80211_TX_RC_SHORT_GI),
                             !!(rate->flags & IEEE80211_TX_RC_40_MHZ_WIDTH) +
                             2*!!(rate->flags & IEEE80211_TX_RC_80_MHZ_WIDTH));
}

/*
 * Look up an MCS group index based on new cfg80211 rate_info.
 */
static int
minstrel_vht_ri_get_group_idx(struct rate_info *rate)
{
        return VHT_GROUP_IDX(rate->nss,
                             !!(rate->flags & RATE_INFO_FLAGS_SHORT_GI),
                             !!(rate->bw & RATE_INFO_BW_40) +
                             2*!!(rate->bw & RATE_INFO_BW_80));
}

static struct minstrel_rate_stats *
minstrel_ht_get_stats(struct minstrel_priv *mp, struct minstrel_ht_sta *mi,
                      struct ieee80211_tx_rate *rate)
{
        int group, idx;

        if (rate->flags & IEEE80211_TX_RC_MCS) {
                group = minstrel_ht_get_group_idx(rate);
                idx = rate->idx % 8;
                goto out;
        }

        if (rate->flags & IEEE80211_TX_RC_VHT_MCS) {
                group = minstrel_vht_get_group_idx(rate);
                idx = ieee80211_rate_get_vht_mcs(rate);
                goto out;
        }

        group = MINSTREL_CCK_GROUP;
        for (idx = 0; idx < ARRAY_SIZE(mp->cck_rates); idx++) {
                if (!(mi->supported[group] & BIT(idx)))
                        continue;

                if (rate->idx != mp->cck_rates[idx])
                        continue;

                /* short preamble */
                if ((mi->supported[group] & BIT(idx + 4)) &&
                    (rate->flags & IEEE80211_TX_RC_USE_SHORT_PREAMBLE))
                        idx += 4;
                goto out;
        }

        group = MINSTREL_OFDM_GROUP;
        for (idx = 0; idx < ARRAY_SIZE(mp->ofdm_rates[0]); idx++)
                if (rate->idx == mp->ofdm_rates[mi->band][idx])
                        goto out;

        idx = 0;
out:
        return &mi->groups[group].rates[idx];
}

/*
 * Get the minstrel rate statistics for specified STA and rate info.
 */
static struct minstrel_rate_stats *
minstrel_ht_ri_get_stats(struct minstrel_priv *mp, struct minstrel_ht_sta *mi,
                          struct ieee80211_rate_status *rate_status)
{
        int group, idx;
        struct rate_info *rate = &rate_status->rate_idx;

        if (rate->flags & RATE_INFO_FLAGS_MCS) {
                group = minstrel_ht_ri_get_group_idx(rate);
                idx = rate->mcs % 8;
                goto out;
        }

        if (rate->flags & RATE_INFO_FLAGS_VHT_MCS) {
                group = minstrel_vht_ri_get_group_idx(rate);
                idx = rate->mcs;
                goto out;
        }

        group = MINSTREL_CCK_GROUP;
        for (idx = 0; idx < ARRAY_SIZE(mp->cck_rates); idx++) {
                if (rate->legacy != minstrel_cck_bitrates[ mp->cck_rates[idx] ])
                        continue;

                /* short preamble */
                if ((mi->supported[group] & BIT(idx + 4)) &&
                                                        mi->use_short_preamble)
                        idx += 4;
                goto out;
        }

        group = MINSTREL_OFDM_GROUP;
        for (idx = 0; idx < ARRAY_SIZE(mp->ofdm_rates[0]); idx++)
                if (rate->legacy == minstrel_ofdm_bitrates[ mp->ofdm_rates[mi->band][idx] ])
                        goto out;

        idx = 0;
out:
        return &mi->groups[group].rates[idx];
}

static inline struct minstrel_rate_stats *
minstrel_get_ratestats(struct minstrel_ht_sta *mi, int index)
{
        return &mi->groups[MI_RATE_GROUP(index)].rates[MI_RATE_IDX(index)];
}

static inline int minstrel_get_duration(int index)
{
        const struct mcs_group *group = &minstrel_mcs_groups[MI_RATE_GROUP(index)];
        unsigned int duration = group->duration[MI_RATE_IDX(index)];

        return duration << group->shift;
}

static unsigned int
minstrel_ht_avg_ampdu_len(struct minstrel_ht_sta *mi)
{
        int duration;

        if (mi->avg_ampdu_len)
                return MINSTREL_TRUNC(mi->avg_ampdu_len);

        if (minstrel_ht_is_legacy_group(MI_RATE_GROUP(mi->max_tp_rate[0])))
                return 1;

        duration = minstrel_get_duration(mi->max_tp_rate[0]);

        if (duration > 400 * 1000)
                return 2;

        if (duration > 250 * 1000)
                return 4;

        if (duration > 150 * 1000)
                return 8;

        return 16;
}

/*
 * Return current throughput based on the average A-MPDU length, taking into
 * account the expected number of retransmissions and their expected length
 */
int
minstrel_ht_get_tp_avg(struct minstrel_ht_sta *mi, int group, int rate,
                       int prob_avg)
{
        unsigned int nsecs = 0, overhead = mi->overhead;
        unsigned int ampdu_len = 1;

        /* do not account throughput if success prob is below 10% */
        if (prob_avg < MINSTREL_FRAC(10, 100))
                return 0;

        if (minstrel_ht_is_legacy_group(group))
                overhead = mi->overhead_legacy;
        else
                ampdu_len = minstrel_ht_avg_ampdu_len(mi);

        nsecs = 1000 * overhead / ampdu_len;
        nsecs += minstrel_mcs_groups[group].duration[rate] <<
                 minstrel_mcs_groups[group].shift;

        /*
         * For the throughput calculation, limit the probability value to 90% to
         * account for collision related packet error rate fluctuation
         * (prob is scaled - see MINSTREL_FRAC above)
         */
        if (prob_avg > MINSTREL_FRAC(90, 100))
                prob_avg = MINSTREL_FRAC(90, 100);

        return MINSTREL_TRUNC(100 * ((prob_avg * 1000000) / nsecs));
}

/*
 * Find & sort topmost throughput rates
 *
 * If multiple rates provide equal throughput the sorting is based on their
 * current success probability. Higher success probability is preferred among
 * MCS groups, CCK rates do not provide aggregation and are therefore at last.
 */
static void
minstrel_ht_sort_best_tp_rates(struct minstrel_ht_sta *mi, u16 index,
                               u16 *tp_list)
{
        int cur_group, cur_idx, cur_tp_avg, cur_prob;
        int tmp_group, tmp_idx, tmp_tp_avg, tmp_prob;
        int j = MAX_THR_RATES;

        cur_group = MI_RATE_GROUP(index);
        cur_idx = MI_RATE_IDX(index);
        cur_prob = mi->groups[cur_group].rates[cur_idx].prob_avg;
        cur_tp_avg = minstrel_ht_get_tp_avg(mi, cur_group, cur_idx, cur_prob);

        do {
                tmp_group = MI_RATE_GROUP(tp_list[j - 1]);
                tmp_idx = MI_RATE_IDX(tp_list[j - 1]);
                tmp_prob = mi->groups[tmp_group].rates[tmp_idx].prob_avg;
                tmp_tp_avg = minstrel_ht_get_tp_avg(mi, tmp_group, tmp_idx,
                                                    tmp_prob);
                if (cur_tp_avg < tmp_tp_avg ||
                    (cur_tp_avg == tmp_tp_avg && cur_prob <= tmp_prob))
                        break;
                j--;
        } while (j > 0);

        if (j < MAX_THR_RATES - 1) {
                memmove(&tp_list[j + 1], &tp_list[j], (sizeof(*tp_list) *
                       (MAX_THR_RATES - (j + 1))));
        }
        if (j < MAX_THR_RATES)
                tp_list[j] = index;
}

/*
 * Find and set the topmost probability rate per sta and per group
 */
static void
minstrel_ht_set_best_prob_rate(struct minstrel_ht_sta *mi, u16 *dest, u16 index)
{
        struct minstrel_mcs_group_data *mg;
        struct minstrel_rate_stats *mrs;
        int tmp_group, tmp_idx, tmp_tp_avg, tmp_prob;
        int max_tp_group, max_tp_idx, max_tp_prob;
        int cur_tp_avg, cur_group, cur_idx;
        int max_gpr_group, max_gpr_idx;
        int max_gpr_tp_avg, max_gpr_prob;

        cur_group = MI_RATE_GROUP(index);
        cur_idx = MI_RATE_IDX(index);
        mg = &mi->groups[cur_group];
        mrs = &mg->rates[cur_idx];

        tmp_group = MI_RATE_GROUP(*dest);
        tmp_idx = MI_RATE_IDX(*dest);
        tmp_prob = mi->groups[tmp_group].rates[tmp_idx].prob_avg;
        tmp_tp_avg = minstrel_ht_get_tp_avg(mi, tmp_group, tmp_idx, tmp_prob);

        /* if max_tp_rate[0] is from MCS_GROUP max_prob_rate get selected from
         * MCS_GROUP as well as CCK_GROUP rates do not allow aggregation */
        max_tp_group = MI_RATE_GROUP(mi->max_tp_rate[0]);
        max_tp_idx = MI_RATE_IDX(mi->max_tp_rate[0]);
        max_tp_prob = mi->groups[max_tp_group].rates[max_tp_idx].prob_avg;

        if (minstrel_ht_is_legacy_group(MI_RATE_GROUP(index)) &&
            !minstrel_ht_is_legacy_group(max_tp_group))
                return;

        /* skip rates faster than max tp rate with lower prob */
        if (minstrel_get_duration(mi->max_tp_rate[0]) > minstrel_get_duration(index) &&
            mrs->prob_avg < max_tp_prob)
                return;

        max_gpr_group = MI_RATE_GROUP(mg->max_group_prob_rate);
        max_gpr_idx = MI_RATE_IDX(mg->max_group_prob_rate);
        max_gpr_prob = mi->groups[max_gpr_group].rates[max_gpr_idx].prob_avg;

        if (mrs->prob_avg > MINSTREL_FRAC(75, 100)) {
                cur_tp_avg = minstrel_ht_get_tp_avg(mi, cur_group, cur_idx,
                                                    mrs->prob_avg);
                if (cur_tp_avg > tmp_tp_avg)
                        *dest = index;

                max_gpr_tp_avg = minstrel_ht_get_tp_avg(mi, max_gpr_group,
                                                        max_gpr_idx,
                                                        max_gpr_prob);
                if (cur_tp_avg > max_gpr_tp_avg)
                        mg->max_group_prob_rate = index;
        } else {
                if (mrs->prob_avg > tmp_prob)
                        *dest = index;
                if (mrs->prob_avg > max_gpr_prob)
                        mg->max_group_prob_rate = index;
        }
}


/*
 * Assign new rate set per sta and use CCK rates only if the fastest
 * rate (max_tp_rate[0]) is from CCK group. This prohibits such sorted
 * rate sets where MCS and CCK rates are mixed, because CCK rates can
 * not use aggregation.
 */
static void
minstrel_ht_assign_best_tp_rates(struct minstrel_ht_sta *mi,
                                 u16 tmp_mcs_tp_rate[MAX_THR_RATES],
                                 u16 tmp_legacy_tp_rate[MAX_THR_RATES])
{
        unsigned int tmp_group, tmp_idx, tmp_cck_tp, tmp_mcs_tp, tmp_prob;
        int i;

        tmp_group = MI_RATE_GROUP(tmp_legacy_tp_rate[0]);
        tmp_idx = MI_RATE_IDX(tmp_legacy_tp_rate[0]);
        tmp_prob = mi->groups[tmp_group].rates[tmp_idx].prob_avg;
        tmp_cck_tp = minstrel_ht_get_tp_avg(mi, tmp_group, tmp_idx, tmp_prob);

        tmp_group = MI_RATE_GROUP(tmp_mcs_tp_rate[0]);
        tmp_idx = MI_RATE_IDX(tmp_mcs_tp_rate[0]);
        tmp_prob = mi->groups[tmp_group].rates[tmp_idx].prob_avg;
        tmp_mcs_tp = minstrel_ht_get_tp_avg(mi, tmp_group, tmp_idx, tmp_prob);

        if (tmp_cck_tp > tmp_mcs_tp) {
                for(i = 0; i < MAX_THR_RATES; i++) {
                        minstrel_ht_sort_best_tp_rates(mi, tmp_legacy_tp_rate[i],
                                                       tmp_mcs_tp_rate);
                }
        }

}

/*
 * Try to increase robustness of max_prob rate by decrease number of
 * streams if possible.
 */
static inline void
minstrel_ht_prob_rate_reduce_streams(struct minstrel_ht_sta *mi)
{
        struct minstrel_mcs_group_data *mg;
        int tmp_max_streams, group, tmp_idx, tmp_prob;
        int tmp_tp = 0;

        if (!mi->sta->deflink.ht_cap.ht_supported)
                return;

        group = MI_RATE_GROUP(mi->max_tp_rate[0]);
        tmp_max_streams = minstrel_mcs_groups[group].streams;
        for (group = 0; group < ARRAY_SIZE(minstrel_mcs_groups); group++) {
                mg = &mi->groups[group];
                if (!mi->supported[group] || group == MINSTREL_CCK_GROUP)
                        continue;

                tmp_idx = MI_RATE_IDX(mg->max_group_prob_rate);
                tmp_prob = mi->groups[group].rates[tmp_idx].prob_avg;

                if (tmp_tp < minstrel_ht_get_tp_avg(mi, group, tmp_idx, tmp_prob) &&
                   (minstrel_mcs_groups[group].streams < tmp_max_streams)) {
                                mi->max_prob_rate = mg->max_group_prob_rate;
                                tmp_tp = minstrel_ht_get_tp_avg(mi, group,
                                                                tmp_idx,
                                                                tmp_prob);
                }
        }
}

static u16
__minstrel_ht_get_sample_rate(struct minstrel_ht_sta *mi,
                              enum minstrel_sample_type type)
{
        u16 *rates = mi->sample[type].sample_rates;
        u16 cur;
        int i;

        for (i = 0; i < MINSTREL_SAMPLE_RATES; i++) {
                if (!rates[i])
                        continue;

                cur = rates[i];
                rates[i] = 0;
                return cur;
        }

        return 0;
}

static inline int
minstrel_ewma(int old, int new, int weight)
{
        int diff, incr;

        diff = new - old;
        incr = (EWMA_DIV - weight) * diff / EWMA_DIV;

        return old + incr;
}

static inline int minstrel_filter_avg_add(u16 *prev_1, u16 *prev_2, s32 in)
{
        s32 out_1 = *prev_1;
        s32 out_2 = *prev_2;
        s32 val;

        if (!in)
                in += 1;

        if (!out_1) {
                val = out_1 = in;
                goto out;
        }

        val = MINSTREL_AVG_COEFF1 * in;
        val += MINSTREL_AVG_COEFF2 * out_1;
        val += MINSTREL_AVG_COEFF3 * out_2;
        val >>= MINSTREL_SCALE;

        if (val > 1 << MINSTREL_SCALE)
                val = 1 << MINSTREL_SCALE;
        if (val < 0)
                val = 1;

out:
        *prev_2 = out_1;
        *prev_1 = val;

        return val;
}

/*
* Recalculate statistics and counters of a given rate
*/
static void
minstrel_ht_calc_rate_stats(struct minstrel_priv *mp,
                            struct minstrel_rate_stats *mrs)
{
        unsigned int cur_prob;

        if (unlikely(mrs->attempts > 0)) {
                cur_prob = MINSTREL_FRAC(mrs->success, mrs->attempts);
                minstrel_filter_avg_add(&mrs->prob_avg,
                                        &mrs->prob_avg_1, cur_prob);
                mrs->att_hist += mrs->attempts;
                mrs->succ_hist += mrs->success;
        }

        mrs->last_success = mrs->success;
        mrs->last_attempts = mrs->attempts;
        mrs->success = 0;
        mrs->attempts = 0;
}

static bool
minstrel_ht_find_sample_rate(struct minstrel_ht_sta *mi, int type, int idx)
{
        int i;

        for (i = 0; i < MINSTREL_SAMPLE_RATES; i++) {
                u16 cur = mi->sample[type].sample_rates[i];

                if (cur == idx)
                        return true;

                if (!cur)
                        break;
        }

        return false;
}

static int
minstrel_ht_move_sample_rates(struct minstrel_ht_sta *mi, int type,
                              u32 fast_rate_dur, u32 slow_rate_dur)
{
        u16 *rates = mi->sample[type].sample_rates;
        int i, j;

        for (i = 0, j = 0; i < MINSTREL_SAMPLE_RATES; i++) {
                u32 duration;
                bool valid = false;
                u16 cur;

                cur = rates[i];
                if (!cur)
                        continue;

                duration = minstrel_get_duration(cur);
                switch (type) {
                case MINSTREL_SAMPLE_TYPE_SLOW:
                        valid = duration > fast_rate_dur &&
                                duration < slow_rate_dur;
                        break;
                case MINSTREL_SAMPLE_TYPE_INC:
                case MINSTREL_SAMPLE_TYPE_JUMP:
                        valid = duration < fast_rate_dur;
                        break;
                default:
                        valid = false;
                        break;
                }

                if (!valid) {
                        rates[i] = 0;
                        continue;
                }

                if (i == j)
                        continue;

                rates[j++] = cur;
                rates[i] = 0;
        }

        return j;
}

static int
minstrel_ht_group_min_rate_offset(struct minstrel_ht_sta *mi, int group,
                                  u32 max_duration)
{
        u16 supported = mi->supported[group];
        int i;

        for (i = 0; i < MCS_GROUP_RATES && supported; i++, supported >>= 1) {
                if (!(supported & BIT(0)))
                        continue;

                if (minstrel_get_duration(MI_RATE(group, i)) >= max_duration)
                        continue;

                return i;
        }

        return -1;
}

/*
 * Incremental update rates:
 * Flip through groups and pick the first group rate that is faster than the
 * highest currently selected rate
 */
static u16
minstrel_ht_next_inc_rate(struct minstrel_ht_sta *mi, u32 fast_rate_dur)
{
        u8 type = MINSTREL_SAMPLE_TYPE_INC;
        int i, index = 0;
        u8 group;

        group = mi->sample[type].sample_group;
        for (i = 0; i < ARRAY_SIZE(minstrel_mcs_groups); i++) {
                group = (group + 1) % ARRAY_SIZE(minstrel_mcs_groups);

                index = minstrel_ht_group_min_rate_offset(mi, group,
                                                          fast_rate_dur);
                if (index < 0)
                        continue;

                index = MI_RATE(group, index & 0xf);
                if (!minstrel_ht_find_sample_rate(mi, type, index))
                        goto out;
        }
        index = 0;

out:
        mi->sample[type].sample_group = group;

        return index;
}

static int
minstrel_ht_next_group_sample_rate(struct minstrel_ht_sta *mi, int group,
                                   u16 supported, int offset)
{
        struct minstrel_mcs_group_data *mg = &mi->groups[group];
        u16 idx;
        int i;

        for (i = 0; i < MCS_GROUP_RATES; i++) {
                idx = sample_table[mg->column][mg->index];
                if (++mg->index >= MCS_GROUP_RATES) {
                        mg->index = 0;
                        if (++mg->column >= ARRAY_SIZE(sample_table))
                                mg->column = 0;
                }

                if (idx < offset)
                        continue;

                if (!(supported & BIT(idx)))
                        continue;

                return MI_RATE(group, idx);
        }

        return -1;
}

/*
 * Jump rates:
 * Sample random rates, use those that are faster than the highest
 * currently selected rate. Rates between the fastest and the slowest
 * get sorted into the slow sample bucket, but only if it has room
 */
static u16
minstrel_ht_next_jump_rate(struct minstrel_ht_sta *mi, u32 fast_rate_dur,
                           u32 slow_rate_dur, int *slow_rate_ofs)
{
        struct minstrel_rate_stats *mrs;
        u32 max_duration = slow_rate_dur;
        int i, index, offset;
        u16 *slow_rates;
        u16 supported;
        u32 duration;
        u8 group;

        if (*slow_rate_ofs >= MINSTREL_SAMPLE_RATES)
                max_duration = fast_rate_dur;

        slow_rates = mi->sample[MINSTREL_SAMPLE_TYPE_SLOW].sample_rates;
        group = mi->sample[MINSTREL_SAMPLE_TYPE_JUMP].sample_group;
        for (i = 0; i < ARRAY_SIZE(minstrel_mcs_groups); i++) {
                u8 type;

                group = (group + 1) % ARRAY_SIZE(minstrel_mcs_groups);

                supported = mi->supported[group];
                if (!supported)
                        continue;

                offset = minstrel_ht_group_min_rate_offset(mi, group,
                                                           max_duration);
                if (offset < 0)
                        continue;

                index = minstrel_ht_next_group_sample_rate(mi, group, supported,
                                                           offset);
                if (index < 0)
                        continue;

                duration = minstrel_get_duration(index);
                if (duration < fast_rate_dur)
                        type = MINSTREL_SAMPLE_TYPE_JUMP;
                else
                        type = MINSTREL_SAMPLE_TYPE_SLOW;

                if (minstrel_ht_find_sample_rate(mi, type, index))
                        continue;

                if (type == MINSTREL_SAMPLE_TYPE_JUMP)
                        goto found;

                if (*slow_rate_ofs >= MINSTREL_SAMPLE_RATES)
                        continue;

                if (duration >= slow_rate_dur)
                        continue;

                /* skip slow rates with high success probability */
                mrs = minstrel_get_ratestats(mi, index);
                if (mrs->prob_avg > MINSTREL_FRAC(95, 100))
                        continue;

                slow_rates[(*slow_rate_ofs)++] = index;
                if (*slow_rate_ofs >= MINSTREL_SAMPLE_RATES)
                        max_duration = fast_rate_dur;
        }
        index = 0;

found:
        mi->sample[MINSTREL_SAMPLE_TYPE_JUMP].sample_group = group;

        return index;
}

static void
minstrel_ht_refill_sample_rates(struct minstrel_ht_sta *mi)
{
        u32 prob_dur = minstrel_get_duration(mi->max_prob_rate);
        u32 tp_dur = minstrel_get_duration(mi->max_tp_rate[0]);
        u32 tp2_dur = minstrel_get_duration(mi->max_tp_rate[1]);
        u32 fast_rate_dur = min(min(tp_dur, tp2_dur), prob_dur);
        u32 slow_rate_dur = max(max(tp_dur, tp2_dur), prob_dur);
        u16 *rates;
        int i, j;

        rates = mi->sample[MINSTREL_SAMPLE_TYPE_INC].sample_rates;
        i = minstrel_ht_move_sample_rates(mi, MINSTREL_SAMPLE_TYPE_INC,
                                          fast_rate_dur, slow_rate_dur);
        while (i < MINSTREL_SAMPLE_RATES) {
                rates[i] = minstrel_ht_next_inc_rate(mi, tp_dur);
                if (!rates[i])
                        break;

                i++;
        }

        rates = mi->sample[MINSTREL_SAMPLE_TYPE_JUMP].sample_rates;
        i = minstrel_ht_move_sample_rates(mi, MINSTREL_SAMPLE_TYPE_JUMP,
                                          fast_rate_dur, slow_rate_dur);
        j = minstrel_ht_move_sample_rates(mi, MINSTREL_SAMPLE_TYPE_SLOW,
                                          fast_rate_dur, slow_rate_dur);
        while (i < MINSTREL_SAMPLE_RATES) {
                rates[i] = minstrel_ht_next_jump_rate(mi, fast_rate_dur,
                                                      slow_rate_dur, &j);
                if (!rates[i])
                        break;

                i++;
        }

        for (i = 0; i < ARRAY_SIZE(mi->sample); i++)
                memcpy(mi->sample[i].cur_sample_rates, mi->sample[i].sample_rates,
                       sizeof(mi->sample[i].cur_sample_rates));
}


/*
 * Update rate statistics and select new primary rates
 *
 * Rules for rate selection:
 *  - max_prob_rate must use only one stream, as a tradeoff between delivery
 *    probability and throughput during strong fluctuations
 *  - as long as the max prob rate has a probability of more than 75%, pick
 *    higher throughput rates, even if the probability is a bit lower
 */
static void
minstrel_ht_update_stats(struct minstrel_priv *mp, struct minstrel_ht_sta *mi)
{
        struct minstrel_mcs_group_data *mg;
        struct minstrel_rate_stats *mrs;
        int group, i, j, cur_prob;
        u16 tmp_mcs_tp_rate[MAX_THR_RATES], tmp_group_tp_rate[MAX_THR_RATES];
        u16 tmp_legacy_tp_rate[MAX_THR_RATES], tmp_max_prob_rate;
        u16 index;
        bool ht_supported = mi->sta->deflink.ht_cap.ht_supported;

        if (mi->ampdu_packets > 0) {
                if (!ieee80211_hw_check(mp->hw, TX_STATUS_NO_AMPDU_LEN))
                        mi->avg_ampdu_len = minstrel_ewma(mi->avg_ampdu_len,
                                MINSTREL_FRAC(mi->ampdu_len, mi->ampdu_packets),
                                              EWMA_LEVEL);
                else
                        mi->avg_ampdu_len = 0;
                mi->ampdu_len = 0;
                mi->ampdu_packets = 0;
        }

        if (mi->supported[MINSTREL_CCK_GROUP])
                group = MINSTREL_CCK_GROUP;
        else if (mi->supported[MINSTREL_OFDM_GROUP])
                group = MINSTREL_OFDM_GROUP;
        else
                group = 0;

        index = MI_RATE(group, 0);
        for (j = 0; j < ARRAY_SIZE(tmp_legacy_tp_rate); j++)
                tmp_legacy_tp_rate[j] = index;

        if (mi->supported[MINSTREL_VHT_GROUP_0])
                group = MINSTREL_VHT_GROUP_0;
        else if (ht_supported)
                group = MINSTREL_HT_GROUP_0;
        else if (mi->supported[MINSTREL_CCK_GROUP])
                group = MINSTREL_CCK_GROUP;
        else
                group = MINSTREL_OFDM_GROUP;

        index = MI_RATE(group, 0);
        tmp_max_prob_rate = index;
        for (j = 0; j < ARRAY_SIZE(tmp_mcs_tp_rate); j++)
                tmp_mcs_tp_rate[j] = index;

        /* Find best rate sets within all MCS groups*/
        for (group = 0; group < ARRAY_SIZE(minstrel_mcs_groups); group++) {
                u16 *tp_rate = tmp_mcs_tp_rate;
                u16 last_prob = 0;

                mg = &mi->groups[group];
                if (!mi->supported[group])
                        continue;

                /* (re)Initialize group rate indexes */
                for(j = 0; j < MAX_THR_RATES; j++)
                        tmp_group_tp_rate[j] = MI_RATE(group, 0);

                if (group == MINSTREL_CCK_GROUP && ht_supported)
                        tp_rate = tmp_legacy_tp_rate;

                for (i = MCS_GROUP_RATES - 1; i >= 0; i--) {
                        if (!(mi->supported[group] & BIT(i)))
                                continue;

                        index = MI_RATE(group, i);

                        mrs = &mg->rates[i];
                        mrs->retry_updated = false;
                        minstrel_ht_calc_rate_stats(mp, mrs);

                        if (mrs->att_hist)
                                last_prob = max(last_prob, mrs->prob_avg);
                        else
                                mrs->prob_avg = max(last_prob, mrs->prob_avg);
                        cur_prob = mrs->prob_avg;

                        if (minstrel_ht_get_tp_avg(mi, group, i, cur_prob) == 0)
                                continue;

                        /* Find max throughput rate set */
                        minstrel_ht_sort_best_tp_rates(mi, index, tp_rate);

                        /* Find max throughput rate set within a group */
                        minstrel_ht_sort_best_tp_rates(mi, index,
                                                       tmp_group_tp_rate);
                }

                memcpy(mg->max_group_tp_rate, tmp_group_tp_rate,
                       sizeof(mg->max_group_tp_rate));
        }

        /* Assign new rate set per sta */
        minstrel_ht_assign_best_tp_rates(mi, tmp_mcs_tp_rate,
                                         tmp_legacy_tp_rate);
        memcpy(mi->max_tp_rate, tmp_mcs_tp_rate, sizeof(mi->max_tp_rate));

        for (group = 0; group < ARRAY_SIZE(minstrel_mcs_groups); group++) {
                if (!mi->supported[group])
                        continue;

                mg = &mi->groups[group];
                mg->max_group_prob_rate = MI_RATE(group, 0);

                for (i = 0; i < MCS_GROUP_RATES; i++) {
                        if (!(mi->supported[group] & BIT(i)))
                                continue;

                        index = MI_RATE(group, i);

                        /* Find max probability rate per group and global */
                        minstrel_ht_set_best_prob_rate(mi, &tmp_max_prob_rate,
                                                       index);
                }
        }

        mi->max_prob_rate = tmp_max_prob_rate;

        /* Try to increase robustness of max_prob_rate*/
        minstrel_ht_prob_rate_reduce_streams(mi);
        minstrel_ht_refill_sample_rates(mi);

#ifdef CONFIG_MAC80211_DEBUGFS
        /* use fixed index if set */
        if (mp->fixed_rate_idx != -1) {
                for (i = 0; i < 4; i++)
                        mi->max_tp_rate[i] = mp->fixed_rate_idx;
                mi->max_prob_rate = mp->fixed_rate_idx;
        }
#endif

        /* Reset update timer */
        mi->last_stats_update = jiffies;
        mi->sample_time = jiffies;
}

static bool
minstrel_ht_txstat_valid(struct minstrel_priv *mp, struct minstrel_ht_sta *mi,
                         struct ieee80211_tx_rate *rate)
{
        int i;

        if (rate->idx < 0)
                return false;

        if (!rate->count)
                return false;

        if (rate->flags & IEEE80211_TX_RC_MCS ||
            rate->flags & IEEE80211_TX_RC_VHT_MCS)
                return true;

        for (i = 0; i < ARRAY_SIZE(mp->cck_rates); i++)
                if (rate->idx == mp->cck_rates[i])
                        return true;

        for (i = 0; i < ARRAY_SIZE(mp->ofdm_rates[0]); i++)
                if (rate->idx == mp->ofdm_rates[mi->band][i])
                        return true;

        return false;
}

/*
 * Check whether rate_status contains valid information.
 */
static bool
minstrel_ht_ri_txstat_valid(struct minstrel_priv *mp,
                            struct minstrel_ht_sta *mi,
                            struct ieee80211_rate_status *rate_status)
{
        int i;

        if (!rate_status)
                return false;
        if (!rate_status->try_count)
                return false;

        if (rate_status->rate_idx.flags & RATE_INFO_FLAGS_MCS ||
            rate_status->rate_idx.flags & RATE_INFO_FLAGS_VHT_MCS)
                return true;

        for (i = 0; i < ARRAY_SIZE(mp->cck_rates); i++) {
                if (rate_status->rate_idx.legacy ==
                    minstrel_cck_bitrates[ mp->cck_rates[i] ])
                        return true;
        }

        for (i = 0; i < ARRAY_SIZE(mp->ofdm_rates); i++) {
                if (rate_status->rate_idx.legacy ==
                    minstrel_ofdm_bitrates[ mp->ofdm_rates[mi->band][i] ])
                        return true;
        }

        return false;
}

static void
minstrel_downgrade_rate(struct minstrel_ht_sta *mi, u16 *idx, bool primary)
{
        int group, orig_group;

        orig_group = group = MI_RATE_GROUP(*idx);
        while (group > 0) {
                group--;

                if (!mi->supported[group])
                        continue;

                if (minstrel_mcs_groups[group].streams >
                    minstrel_mcs_groups[orig_group].streams)
                        continue;

                if (primary)
                        *idx = mi->groups[group].max_group_tp_rate[0];
                else
                        *idx = mi->groups[group].max_group_tp_rate[1];
                break;
        }
}

static void
minstrel_ht_tx_status(void *priv, struct ieee80211_supported_band *sband,
                      void *priv_sta, struct ieee80211_tx_status *st)
{
        struct ieee80211_tx_info *info = st->info;
        struct minstrel_ht_sta *mi = priv_sta;
        struct ieee80211_tx_rate *ar = info->status.rates;
        struct minstrel_rate_stats *rate, *rate2;
        struct minstrel_priv *mp = priv;
        u32 update_interval = mp->update_interval;
        bool last, update = false;
        int i;

        /* Ignore packet that was sent with noAck flag */
        if (info->flags & IEEE80211_TX_CTL_NO_ACK)
                return;

        /* This packet was aggregated but doesn't carry status info */
        if ((info->flags & IEEE80211_TX_CTL_AMPDU) &&
            !(info->flags & IEEE80211_TX_STAT_AMPDU))
                return;

        if (!(info->flags & IEEE80211_TX_STAT_AMPDU)) {
                info->status.ampdu_ack_len =
                        (info->flags & IEEE80211_TX_STAT_ACK ? 1 : 0);
                info->status.ampdu_len = 1;
        }

        /* wraparound */
        if (mi->total_packets >= ~0 - info->status.ampdu_len) {
                mi->total_packets = 0;
                mi->sample_packets = 0;
        }

        mi->total_packets += info->status.ampdu_len;
        if (info->flags & IEEE80211_TX_CTL_RATE_CTRL_PROBE)
                mi->sample_packets += info->status.ampdu_len;

        mi->ampdu_packets++;
        mi->ampdu_len += info->status.ampdu_len;

        if (st->rates && st->n_rates) {
                last = !minstrel_ht_ri_txstat_valid(mp, mi, &(st->rates[0]));
                for (i = 0; !last; i++) {
                        last = (i == st->n_rates - 1) ||
                                !minstrel_ht_ri_txstat_valid(mp, mi,
                                                        &(st->rates[i + 1]));

                        rate = minstrel_ht_ri_get_stats(mp, mi,
                                                        &(st->rates[i]));

                        if (last)
                                rate->success += info->status.ampdu_ack_len;

                        rate->attempts += st->rates[i].try_count *
                                          info->status.ampdu_len;
                }
        } else {
                last = !minstrel_ht_txstat_valid(mp, mi, &ar[0]);
                for (i = 0; !last; i++) {
                        last = (i == IEEE80211_TX_MAX_RATES - 1) ||
                                !minstrel_ht_txstat_valid(mp, mi, &ar[i + 1]);

                        rate = minstrel_ht_get_stats(mp, mi, &ar[i]);
                        if (last)
                                rate->success += info->status.ampdu_ack_len;

                        rate->attempts += ar[i].count * info->status.ampdu_len;
                }
        }

        if (mp->hw->max_rates > 1) {
                /*
                 * check for sudden death of spatial multiplexing,
                 * downgrade to a lower number of streams if necessary.
                 */
                rate = minstrel_get_ratestats(mi, mi->max_tp_rate[0]);
                if (rate->attempts > 30 &&
                    rate->success < rate->attempts / 4) {
                        minstrel_downgrade_rate(mi, &mi->max_tp_rate[0], true);
                        update = true;
                }

                rate2 = minstrel_get_ratestats(mi, mi->max_tp_rate[1]);
                if (rate2->attempts > 30 &&
                    rate2->success < rate2->attempts / 4) {
                        minstrel_downgrade_rate(mi, &mi->max_tp_rate[1], false);
                        update = true;
                }
        }

        if (time_after(jiffies, mi->last_stats_update + update_interval)) {
                update = true;
                minstrel_ht_update_stats(mp, mi);
        }

        if (update)
                minstrel_ht_update_rates(mp, mi);
}

static void
minstrel_calc_retransmit(struct minstrel_priv *mp, struct minstrel_ht_sta *mi,
                         int index)
{
        struct minstrel_rate_stats *mrs;
        unsigned int tx_time, tx_time_rtscts, tx_time_data;
        unsigned int cw = mp->cw_min;
        unsigned int ctime = 0;
        unsigned int t_slot = 9; /* FIXME */
        unsigned int ampdu_len = minstrel_ht_avg_ampdu_len(mi);
        unsigned int overhead = 0, overhead_rtscts = 0;

        mrs = minstrel_get_ratestats(mi, index);
        if (mrs->prob_avg < MINSTREL_FRAC(1, 10)) {
                mrs->retry_count = 1;
                mrs->retry_count_rtscts = 1;
                return;
        }

        mrs->retry_count = 2;
        mrs->retry_count_rtscts = 2;
        mrs->retry_updated = true;

        tx_time_data = minstrel_get_duration(index) * ampdu_len / 1000;

        /* Contention time for first 2 tries */
        ctime = (t_slot * cw) >> 1;
        cw = min((cw << 1) | 1, mp->cw_max);
        ctime += (t_slot * cw) >> 1;
        cw = min((cw << 1) | 1, mp->cw_max);

        if (minstrel_ht_is_legacy_group(MI_RATE_GROUP(index))) {
                overhead = mi->overhead_legacy;
                overhead_rtscts = mi->overhead_legacy_rtscts;
        } else {
                overhead = mi->overhead;
                overhead_rtscts = mi->overhead_rtscts;
        }

        /* Total TX time for data and Contention after first 2 tries */
        tx_time = ctime + 2 * (overhead + tx_time_data);
        tx_time_rtscts = ctime + 2 * (overhead_rtscts + tx_time_data);

        /* See how many more tries we can fit inside segment size */
        do {
                /* Contention time for this try */
                ctime = (t_slot * cw) >> 1;
                cw = min((cw << 1) | 1, mp->cw_max);

                /* Total TX time after this try */
                tx_time += ctime + overhead + tx_time_data;
                tx_time_rtscts += ctime + overhead_rtscts + tx_time_data;

                if (tx_time_rtscts < mp->segment_size)
                        mrs->retry_count_rtscts++;
        } while ((tx_time < mp->segment_size) &&
                 (++mrs->retry_count < mp->max_retry));
}


static void
minstrel_ht_set_rate(struct minstrel_priv *mp, struct minstrel_ht_sta *mi,
                     struct ieee80211_sta_rates *ratetbl, int offset, int index)
{
        int group_idx = MI_RATE_GROUP(index);
        const struct mcs_group *group = &minstrel_mcs_groups[group_idx];
        struct minstrel_rate_stats *mrs;
        u8 idx;
        u16 flags = group->flags;

        mrs = minstrel_get_ratestats(mi, index);
        if (!mrs->retry_updated)
                minstrel_calc_retransmit(mp, mi, index);

        if (mrs->prob_avg < MINSTREL_FRAC(20, 100) || !mrs->retry_count) {
                ratetbl->rate[offset].count = 2;
                ratetbl->rate[offset].count_rts = 2;
                ratetbl->rate[offset].count_cts = 2;
        } else {
                ratetbl->rate[offset].count = mrs->retry_count;
                ratetbl->rate[offset].count_cts = mrs->retry_count;
                ratetbl->rate[offset].count_rts = mrs->retry_count_rtscts;
        }

        index = MI_RATE_IDX(index);
        if (group_idx == MINSTREL_CCK_GROUP)
                idx = mp->cck_rates[index % ARRAY_SIZE(mp->cck_rates)];
        else if (group_idx == MINSTREL_OFDM_GROUP)
                idx = mp->ofdm_rates[mi->band][index %
                                               ARRAY_SIZE(mp->ofdm_rates[0])];
        else if (flags & IEEE80211_TX_RC_VHT_MCS)
                idx = ((group->streams - 1) << 4) |
                      (index & 0xF);
        else
                idx = index + (group->streams - 1) * 8;

        /* enable RTS/CTS if needed:
         *  - if station is in dynamic SMPS (and streams > 1)
         *  - for fallback rates, to increase chances of getting through
         */
        if (offset > 0 ||
            (mi->sta->deflink.smps_mode == IEEE80211_SMPS_DYNAMIC &&
             group->streams > 1)) {
                ratetbl->rate[offset].count = ratetbl->rate[offset].count_rts;
                flags |= IEEE80211_TX_RC_USE_RTS_CTS;
        }

        ratetbl->rate[offset].idx = idx;
        ratetbl->rate[offset].flags = flags;
}

static inline int
minstrel_ht_get_prob_avg(struct minstrel_ht_sta *mi, int rate)
{
        int group = MI_RATE_GROUP(rate);
        rate = MI_RATE_IDX(rate);
        return mi->groups[group].rates[rate].prob_avg;
}

static int
minstrel_ht_get_max_amsdu_len(struct minstrel_ht_sta *mi)
{
        int group = MI_RATE_GROUP(mi->max_prob_rate);
        const struct mcs_group *g = &minstrel_mcs_groups[group];
        int rate = MI_RATE_IDX(mi->max_prob_rate);
        unsigned int duration;

        /* Disable A-MSDU if max_prob_rate is bad */
        if (mi->groups[group].rates[rate].prob_avg < MINSTREL_FRAC(50, 100))
                return 1;

        duration = g->duration[rate];
        duration <<= g->shift;

        /* If the rate is slower than single-stream MCS1, make A-MSDU limit small */
        if (duration > MCS_DURATION(1, 0, 52))
                return 500;

        /*
         * If the rate is slower than single-stream MCS4, limit A-MSDU to usual
         * data packet size
         */
        if (duration > MCS_DURATION(1, 0, 104))
                return 1600;

        /*
         * If the rate is slower than single-stream MCS7, or if the max throughput
         * rate success probability is less than 75%, limit A-MSDU to twice the usual
         * data packet size
         */
        if (duration > MCS_DURATION(1, 0, 260) ||
            (minstrel_ht_get_prob_avg(mi, mi->max_tp_rate[0]) <
             MINSTREL_FRAC(75, 100)))
                return 3200;

        /*
         * HT A-MPDU limits maximum MPDU size under BA agreement to 4095 bytes.
         * Since aggregation sessions are started/stopped without txq flush, use
         * the limit here to avoid the complexity of having to de-aggregate
         * packets in the queue.
         */
        if (!mi->sta->deflink.vht_cap.vht_supported)
                return IEEE80211_MAX_MPDU_LEN_HT_BA;

        /* unlimited */
        return 0;
}

static void
minstrel_ht_update_rates(struct minstrel_priv *mp, struct minstrel_ht_sta *mi)
{
        struct ieee80211_sta_rates *rates;
        int i = 0;
        int max_rates = min_t(int, mp->hw->max_rates, IEEE80211_TX_RATE_TABLE_SIZE);

        rates = kzalloc(sizeof(*rates), GFP_ATOMIC);
        if (!rates)
                return;

        /* Start with max_tp_rate[0] */
        minstrel_ht_set_rate(mp, mi, rates, i++, mi->max_tp_rate[0]);

        /* Fill up remaining, keep one entry for max_probe_rate */
        for (; i < (max_rates - 1); i++)
                minstrel_ht_set_rate(mp, mi, rates, i, mi->max_tp_rate[i]);

        if (i < max_rates)
                minstrel_ht_set_rate(mp, mi, rates, i++, mi->max_prob_rate);

        if (i < IEEE80211_TX_RATE_TABLE_SIZE)
                rates->rate[i].idx = -1;

        mi->sta->deflink.agg.max_rc_amsdu_len = minstrel_ht_get_max_amsdu_len(mi);
        ieee80211_sta_recalc_aggregates(mi->sta);
        rate_control_set_rates(mp->hw, mi->sta, rates);
}

static u16
minstrel_ht_get_sample_rate(struct minstrel_priv *mp, struct minstrel_ht_sta *mi)
{
        u8 seq;

        if (mp->hw->max_rates > 1) {
                seq = mi->sample_seq;
                mi->sample_seq = (seq + 1) % ARRAY_SIZE(minstrel_sample_seq);
                seq = minstrel_sample_seq[seq];
        } else {
                seq = MINSTREL_SAMPLE_TYPE_INC;
        }

        return __minstrel_ht_get_sample_rate(mi, seq);
}

static void
minstrel_ht_get_rate(void *priv, struct ieee80211_sta *sta, void *priv_sta,
                     struct ieee80211_tx_rate_control *txrc)
{
        const struct mcs_group *sample_group;
        struct ieee80211_tx_info *info = IEEE80211_SKB_CB(txrc->skb);
        struct ieee80211_tx_rate *rate = &info->status.rates[0];
        struct minstrel_ht_sta *mi = priv_sta;
        struct minstrel_priv *mp = priv;
        u16 sample_idx;

        info->flags |= mi->tx_flags;

#ifdef CONFIG_MAC80211_DEBUGFS
        if (mp->fixed_rate_idx != -1)
                return;
#endif

        /* Don't use EAPOL frames for sampling on non-mrr hw */
        if (mp->hw->max_rates == 1 &&
            (info->control.flags & IEEE80211_TX_CTRL_PORT_CTRL_PROTO))
                return;

        if (time_is_after_jiffies(mi->sample_time))
                return;

        mi->sample_time = jiffies + MINSTREL_SAMPLE_INTERVAL;
        sample_idx = minstrel_ht_get_sample_rate(mp, mi);
        if (!sample_idx)
                return;

        sample_group = &minstrel_mcs_groups[MI_RATE_GROUP(sample_idx)];
        sample_idx = MI_RATE_IDX(sample_idx);

        if (sample_group == &minstrel_mcs_groups[MINSTREL_CCK_GROUP] &&
            (sample_idx >= 4) != txrc->short_preamble)
                return;

        info->flags |= IEEE80211_TX_CTL_RATE_CTRL_PROBE;
        rate->count = 1;

        if (sample_group == &minstrel_mcs_groups[MINSTREL_CCK_GROUP]) {
                int idx = sample_idx % ARRAY_SIZE(mp->cck_rates);
                rate->idx = mp->cck_rates[idx];
        } else if (sample_group == &minstrel_mcs_groups[MINSTREL_OFDM_GROUP]) {
                int idx = sample_idx % ARRAY_SIZE(mp->ofdm_rates[0]);
                rate->idx = mp->ofdm_rates[mi->band][idx];
        } else if (sample_group->flags & IEEE80211_TX_RC_VHT_MCS) {
                ieee80211_rate_set_vht(rate, MI_RATE_IDX(sample_idx),
                                       sample_group->streams);
        } else {
                rate->idx = sample_idx + (sample_group->streams - 1) * 8;
        }

        rate->flags = sample_group->flags;
}

static void
minstrel_ht_update_cck(struct minstrel_priv *mp, struct minstrel_ht_sta *mi,
                       struct ieee80211_supported_band *sband,
                       struct ieee80211_sta *sta)
{
        int i;

        if (sband->band != NL80211_BAND_2GHZ)
                return;

        if (sta->deflink.ht_cap.ht_supported &&
            !ieee80211_hw_check(mp->hw, SUPPORTS_HT_CCK_RATES))
                return;

        for (i = 0; i < 4; i++) {
                if (mp->cck_rates[i] == 0xff ||
                    !rate_supported(sta, sband->band, mp->cck_rates[i]))
                        continue;

                mi->supported[MINSTREL_CCK_GROUP] |= BIT(i);
                if (sband->bitrates[i].flags & IEEE80211_RATE_SHORT_PREAMBLE)
                        mi->supported[MINSTREL_CCK_GROUP] |= BIT(i + 4);
        }
}

static void
minstrel_ht_update_ofdm(struct minstrel_priv *mp, struct minstrel_ht_sta *mi,
                        struct ieee80211_supported_band *sband,
                        struct ieee80211_sta *sta)
{
        const u8 *rates;
        int i;

        if (sta->deflink.ht_cap.ht_supported)
                return;

        rates = mp->ofdm_rates[sband->band];
        for (i = 0; i < ARRAY_SIZE(mp->ofdm_rates[0]); i++) {
                if (rates[i] == 0xff ||
                    !rate_supported(sta, sband->band, rates[i]))
                        continue;

                mi->supported[MINSTREL_OFDM_GROUP] |= BIT(i);
        }
}

static void
minstrel_ht_update_caps(void *priv, struct ieee80211_supported_band *sband,
                        struct cfg80211_chan_def *chandef,
                        struct ieee80211_sta *sta, void *priv_sta)
{
        struct minstrel_priv *mp = priv;
        struct minstrel_ht_sta *mi = priv_sta;
        struct ieee80211_mcs_info *mcs = &sta->deflink.ht_cap.mcs;
        u16 ht_cap = sta->deflink.ht_cap.cap;
        struct ieee80211_sta_vht_cap *vht_cap = &sta->deflink.vht_cap;
        const struct ieee80211_rate *ctl_rate;
        struct sta_info *sta_info;
        bool ldpc, erp;
        int use_vht;
        int ack_dur;
        int stbc;
        int i;

        BUILD_BUG_ON(ARRAY_SIZE(minstrel_mcs_groups) != MINSTREL_GROUPS_NB);

        if (vht_cap->vht_supported)
                use_vht = vht_cap->vht_mcs.tx_mcs_map != cpu_to_le16(~0);
        else
                use_vht = 0;

        memset(mi, 0, sizeof(*mi));

        mi->sta = sta;
        mi->band = sband->band;
        mi->last_stats_update = jiffies;

        ack_dur = ieee80211_frame_duration(sband->band, 10, 60, 1, 1);
        mi->overhead = ieee80211_frame_duration(sband->band, 0, 60, 1, 1);
        mi->overhead += ack_dur;
        mi->overhead_rtscts = mi->overhead + 2 * ack_dur;

        ctl_rate = &sband->bitrates[rate_lowest_index(sband, sta)];
        erp = ctl_rate->flags & IEEE80211_RATE_ERP_G;
        ack_dur = ieee80211_frame_duration(sband->band, 10,
                                           ctl_rate->bitrate, erp, 1);
        mi->overhead_legacy = ack_dur;
        mi->overhead_legacy_rtscts = mi->overhead_legacy + 2 * ack_dur;

        mi->avg_ampdu_len = MINSTREL_FRAC(1, 1);

        if (!use_vht) {
                stbc = (ht_cap & IEEE80211_HT_CAP_RX_STBC) >>
                        IEEE80211_HT_CAP_RX_STBC_SHIFT;

                ldpc = ht_cap & IEEE80211_HT_CAP_LDPC_CODING;
        } else {
                stbc = (vht_cap->cap & IEEE80211_VHT_CAP_RXSTBC_MASK) >>
                        IEEE80211_VHT_CAP_RXSTBC_SHIFT;

                ldpc = vht_cap->cap & IEEE80211_VHT_CAP_RXLDPC;
        }

        mi->tx_flags |= stbc << IEEE80211_TX_CTL_STBC_SHIFT;
        if (ldpc)
                mi->tx_flags |= IEEE80211_TX_CTL_LDPC;

        for (i = 0; i < ARRAY_SIZE(mi->groups); i++) {
                u32 gflags = minstrel_mcs_groups[i].flags;
                int bw, nss;

                mi->supported[i] = 0;
                if (minstrel_ht_is_legacy_group(i))
                        continue;

                if (gflags & IEEE80211_TX_RC_SHORT_GI) {
                        if (gflags & IEEE80211_TX_RC_40_MHZ_WIDTH) {
                                if (!(ht_cap & IEEE80211_HT_CAP_SGI_40))
                                        continue;
                        } else {
                                if (!(ht_cap & IEEE80211_HT_CAP_SGI_20))
                                        continue;
                        }
                }

                if (gflags & IEEE80211_TX_RC_40_MHZ_WIDTH &&
                    sta->deflink.bandwidth < IEEE80211_STA_RX_BW_40)
                        continue;

                nss = minstrel_mcs_groups[i].streams;

                /* Mark MCS > 7 as unsupported if STA is in static SMPS mode */
                if (sta->deflink.smps_mode == IEEE80211_SMPS_STATIC && nss > 1)
                        continue;

                /* HT rate */
                if (gflags & IEEE80211_TX_RC_MCS) {
                        if (use_vht && minstrel_vht_only)
                                continue;

                        mi->supported[i] = mcs->rx_mask[nss - 1];
                        continue;
                }

                /* VHT rate */
                if (!vht_cap->vht_supported ||
                    WARN_ON(!(gflags & IEEE80211_TX_RC_VHT_MCS)) ||
                    WARN_ON(gflags & IEEE80211_TX_RC_160_MHZ_WIDTH))
                        continue;

                if (gflags & IEEE80211_TX_RC_80_MHZ_WIDTH) {
                        if (sta->deflink.bandwidth < IEEE80211_STA_RX_BW_80 ||
                            ((gflags & IEEE80211_TX_RC_SHORT_GI) &&
                             !(vht_cap->cap & IEEE80211_VHT_CAP_SHORT_GI_80))) {
                                continue;
                        }
                }

                if (gflags & IEEE80211_TX_RC_40_MHZ_WIDTH)
                        bw = BW_40;
                else if (gflags & IEEE80211_TX_RC_80_MHZ_WIDTH)
                        bw = BW_80;
                else
                        bw = BW_20;

                mi->supported[i] = minstrel_get_valid_vht_rates(bw, nss,
                                vht_cap->vht_mcs.tx_mcs_map);
        }

        sta_info = container_of(sta, struct sta_info, sta);
        mi->use_short_preamble = test_sta_flag(sta_info, WLAN_STA_SHORT_PREAMBLE) &&
                                 sta_info->sdata->vif.bss_conf.use_short_preamble;

        minstrel_ht_update_cck(mp, mi, sband, sta);
        minstrel_ht_update_ofdm(mp, mi, sband, sta);

        /* create an initial rate table with the lowest supported rates */
        minstrel_ht_update_stats(mp, mi);
        minstrel_ht_update_rates(mp, mi);
}

static void
minstrel_ht_rate_init(void *priv, struct ieee80211_supported_band *sband,
                      struct cfg80211_chan_def *chandef,
                      struct ieee80211_sta *sta, void *priv_sta)
{
        minstrel_ht_update_caps(priv, sband, chandef, sta, priv_sta);
}

static void
minstrel_ht_rate_update(void *priv, struct ieee80211_supported_band *sband,
                        struct cfg80211_chan_def *chandef,
                        struct ieee80211_sta *sta, void *priv_sta,
                        u32 changed)
{
        minstrel_ht_update_caps(priv, sband, chandef, sta, priv_sta);
}

static void *
minstrel_ht_alloc_sta(void *priv, struct ieee80211_sta *sta, gfp_t gfp)
{
        struct ieee80211_supported_band *sband;
        struct minstrel_ht_sta *mi;
        struct minstrel_priv *mp = priv;
        struct ieee80211_hw *hw = mp->hw;
        int max_rates = 0;
        int i;

        for (i = 0; i < NUM_NL80211_BANDS; i++) {
                sband = hw->wiphy->bands[i];
                if (sband && sband->n_bitrates > max_rates)
                        max_rates = sband->n_bitrates;
        }

        return kzalloc(sizeof(*mi), gfp);
}

static void
minstrel_ht_free_sta(void *priv, struct ieee80211_sta *sta, void *priv_sta)
{
        kfree(priv_sta);
}

static void
minstrel_ht_fill_rate_array(u8 *dest, struct ieee80211_supported_band *sband,
                            const s16 *bitrates, int n_rates)
{
        int i, j;

        for (i = 0; i < sband->n_bitrates; i++) {
                struct ieee80211_rate *rate = &sband->bitrates[i];

                for (j = 0; j < n_rates; j++) {
                        if (rate->bitrate != bitrates[j])
                                continue;

                        dest[j] = i;
                        break;
                }
        }
}

static void
minstrel_ht_init_cck_rates(struct minstrel_priv *mp)
{
        static const s16 bitrates[4] = { 10, 20, 55, 110 };
        struct ieee80211_supported_band *sband;

        memset(mp->cck_rates, 0xff, sizeof(mp->cck_rates));
        sband = mp->hw->wiphy->bands[NL80211_BAND_2GHZ];
        if (!sband)
                return;

        BUILD_BUG_ON(ARRAY_SIZE(mp->cck_rates) != ARRAY_SIZE(bitrates));
        minstrel_ht_fill_rate_array(mp->cck_rates, sband,
                                    minstrel_cck_bitrates,
                                    ARRAY_SIZE(minstrel_cck_bitrates));
}

static void
minstrel_ht_init_ofdm_rates(struct minstrel_priv *mp, enum nl80211_band band)
{
        static const s16 bitrates[8] = { 60, 90, 120, 180, 240, 360, 480, 540 };
        struct ieee80211_supported_band *sband;

        memset(mp->ofdm_rates[band], 0xff, sizeof(mp->ofdm_rates[band]));
        sband = mp->hw->wiphy->bands[band];
        if (!sband)
                return;

        BUILD_BUG_ON(ARRAY_SIZE(mp->ofdm_rates[band]) != ARRAY_SIZE(bitrates));
        minstrel_ht_fill_rate_array(mp->ofdm_rates[band], sband,
                                    minstrel_ofdm_bitrates,
                                    ARRAY_SIZE(minstrel_ofdm_bitrates));
}

static void *
minstrel_ht_alloc(struct ieee80211_hw *hw)
{
        struct minstrel_priv *mp;
        int i;

        mp = kzalloc(sizeof(struct minstrel_priv), GFP_ATOMIC);
        if (!mp)
                return NULL;

        /* contention window settings
         * Just an approximation. Using the per-queue values would complicate
         * the calculations and is probably unnecessary */
        mp->cw_min = 15;
        mp->cw_max = 1023;

        /* maximum time that the hw is allowed to stay in one MRR segment */
        mp->segment_size = 6000;

        if (hw->max_rate_tries > 0)
                mp->max_retry = hw->max_rate_tries;
        else
                /* safe default, does not necessarily have to match hw properties */
                mp->max_retry = 7;

        mp->hw = hw;
        mp->update_interval = HZ / 20;

        minstrel_ht_init_cck_rates(mp);
        for (i = 0; i < ARRAY_SIZE(mp->hw->wiphy->bands); i++)
            minstrel_ht_init_ofdm_rates(mp, i);

        return mp;
}

#ifdef CONFIG_MAC80211_DEBUGFS
static void minstrel_ht_add_debugfs(struct ieee80211_hw *hw, void *priv,
                                    struct dentry *debugfsdir)
{
        struct minstrel_priv *mp = priv;

        mp->fixed_rate_idx = (u32) -1;
        debugfs_create_u32("fixed_rate_idx", S_IRUGO | S_IWUGO, debugfsdir,
                           &mp->fixed_rate_idx);
}
#endif

static void
minstrel_ht_free(void *priv)
{
        kfree(priv);
}

static u32 minstrel_ht_get_expected_throughput(void *priv_sta)
{
        struct minstrel_ht_sta *mi = priv_sta;
        int i, j, prob, tp_avg;

        i = MI_RATE_GROUP(mi->max_tp_rate[0]);
        j = MI_RATE_IDX(mi->max_tp_rate[0]);
        prob = mi->groups[i].rates[j].prob_avg;

        /* convert tp_avg from pkt per second in kbps */
        tp_avg = minstrel_ht_get_tp_avg(mi, i, j, prob) * 10;
        tp_avg = tp_avg * AVG_PKT_SIZE * 8 / 1024;

        return tp_avg;
}

static const struct rate_control_ops mac80211_minstrel_ht = {
        .name = "minstrel_ht",
        .capa = RATE_CTRL_CAPA_AMPDU_TRIGGER,
        .tx_status_ext = minstrel_ht_tx_status,
        .get_rate = minstrel_ht_get_rate,
        .rate_init = minstrel_ht_rate_init,
        .rate_update = minstrel_ht_rate_update,
        .alloc_sta = minstrel_ht_alloc_sta,
        .free_sta = minstrel_ht_free_sta,
        .alloc = minstrel_ht_alloc,
        .free = minstrel_ht_free,
#ifdef CONFIG_MAC80211_DEBUGFS
        .add_debugfs = minstrel_ht_add_debugfs,
        .add_sta_debugfs = minstrel_ht_add_sta_debugfs,
#endif
        .get_expected_throughput = minstrel_ht_get_expected_throughput,
};


static void __init init_sample_table(void)
{
        int col, i, new_idx;
        u8 rnd[MCS_GROUP_RATES];

        memset(sample_table, 0xff, sizeof(sample_table));
        for (col = 0; col < SAMPLE_COLUMNS; col++) {
                get_random_bytes(rnd, sizeof(rnd));
                for (i = 0; i < MCS_GROUP_RATES; i++) {
                        new_idx = (i + rnd[i]) % MCS_GROUP_RATES;
                        while (sample_table[col][new_idx] != 0xff)
                                new_idx = (new_idx + 1) % MCS_GROUP_RATES;

                        sample_table[col][new_idx] = i;
                }
        }
}

int __init
rc80211_minstrel_init(void)
{
        init_sample_table();
        return ieee80211_rate_control_register(&mac80211_minstrel_ht);
}

void
rc80211_minstrel_exit(void)
{
        ieee80211_rate_control_unregister(&mac80211_minstrel_ht);
}











































































































































































































































































































































































































































































































































































































































































































































   19 


   19 





























1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * Checksum updating actions
 *
 * Copyright (c) 2010 Gregoire Baron <baronchon@n7mm.org>
 */

#include <linux/types.h>
#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/spinlock.h>

#include <linux/netlink.h>
#include <net/netlink.h>
#include <linux/rtnetlink.h>

#include <linux/skbuff.h>

#include <net/ip.h>
#include <net/ipv6.h>
#include <net/icmp.h>
#include <linux/icmpv6.h>
#include <linux/igmp.h>
#include <net/tcp.h>
#include <net/udp.h>
#include <net/ip6_checksum.h>
#include <net/sctp/checksum.h>

#include <net/act_api.h>
#include <net/pkt_cls.h>

#include <linux/tc_act/tc_csum.h>
#include <net/tc_act/tc_csum.h>
#include <net/tc_wrapper.h>

static const struct nla_policy csum_policy[TCA_CSUM_MAX + 1] = {
        [TCA_CSUM_PARMS] = { .len = sizeof(struct tc_csum), },
};

static struct tc_action_ops act_csum_ops;

static int tcf_csum_init(struct net *net, struct nlattr *nla,
                         struct nlattr *est, struct tc_action **a,
                         struct tcf_proto *tp,
                         u32 flags, struct netlink_ext_ack *extack)
{
        struct tc_action_net *tn = net_generic(net, act_csum_ops.net_id);
        bool bind = flags & TCA_ACT_FLAGS_BIND;
        struct tcf_csum_params *params_new;
        struct nlattr *tb[TCA_CSUM_MAX + 1];
        struct tcf_chain *goto_ch = NULL;
        struct tc_csum *parm;
        struct tcf_csum *p;
        int ret = 0, err;
        u32 index;

        if (nla == NULL)
                return -EINVAL;

        err = nla_parse_nested_deprecated(tb, TCA_CSUM_MAX, nla, csum_policy,
                                          NULL);
        if (err < 0)
                return err;

        if (tb[TCA_CSUM_PARMS] == NULL)
                return -EINVAL;
        parm = nla_data(tb[TCA_CSUM_PARMS]);
        index = parm->index;
        err = tcf_idr_check_alloc(tn, &index, a, bind);
        if (!err) {
                ret = tcf_idr_create_from_flags(tn, index, est, a,
                                                &act_csum_ops, bind, flags);
                if (ret) {
                        tcf_idr_cleanup(tn, index);
                        return ret;
                }
                ret = ACT_P_CREATED;
        } else if (err > 0) {
                if (bind) /* dont override defaults */
                        return ACT_P_BOUND;
                if (!(flags & TCA_ACT_FLAGS_REPLACE)) {
                        tcf_idr_release(*a, bind);
                        return -EEXIST;
                }
        } else {
                return err;
        }

        err = tcf_action_check_ctrlact(parm->action, tp, &goto_ch, extack);
        if (err < 0)
                goto release_idr;

        p = to_tcf_csum(*a);

        params_new = kzalloc(sizeof(*params_new), GFP_KERNEL);
        if (unlikely(!params_new)) {
                err = -ENOMEM;
                goto put_chain;
        }
        params_new->update_flags = parm->update_flags;

        spin_lock_bh(&p->tcf_lock);
        goto_ch = tcf_action_set_ctrlact(*a, parm->action, goto_ch);
        params_new = rcu_replace_pointer(p->params, params_new,
                                         lockdep_is_held(&p->tcf_lock));
        spin_unlock_bh(&p->tcf_lock);

        if (goto_ch)
                tcf_chain_put_by_act(goto_ch);
        if (params_new)
                kfree_rcu(params_new, rcu);

        return ret;
put_chain:
        if (goto_ch)
                tcf_chain_put_by_act(goto_ch);
release_idr:
        tcf_idr_release(*a, bind);
        return err;
}

/**
 * tcf_csum_skb_nextlayer - Get next layer pointer
 * @skb: sk_buff to use
 * @ihl: previous summed headers length
 * @ipl: complete packet length
 * @jhl: next header length
 *
 * Check the expected next layer availability in the specified sk_buff.
 * Return the next layer pointer if pass, NULL otherwise.
 */
static void *tcf_csum_skb_nextlayer(struct sk_buff *skb,
                                    unsigned int ihl, unsigned int ipl,
                                    unsigned int jhl)
{
        int ntkoff = skb_network_offset(skb);
        int hl = ihl + jhl;

        if (!pskb_may_pull(skb, ipl + ntkoff) || (ipl < hl) ||
            skb_try_make_writable(skb, hl + ntkoff))
                return NULL;
        else
                return (void *)(skb_network_header(skb) + ihl);
}

static int tcf_csum_ipv4_icmp(struct sk_buff *skb, unsigned int ihl,
                              unsigned int ipl)
{
        struct icmphdr *icmph;

        icmph = tcf_csum_skb_nextlayer(skb, ihl, ipl, sizeof(*icmph));
        if (icmph == NULL)
                return 0;

        icmph->checksum = 0;
        skb->csum = csum_partial(icmph, ipl - ihl, 0);
        icmph->checksum = csum_fold(skb->csum);

        skb->ip_summed = CHECKSUM_NONE;

        return 1;
}

static int tcf_csum_ipv4_igmp(struct sk_buff *skb,
                              unsigned int ihl, unsigned int ipl)
{
        struct igmphdr *igmph;

        igmph = tcf_csum_skb_nextlayer(skb, ihl, ipl, sizeof(*igmph));
        if (igmph == NULL)
                return 0;

        igmph->csum = 0;
        skb->csum = csum_partial(igmph, ipl - ihl, 0);
        igmph->csum = csum_fold(skb->csum);

        skb->ip_summed = CHECKSUM_NONE;

        return 1;
}

static int tcf_csum_ipv6_icmp(struct sk_buff *skb, unsigned int ihl,
                              unsigned int ipl)
{
        struct icmp6hdr *icmp6h;
        const struct ipv6hdr *ip6h;

        icmp6h = tcf_csum_skb_nextlayer(skb, ihl, ipl, sizeof(*icmp6h));
        if (icmp6h == NULL)
                return 0;

        ip6h = ipv6_hdr(skb);
        icmp6h->icmp6_cksum = 0;
        skb->csum = csum_partial(icmp6h, ipl - ihl, 0);
        icmp6h->icmp6_cksum = csum_ipv6_magic(&ip6h->saddr, &ip6h->daddr,
                                              ipl - ihl, IPPROTO_ICMPV6,
                                              skb->csum);

        skb->ip_summed = CHECKSUM_NONE;

        return 1;
}

static int tcf_csum_ipv4_tcp(struct sk_buff *skb, unsigned int ihl,
                             unsigned int ipl)
{
        struct tcphdr *tcph;
        const struct iphdr *iph;

        if (skb_is_gso(skb) && skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4)
                return 1;

        tcph = tcf_csum_skb_nextlayer(skb, ihl, ipl, sizeof(*tcph));
        if (tcph == NULL)
                return 0;

        iph = ip_hdr(skb);
        tcph->check = 0;
        skb->csum = csum_partial(tcph, ipl - ihl, 0);
        tcph->check = tcp_v4_check(ipl - ihl,
                                   iph->saddr, iph->daddr, skb->csum);

        skb->ip_summed = CHECKSUM_NONE;

        return 1;
}

static int tcf_csum_ipv6_tcp(struct sk_buff *skb, unsigned int ihl,
                             unsigned int ipl)
{
        struct tcphdr *tcph;
        const struct ipv6hdr *ip6h;

        if (skb_is_gso(skb) && skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6)
                return 1;

        tcph = tcf_csum_skb_nextlayer(skb, ihl, ipl, sizeof(*tcph));
        if (tcph == NULL)
                return 0;

        ip6h = ipv6_hdr(skb);
        tcph->check = 0;
        skb->csum = csum_partial(tcph, ipl - ihl, 0);
        tcph->check = csum_ipv6_magic(&ip6h->saddr, &ip6h->daddr,
                                      ipl - ihl, IPPROTO_TCP,
                                      skb->csum);

        skb->ip_summed = CHECKSUM_NONE;

        return 1;
}

static int tcf_csum_ipv4_udp(struct sk_buff *skb, unsigned int ihl,
                             unsigned int ipl, int udplite)
{
        struct udphdr *udph;
        const struct iphdr *iph;
        u16 ul;

        if (skb_is_gso(skb) && skb_shinfo(skb)->gso_type & SKB_GSO_UDP)
                return 1;

        /*
         * Support both UDP and UDPLITE checksum algorithms, Don't use
         * udph->len to get the real length without any protocol check,
         * UDPLITE uses udph->len for another thing,
         * Use iph->tot_len, or just ipl.
         */

        udph = tcf_csum_skb_nextlayer(skb, ihl, ipl, sizeof(*udph));
        if (udph == NULL)
                return 0;

        iph = ip_hdr(skb);
        ul = ntohs(udph->len);

        if (udplite || udph->check) {

                udph->check = 0;

                if (udplite) {
                        if (ul == 0)
                                skb->csum = csum_partial(udph, ipl - ihl, 0);
                        else if ((ul >= sizeof(*udph)) && (ul <= ipl - ihl))
                                skb->csum = csum_partial(udph, ul, 0);
                        else
                                goto ignore_obscure_skb;
                } else {
                        if (ul != ipl - ihl)
                                goto ignore_obscure_skb;

                        skb->csum = csum_partial(udph, ul, 0);
                }

                udph->check = csum_tcpudp_magic(iph->saddr, iph->daddr,
                                                ul, iph->protocol,
                                                skb->csum);

                if (!udph->check)
                        udph->check = CSUM_MANGLED_0;
        }

        skb->ip_summed = CHECKSUM_NONE;

ignore_obscure_skb:
        return 1;
}

static int tcf_csum_ipv6_udp(struct sk_buff *skb, unsigned int ihl,
                             unsigned int ipl, int udplite)
{
        struct udphdr *udph;
        const struct ipv6hdr *ip6h;
        u16 ul;

        if (skb_is_gso(skb) && skb_shinfo(skb)->gso_type & SKB_GSO_UDP)
                return 1;

        /*
         * Support both UDP and UDPLITE checksum algorithms, Don't use
         * udph->len to get the real length without any protocol check,
         * UDPLITE uses udph->len for another thing,
         * Use ip6h->payload_len + sizeof(*ip6h) ... , or just ipl.
         */

        udph = tcf_csum_skb_nextlayer(skb, ihl, ipl, sizeof(*udph));
        if (udph == NULL)
                return 0;

        ip6h = ipv6_hdr(skb);
        ul = ntohs(udph->len);

        udph->check = 0;

        if (udplite) {
                if (ul == 0)
                        skb->csum = csum_partial(udph, ipl - ihl, 0);

                else if ((ul >= sizeof(*udph)) && (ul <= ipl - ihl))
                        skb->csum = csum_partial(udph, ul, 0);

                else
                        goto ignore_obscure_skb;
        } else {
                if (ul != ipl - ihl)
                        goto ignore_obscure_skb;

                skb->csum = csum_partial(udph, ul, 0);
        }

        udph->check = csum_ipv6_magic(&ip6h->saddr, &ip6h->daddr, ul,
                                      udplite ? IPPROTO_UDPLITE : IPPROTO_UDP,
                                      skb->csum);

        if (!udph->check)
                udph->check = CSUM_MANGLED_0;

        skb->ip_summed = CHECKSUM_NONE;

ignore_obscure_skb:
        return 1;
}

static int tcf_csum_sctp(struct sk_buff *skb, unsigned int ihl,
                         unsigned int ipl)
{
        struct sctphdr *sctph;

        if (skb_is_gso(skb) && skb_is_gso_sctp(skb))
                return 1;

        sctph = tcf_csum_skb_nextlayer(skb, ihl, ipl, sizeof(*sctph));
        if (!sctph)
                return 0;

        sctph->checksum = sctp_compute_cksum(skb,
                                             skb_network_offset(skb) + ihl);
        skb_reset_csum_not_inet(skb);

        return 1;
}

static int tcf_csum_ipv4(struct sk_buff *skb, u32 update_flags)
{
        const struct iphdr *iph;
        int ntkoff;

        ntkoff = skb_network_offset(skb);

        if (!pskb_may_pull(skb, sizeof(*iph) + ntkoff))
                goto fail;

        iph = ip_hdr(skb);

        switch (iph->frag_off & htons(IP_OFFSET) ? 0 : iph->protocol) {
        case IPPROTO_ICMP:
                if (update_flags & TCA_CSUM_UPDATE_FLAG_ICMP)
                        if (!tcf_csum_ipv4_icmp(skb, iph->ihl * 4,
                                                ntohs(iph->tot_len)))
                                goto fail;
                break;
        case IPPROTO_IGMP:
                if (update_flags & TCA_CSUM_UPDATE_FLAG_IGMP)
                        if (!tcf_csum_ipv4_igmp(skb, iph->ihl * 4,
                                                ntohs(iph->tot_len)))
                                goto fail;
                break;
        case IPPROTO_TCP:
                if (update_flags & TCA_CSUM_UPDATE_FLAG_TCP)
                        if (!tcf_csum_ipv4_tcp(skb, iph->ihl * 4,
                                               ntohs(iph->tot_len)))
                                goto fail;
                break;
        case IPPROTO_UDP:
                if (update_flags & TCA_CSUM_UPDATE_FLAG_UDP)
                        if (!tcf_csum_ipv4_udp(skb, iph->ihl * 4,
                                               ntohs(iph->tot_len), 0))
                                goto fail;
                break;
        case IPPROTO_UDPLITE:
                if (update_flags & TCA_CSUM_UPDATE_FLAG_UDPLITE)
                        if (!tcf_csum_ipv4_udp(skb, iph->ihl * 4,
                                               ntohs(iph->tot_len), 1))
                                goto fail;
                break;
        case IPPROTO_SCTP:
                if ((update_flags & TCA_CSUM_UPDATE_FLAG_SCTP) &&
                    !tcf_csum_sctp(skb, iph->ihl * 4, ntohs(iph->tot_len)))
                        goto fail;
                break;
        }

        if (update_flags & TCA_CSUM_UPDATE_FLAG_IPV4HDR) {
                if (skb_try_make_writable(skb, sizeof(*iph) + ntkoff))
                        goto fail;

                ip_send_check(ip_hdr(skb));
        }

        return 1;

fail:
        return 0;
}

static int tcf_csum_ipv6_hopopts(struct ipv6_opt_hdr *ip6xh, unsigned int ixhl,
                                 unsigned int *pl)
{
        int off, len, optlen;
        unsigned char *xh = (void *)ip6xh;

        off = sizeof(*ip6xh);
        len = ixhl - off;

        while (len > 1) {
                switch (xh[off]) {
                case IPV6_TLV_PAD1:
                        optlen = 1;
                        break;
                case IPV6_TLV_JUMBO:
                        optlen = xh[off + 1] + 2;
                        if (optlen != 6 || len < 6 || (off & 3) != 2)
                                /* wrong jumbo option length/alignment */
                                return 0;
                        *pl = ntohl(*(__be32 *)(xh + off + 2));
                        goto done;
                default:
                        optlen = xh[off + 1] + 2;
                        if (optlen > len)
                                /* ignore obscure options */
                                goto done;
                        break;
                }
                off += optlen;
                len -= optlen;
        }

done:
        return 1;
}

static int tcf_csum_ipv6(struct sk_buff *skb, u32 update_flags)
{
        struct ipv6hdr *ip6h;
        struct ipv6_opt_hdr *ip6xh;
        unsigned int hl, ixhl;
        unsigned int pl;
        int ntkoff;
        u8 nexthdr;

        ntkoff = skb_network_offset(skb);

        hl = sizeof(*ip6h);

        if (!pskb_may_pull(skb, hl + ntkoff))
                goto fail;

        ip6h = ipv6_hdr(skb);

        pl = ntohs(ip6h->payload_len);
        nexthdr = ip6h->nexthdr;

        do {
                switch (nexthdr) {
                case NEXTHDR_FRAGMENT:
                        goto ignore_skb;
                case NEXTHDR_ROUTING:
                case NEXTHDR_HOP:
                case NEXTHDR_DEST:
                        if (!pskb_may_pull(skb, hl + sizeof(*ip6xh) + ntkoff))
                                goto fail;
                        ip6xh = (void *)(skb_network_header(skb) + hl);
                        ixhl = ipv6_optlen(ip6xh);
                        if (!pskb_may_pull(skb, hl + ixhl + ntkoff))
                                goto fail;
                        ip6xh = (void *)(skb_network_header(skb) + hl);
                        if ((nexthdr == NEXTHDR_HOP) &&
                            !(tcf_csum_ipv6_hopopts(ip6xh, ixhl, &pl)))
                                goto fail;
                        nexthdr = ip6xh->nexthdr;
                        hl += ixhl;
                        break;
                case IPPROTO_ICMPV6:
                        if (update_flags & TCA_CSUM_UPDATE_FLAG_ICMP)
                                if (!tcf_csum_ipv6_icmp(skb,
                                                        hl, pl + sizeof(*ip6h)))
                                        goto fail;
                        goto done;
                case IPPROTO_TCP:
                        if (update_flags & TCA_CSUM_UPDATE_FLAG_TCP)
                                if (!tcf_csum_ipv6_tcp(skb,
                                                       hl, pl + sizeof(*ip6h)))
                                        goto fail;
                        goto done;
                case IPPROTO_UDP:
                        if (update_flags & TCA_CSUM_UPDATE_FLAG_UDP)
                                if (!tcf_csum_ipv6_udp(skb, hl,
                                                       pl + sizeof(*ip6h), 0))
                                        goto fail;
                        goto done;
                case IPPROTO_UDPLITE:
                        if (update_flags & TCA_CSUM_UPDATE_FLAG_UDPLITE)
                                if (!tcf_csum_ipv6_udp(skb, hl,
                                                       pl + sizeof(*ip6h), 1))
                                        goto fail;
                        goto done;
                case IPPROTO_SCTP:
                        if ((update_flags & TCA_CSUM_UPDATE_FLAG_SCTP) &&
                            !tcf_csum_sctp(skb, hl, pl + sizeof(*ip6h)))
                                goto fail;
                        goto done;
                default:
                        goto ignore_skb;
                }
        } while (pskb_may_pull(skb, hl + 1 + ntkoff));

done:
ignore_skb:
        return 1;

fail:
        return 0;
}

TC_INDIRECT_SCOPE int tcf_csum_act(struct sk_buff *skb,
                                   const struct tc_action *a,
                                   struct tcf_result *res)
{
        struct tcf_csum *p = to_tcf_csum(a);
        bool orig_vlan_tag_present = false;
        unsigned int vlan_hdr_count = 0;
        struct tcf_csum_params *params;
        u32 update_flags;
        __be16 protocol;
        int action;

        params = rcu_dereference_bh(p->params);

        tcf_lastuse_update(&p->tcf_tm);
        tcf_action_update_bstats(&p->common, skb);

        action = READ_ONCE(p->tcf_action);
        if (unlikely(action == TC_ACT_SHOT))
                goto drop;

        update_flags = params->update_flags;
        protocol = skb_protocol(skb, false);
again:
        switch (protocol) {
        case cpu_to_be16(ETH_P_IP):
                if (!tcf_csum_ipv4(skb, update_flags))
                        goto drop;
                break;
        case cpu_to_be16(ETH_P_IPV6):
                if (!tcf_csum_ipv6(skb, update_flags))
                        goto drop;
                break;
        case cpu_to_be16(ETH_P_8021AD):
                fallthrough;
        case cpu_to_be16(ETH_P_8021Q):
                if (skb_vlan_tag_present(skb) && !orig_vlan_tag_present) {
                        protocol = skb->protocol;
                        orig_vlan_tag_present = true;
                } else {
                        struct vlan_hdr *vlan = (struct vlan_hdr *)skb->data;

                        protocol = vlan->h_vlan_encapsulated_proto;
                        skb_pull(skb, VLAN_HLEN);
                        skb_reset_network_header(skb);
                        vlan_hdr_count++;
                }
                goto again;
        }

out:
        /* Restore the skb for the pulled VLAN tags */
        while (vlan_hdr_count--) {
                skb_push(skb, VLAN_HLEN);
                skb_reset_network_header(skb);
        }

        return action;

drop:
        tcf_action_inc_drop_qstats(&p->common);
        action = TC_ACT_SHOT;
        goto out;
}

static int tcf_csum_dump(struct sk_buff *skb, struct tc_action *a, int bind,
                         int ref)
{
        unsigned char *b = skb_tail_pointer(skb);
        struct tcf_csum *p = to_tcf_csum(a);
        struct tcf_csum_params *params;
        struct tc_csum opt = {
                .index   = p->tcf_index,
                .refcnt  = refcount_read(&p->tcf_refcnt) - ref,
                .bindcnt = atomic_read(&p->tcf_bindcnt) - bind,
        };
        struct tcf_t t;

        spin_lock_bh(&p->tcf_lock);
        params = rcu_dereference_protected(p->params,
                                           lockdep_is_held(&p->tcf_lock));
        opt.action = p->tcf_action;
        opt.update_flags = params->update_flags;

        if (nla_put(skb, TCA_CSUM_PARMS, sizeof(opt), &opt))
                goto nla_put_failure;

        tcf_tm_dump(&t, &p->tcf_tm);
        if (nla_put_64bit(skb, TCA_CSUM_TM, sizeof(t), &t, TCA_CSUM_PAD))
                goto nla_put_failure;
        spin_unlock_bh(&p->tcf_lock);

        return skb->len;

nla_put_failure:
        spin_unlock_bh(&p->tcf_lock);
        nlmsg_trim(skb, b);
        return -1;
}

static void tcf_csum_cleanup(struct tc_action *a)
{
        struct tcf_csum *p = to_tcf_csum(a);
        struct tcf_csum_params *params;

        params = rcu_dereference_protected(p->params, 1);
        if (params)
                kfree_rcu(params, rcu);
}

static size_t tcf_csum_get_fill_size(const struct tc_action *act)
{
        return nla_total_size(sizeof(struct tc_csum));
}

static int tcf_csum_offload_act_setup(struct tc_action *act, void *entry_data,
                                      u32 *index_inc, bool bind,
                                      struct netlink_ext_ack *extack)
{
        if (bind) {
                struct flow_action_entry *entry = entry_data;

                entry->id = FLOW_ACTION_CSUM;
                entry->csum_flags = tcf_csum_update_flags(act);
                *index_inc = 1;
        } else {
                struct flow_offload_action *fl_action = entry_data;

                fl_action->id = FLOW_ACTION_CSUM;
        }

        return 0;
}

static struct tc_action_ops act_csum_ops = {
        .kind                = "csum",
        .id                = TCA_ID_CSUM,
        .owner                = THIS_MODULE,
        .act                = tcf_csum_act,
        .dump                = tcf_csum_dump,
        .init                = tcf_csum_init,
        .cleanup        = tcf_csum_cleanup,
        .get_fill_size  = tcf_csum_get_fill_size,
        .offload_act_setup = tcf_csum_offload_act_setup,
        .size                = sizeof(struct tcf_csum),
};
MODULE_ALIAS_NET_ACT("csum");

static __net_init int csum_init_net(struct net *net)
{
        struct tc_action_net *tn = net_generic(net, act_csum_ops.net_id);

        return tc_action_net_init(net, tn, &act_csum_ops);
}

static void __net_exit csum_exit_net(struct list_head *net_list)
{
        tc_action_net_exit(net_list, act_csum_ops.net_id);
}

static struct pernet_operations csum_net_ops = {
        .init = csum_init_net,
        .exit_batch = csum_exit_net,
        .id   = &act_csum_ops.net_id,
        .size = sizeof(struct tc_action_net),
};

MODULE_DESCRIPTION("Checksum updating actions");
MODULE_LICENSE("GPL");

static int __init csum_init_module(void)
{
        return tcf_register_action(&act_csum_ops, &csum_net_ops);
}

static void __exit csum_cleanup_module(void)
{
        tcf_unregister_action(&act_csum_ops, &csum_net_ops);
}

module_init(csum_init_module);
module_exit(csum_cleanup_module);














































  156 




  125 






    3 
   38 

   38 






















  231 














  157 

   77 



  231 


  152 
   87 















   17 
   16 
    1 
   17 






















































  244 









  240 






  244 
    1 

  241 














   25 







  217 
   13 




























   62 





   74 
















    2 

   11 
   11 

















   13 



    3 


   31 



   34 
    1 










    5 
    5 


















   49 




    1 





   12 

   36 

   12 








   14 


   16 


    3 

   13 































































































   37 





































   10 






    7 





   17 






   10 






   16 






    5 






   10 




   10 



    9 





   14 





    1 





    3 







   12 



   12 



    1 



   13 






    7 





    9 


































































































    9 
















































    1 











    5 


    5 




    6 



    1 










    5 
    1 


    4 














   17 





   17 


    6 

   10 






























































































    4 



  254 







  251 

  252 






    2 


   37 









   38 

   37 






    2 











    9 







1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
// SPDX-License-Identifier: GPL-2.0
/*
 *  linux/fs/stat.c
 *
 *  Copyright (C) 1991, 1992  Linus Torvalds
 */

#include <linux/blkdev.h>
#include <linux/export.h>
#include <linux/mm.h>
#include <linux/errno.h>
#include <linux/file.h>
#include <linux/highuid.h>
#include <linux/fs.h>
#include <linux/namei.h>
#include <linux/security.h>
#include <linux/cred.h>
#include <linux/syscalls.h>
#include <linux/pagemap.h>
#include <linux/compat.h>
#include <linux/iversion.h>

#include <linux/uaccess.h>
#include <asm/unistd.h>

#include <trace/events/timestamp.h>

#include "internal.h"
#include "mount.h"

/**
 * fill_mg_cmtime - Fill in the mtime and ctime and flag ctime as QUERIED
 * @stat: where to store the resulting values
 * @request_mask: STATX_* values requested
 * @inode: inode from which to grab the c/mtime
 *
 * Given @inode, grab the ctime and mtime out if it and store the result
 * in @stat. When fetching the value, flag it as QUERIED (if not already)
 * so the next write will record a distinct timestamp.
 *
 * NB: The QUERIED flag is tracked in the ctime, but we set it there even
 * if only the mtime was requested, as that ensures that the next mtime
 * change will be distinct.
 */
void fill_mg_cmtime(struct kstat *stat, u32 request_mask, struct inode *inode)
{
        atomic_t *pcn = (atomic_t *)&inode->i_ctime_nsec;

        /* If neither time was requested, then don't report them */
        if (!(request_mask & (STATX_CTIME|STATX_MTIME))) {
                stat->result_mask &= ~(STATX_CTIME|STATX_MTIME);
                return;
        }

        stat->mtime = inode_get_mtime(inode);
        stat->ctime.tv_sec = inode->i_ctime_sec;
        stat->ctime.tv_nsec = (u32)atomic_read(pcn);
        if (!(stat->ctime.tv_nsec & I_CTIME_QUERIED))
                stat->ctime.tv_nsec = ((u32)atomic_fetch_or(I_CTIME_QUERIED, pcn));
        stat->ctime.tv_nsec &= ~I_CTIME_QUERIED;
        trace_fill_mg_cmtime(inode, &stat->ctime, &stat->mtime);
}
EXPORT_SYMBOL(fill_mg_cmtime);

/**
 * generic_fillattr - Fill in the basic attributes from the inode struct
 * @idmap:                idmap of the mount the inode was found from
 * @request_mask:        statx request_mask
 * @inode:                Inode to use as the source
 * @stat:                Where to fill in the attributes
 *
 * Fill in the basic attributes in the kstat structure from data that's to be
 * found on the VFS inode structure.  This is the default if no getattr inode
 * operation is supplied.
 *
 * If the inode has been found through an idmapped mount the idmap of
 * the vfsmount must be passed through @idmap. This function will then
 * take care to map the inode according to @idmap before filling in the
 * uid and gid filds. On non-idmapped mounts or if permission checking is to be
 * performed on the raw inode simply pass @nop_mnt_idmap.
 */
void generic_fillattr(struct mnt_idmap *idmap, u32 request_mask,
                      struct inode *inode, struct kstat *stat)
{
        vfsuid_t vfsuid = i_uid_into_vfsuid(idmap, inode);
        vfsgid_t vfsgid = i_gid_into_vfsgid(idmap, inode);

        stat->dev = inode->i_sb->s_dev;
        stat->ino = inode->i_ino;
        stat->mode = inode->i_mode;
        stat->nlink = inode->i_nlink;
        stat->uid = vfsuid_into_kuid(vfsuid);
        stat->gid = vfsgid_into_kgid(vfsgid);
        stat->rdev = inode->i_rdev;
        stat->size = i_size_read(inode);
        stat->atime = inode_get_atime(inode);

        if (is_mgtime(inode)) {
                fill_mg_cmtime(stat, request_mask, inode);
        } else {
                stat->ctime = inode_get_ctime(inode);
                stat->mtime = inode_get_mtime(inode);
        }

        stat->blksize = i_blocksize(inode);
        stat->blocks = inode->i_blocks;

        if ((request_mask & STATX_CHANGE_COOKIE) && IS_I_VERSION(inode)) {
                stat->result_mask |= STATX_CHANGE_COOKIE;
                stat->change_cookie = inode_query_iversion(inode);
        }

}
EXPORT_SYMBOL(generic_fillattr);

/**
 * generic_fill_statx_attr - Fill in the statx attributes from the inode flags
 * @inode:        Inode to use as the source
 * @stat:        Where to fill in the attribute flags
 *
 * Fill in the STATX_ATTR_* flags in the kstat structure for properties of the
 * inode that are published on i_flags and enforced by the VFS.
 */
void generic_fill_statx_attr(struct inode *inode, struct kstat *stat)
{
        if (inode->i_flags & S_IMMUTABLE)
                stat->attributes |= STATX_ATTR_IMMUTABLE;
        if (inode->i_flags & S_APPEND)
                stat->attributes |= STATX_ATTR_APPEND;
        stat->attributes_mask |= KSTAT_ATTR_VFS_FLAGS;
}
EXPORT_SYMBOL(generic_fill_statx_attr);

/**
 * generic_fill_statx_atomic_writes - Fill in atomic writes statx attributes
 * @stat:        Where to fill in the attribute flags
 * @unit_min:        Minimum supported atomic write length in bytes
 * @unit_max:        Maximum supported atomic write length in bytes
 * @unit_max_opt: Optimised maximum supported atomic write length in bytes
 *
 * Fill in the STATX{_ATTR}_WRITE_ATOMIC flags in the kstat structure from
 * atomic write unit_min and unit_max values.
 */
void generic_fill_statx_atomic_writes(struct kstat *stat,
                                      unsigned int unit_min,
                                      unsigned int unit_max,
                                      unsigned int unit_max_opt)
{
        /* Confirm that the request type is known */
        stat->result_mask |= STATX_WRITE_ATOMIC;

        /* Confirm that the file attribute type is known */
        stat->attributes_mask |= STATX_ATTR_WRITE_ATOMIC;

        if (unit_min) {
                stat->atomic_write_unit_min = unit_min;
                stat->atomic_write_unit_max = unit_max;
                stat->atomic_write_unit_max_opt = unit_max_opt;
                /* Initially only allow 1x segment */
                stat->atomic_write_segments_max = 1;

                /* Confirm atomic writes are actually supported */
                stat->attributes |= STATX_ATTR_WRITE_ATOMIC;
        }
}
EXPORT_SYMBOL_GPL(generic_fill_statx_atomic_writes);

/**
 * vfs_getattr_nosec - getattr without security checks
 * @path: file to get attributes from
 * @stat: structure to return attributes in
 * @request_mask: STATX_xxx flags indicating what the caller wants
 * @query_flags: Query mode (AT_STATX_SYNC_TYPE)
 *
 * Get attributes without calling security_inode_getattr.
 *
 * Currently the only caller other than vfs_getattr is internal to the
 * filehandle lookup code, which uses only the inode number and returns no
 * attributes to any user.  Any other code probably wants vfs_getattr.
 */
int vfs_getattr_nosec(const struct path *path, struct kstat *stat,
                      u32 request_mask, unsigned int query_flags)
{
        struct mnt_idmap *idmap;
        struct inode *inode = d_backing_inode(path->dentry);

        memset(stat, 0, sizeof(*stat));
        stat->result_mask |= STATX_BASIC_STATS;
        query_flags &= AT_STATX_SYNC_TYPE;

        /* allow the fs to override these if it really wants to */
        /* SB_NOATIME means filesystem supplies dummy atime value */
        if (inode->i_sb->s_flags & SB_NOATIME)
                stat->result_mask &= ~STATX_ATIME;

        /*
         * Note: If you add another clause to set an attribute flag, please
         * update attributes_mask below.
         */
        if (IS_AUTOMOUNT(inode))
                stat->attributes |= STATX_ATTR_AUTOMOUNT;

        if (IS_DAX(inode))
                stat->attributes |= STATX_ATTR_DAX;

        stat->attributes_mask |= (STATX_ATTR_AUTOMOUNT |
                                  STATX_ATTR_DAX);

        idmap = mnt_idmap(path->mnt);
        if (inode->i_op->getattr) {
                int ret;

                ret = inode->i_op->getattr(idmap, path, stat, request_mask,
                                query_flags);
                if (ret)
                        return ret;
        } else {
                generic_fillattr(idmap, request_mask, inode, stat);
        }

        /*
         * If this is a block device inode, override the filesystem attributes
         * with the block device specific parameters that need to be obtained
         * from the bdev backing inode.
         */
        if (S_ISBLK(stat->mode))
                bdev_statx(path, stat, request_mask);

        return 0;
}
EXPORT_SYMBOL(vfs_getattr_nosec);

/*
 * vfs_getattr - Get the enhanced basic attributes of a file
 * @path: The file of interest
 * @stat: Where to return the statistics
 * @request_mask: STATX_xxx flags indicating what the caller wants
 * @query_flags: Query mode (AT_STATX_SYNC_TYPE)
 *
 * Ask the filesystem for a file's attributes.  The caller must indicate in
 * request_mask and query_flags to indicate what they want.
 *
 * If the file is remote, the filesystem can be forced to update the attributes
 * from the backing store by passing AT_STATX_FORCE_SYNC in query_flags or can
 * suppress the update by passing AT_STATX_DONT_SYNC.
 *
 * Bits must have been set in request_mask to indicate which attributes the
 * caller wants retrieving.  Any such attribute not requested may be returned
 * anyway, but the value may be approximate, and, if remote, may not have been
 * synchronised with the server.
 *
 * 0 will be returned on success, and a -ve error code if unsuccessful.
 */
int vfs_getattr(const struct path *path, struct kstat *stat,
                u32 request_mask, unsigned int query_flags)
{
        int retval;

        retval = security_inode_getattr(path);
        if (unlikely(retval))
                return retval;
        return vfs_getattr_nosec(path, stat, request_mask, query_flags);
}
EXPORT_SYMBOL(vfs_getattr);

/**
 * vfs_fstat - Get the basic attributes by file descriptor
 * @fd: The file descriptor referring to the file of interest
 * @stat: The result structure to fill in.
 *
 * This function is a wrapper around vfs_getattr().  The main difference is
 * that it uses a file descriptor to determine the file location.
 *
 * 0 will be returned on success, and a -ve error code if unsuccessful.
 */
int vfs_fstat(int fd, struct kstat *stat)
{
        CLASS(fd_raw, f)(fd);
        if (fd_empty(f))
                return -EBADF;
        return vfs_getattr(&fd_file(f)->f_path, stat, STATX_BASIC_STATS, 0);
}

static int statx_lookup_flags(int flags)
{
        int lookup_flags = 0;

        if (!(flags & AT_SYMLINK_NOFOLLOW))
                lookup_flags |= LOOKUP_FOLLOW;
        if (!(flags & AT_NO_AUTOMOUNT))
                lookup_flags |= LOOKUP_AUTOMOUNT;

        return lookup_flags;
}

static int vfs_statx_path(struct path *path, int flags, struct kstat *stat,
                          u32 request_mask)
{
        int error = vfs_getattr(path, stat, request_mask, flags);
        if (error)
                return error;

        if (request_mask & STATX_MNT_ID_UNIQUE) {
                stat->mnt_id = real_mount(path->mnt)->mnt_id_unique;
                stat->result_mask |= STATX_MNT_ID_UNIQUE;
        } else {
                stat->mnt_id = real_mount(path->mnt)->mnt_id;
                stat->result_mask |= STATX_MNT_ID;
        }

        if (path_mounted(path))
                stat->attributes |= STATX_ATTR_MOUNT_ROOT;
        stat->attributes_mask |= STATX_ATTR_MOUNT_ROOT;
        return 0;
}

static int vfs_statx_fd(int fd, int flags, struct kstat *stat,
                          u32 request_mask)
{
        CLASS(fd_raw, f)(fd);
        if (fd_empty(f))
                return -EBADF;
        return vfs_statx_path(&fd_file(f)->f_path, flags, stat, request_mask);
}

/**
 * vfs_statx - Get basic and extra attributes by filename
 * @dfd: A file descriptor representing the base dir for a relative filename
 * @filename: The name of the file of interest
 * @flags: Flags to control the query
 * @stat: The result structure to fill in.
 * @request_mask: STATX_xxx flags indicating what the caller wants
 *
 * This function is a wrapper around vfs_getattr().  The main difference is
 * that it uses a filename and base directory to determine the file location.
 * Additionally, the use of AT_SYMLINK_NOFOLLOW in flags will prevent a symlink
 * at the given name from being referenced.
 *
 * 0 will be returned on success, and a -ve error code if unsuccessful.
 */
static int vfs_statx(int dfd, struct filename *filename, int flags,
              struct kstat *stat, u32 request_mask)
{
        struct path path;
        unsigned int lookup_flags = statx_lookup_flags(flags);
        int error;

        if (flags & ~(AT_SYMLINK_NOFOLLOW | AT_NO_AUTOMOUNT | AT_EMPTY_PATH |
                      AT_STATX_SYNC_TYPE))
                return -EINVAL;

retry:
        error = filename_lookup(dfd, filename, lookup_flags, &path, NULL);
        if (error)
                return error;
        error = vfs_statx_path(&path, flags, stat, request_mask);
        path_put(&path);
        if (retry_estale(error, lookup_flags)) {
                lookup_flags |= LOOKUP_REVAL;
                goto retry;
        }
        return error;
}

int vfs_fstatat(int dfd, const char __user *filename,
                              struct kstat *stat, int flags)
{
        int ret;
        int statx_flags = flags | AT_NO_AUTOMOUNT;
        struct filename *name = getname_maybe_null(filename, flags);

        if (!name && dfd >= 0)
                return vfs_fstat(dfd, stat);

        ret = vfs_statx(dfd, name, statx_flags, stat, STATX_BASIC_STATS);
        putname(name);

        return ret;
}

#ifdef __ARCH_WANT_OLD_STAT

/*
 * For backward compatibility?  Maybe this should be moved
 * into arch/i386 instead?
 */
static int cp_old_stat(struct kstat *stat, struct __old_kernel_stat __user * statbuf)
{
        static int warncount = 5;
        struct __old_kernel_stat tmp;

        if (warncount > 0) {
                warncount--;
                printk(KERN_WARNING "VFS: Warning: %s using old stat() call. Recompile your binary.\n",
                        current->comm);
        } else if (warncount < 0) {
                /* it's laughable, but... */
                warncount = 0;
        }

        memset(&tmp, 0, sizeof(struct __old_kernel_stat));
        tmp.st_dev = old_encode_dev(stat->dev);
        tmp.st_ino = stat->ino;
        if (sizeof(tmp.st_ino) < sizeof(stat->ino) && tmp.st_ino != stat->ino)
                return -EOVERFLOW;
        tmp.st_mode = stat->mode;
        tmp.st_nlink = stat->nlink;
        if (tmp.st_nlink != stat->nlink)
                return -EOVERFLOW;
        SET_UID(tmp.st_uid, from_kuid_munged(current_user_ns(), stat->uid));
        SET_GID(tmp.st_gid, from_kgid_munged(current_user_ns(), stat->gid));
        tmp.st_rdev = old_encode_dev(stat->rdev);
#if BITS_PER_LONG == 32
        if (stat->size > MAX_NON_LFS)
                return -EOVERFLOW;
#endif
        tmp.st_size = stat->size;
        tmp.st_atime = stat->atime.tv_sec;
        tmp.st_mtime = stat->mtime.tv_sec;
        tmp.st_ctime = stat->ctime.tv_sec;
        return copy_to_user(statbuf,&tmp,sizeof(tmp)) ? -EFAULT : 0;
}

SYSCALL_DEFINE2(stat, const char __user *, filename,
                struct __old_kernel_stat __user *, statbuf)
{
        struct kstat stat;
        int error;

        error = vfs_stat(filename, &stat);
        if (unlikely(error))
                return error;

        return cp_old_stat(&stat, statbuf);
}

SYSCALL_DEFINE2(lstat, const char __user *, filename,
                struct __old_kernel_stat __user *, statbuf)
{
        struct kstat stat;
        int error;

        error = vfs_lstat(filename, &stat);
        if (unlikely(error))
                return error;

        return cp_old_stat(&stat, statbuf);
}

SYSCALL_DEFINE2(fstat, unsigned int, fd, struct __old_kernel_stat __user *, statbuf)
{
        struct kstat stat;
        int error;

        error = vfs_fstat(fd, &stat);
        if (unlikely(error))
                return error;

        return cp_old_stat(&stat, statbuf);
}

#endif /* __ARCH_WANT_OLD_STAT */

#ifdef __ARCH_WANT_NEW_STAT

#ifndef INIT_STRUCT_STAT_PADDING
#  define INIT_STRUCT_STAT_PADDING(st) memset(&st, 0, sizeof(st))
#endif

static int cp_new_stat(struct kstat *stat, struct stat __user *statbuf)
{
        struct stat tmp;

        if (sizeof(tmp.st_dev) < 4 && !old_valid_dev(stat->dev))
                return -EOVERFLOW;
        if (sizeof(tmp.st_rdev) < 4 && !old_valid_dev(stat->rdev))
                return -EOVERFLOW;
#if BITS_PER_LONG == 32
        if (stat->size > MAX_NON_LFS)
                return -EOVERFLOW;
#endif

        INIT_STRUCT_STAT_PADDING(tmp);
        tmp.st_dev = new_encode_dev(stat->dev);
        tmp.st_ino = stat->ino;
        if (sizeof(tmp.st_ino) < sizeof(stat->ino) && tmp.st_ino != stat->ino)
                return -EOVERFLOW;
        tmp.st_mode = stat->mode;
        tmp.st_nlink = stat->nlink;
        if (tmp.st_nlink != stat->nlink)
                return -EOVERFLOW;
        SET_UID(tmp.st_uid, from_kuid_munged(current_user_ns(), stat->uid));
        SET_GID(tmp.st_gid, from_kgid_munged(current_user_ns(), stat->gid));
        tmp.st_rdev = new_encode_dev(stat->rdev);
        tmp.st_size = stat->size;
        tmp.st_atime = stat->atime.tv_sec;
        tmp.st_mtime = stat->mtime.tv_sec;
        tmp.st_ctime = stat->ctime.tv_sec;
#ifdef STAT_HAVE_NSEC
        tmp.st_atime_nsec = stat->atime.tv_nsec;
        tmp.st_mtime_nsec = stat->mtime.tv_nsec;
        tmp.st_ctime_nsec = stat->ctime.tv_nsec;
#endif
        tmp.st_blocks = stat->blocks;
        tmp.st_blksize = stat->blksize;
        return copy_to_user(statbuf,&tmp,sizeof(tmp)) ? -EFAULT : 0;
}

SYSCALL_DEFINE2(newstat, const char __user *, filename,
                struct stat __user *, statbuf)
{
        struct kstat stat;
        int error;

        error = vfs_stat(filename, &stat);
        if (unlikely(error))
                return error;

        return cp_new_stat(&stat, statbuf);
}

SYSCALL_DEFINE2(newlstat, const char __user *, filename,
                struct stat __user *, statbuf)
{
        struct kstat stat;
        int error;

        error = vfs_lstat(filename, &stat);
        if (unlikely(error))
                return error;

        return cp_new_stat(&stat, statbuf);
}

#if !defined(__ARCH_WANT_STAT64) || defined(__ARCH_WANT_SYS_NEWFSTATAT)
SYSCALL_DEFINE4(newfstatat, int, dfd, const char __user *, filename,
                struct stat __user *, statbuf, int, flag)
{
        struct kstat stat;
        int error;

        error = vfs_fstatat(dfd, filename, &stat, flag);
        if (unlikely(error))
                return error;

        return cp_new_stat(&stat, statbuf);
}
#endif

SYSCALL_DEFINE2(newfstat, unsigned int, fd, struct stat __user *, statbuf)
{
        struct kstat stat;
        int error;

        error = vfs_fstat(fd, &stat);
        if (unlikely(error))
                return error;

        return cp_new_stat(&stat, statbuf);
}
#endif

static int do_readlinkat(int dfd, const char __user *pathname,
                         char __user *buf, int bufsiz)
{
        struct path path;
        struct filename *name;
        int error;
        unsigned int lookup_flags = LOOKUP_EMPTY;

        if (bufsiz <= 0)
                return -EINVAL;

retry:
        name = getname_flags(pathname, lookup_flags);
        error = filename_lookup(dfd, name, lookup_flags, &path, NULL);
        if (unlikely(error)) {
                putname(name);
                return error;
        }

        /*
         * AFS mountpoints allow readlink(2) but are not symlinks
         */
        if (d_is_symlink(path.dentry) ||
            d_backing_inode(path.dentry)->i_op->readlink) {
                error = security_inode_readlink(path.dentry);
                if (!error) {
                        touch_atime(&path);
                        error = vfs_readlink(path.dentry, buf, bufsiz);
                }
        } else {
                error = (name->name[0] == '\0') ? -ENOENT : -EINVAL;
        }
        path_put(&path);
        putname(name);
        if (retry_estale(error, lookup_flags)) {
                lookup_flags |= LOOKUP_REVAL;
                goto retry;
        }
        return error;
}

SYSCALL_DEFINE4(readlinkat, int, dfd, const char __user *, pathname,
                char __user *, buf, int, bufsiz)
{
        return do_readlinkat(dfd, pathname, buf, bufsiz);
}

SYSCALL_DEFINE3(readlink, const char __user *, path, char __user *, buf,
                int, bufsiz)
{
        return do_readlinkat(AT_FDCWD, path, buf, bufsiz);
}


/* ---------- LFS-64 ----------- */
#if defined(__ARCH_WANT_STAT64) || defined(__ARCH_WANT_COMPAT_STAT64)

#ifndef INIT_STRUCT_STAT64_PADDING
#  define INIT_STRUCT_STAT64_PADDING(st) memset(&st, 0, sizeof(st))
#endif

static long cp_new_stat64(struct kstat *stat, struct stat64 __user *statbuf)
{
        struct stat64 tmp;

        INIT_STRUCT_STAT64_PADDING(tmp);
#ifdef CONFIG_MIPS
        /* mips has weird padding, so we don't get 64 bits there */
        tmp.st_dev = new_encode_dev(stat->dev);
        tmp.st_rdev = new_encode_dev(stat->rdev);
#else
        tmp.st_dev = huge_encode_dev(stat->dev);
        tmp.st_rdev = huge_encode_dev(stat->rdev);
#endif
        tmp.st_ino = stat->ino;
        if (sizeof(tmp.st_ino) < sizeof(stat->ino) && tmp.st_ino != stat->ino)
                return -EOVERFLOW;
#ifdef STAT64_HAS_BROKEN_ST_INO
        tmp.__st_ino = stat->ino;
#endif
        tmp.st_mode = stat->mode;
        tmp.st_nlink = stat->nlink;
        tmp.st_uid = from_kuid_munged(current_user_ns(), stat->uid);
        tmp.st_gid = from_kgid_munged(current_user_ns(), stat->gid);
        tmp.st_atime = stat->atime.tv_sec;
        tmp.st_atime_nsec = stat->atime.tv_nsec;
        tmp.st_mtime = stat->mtime.tv_sec;
        tmp.st_mtime_nsec = stat->mtime.tv_nsec;
        tmp.st_ctime = stat->ctime.tv_sec;
        tmp.st_ctime_nsec = stat->ctime.tv_nsec;
        tmp.st_size = stat->size;
        tmp.st_blocks = stat->blocks;
        tmp.st_blksize = stat->blksize;
        return copy_to_user(statbuf,&tmp,sizeof(tmp)) ? -EFAULT : 0;
}

SYSCALL_DEFINE2(stat64, const char __user *, filename,
                struct stat64 __user *, statbuf)
{
        struct kstat stat;
        int error = vfs_stat(filename, &stat);

        if (!error)
                error = cp_new_stat64(&stat, statbuf);

        return error;
}

SYSCALL_DEFINE2(lstat64, const char __user *, filename,
                struct stat64 __user *, statbuf)
{
        struct kstat stat;
        int error = vfs_lstat(filename, &stat);

        if (!error)
                error = cp_new_stat64(&stat, statbuf);

        return error;
}

SYSCALL_DEFINE2(fstat64, unsigned long, fd, struct stat64 __user *, statbuf)
{
        struct kstat stat;
        int error = vfs_fstat(fd, &stat);

        if (!error)
                error = cp_new_stat64(&stat, statbuf);

        return error;
}

SYSCALL_DEFINE4(fstatat64, int, dfd, const char __user *, filename,
                struct stat64 __user *, statbuf, int, flag)
{
        struct kstat stat;
        int error;

        error = vfs_fstatat(dfd, filename, &stat, flag);
        if (error)
                return error;
        return cp_new_stat64(&stat, statbuf);
}
#endif /* __ARCH_WANT_STAT64 || __ARCH_WANT_COMPAT_STAT64 */

static noinline_for_stack int
cp_statx(const struct kstat *stat, struct statx __user *buffer)
{
        struct statx tmp;

        memset(&tmp, 0, sizeof(tmp));

        /* STATX_CHANGE_COOKIE is kernel-only for now */
        tmp.stx_mask = stat->result_mask & ~STATX_CHANGE_COOKIE;
        tmp.stx_blksize = stat->blksize;
        /* STATX_ATTR_CHANGE_MONOTONIC is kernel-only for now */
        tmp.stx_attributes = stat->attributes & ~STATX_ATTR_CHANGE_MONOTONIC;
        tmp.stx_nlink = stat->nlink;
        tmp.stx_uid = from_kuid_munged(current_user_ns(), stat->uid);
        tmp.stx_gid = from_kgid_munged(current_user_ns(), stat->gid);
        tmp.stx_mode = stat->mode;
        tmp.stx_ino = stat->ino;
        tmp.stx_size = stat->size;
        tmp.stx_blocks = stat->blocks;
        tmp.stx_attributes_mask = stat->attributes_mask;
        tmp.stx_atime.tv_sec = stat->atime.tv_sec;
        tmp.stx_atime.tv_nsec = stat->atime.tv_nsec;
        tmp.stx_btime.tv_sec = stat->btime.tv_sec;
        tmp.stx_btime.tv_nsec = stat->btime.tv_nsec;
        tmp.stx_ctime.tv_sec = stat->ctime.tv_sec;
        tmp.stx_ctime.tv_nsec = stat->ctime.tv_nsec;
        tmp.stx_mtime.tv_sec = stat->mtime.tv_sec;
        tmp.stx_mtime.tv_nsec = stat->mtime.tv_nsec;
        tmp.stx_rdev_major = MAJOR(stat->rdev);
        tmp.stx_rdev_minor = MINOR(stat->rdev);
        tmp.stx_dev_major = MAJOR(stat->dev);
        tmp.stx_dev_minor = MINOR(stat->dev);
        tmp.stx_mnt_id = stat->mnt_id;
        tmp.stx_dio_mem_align = stat->dio_mem_align;
        tmp.stx_dio_offset_align = stat->dio_offset_align;
        tmp.stx_dio_read_offset_align = stat->dio_read_offset_align;
        tmp.stx_subvol = stat->subvol;
        tmp.stx_atomic_write_unit_min = stat->atomic_write_unit_min;
        tmp.stx_atomic_write_unit_max = stat->atomic_write_unit_max;
        tmp.stx_atomic_write_segments_max = stat->atomic_write_segments_max;
        tmp.stx_atomic_write_unit_max_opt = stat->atomic_write_unit_max_opt;

        return copy_to_user(buffer, &tmp, sizeof(tmp)) ? -EFAULT : 0;
}

int do_statx(int dfd, struct filename *filename, unsigned int flags,
             unsigned int mask, struct statx __user *buffer)
{
        struct kstat stat;
        int error;

        if (mask & STATX__RESERVED)
                return -EINVAL;
        if ((flags & AT_STATX_SYNC_TYPE) == AT_STATX_SYNC_TYPE)
                return -EINVAL;

        /*
         * STATX_CHANGE_COOKIE is kernel-only for now. Ignore requests
         * from userland.
         */
        mask &= ~STATX_CHANGE_COOKIE;

        error = vfs_statx(dfd, filename, flags, &stat, mask);
        if (error)
                return error;

        return cp_statx(&stat, buffer);
}

int do_statx_fd(int fd, unsigned int flags, unsigned int mask,
             struct statx __user *buffer)
{
        struct kstat stat;
        int error;

        if (mask & STATX__RESERVED)
                return -EINVAL;
        if ((flags & AT_STATX_SYNC_TYPE) == AT_STATX_SYNC_TYPE)
                return -EINVAL;

        /*
         * STATX_CHANGE_COOKIE is kernel-only for now. Ignore requests
         * from userland.
         */
        mask &= ~STATX_CHANGE_COOKIE;

        error = vfs_statx_fd(fd, flags, &stat, mask);
        if (error)
                return error;

        return cp_statx(&stat, buffer);
}

/**
 * sys_statx - System call to get enhanced stats
 * @dfd: Base directory to pathwalk from *or* fd to stat.
 * @filename: File to stat or either NULL or "" with AT_EMPTY_PATH
 * @flags: AT_* flags to control pathwalk.
 * @mask: Parts of statx struct actually required.
 * @buffer: Result buffer.
 *
 * Note that fstat() can be emulated by setting dfd to the fd of interest,
 * supplying "" (or preferably NULL) as the filename and setting AT_EMPTY_PATH
 * in the flags.
 */
SYSCALL_DEFINE5(statx,
                int, dfd, const char __user *, filename, unsigned, flags,
                unsigned int, mask,
                struct statx __user *, buffer)
{
        int ret;
        struct filename *name = getname_maybe_null(filename, flags);

        if (!name && dfd >= 0)
                return do_statx_fd(dfd, flags & ~AT_NO_AUTOMOUNT, mask, buffer);

        ret = do_statx(dfd, name, flags, mask, buffer);
        putname(name);

        return ret;
}

#if defined(CONFIG_COMPAT) && defined(__ARCH_WANT_COMPAT_STAT)
static int cp_compat_stat(struct kstat *stat, struct compat_stat __user *ubuf)
{
        struct compat_stat tmp;

        if (sizeof(tmp.st_dev) < 4 && !old_valid_dev(stat->dev))
                return -EOVERFLOW;
        if (sizeof(tmp.st_rdev) < 4 && !old_valid_dev(stat->rdev))
                return -EOVERFLOW;

        memset(&tmp, 0, sizeof(tmp));
        tmp.st_dev = new_encode_dev(stat->dev);
        tmp.st_ino = stat->ino;
        if (sizeof(tmp.st_ino) < sizeof(stat->ino) && tmp.st_ino != stat->ino)
                return -EOVERFLOW;
        tmp.st_mode = stat->mode;
        tmp.st_nlink = stat->nlink;
        if (tmp.st_nlink != stat->nlink)
                return -EOVERFLOW;
        SET_UID(tmp.st_uid, from_kuid_munged(current_user_ns(), stat->uid));
        SET_GID(tmp.st_gid, from_kgid_munged(current_user_ns(), stat->gid));
        tmp.st_rdev = new_encode_dev(stat->rdev);
        if ((u64) stat->size > MAX_NON_LFS)
                return -EOVERFLOW;
        tmp.st_size = stat->size;
        tmp.st_atime = stat->atime.tv_sec;
        tmp.st_atime_nsec = stat->atime.tv_nsec;
        tmp.st_mtime = stat->mtime.tv_sec;
        tmp.st_mtime_nsec = stat->mtime.tv_nsec;
        tmp.st_ctime = stat->ctime.tv_sec;
        tmp.st_ctime_nsec = stat->ctime.tv_nsec;
        tmp.st_blocks = stat->blocks;
        tmp.st_blksize = stat->blksize;
        return copy_to_user(ubuf, &tmp, sizeof(tmp)) ? -EFAULT : 0;
}

COMPAT_SYSCALL_DEFINE2(newstat, const char __user *, filename,
                       struct compat_stat __user *, statbuf)
{
        struct kstat stat;
        int error;

        error = vfs_stat(filename, &stat);
        if (error)
                return error;
        return cp_compat_stat(&stat, statbuf);
}

COMPAT_SYSCALL_DEFINE2(newlstat, const char __user *, filename,
                       struct compat_stat __user *, statbuf)
{
        struct kstat stat;
        int error;

        error = vfs_lstat(filename, &stat);
        if (error)
                return error;
        return cp_compat_stat(&stat, statbuf);
}

#ifndef __ARCH_WANT_STAT64
COMPAT_SYSCALL_DEFINE4(newfstatat, unsigned int, dfd,
                       const char __user *, filename,
                       struct compat_stat __user *, statbuf, int, flag)
{
        struct kstat stat;
        int error;

        error = vfs_fstatat(dfd, filename, &stat, flag);
        if (error)
                return error;
        return cp_compat_stat(&stat, statbuf);
}
#endif

COMPAT_SYSCALL_DEFINE2(newfstat, unsigned int, fd,
                       struct compat_stat __user *, statbuf)
{
        struct kstat stat;
        int error = vfs_fstat(fd, &stat);

        if (!error)
                error = cp_compat_stat(&stat, statbuf);
        return error;
}
#endif

/* Caller is here responsible for sufficient locking (ie. inode->i_lock) */
void __inode_add_bytes(struct inode *inode, loff_t bytes)
{
        inode->i_blocks += bytes >> 9;
        bytes &= 511;
        inode->i_bytes += bytes;
        if (inode->i_bytes >= 512) {
                inode->i_blocks++;
                inode->i_bytes -= 512;
        }
}
EXPORT_SYMBOL(__inode_add_bytes);

void inode_add_bytes(struct inode *inode, loff_t bytes)
{
        spin_lock(&inode->i_lock);
        __inode_add_bytes(inode, bytes);
        spin_unlock(&inode->i_lock);
}

EXPORT_SYMBOL(inode_add_bytes);

void __inode_sub_bytes(struct inode *inode, loff_t bytes)
{
        inode->i_blocks -= bytes >> 9;
        bytes &= 511;
        if (inode->i_bytes < bytes) {
                inode->i_blocks--;
                inode->i_bytes += 512;
        }
        inode->i_bytes -= bytes;
}

EXPORT_SYMBOL(__inode_sub_bytes);

void inode_sub_bytes(struct inode *inode, loff_t bytes)
{
        spin_lock(&inode->i_lock);
        __inode_sub_bytes(inode, bytes);
        spin_unlock(&inode->i_lock);
}

EXPORT_SYMBOL(inode_sub_bytes);

loff_t inode_get_bytes(struct inode *inode)
{
        loff_t ret;

        spin_lock(&inode->i_lock);
        ret = __inode_get_bytes(inode);
        spin_unlock(&inode->i_lock);
        return ret;
}

EXPORT_SYMBOL(inode_get_bytes);

void inode_set_bytes(struct inode *inode, loff_t bytes)
{
        /* Caller is here responsible for sufficient locking
         * (ie. inode->i_lock) */
        inode->i_blocks = bytes >> 9;
        inode->i_bytes = bytes & 511;
}

EXPORT_SYMBOL(inode_set_bytes);















































































































































































































































































































































































































































































































    1 



























































































































































































































































































































































































































































































































































































































































































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
/* SPDX-License-Identifier: GPL-2.0-only */
/*
 * Media entity
 *
 * Copyright (C) 2010 Nokia Corporation
 *
 * Contacts: Laurent Pinchart <laurent.pinchart@ideasonboard.com>
 *             Sakari Ailus <sakari.ailus@iki.fi>
 */

#ifndef _MEDIA_ENTITY_H
#define _MEDIA_ENTITY_H

#include <linux/bitmap.h>
#include <linux/bug.h>
#include <linux/container_of.h>
#include <linux/fwnode.h>
#include <linux/list.h>
#include <linux/media.h>
#include <linux/minmax.h>
#include <linux/types.h>

/* Enums used internally at the media controller to represent graphs */

/**
 * enum media_gobj_type - type of a graph object
 *
 * @MEDIA_GRAPH_ENTITY:                Identify a media entity
 * @MEDIA_GRAPH_PAD:                Identify a media pad
 * @MEDIA_GRAPH_LINK:                Identify a media link
 * @MEDIA_GRAPH_INTF_DEVNODE:        Identify a media Kernel API interface via
 *                                a device node
 */
enum media_gobj_type {
        MEDIA_GRAPH_ENTITY,
        MEDIA_GRAPH_PAD,
        MEDIA_GRAPH_LINK,
        MEDIA_GRAPH_INTF_DEVNODE,
};

#define MEDIA_BITS_PER_TYPE                8
#define MEDIA_BITS_PER_ID                (32 - MEDIA_BITS_PER_TYPE)
#define MEDIA_ID_MASK                         GENMASK_ULL(MEDIA_BITS_PER_ID - 1, 0)

/* Structs to represent the objects that belong to a media graph */

/**
 * struct media_gobj - Define a graph object.
 *
 * @mdev:        Pointer to the struct &media_device that owns the object
 * @id:                Non-zero object ID identifier. The ID should be unique
 *                inside a media_device, as it is composed by
 *                %MEDIA_BITS_PER_TYPE to store the type plus
 *                %MEDIA_BITS_PER_ID to store the ID
 * @list:        List entry stored in one of the per-type mdev object lists
 *
 * All objects on the media graph should have this struct embedded
 */
struct media_gobj {
        struct media_device        *mdev;
        u32                        id;
        struct list_head        list;
};

#define MEDIA_ENTITY_ENUM_MAX_DEPTH        16

/**
 * struct media_entity_enum - An enumeration of media entities.
 *
 * @bmap:        Bit map in which each bit represents one entity at struct
 *                media_entity->internal_idx.
 * @idx_max:        Number of bits in bmap
 */
struct media_entity_enum {
        unsigned long *bmap;
        int idx_max;
};

/**
 * struct media_graph - Media graph traversal state
 *
 * @stack:                Graph traversal stack; the stack contains information
 *                        on the path the media entities to be walked and the
 *                        links through which they were reached.
 * @stack.entity:        pointer to &struct media_entity at the graph.
 * @stack.link:                pointer to &struct list_head.
 * @ent_enum:                Visited entities
 * @top:                The top of the stack
 */
struct media_graph {
        struct {
                struct media_entity *entity;
                struct list_head *link;
        } stack[MEDIA_ENTITY_ENUM_MAX_DEPTH];

        struct media_entity_enum ent_enum;
        int top;
};

/**
 * struct media_pipeline - Media pipeline related information
 *
 * @allocated:                Media pipeline allocated and freed by the framework
 * @mdev:                The media device the pipeline is part of
 * @pads:                List of media_pipeline_pad
 * @start_count:        Media pipeline start - stop count
 */
struct media_pipeline {
        bool allocated;
        struct media_device *mdev;
        struct list_head pads;
        int start_count;
};

/**
 * struct media_pipeline_pad - A pad part of a media pipeline
 *
 * @list:                Entry in the media_pad pads list
 * @pipe:                The media_pipeline that the pad is part of
 * @pad:                The media pad
 *
 * This structure associate a pad with a media pipeline. Instances of
 * media_pipeline_pad are created by media_pipeline_start() when it builds the
 * pipeline, and stored in the &media_pad.pads list. media_pipeline_stop()
 * removes the entries from the list and deletes them.
 */
struct media_pipeline_pad {
        struct list_head list;
        struct media_pipeline *pipe;
        struct media_pad *pad;
};

/**
 * struct media_pipeline_pad_iter - Iterator for media_pipeline_for_each_pad
 *
 * @cursor: The current element
 */
struct media_pipeline_pad_iter {
        struct list_head *cursor;
};

/**
 * struct media_pipeline_entity_iter - Iterator for media_pipeline_for_each_entity
 *
 * @ent_enum: The entity enumeration tracker
 * @cursor: The current element
 */
struct media_pipeline_entity_iter {
        struct media_entity_enum ent_enum;
        struct list_head *cursor;
};

/**
 * struct media_link - A link object part of a media graph.
 *
 * @graph_obj:        Embedded structure containing the media object common data
 * @list:        Linked list associated with an entity or an interface that
 *                owns the link.
 * @gobj0:        Part of a union. Used to get the pointer for the first
 *                graph_object of the link.
 * @source:        Part of a union. Used only if the first object (gobj0) is
 *                a pad. In that case, it represents the source pad.
 * @intf:        Part of a union. Used only if the first object (gobj0) is
 *                an interface.
 * @gobj1:        Part of a union. Used to get the pointer for the second
 *                graph_object of the link.
 * @sink:        Part of a union. Used only if the second object (gobj1) is
 *                a pad. In that case, it represents the sink pad.
 * @entity:        Part of a union. Used only if the second object (gobj1) is
 *                an entity.
 * @reverse:        Pointer to the link for the reverse direction of a pad to pad
 *                link.
 * @flags:        Link flags, as defined in uapi/media.h (MEDIA_LNK_FL_*)
 * @is_backlink: Indicate if the link is a backlink.
 */
struct media_link {
        struct media_gobj graph_obj;
        struct list_head list;
        union {
                struct media_gobj *gobj0;
                struct media_pad *source;
                struct media_interface *intf;
        };
        union {
                struct media_gobj *gobj1;
                struct media_pad *sink;
                struct media_entity *entity;
        };
        struct media_link *reverse;
        unsigned long flags;
        bool is_backlink;
};

/**
 * enum media_pad_signal_type - type of the signal inside a media pad
 *
 * @PAD_SIGNAL_DEFAULT:
 *        Default signal. Use this when all inputs or all outputs are
 *        uniquely identified by the pad number.
 * @PAD_SIGNAL_ANALOG:
 *        The pad contains an analog signal. It can be Radio Frequency,
 *        Intermediate Frequency, a baseband signal or sub-carriers.
 *        Tuner inputs, IF-PLL demodulators, composite and s-video signals
 *        should use it.
 * @PAD_SIGNAL_DV:
 *        Contains a digital video signal, with can be a bitstream of samples
 *        taken from an analog TV video source. On such case, it usually
 *        contains the VBI data on it.
 * @PAD_SIGNAL_AUDIO:
 *        Contains an Intermediate Frequency analog signal from an audio
 *        sub-carrier or an audio bitstream. IF signals are provided by tuners
 *        and consumed by        audio AM/FM decoders. Bitstream audio is provided by
 *        an audio decoder.
 */
enum media_pad_signal_type {
        PAD_SIGNAL_DEFAULT = 0,
        PAD_SIGNAL_ANALOG,
        PAD_SIGNAL_DV,
        PAD_SIGNAL_AUDIO,
};

/**
 * struct media_pad - A media pad graph object.
 *
 * @graph_obj:        Embedded structure containing the media object common data
 * @entity:        Entity this pad belongs to
 * @index:        Pad index in the entity pads array, numbered from 0 to n
 * @num_links:        Number of links connected to this pad
 * @sig_type:        Type of the signal inside a media pad
 * @flags:        Pad flags, as defined in
 *                :ref:`include/uapi/linux/media.h <media_header>`
 *                (seek for ``MEDIA_PAD_FL_*``)
 * @pipe:        Pipeline this pad belongs to. Use media_entity_pipeline() to
 *                access this field.
 */
struct media_pad {
        struct media_gobj graph_obj;        /* must be first field in struct */
        struct media_entity *entity;
        u16 index;
        u16 num_links;
        enum media_pad_signal_type sig_type;
        unsigned long flags;

        /*
         * The fields below are private, and should only be accessed via
         * appropriate functions.
         */
        struct media_pipeline *pipe;
};

/**
 * struct media_entity_operations - Media entity operations
 * @get_fwnode_pad:        Return the pad number based on a fwnode endpoint or
 *                        a negative value on error. This operation can be used
 *                        to map a fwnode to a media pad number. Optional.
 * @link_setup:                Notify the entity of link changes. The operation can
 *                        return an error, in which case link setup will be
 *                        cancelled. Optional.
 * @link_validate:        Return whether a link is valid from the entity point of
 *                        view. The media_pipeline_start() function
 *                        validates all links by calling this operation. Optional.
 * @has_pad_interdep:        Return whether two pads of the entity are
 *                        interdependent. If two pads are interdependent they are
 *                        part of the same pipeline and enabling one of the pads
 *                        means that the other pad will become "locked" and
 *                        doesn't allow configuration changes. pad0 and pad1 are
 *                        guaranteed to not both be sinks or sources. Never call
 *                        the .has_pad_interdep() operation directly, always use
 *                        media_entity_has_pad_interdep().
 *                        Optional: If the operation isn't implemented all pads
 *                        will be considered as interdependent.
 *
 * .. note::
 *
 *    Those these callbacks are called with struct &media_device.graph_mutex
 *    mutex held.
 */
struct media_entity_operations {
        int (*get_fwnode_pad)(struct media_entity *entity,
                              struct fwnode_endpoint *endpoint);
        int (*link_setup)(struct media_entity *entity,
                          const struct media_pad *local,
                          const struct media_pad *remote, u32 flags);
        int (*link_validate)(struct media_link *link);
        bool (*has_pad_interdep)(struct media_entity *entity, unsigned int pad0,
                                 unsigned int pad1);
};

/**
 * enum media_entity_type - Media entity type
 *
 * @MEDIA_ENTITY_TYPE_BASE:
 *        The entity isn't embedded in another subsystem structure.
 * @MEDIA_ENTITY_TYPE_VIDEO_DEVICE:
 *        The entity is embedded in a struct video_device instance.
 * @MEDIA_ENTITY_TYPE_V4L2_SUBDEV:
 *        The entity is embedded in a struct v4l2_subdev instance.
 *
 * Media entity objects are often not instantiated directly, but the media
 * entity structure is inherited by (through embedding) other subsystem-specific
 * structures. The media entity type identifies the type of the subclass
 * structure that implements a media entity instance.
 *
 * This allows runtime type identification of media entities and safe casting to
 * the correct object type. For instance, a media entity structure instance
 * embedded in a v4l2_subdev structure instance will have the type
 * %MEDIA_ENTITY_TYPE_V4L2_SUBDEV and can safely be cast to a &v4l2_subdev
 * structure using the container_of() macro.
 */
enum media_entity_type {
        MEDIA_ENTITY_TYPE_BASE,
        MEDIA_ENTITY_TYPE_VIDEO_DEVICE,
        MEDIA_ENTITY_TYPE_V4L2_SUBDEV,
};

/**
 * struct media_entity - A media entity graph object.
 *
 * @graph_obj:        Embedded structure containing the media object common data.
 * @name:        Entity name.
 * @obj_type:        Type of the object that implements the media_entity.
 * @function:        Entity main function, as defined in
 *                :ref:`include/uapi/linux/media.h <media_header>`
 *                (seek for ``MEDIA_ENT_F_*``)
 * @flags:        Entity flags, as defined in
 *                :ref:`include/uapi/linux/media.h <media_header>`
 *                (seek for ``MEDIA_ENT_FL_*``)
 * @num_pads:        Number of sink and source pads.
 * @num_links:        Total number of links, forward and back, enabled and disabled.
 * @num_backlinks: Number of backlinks
 * @internal_idx: An unique internal entity specific number. The numbers are
 *                re-used if entities are unregistered or registered again.
 * @pads:        Pads array with the size defined by @num_pads.
 * @links:        List of data links.
 * @ops:        Entity operations.
 * @use_count:        Use count for the entity.
 * @info:        Union with devnode information.  Kept just for backward
 *                compatibility.
 * @info.dev:        Contains device major and minor info.
 * @info.dev.major: device node major, if the device is a devnode.
 * @info.dev.minor: device node minor, if the device is a devnode.
 *
 * .. note::
 *
 *    The @use_count reference count must never be negative, but is a signed
 *    integer on purpose: a simple ``WARN_ON(<0)`` check can be used to detect
 *    reference count bugs that would make it negative.
 */
struct media_entity {
        struct media_gobj graph_obj;        /* must be first field in struct */
        const char *name;
        enum media_entity_type obj_type;
        u32 function;
        unsigned long flags;

        u16 num_pads;
        u16 num_links;
        u16 num_backlinks;
        int internal_idx;

        struct media_pad *pads;
        struct list_head links;

        const struct media_entity_operations *ops;

        int use_count;

        union {
                struct {
                        u32 major;
                        u32 minor;
                } dev;
        } info;
};

/**
 * media_entity_for_each_pad - Iterate on all pads in an entity
 * @entity: The entity the pads belong to
 * @iter: The iterator pad
 *
 * Iterate on all pads in a media entity.
 */
#define media_entity_for_each_pad(entity, iter)                        \
        for (iter = (entity)->pads;                                \
             iter < &(entity)->pads[(entity)->num_pads];        \
             ++iter)

/**
 * struct media_interface - A media interface graph object.
 *
 * @graph_obj:                embedded graph object
 * @links:                List of links pointing to graph entities
 * @type:                Type of the interface as defined in
 *                        :ref:`include/uapi/linux/media.h <media_header>`
 *                        (seek for ``MEDIA_INTF_T_*``)
 * @flags:                Interface flags as defined in
 *                        :ref:`include/uapi/linux/media.h <media_header>`
 *                        (seek for ``MEDIA_INTF_FL_*``)
 *
 * .. note::
 *
 *    Currently, no flags for &media_interface is defined.
 */
struct media_interface {
        struct media_gobj                graph_obj;
        struct list_head                links;
        u32                                type;
        u32                                flags;
};

/**
 * struct media_intf_devnode - A media interface via a device node.
 *
 * @intf:        embedded interface object
 * @major:        Major number of a device node
 * @minor:        Minor number of a device node
 */
struct media_intf_devnode {
        struct media_interface                intf;

        /* Should match the fields at media_v2_intf_devnode */
        u32                                major;
        u32                                minor;
};

/**
 * media_entity_id() - return the media entity graph object id
 *
 * @entity:        pointer to &media_entity
 */
static inline u32 media_entity_id(struct media_entity *entity)
{
        return entity->graph_obj.id;
}

/**
 * media_type() - return the media object type
 *
 * @gobj:        Pointer to the struct &media_gobj graph object
 */
static inline enum media_gobj_type media_type(struct media_gobj *gobj)
{
        return gobj->id >> MEDIA_BITS_PER_ID;
}

/**
 * media_id() - return the media object ID
 *
 * @gobj:        Pointer to the struct &media_gobj graph object
 */
static inline u32 media_id(struct media_gobj *gobj)
{
        return gobj->id & MEDIA_ID_MASK;
}

/**
 * media_gobj_gen_id() - encapsulates type and ID on at the object ID
 *
 * @type:        object type as define at enum &media_gobj_type.
 * @local_id:        next ID, from struct &media_device.id.
 */
static inline u32 media_gobj_gen_id(enum media_gobj_type type, u64 local_id)
{
        u32 id;

        id = type << MEDIA_BITS_PER_ID;
        id |= local_id & MEDIA_ID_MASK;

        return id;
}

/**
 * is_media_entity_v4l2_video_device() - Check if the entity is a video_device
 * @entity:        pointer to entity
 *
 * Return: %true if the entity is an instance of a video_device object and can
 * safely be cast to a struct video_device using the container_of() macro, or
 * %false otherwise.
 */
static inline bool is_media_entity_v4l2_video_device(struct media_entity *entity)
{
        return entity && entity->obj_type == MEDIA_ENTITY_TYPE_VIDEO_DEVICE;
}

/**
 * is_media_entity_v4l2_subdev() - Check if the entity is a v4l2_subdev
 * @entity:        pointer to entity
 *
 * Return: %true if the entity is an instance of a &v4l2_subdev object and can
 * safely be cast to a struct &v4l2_subdev using the container_of() macro, or
 * %false otherwise.
 */
static inline bool is_media_entity_v4l2_subdev(struct media_entity *entity)
{
        return entity && entity->obj_type == MEDIA_ENTITY_TYPE_V4L2_SUBDEV;
}

/**
 * media_entity_enum_init - Initialise an entity enumeration
 *
 * @ent_enum: Entity enumeration to be initialised
 * @mdev: The related media device
 *
 * Return: zero on success or a negative error code.
 */
__must_check int media_entity_enum_init(struct media_entity_enum *ent_enum,
                                        struct media_device *mdev);

/**
 * media_entity_enum_cleanup - Release resources of an entity enumeration
 *
 * @ent_enum: Entity enumeration to be released
 */
void media_entity_enum_cleanup(struct media_entity_enum *ent_enum);

/**
 * media_entity_enum_zero - Clear the entire enum
 *
 * @ent_enum: Entity enumeration to be cleared
 */
static inline void media_entity_enum_zero(struct media_entity_enum *ent_enum)
{
        bitmap_zero(ent_enum->bmap, ent_enum->idx_max);
}

/**
 * media_entity_enum_set - Mark a single entity in the enum
 *
 * @ent_enum: Entity enumeration
 * @entity: Entity to be marked
 */
static inline void media_entity_enum_set(struct media_entity_enum *ent_enum,
                                         struct media_entity *entity)
{
        if (WARN_ON(entity->internal_idx >= ent_enum->idx_max))
                return;

        __set_bit(entity->internal_idx, ent_enum->bmap);
}

/**
 * media_entity_enum_clear - Unmark a single entity in the enum
 *
 * @ent_enum: Entity enumeration
 * @entity: Entity to be unmarked
 */
static inline void media_entity_enum_clear(struct media_entity_enum *ent_enum,
                                           struct media_entity *entity)
{
        if (WARN_ON(entity->internal_idx >= ent_enum->idx_max))
                return;

        __clear_bit(entity->internal_idx, ent_enum->bmap);
}

/**
 * media_entity_enum_test - Test whether the entity is marked
 *
 * @ent_enum: Entity enumeration
 * @entity: Entity to be tested
 *
 * Returns %true if the entity was marked.
 */
static inline bool media_entity_enum_test(struct media_entity_enum *ent_enum,
                                          struct media_entity *entity)
{
        if (WARN_ON(entity->internal_idx >= ent_enum->idx_max))
                return true;

        return test_bit(entity->internal_idx, ent_enum->bmap);
}

/**
 * media_entity_enum_test_and_set - Test whether the entity is marked,
 *        and mark it
 *
 * @ent_enum: Entity enumeration
 * @entity: Entity to be tested
 *
 * Returns %true if the entity was marked, and mark it before doing so.
 */
static inline bool
media_entity_enum_test_and_set(struct media_entity_enum *ent_enum,
                               struct media_entity *entity)
{
        if (WARN_ON(entity->internal_idx >= ent_enum->idx_max))
                return true;

        return __test_and_set_bit(entity->internal_idx, ent_enum->bmap);
}

/**
 * media_entity_enum_empty - Test whether the entire enum is empty
 *
 * @ent_enum: Entity enumeration
 *
 * Return: %true if the entity was empty.
 */
static inline bool media_entity_enum_empty(struct media_entity_enum *ent_enum)
{
        return bitmap_empty(ent_enum->bmap, ent_enum->idx_max);
}

/**
 * media_entity_enum_intersects - Test whether two enums intersect
 *
 * @ent_enum1: First entity enumeration
 * @ent_enum2: Second entity enumeration
 *
 * Return: %true if entity enumerations @ent_enum1 and @ent_enum2 intersect,
 * otherwise %false.
 */
static inline bool media_entity_enum_intersects(
        struct media_entity_enum *ent_enum1,
        struct media_entity_enum *ent_enum2)
{
        WARN_ON(ent_enum1->idx_max != ent_enum2->idx_max);

        return bitmap_intersects(ent_enum1->bmap, ent_enum2->bmap,
                                 min(ent_enum1->idx_max, ent_enum2->idx_max));
}

/**
 * gobj_to_entity - returns the struct &media_entity pointer from the
 *        @gobj contained on it.
 *
 * @gobj: Pointer to the struct &media_gobj graph object
 */
#define gobj_to_entity(gobj) \
                container_of(gobj, struct media_entity, graph_obj)

/**
 * gobj_to_pad - returns the struct &media_pad pointer from the
 *        @gobj contained on it.
 *
 * @gobj: Pointer to the struct &media_gobj graph object
 */
#define gobj_to_pad(gobj) \
                container_of(gobj, struct media_pad, graph_obj)

/**
 * gobj_to_link - returns the struct &media_link pointer from the
 *        @gobj contained on it.
 *
 * @gobj: Pointer to the struct &media_gobj graph object
 */
#define gobj_to_link(gobj) \
                container_of(gobj, struct media_link, graph_obj)

/**
 * gobj_to_intf - returns the struct &media_interface pointer from the
 *        @gobj contained on it.
 *
 * @gobj: Pointer to the struct &media_gobj graph object
 */
#define gobj_to_intf(gobj) \
                container_of(gobj, struct media_interface, graph_obj)

/**
 * intf_to_devnode - returns the struct media_intf_devnode pointer from the
 *        @intf contained on it.
 *
 * @intf: Pointer to struct &media_intf_devnode
 */
#define intf_to_devnode(intf) \
                container_of(intf, struct media_intf_devnode, intf)

/**
 *  media_gobj_create - Initialize a graph object
 *
 * @mdev:        Pointer to the &media_device that contains the object
 * @type:        Type of the object
 * @gobj:        Pointer to the struct &media_gobj graph object
 *
 * This routine initializes the embedded struct &media_gobj inside a
 * media graph object. It is called automatically if ``media_*_create``
 * function calls are used. However, if the object (entity, link, pad,
 * interface) is embedded on some other object, this function should be
 * called before registering the object at the media controller.
 */
void media_gobj_create(struct media_device *mdev,
                    enum media_gobj_type type,
                    struct media_gobj *gobj);

/**
 *  media_gobj_destroy - Stop using a graph object on a media device
 *
 * @gobj:        Pointer to the struct &media_gobj graph object
 *
 * This should be called by all routines like media_device_unregister()
 * that remove/destroy media graph objects.
 */
void media_gobj_destroy(struct media_gobj *gobj);

/**
 * media_entity_pads_init() - Initialize the entity pads
 *
 * @entity:        entity where the pads belong
 * @num_pads:        total number of sink and source pads
 * @pads:        Array of @num_pads pads.
 *
 * The pads array is managed by the entity driver and passed to
 * media_entity_pads_init() where its pointer will be stored in the
 * &media_entity structure.
 *
 * If no pads are needed, drivers could either directly fill
 * &media_entity->num_pads with 0 and &media_entity->pads with %NULL or call
 * this function that will do the same.
 *
 * As the number of pads is known in advance, the pads array is not allocated
 * dynamically but is managed by the entity driver. Most drivers will embed the
 * pads array in a driver-specific structure, avoiding dynamic allocation.
 *
 * Drivers must set the direction of every pad in the pads array before calling
 * media_entity_pads_init(). The function will initialize the other pads fields.
 */
int media_entity_pads_init(struct media_entity *entity, u16 num_pads,
                      struct media_pad *pads);

/**
 * media_entity_cleanup() - free resources associated with an entity
 *
 * @entity:        entity where the pads belong
 *
 * This function must be called during the cleanup phase after unregistering
 * the entity (currently, it does nothing).
 *
 * Calling media_entity_cleanup() on a media_entity whose memory has been
 * zeroed but that has not been initialized with media_entity_pad_init() is
 * valid and is a no-op.
 */
#if IS_ENABLED(CONFIG_MEDIA_CONTROLLER)
static inline void media_entity_cleanup(struct media_entity *entity) {}
#else
#define media_entity_cleanup(entity) do { } while (false)
#endif

/**
 * media_get_pad_index() - retrieves a pad index from an entity
 *
 * @entity:        entity where the pads belong
 * @pad_type:        the type of the pad, one of MEDIA_PAD_FL_* pad types
 * @sig_type:        type of signal of the pad to be search
 *
 * This helper function finds the first pad index inside an entity that
 * satisfies both @is_sink and @sig_type conditions.
 *
 * Return:
 *
 * On success, return the pad number. If the pad was not found or the media
 * entity is a NULL pointer, return -EINVAL.
 */
int media_get_pad_index(struct media_entity *entity, u32 pad_type,
                        enum media_pad_signal_type sig_type);

/**
 * media_create_pad_link() - creates a link between two entities.
 *
 * @source:        pointer to &media_entity of the source pad.
 * @source_pad:        number of the source pad in the pads array
 * @sink:        pointer to &media_entity of the sink pad.
 * @sink_pad:        number of the sink pad in the pads array.
 * @flags:        Link flags, as defined in
 *                :ref:`include/uapi/linux/media.h <media_header>`
 *                ( seek for ``MEDIA_LNK_FL_*``)
 *
 * Valid values for flags:
 *
 * %MEDIA_LNK_FL_ENABLED
 *   Indicates that the link is enabled and can be used to transfer media data.
 *   When two or more links target a sink pad, only one of them can be
 *   enabled at a time.
 *
 * %MEDIA_LNK_FL_IMMUTABLE
 *   Indicates that the link enabled state can't be modified at runtime. If
 *   %MEDIA_LNK_FL_IMMUTABLE is set, then %MEDIA_LNK_FL_ENABLED must also be
 *   set, since an immutable link is always enabled.
 *
 * .. note::
 *
 *    Before calling this function, media_entity_pads_init() and
 *    media_device_register_entity() should be called previously for both ends.
 */
__must_check int media_create_pad_link(struct media_entity *source,
                        u16 source_pad, struct media_entity *sink,
                        u16 sink_pad, u32 flags);

/**
 * media_create_pad_links() - creates a link between two entities.
 *
 * @mdev: Pointer to the media_device that contains the object
 * @source_function: Function of the source entities. Used only if @source is
 *        NULL.
 * @source: pointer to &media_entity of the source pad. If NULL, it will use
 *        all entities that matches the @sink_function.
 * @source_pad: number of the source pad in the pads array
 * @sink_function: Function of the sink entities. Used only if @sink is NULL.
 * @sink: pointer to &media_entity of the sink pad. If NULL, it will use
 *        all entities that matches the @sink_function.
 * @sink_pad: number of the sink pad in the pads array.
 * @flags: Link flags, as defined in include/uapi/linux/media.h.
 * @allow_both_undefined: if %true, then both @source and @sink can be NULL.
 *        In such case, it will create a crossbar between all entities that
 *        matches @source_function to all entities that matches @sink_function.
 *        If %false, it will return 0 and won't create any link if both @source
 *        and @sink are NULL.
 *
 * Valid values for flags:
 *
 * A %MEDIA_LNK_FL_ENABLED flag indicates that the link is enabled and can be
 *        used to transfer media data. If multiple links are created and this
 *        flag is passed as an argument, only the first created link will have
 *        this flag.
 *
 * A %MEDIA_LNK_FL_IMMUTABLE flag indicates that the link enabled state can't
 *        be modified at runtime. If %MEDIA_LNK_FL_IMMUTABLE is set, then
 *        %MEDIA_LNK_FL_ENABLED must also be set since an immutable link is
 *        always enabled.
 *
 * It is common for some devices to have multiple source and/or sink entities
 * of the same type that should be linked. While media_create_pad_link()
 * creates link by link, this function is meant to allow 1:n, n:1 and even
 * cross-bar (n:n) links.
 *
 * .. note::
 *
 *    Before calling this function, media_entity_pads_init() and
 *    media_device_register_entity() should be called previously for the
 *    entities to be linked.
 */
int media_create_pad_links(const struct media_device *mdev,
                           const u32 source_function,
                           struct media_entity *source,
                           const u16 source_pad,
                           const u32 sink_function,
                           struct media_entity *sink,
                           const u16 sink_pad,
                           u32 flags,
                           const bool allow_both_undefined);

void __media_entity_remove_links(struct media_entity *entity);

/**
 * media_entity_remove_links() - remove all links associated with an entity
 *
 * @entity:        pointer to &media_entity
 *
 * .. note::
 *
 *    This is called automatically when an entity is unregistered via
 *    media_device_register_entity().
 */
void media_entity_remove_links(struct media_entity *entity);

/**
 * __media_entity_setup_link - Configure a media link without locking
 * @link: The link being configured
 * @flags: Link configuration flags
 *
 * The bulk of link setup is handled by the two entities connected through the
 * link. This function notifies both entities of the link configuration change.
 *
 * If the link is immutable or if the current and new configuration are
 * identical, return immediately.
 *
 * The user is expected to hold link->source->parent->mutex. If not,
 * media_entity_setup_link() should be used instead.
 */
int __media_entity_setup_link(struct media_link *link, u32 flags);

/**
 * media_entity_setup_link() - changes the link flags properties in runtime
 *
 * @link:        pointer to &media_link
 * @flags:        the requested new link flags
 *
 * The only configurable property is the %MEDIA_LNK_FL_ENABLED link flag
 * to enable/disable a link. Links marked with the
 * %MEDIA_LNK_FL_IMMUTABLE link flag can not be enabled or disabled.
 *
 * When a link is enabled or disabled, the media framework calls the
 * link_setup operation for the two entities at the source and sink of the
 * link, in that order. If the second link_setup call fails, another
 * link_setup call is made on the first entity to restore the original link
 * flags.
 *
 * Media device drivers can be notified of link setup operations by setting the
 * &media_device.link_notify pointer to a callback function. If provided, the
 * notification callback will be called before enabling and after disabling
 * links.
 *
 * Entity drivers must implement the link_setup operation if any of their links
 * is non-immutable. The operation must either configure the hardware or store
 * the configuration information to be applied later.
 *
 * Link configuration must not have any side effect on other links. If an
 * enabled link at a sink pad prevents another link at the same pad from
 * being enabled, the link_setup operation must return %-EBUSY and can't
 * implicitly disable the first enabled link.
 *
 * .. note::
 *
 *    The valid values of the flags for the link is the same as described
 *    on media_create_pad_link(), for pad to pad links or the same as described
 *    on media_create_intf_link(), for interface to entity links.
 */
int media_entity_setup_link(struct media_link *link, u32 flags);

/**
 * media_entity_find_link - Find a link between two pads
 * @source: Source pad
 * @sink: Sink pad
 *
 * Return: returns a pointer to the link between the two entities. If no
 * such link exists, return %NULL.
 */
struct media_link *media_entity_find_link(struct media_pad *source,
                struct media_pad *sink);

/**
 * media_pad_remote_pad_first - Find the first pad at the remote end of a link
 * @pad: Pad at the local end of the link
 *
 * Search for a remote pad connected to the given pad by iterating over all
 * links originating or terminating at that pad until an enabled link is found.
 *
 * Return: returns a pointer to the pad at the remote end of the first found
 * enabled link, or %NULL if no enabled link has been found.
 */
struct media_pad *media_pad_remote_pad_first(const struct media_pad *pad);

/**
 * media_pad_remote_pad_unique - Find a remote pad connected to a pad
 * @pad: The pad
 *
 * Search for and return a remote pad connected to @pad through an enabled
 * link. If multiple (or no) remote pads are found, an error is returned.
 *
 * The uniqueness constraint makes this helper function suitable for entities
 * that support a single active source at a time on a given pad.
 *
 * Return: A pointer to the remote pad, or one of the following error pointers
 * if an error occurs:
 *
 * * -ENOTUNIQ - Multiple links are enabled
 * * -ENOLINK - No connected pad found
 */
struct media_pad *media_pad_remote_pad_unique(const struct media_pad *pad);

/**
 * media_entity_remote_pad_unique - Find a remote pad connected to an entity
 * @entity: The entity
 * @type: The type of pad to find (MEDIA_PAD_FL_SINK or MEDIA_PAD_FL_SOURCE)
 *
 * Search for and return a remote pad of @type connected to @entity through an
 * enabled link. If multiple (or no) remote pads match these criteria, an error
 * is returned.
 *
 * The uniqueness constraint makes this helper function suitable for entities
 * that support a single active source or sink at a time.
 *
 * Return: A pointer to the remote pad, or one of the following error pointers
 * if an error occurs:
 *
 * * -ENOTUNIQ - Multiple links are enabled
 * * -ENOLINK - No connected pad found
 */
struct media_pad *
media_entity_remote_pad_unique(const struct media_entity *entity,
                               unsigned int type);

/**
 * media_entity_remote_source_pad_unique - Find a remote source pad connected to
 *        an entity
 * @entity: The entity
 *
 * Search for and return a remote source pad connected to @entity through an
 * enabled link. If multiple (or no) remote pads match these criteria, an error
 * is returned.
 *
 * The uniqueness constraint makes this helper function suitable for entities
 * that support a single active source at a time.
 *
 * Return: A pointer to the remote pad, or one of the following error pointers
 * if an error occurs:
 *
 * * -ENOTUNIQ - Multiple links are enabled
 * * -ENOLINK - No connected pad found
 */
static inline struct media_pad *
media_entity_remote_source_pad_unique(const struct media_entity *entity)
{
        return media_entity_remote_pad_unique(entity, MEDIA_PAD_FL_SOURCE);
}

/**
 * media_pad_is_streaming - Test if a pad is part of a streaming pipeline
 * @pad: The pad
 *
 * Return: True if the pad is part of a pipeline started with the
 * media_pipeline_start() function, false otherwise.
 */
static inline bool media_pad_is_streaming(const struct media_pad *pad)
{
        return pad->pipe;
}

/**
 * media_entity_is_streaming - Test if an entity is part of a streaming pipeline
 * @entity: The entity
 *
 * Return: True if the entity is part of a pipeline started with the
 * media_pipeline_start() function, false otherwise.
 */
static inline bool media_entity_is_streaming(const struct media_entity *entity)
{
        struct media_pad *pad;

        media_entity_for_each_pad(entity, pad) {
                if (media_pad_is_streaming(pad))
                        return true;
        }

        return false;
}

/**
 * media_entity_pipeline - Get the media pipeline an entity is part of
 * @entity: The entity
 *
 * DEPRECATED: use media_pad_pipeline() instead.
 *
 * This function returns the media pipeline that an entity has been associated
 * with when constructing the pipeline with media_pipeline_start(). The pointer
 * remains valid until media_pipeline_stop() is called.
 *
 * In general, entities can be part of multiple pipelines, when carrying
 * multiple streams (either on different pads, or on the same pad using
 * multiplexed streams). This function is to be used only for entities that
 * do not support multiple pipelines.
 *
 * Return: The media_pipeline the entity is part of, or NULL if the entity is
 * not part of any pipeline.
 */
struct media_pipeline *media_entity_pipeline(struct media_entity *entity);

/**
 * media_pad_pipeline - Get the media pipeline a pad is part of
 * @pad: The pad
 *
 * This function returns the media pipeline that a pad has been associated
 * with when constructing the pipeline with media_pipeline_start(). The pointer
 * remains valid until media_pipeline_stop() is called.
 *
 * Return: The media_pipeline the pad is part of, or NULL if the pad is
 * not part of any pipeline.
 */
struct media_pipeline *media_pad_pipeline(struct media_pad *pad);

/**
 * media_entity_get_fwnode_pad - Get pad number from fwnode
 *
 * @entity: The entity
 * @fwnode: Pointer to the fwnode_handle which should be used to find the pad
 * @direction_flags: Expected direction of the pad, as defined in
 *                     :ref:`include/uapi/linux/media.h <media_header>`
 *                     (seek for ``MEDIA_PAD_FL_*``)
 *
 * This function can be used to resolve the media pad number from
 * a fwnode. This is useful for devices which use more complex
 * mappings of media pads.
 *
 * If the entity does not implement the get_fwnode_pad() operation
 * then this function searches the entity for the first pad that
 * matches the @direction_flags.
 *
 * Return: returns the pad number on success or a negative error code.
 */
int media_entity_get_fwnode_pad(struct media_entity *entity,
                                const struct fwnode_handle *fwnode,
                                unsigned long direction_flags);

/**
 * media_graph_walk_init - Allocate resources used by graph walk.
 *
 * @graph: Media graph structure that will be used to walk the graph
 * @mdev: Pointer to the &media_device that contains the object
 *
 * This function is deprecated, use media_pipeline_for_each_pad() instead.
 *
 * The caller is required to hold the media_device graph_mutex during the graph
 * walk until the graph state is released.
 *
 * Returns zero on success or a negative error code otherwise.
 */
__must_check int media_graph_walk_init(
        struct media_graph *graph, struct media_device *mdev);

/**
 * media_graph_walk_cleanup - Release resources used by graph walk.
 *
 * @graph: Media graph structure that will be used to walk the graph
 *
 * This function is deprecated, use media_pipeline_for_each_pad() instead.
 */
void media_graph_walk_cleanup(struct media_graph *graph);

/**
 * media_graph_walk_start - Start walking the media graph at a
 *        given entity
 *
 * @graph: Media graph structure that will be used to walk the graph
 * @entity: Starting entity
 *
 * This function is deprecated, use media_pipeline_for_each_pad() instead.
 *
 * Before using this function, media_graph_walk_init() must be
 * used to allocate resources used for walking the graph. This
 * function initializes the graph traversal structure to walk the
 * entities graph starting at the given entity. The traversal
 * structure must not be modified by the caller during graph
 * traversal. After the graph walk, the resources must be released
 * using media_graph_walk_cleanup().
 */
void media_graph_walk_start(struct media_graph *graph,
                            struct media_entity *entity);

/**
 * media_graph_walk_next - Get the next entity in the graph
 * @graph: Media graph structure
 *
 * This function is deprecated, use media_pipeline_for_each_pad() instead.
 *
 * Perform a depth-first traversal of the given media entities graph.
 *
 * The graph structure must have been previously initialized with a call to
 * media_graph_walk_start().
 *
 * Return: returns the next entity in the graph or %NULL if the whole graph
 * have been traversed.
 */
struct media_entity *media_graph_walk_next(struct media_graph *graph);

/**
 * media_pipeline_start - Mark a pipeline as streaming
 * @origin: Starting pad
 * @pipe: Media pipeline to be assigned to all pads in the pipeline.
 *
 * Mark all pads connected to pad @origin through enabled links, either
 * directly or indirectly, as streaming. The given pipeline object is assigned
 * to every pad in the pipeline and stored in the media_pad pipe field.
 *
 * Calls to this function can be nested, in which case the same number of
 * media_pipeline_stop() calls will be required to stop streaming. The
 * pipeline pointer must be identical for all nested calls to
 * media_pipeline_start().
 */
__must_check int media_pipeline_start(struct media_pad *origin,
                                      struct media_pipeline *pipe);
/**
 * __media_pipeline_start - Mark a pipeline as streaming
 *
 * @origin: Starting pad
 * @pipe: Media pipeline to be assigned to all pads in the pipeline.
 *
 * ..note:: This is the non-locking version of media_pipeline_start()
 */
__must_check int __media_pipeline_start(struct media_pad *origin,
                                        struct media_pipeline *pipe);

/**
 * media_pipeline_stop - Mark a pipeline as not streaming
 * @pad: Starting pad
 *
 * Mark all pads connected to a given pad through enabled links, either
 * directly or indirectly, as not streaming. The media_pad pipe field is
 * reset to %NULL.
 *
 * If multiple calls to media_pipeline_start() have been made, the same
 * number of calls to this function are required to mark the pipeline as not
 * streaming.
 */
void media_pipeline_stop(struct media_pad *pad);

/**
 * __media_pipeline_stop - Mark a pipeline as not streaming
 *
 * @pad: Starting pad
 *
 * .. note:: This is the non-locking version of media_pipeline_stop()
 */
void __media_pipeline_stop(struct media_pad *pad);

struct media_pad *
__media_pipeline_pad_iter_next(struct media_pipeline *pipe,
                               struct media_pipeline_pad_iter *iter,
                               struct media_pad *pad);

/**
 * media_pipeline_for_each_pad - Iterate on all pads in a media pipeline
 * @pipe: The pipeline
 * @iter: The iterator (struct media_pipeline_pad_iter)
 * @pad: The iterator pad
 *
 * Iterate on all pads in a media pipeline. This is only valid after the
 * pipeline has been built with media_pipeline_start() and before it gets
 * destroyed with media_pipeline_stop().
 */
#define media_pipeline_for_each_pad(pipe, iter, pad)                        \
        for (pad = __media_pipeline_pad_iter_next((pipe), iter, NULL);        \
             pad != NULL;                                                \
             pad = __media_pipeline_pad_iter_next((pipe), iter, pad))

/**
 * media_pipeline_entity_iter_init - Initialize a pipeline entity iterator
 * @pipe: The pipeline
 * @iter: The iterator
 *
 * This function must be called to initialize the iterator before using it in a
 * media_pipeline_for_each_entity() loop. The iterator must be destroyed by a
 * call to media_pipeline_entity_iter_cleanup after the loop (including in code
 * paths that break from the loop).
 *
 * The same iterator can be used in multiple consecutive loops without being
 * destroyed and reinitialized.
 *
 * Return: 0 on success or a negative error code otherwise.
 */
int media_pipeline_entity_iter_init(struct media_pipeline *pipe,
                                    struct media_pipeline_entity_iter *iter);

/**
 * media_pipeline_entity_iter_cleanup - Destroy a pipeline entity iterator
 * @iter: The iterator
 *
 * This function must be called to destroy iterators initialized with
 * media_pipeline_entity_iter_init().
 */
void media_pipeline_entity_iter_cleanup(struct media_pipeline_entity_iter *iter);

struct media_entity *
__media_pipeline_entity_iter_next(struct media_pipeline *pipe,
                                  struct media_pipeline_entity_iter *iter,
                                  struct media_entity *entity);

/**
 * media_pipeline_for_each_entity - Iterate on all entities in a media pipeline
 * @pipe: The pipeline
 * @iter: The iterator (struct media_pipeline_entity_iter)
 * @entity: The iterator entity
 *
 * Iterate on all entities in a media pipeline. This is only valid after the
 * pipeline has been built with media_pipeline_start() and before it gets
 * destroyed with media_pipeline_stop(). The iterator must be initialized with
 * media_pipeline_entity_iter_init() before iteration, and destroyed with
 * media_pipeline_entity_iter_cleanup() after (including in code paths that
 * break from the loop).
 */
#define media_pipeline_for_each_entity(pipe, iter, entity)                        \
        for (entity = __media_pipeline_entity_iter_next((pipe), iter, NULL);        \
             entity != NULL;                                                        \
             entity = __media_pipeline_entity_iter_next((pipe), iter, entity))

/**
 * media_pipeline_alloc_start - Mark a pipeline as streaming
 * @pad: Starting pad
 *
 * media_pipeline_alloc_start() is similar to media_pipeline_start() but instead
 * of working on a given pipeline the function will use an existing pipeline if
 * the pad is already part of a pipeline, or allocate a new pipeline.
 *
 * Calls to media_pipeline_alloc_start() must be matched with
 * media_pipeline_stop().
 */
__must_check int media_pipeline_alloc_start(struct media_pad *pad);

/**
 * media_devnode_create() - creates and initializes a device node interface
 *
 * @mdev:        pointer to struct &media_device
 * @type:        type of the interface, as given by
 *                :ref:`include/uapi/linux/media.h <media_header>`
 *                ( seek for ``MEDIA_INTF_T_*``) macros.
 * @flags:        Interface flags, as defined in
 *                :ref:`include/uapi/linux/media.h <media_header>`
 *                ( seek for ``MEDIA_INTF_FL_*``)
 * @major:        Device node major number.
 * @minor:        Device node minor number.
 *
 * Return: if succeeded, returns a pointer to the newly allocated
 *        &media_intf_devnode pointer.
 *
 * .. note::
 *
 *    Currently, no flags for &media_interface is defined.
 */
struct media_intf_devnode *
__must_check media_devnode_create(struct media_device *mdev,
                                  u32 type, u32 flags,
                                  u32 major, u32 minor);
/**
 * media_devnode_remove() - removes a device node interface
 *
 * @devnode:        pointer to &media_intf_devnode to be freed.
 *
 * When a device node interface is removed, all links to it are automatically
 * removed.
 */
void media_devnode_remove(struct media_intf_devnode *devnode);

/**
 * media_create_intf_link() - creates a link between an entity and an interface
 *
 * @entity:        pointer to %media_entity
 * @intf:        pointer to %media_interface
 * @flags:        Link flags, as defined in
 *                :ref:`include/uapi/linux/media.h <media_header>`
 *                ( seek for ``MEDIA_LNK_FL_*``)
 *
 *
 * Valid values for flags:
 *
 * %MEDIA_LNK_FL_ENABLED
 *   Indicates that the interface is connected to the entity hardware.
 *   That's the default value for interfaces. An interface may be disabled if
 *   the hardware is busy due to the usage of some other interface that it is
 *   currently controlling the hardware.
 *
 *   A typical example is an hybrid TV device that handle only one type of
 *   stream on a given time. So, when the digital TV is streaming,
 *   the V4L2 interfaces won't be enabled, as such device is not able to
 *   also stream analog TV or radio.
 *
 * .. note::
 *
 *    Before calling this function, media_devnode_create() should be called for
 *    the interface and media_device_register_entity() should be called for the
 *    interface that will be part of the link.
 */
struct media_link *
__must_check media_create_intf_link(struct media_entity *entity,
                                    struct media_interface *intf,
                                    u32 flags);
/**
 * __media_remove_intf_link() - remove a single interface link
 *
 * @link:        pointer to &media_link.
 *
 * .. note:: This is an unlocked version of media_remove_intf_link()
 */
void __media_remove_intf_link(struct media_link *link);

/**
 * media_remove_intf_link() - remove a single interface link
 *
 * @link:        pointer to &media_link.
 *
 * .. note:: Prefer to use this one, instead of __media_remove_intf_link()
 */
void media_remove_intf_link(struct media_link *link);

/**
 * __media_remove_intf_links() - remove all links associated with an interface
 *
 * @intf:        pointer to &media_interface
 *
 * .. note:: This is an unlocked version of media_remove_intf_links().
 */
void __media_remove_intf_links(struct media_interface *intf);

/**
 * media_remove_intf_links() - remove all links associated with an interface
 *
 * @intf:        pointer to &media_interface
 *
 * .. note::
 *
 *   #) This is called automatically when an entity is unregistered via
 *      media_device_register_entity() and by media_devnode_remove().
 *
 *   #) Prefer to use this one, instead of __media_remove_intf_links().
 */
void media_remove_intf_links(struct media_interface *intf);

/**
 * media_entity_call - Calls a struct media_entity_operations operation on
 *        an entity
 *
 * @entity: entity where the @operation will be called
 * @operation: type of the operation. Should be the name of a member of
 *        struct &media_entity_operations.
 *
 * This helper function will check if @operation is not %NULL. On such case,
 * it will issue a call to @operation\(@entity, @args\).
 */

#define media_entity_call(entity, operation, args...)                        \
        (((entity)->ops && (entity)->ops->operation) ?                        \
         (entity)->ops->operation((entity) , ##args) : -ENOIOCTLCMD)

/**
 * media_create_ancillary_link() - create an ancillary link between two
 *                                   instances of &media_entity
 *
 * @primary:        pointer to the primary &media_entity
 * @ancillary:        pointer to the ancillary &media_entity
 *
 * Create an ancillary link between two entities, indicating that they
 * represent two connected pieces of hardware that form a single logical unit.
 * A typical example is a camera lens controller being linked to the sensor that
 * it is supporting.
 *
 * The function sets both MEDIA_LNK_FL_ENABLED and MEDIA_LNK_FL_IMMUTABLE for
 * the new link.
 */
struct media_link *
media_create_ancillary_link(struct media_entity *primary,
                            struct media_entity *ancillary);

/**
 * __media_entity_next_link() - Iterate through a &media_entity's links
 *
 * @entity:        pointer to the &media_entity
 * @link:        pointer to a &media_link to hold the iterated values
 * @link_type:        one of the MEDIA_LNK_FL_LINK_TYPE flags
 *
 * Return the next link against an entity matching a specific link type. This
 * allows iteration through an entity's links whilst guaranteeing all of the
 * returned links are of the given type.
 */
struct media_link *__media_entity_next_link(struct media_entity *entity,
                                            struct media_link *link,
                                            unsigned long link_type);

/**
 * for_each_media_entity_data_link() - Iterate through an entity's data links
 *
 * @entity:        pointer to the &media_entity
 * @link:        pointer to a &media_link to hold the iterated values
 *
 * Iterate over a &media_entity's data links
 */
#define for_each_media_entity_data_link(entity, link)                        \
        for (link = __media_entity_next_link(entity, NULL,                \
                                             MEDIA_LNK_FL_DATA_LINK);        \
             link;                                                        \
             link = __media_entity_next_link(entity, link,                \
                                             MEDIA_LNK_FL_DATA_LINK))

#endif















































































 2640 
  350 









   71 














 3509 



































 3502 



 3506 









 2640 












 2004 





    1 






 2268 
















































































































  100 



  133 


  134 
  132 


















  136 

  136 



















  134 







  132 






















 1194 





 1201 
  938 
  267 

 1199 






























































































































































































































































































































  357 


  333 



   40 



   43 


  358 








  178 





  134 


  278 

  329 

   28 

  359 







    6 


    6 

    6 

    1 
    6 
    6 






    6 






    6 
















































  495 
  498 
  497 


  496 


  495 





  235 



  235 


  236 

  235 












  476 




    4 

   93 

    1 




  105 




   45 










   22 

   15 

  472 

   28 


  204 

    2 




  422 




  123 




  502 




    7 


  506 



  505 







  130 





    3 






   22 








  642 
    3 

  671 












  253 




  241 






   17 
   16 

















    3 







   11 



    6 









  195 






  186 
   14 




  193 


    3 








    6 



    8 



    4 




    4 


    1 

    2 


    1 









    1 




    4 














   48 
   58 











































  495 
    1 


























































  493 




















  496 



  495 


  495 





















  486 
   17 















  497 


































  495 
  493 
  498 
  499 















  497 































































  639 


  640 

















  638 




  636 












  640 




  628 





   63 






  639 




  630 





   61 







  640 





   39 







   50 


  638 

























































































































































  638 



  639 


    1 



    1 






    1 












  639 





















  635 
  640 
  637 

  639 





















  642 
  642 
















  640 






    9 




  642 



  640 



  637 





    7 


  639 









    7 













   46 




















  639 



  636 


  611 

  141 

  638 








  640 







  634 




  639 





  638 


  460 


  349 









  640 





  636 




   16 


  642 



  640 



   17 










  610 
   63 


  603 












  634 










  641 




    3 


  634 







  640 





  637 



    7 



  637 




















  638 




    1 


  628 

   62 







    2 





  635 




    4 






  638 













  637 





  639 
   17 











  640 



    5 


    8 

  639 
    9 
    4 
    4 
    5 
    6 
    4 
    4 
    6 
    5 

    8 

    6 

    5 

    8 

    6 

    6 

    6 

    7 

    7 


    6 

    1 
    7 



    8 

   12 




    7 


  625 
   43 



  169 
  559 







    1 


    7 


   60 








   13 


  636 
  506 



  636 


  637 



  639 



  640 
    9 

  637 
  640 

   66 

   11 

    3 



  543 

  163 


  163 




    2 


    4 


  637 





  144 



























































































































    1 













  144 










    1 








  145 









  145 
  145 














    6 



    6 
























    1 



    4 

    4 



    1 



    1 














  146 




  153 

















  153 
    5 
   11 





  142 
  142 








    1 






   15 


    1 


    1 


   20 







  140 
    1 



  145 
  145 






  145 





  145 



  146 

  145 



































    2 

    2 






    4 

    4 



















    4 


  136 


    1 








  140 


  141 
    4 


  135 
















    2 






    1 


  140 



























  116 
  117 



  115 



  113 

    1 



  113 
    1 

    1 



  112 





  111 

    1 



  111 

    1 



  101 



    8 







    1 


    5 



    6 



    1 


























    3 












    3 













    3 






























    1 



    1 


    1 










    1 











    1 












    1 












    1 













    1 





















    1 















   39 









   38 
    2 



    8 

    8 
    8 






    1 

    5 


    1 
   37 

   37 
   37 

















    5 







    2 



    2 









    1 















    2 




    2 













  116 





    8 



   11 

   98 

  109 













  109 

  108 




























  104 


    5 



    5 





    3 


    2 




   92 



   14 


   99 
    6 








   95 

    1 

   10 


   91 


   14 




  104 
    1 



  105 
  104 

  103 



   65 

   17 

   28 


   87 

    1 

    3 


   89 








   91 








   90 



    1 




   89 



    2 




   91 








   89 


    1 
    1 




   86 
    4 

   90 
    1 






   83 




    2 
    2 








    1 

    3 
    1 






   83 






    1 



























   82 









    1 






   76 



    7 










    4 









   83 


    5 




   69 






    1 


    2 






    1 



    1 





    3 


    3 



















   40 
   47 
   27 

   64 
   12 



  109 







    4 




    1 








   42 








    1 

    1 


   41 





    4 




   37 
   37 

   37 


   34 
    1 
    1 



    1 
   35 



   37 






    2 




    1 


    1 










    1 





















    1 


   32 








   42 









    1 

    1 


    2 



   36 



    1 





   33 
    3 
    1 

    3 
   33 
    3 

    1 
    2 





   37 

















































































































































    1 


    1 


   78 










   77 












































    6 
    1 

    5 


































































































































   89 

    1 



    1 






   80 

   15 



    1 


    6 








    1 









  103 













    2 

    1 


  101 








    1 


   99 















  100 




































    1 



  100 

   98 




    1 












   98 
   99 
   96 




   96 
















    1 




    7 


    4 

    4 





    1 


    3 
    3 


    1 






    2 









   16 










   16 
    8 






    6 



    1 


    5 
    1 




    3 
    1 
    2 




    3 
























    3 

    6 

















































    5 








    1 
    1 








    2 
    1 




    1 


    1 



















    1 





    4 





   52 














   52 


    7 
    2 


   26 





    2 
    2 



    2 

    2 



   47 








   48 




   20 





   48 
   48 


   47 
   21 






    8 
   46 

















  493 









  491 
    7 






  490 













  123 
  172 

  428 









   10 





  462 




  391 
  394 





    2 
    2 








    8 



    3 














    8 





















    3 















    3 













    3 




















    2 


    1 


    3 








    5 


    7 

    1 





    1 
    1 









    9 








    1 






    1 









    2 
    2 










    5 






    5 


















    2 




    3 


    4 

    3 




































   11 














    6 





    5 

    1 




    1 





    1 



    3 
    2 
    2 





    2 




    1 






    4 





















    1 












    1 



    2 
























    5 
    1 






    5 























    5 


    3 



    5 


    5 
















    1 



    3 
    2 
    5 





    1 



    4 
    4 





    1 


    1 






    1 


    1 





















    4 




    1 


    1 


    1 










   18 












   11 


    7 

   11 


    6 

    1 


    1 


    7 
    6 



    4 
    2 



    1 



    1 





    1 



    3 





    5 


    4 
    3 





    5 


















    1 



    4 

    4 




    1 





    1 




    9 
    9 


    1 

    1 



    1 


    2 



    3 





    1 









   16 












   16 





    3 






    1 








































    1 





















































































































































    1 



    3 
    2 
    5 



    1 




    5 



    1 



    3 
    3 




    3 










    3 


   12 









   12 

    9 


    5 
    5 



    5 
    1 



    4 









    5 
    1 




    4 











    5 


























































   27 







    1 








    1 




   11 
   13 
   12 
    1 



    1 


   11 
    2 





    1 


    4 






   10 


   11 


   11 
    1 















   11 
    1 






   16 








    1 








    1 




    1 
   10 

    1 



    2 




    2 


    4 





    7 


    3 


    4 
    1 















    3 
    2 






   16 







   10 







   10 







   10 


























    3 






















   10 






















    1 


    1 


    1 






    2 












   10 




    1 


    3 








    3 














    1 


   10 












   10 















   10 
























































    3 









    3 






















   17 







   17 












   15 





    1 




   10 



   17 


    7 



    2 








    5 



   17 





    7 

    5 



    1 






    3 

    7 



   17 







    1 




   10 











   17 




    1 




   13 
   13 
   13 














    2 



   13 


   13 












    6 
    6 

    7 






    4 






    1 



    2 
    1 






    1 






    2 




    2 






    1 



    3 


    1 





    3 
    3 
    3 

    3 





    3 






































    2 















   19 










    2 


   15 
    2 















    3 



   29 





    7 
    7 


    3 
    3 








   13 








   13 





    2 




    1 



    1 




    1 














    4 






   21 















   21 





    2 





    1 


   13 









   11 





   13 










































   11 









   11 






    1 



    2 




    1 



    1 





    1 


    1 































    1 




    1 


    1 
    1 











    6 








    2 
    4 







    3 
    1 

    2 








    2 




























































    4 






    1 

    1 




    1 









    1 















    6 



    1 




    1 















    1 






    1 
    1 



    1 




    1 


    1 

















    9 






    1 

    5 




    1 





    1 



    1 























































    6 








    4 



    2 






    2 





    1 



    1 























  356 







   49 



  350 
    7 






  196 
   11 

  148 
   49 










   77 











 1203 











    1 





    1 





 1198 


 1198 
  933 





  266 


   15 






  212 
   52 






  265 



  185 





   77 









  932 


  135 










  931 

    1 








  565 








  367 







    3 



    3 




 1204 




    3 
    2 


    3 







  505 


  495 













  174 














   18 











   19 


























































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965
2966
2967
2968
2969
2970
2971
2972
2973
2974
2975
2976
2977
2978
2979
2980
2981
2982
2983
2984
2985
2986
2987
2988
2989
2990
2991
2992
2993
2994
2995
2996
2997
2998
2999
3000
3001
3002
3003
3004
3005
3006
3007
3008
3009
3010
3011
3012
3013
3014
3015
3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029
3030
3031
3032
3033
3034
3035
3036
3037
3038
3039
3040
3041
3042
3043
3044
3045
3046
3047
3048
3049
3050
3051
3052
3053
3054
3055
3056
3057
3058
3059
3060
3061
3062
3063
3064
3065
3066
3067
3068
3069
3070
3071
3072
3073
3074
3075
3076
3077
3078
3079
3080
3081
3082
3083
3084
3085
3086
3087
3088
3089
3090
3091
3092
3093
3094
3095
3096
3097
3098
3099
3100
3101
3102
3103
3104
3105
3106
3107
3108
3109
3110
3111
3112
3113
3114
3115
3116
3117
3118
3119
3120
3121
3122
3123
3124
3125
3126
3127
3128
3129
3130
3131
3132
3133
3134
3135
3136
3137
3138
3139
3140
3141
3142
3143
3144
3145
3146
3147
3148
3149
3150
3151
3152
3153
3154
3155
3156
3157
3158
3159
3160
3161
3162
3163
3164
3165
3166
3167
3168
3169
3170
3171
3172
3173
3174
3175
3176
3177
3178
3179
3180
3181
3182
3183
3184
3185
3186
3187
3188
3189
3190
3191
3192
3193
3194
3195
3196
3197
3198
3199
3200
3201
3202
3203
3204
3205
3206
3207
3208
3209
3210
3211
3212
3213
3214
3215
3216
3217
3218
3219
3220
3221
3222
3223
3224
3225
3226
3227
3228
3229
3230
3231
3232
3233
3234
3235
3236
3237
3238
3239
3240
3241
3242
3243
3244
3245
3246
3247
3248
3249
3250
3251
3252
3253
3254
3255
3256
3257
3258
3259
3260
3261
3262
3263
3264
3265
3266
3267
3268
3269
3270
3271
3272
3273
3274
3275
3276
3277
3278
3279
3280
3281
3282
3283
3284
3285
3286
3287
3288
3289
3290
3291
3292
3293
3294
3295
3296
3297
3298
3299
3300
3301
3302
3303
3304
3305
3306
3307
3308
3309
3310
3311
3312
3313
3314
3315
3316
3317
3318
3319
3320
3321
3322
3323
3324
3325
3326
3327
3328
3329
3330
3331
3332
3333
3334
3335
3336
3337
3338
3339
3340
3341
3342
3343
3344
3345
3346
3347
3348
3349
3350
3351
3352
3353
3354
3355
3356
3357
3358
3359
3360
3361
3362
3363
3364
3365
3366
3367
3368
3369
3370
3371
3372
3373
3374
3375
3376
3377
3378
3379
3380
3381
3382
3383
3384
3385
3386
3387
3388
3389
3390
3391
3392
3393
3394
3395
3396
3397
3398
3399
3400
3401
3402
3403
3404
3405
3406
3407
3408
3409
3410
3411
3412
3413
3414
3415
3416
3417
3418
3419
3420
3421
3422
3423
3424
3425
3426
3427
3428
3429
3430
3431
3432
3433
3434
3435
3436
3437
3438
3439
3440
3441
3442
3443
3444
3445
3446
3447
3448
3449
3450
3451
3452
3453
3454
3455
3456
3457
3458
3459
3460
3461
3462
3463
3464
3465
3466
3467
3468
3469
3470
3471
3472
3473
3474
3475
3476
3477
3478
3479
3480
3481
3482
3483
3484
3485
3486
3487
3488
3489
3490
3491
3492
3493
3494
3495
3496
3497
3498
3499
3500
3501
3502
3503
3504
3505
3506
3507
3508
3509
3510
3511
3512
3513
3514
3515
3516
3517
3518
3519
3520
3521
3522
3523
3524
3525
3526
3527
3528
3529
3530
3531
3532
3533
3534
3535
3536
3537
3538
3539
3540
3541
3542
3543
3544
3545
3546
3547
3548
3549
3550
3551
3552
3553
3554
3555
3556
3557
3558
3559
3560
3561
3562
3563
3564
3565
3566
3567
3568
3569
3570
3571
3572
3573
3574
3575
3576
3577
3578
3579
3580
3581
3582
3583
3584
3585
3586
3587
3588
3589
3590
3591
3592
3593
3594
3595
3596
3597
3598
3599
3600
3601
3602
3603
3604
3605
3606
3607
3608
3609
3610
3611
3612
3613
3614
3615
3616
3617
3618
3619
3620
3621
3622
3623
3624
3625
3626
3627
3628
3629
3630
3631
3632
3633
3634
3635
3636
3637
3638
3639
3640
3641
3642
3643
3644
3645
3646
3647
3648
3649
3650
3651
3652
3653
3654
3655
3656
3657
3658
3659
3660
3661
3662
3663
3664
3665
3666
3667
3668
3669
3670
3671
3672
3673
3674
3675
3676
3677
3678
3679
3680
3681
3682
3683
3684
3685
3686
3687
3688
3689
3690
3691
3692
3693
3694
3695
3696
3697
3698
3699
3700
3701
3702
3703
3704
3705
3706
3707
3708
3709
3710
3711
3712
3713
3714
3715
3716
3717
3718
3719
3720
3721
3722
3723
3724
3725
3726
3727
3728
3729
3730
3731
3732
3733
3734
3735
3736
3737
3738
3739
3740
3741
3742
3743
3744
3745
3746
3747
3748
3749
3750
3751
3752
3753
3754
3755
3756
3757
3758
3759
3760
3761
3762
3763
3764
3765
3766
3767
3768
3769
3770
3771
3772
3773
3774
3775
3776
3777
3778
3779
3780
3781
3782
3783
3784
3785
3786
3787
3788
3789
3790
3791
3792
3793
3794
3795
3796
3797
3798
3799
3800
3801
3802
3803
3804
3805
3806
3807
3808
3809
3810
3811
3812
3813
3814
3815
3816
3817
3818
3819
3820
3821
3822
3823
3824
3825
3826
3827
3828
3829
3830
3831
3832
3833
3834
3835
3836
3837
3838
3839
3840
3841
3842
3843
3844
3845
3846
3847
3848
3849
3850
3851
3852
3853
3854
3855
3856
3857
3858
3859
3860
3861
3862
3863
3864
3865
3866
3867
3868
3869
3870
3871
3872
3873
3874
3875
3876
3877
3878
3879
3880
3881
3882
3883
3884
3885
3886
3887
3888
3889
3890
3891
3892
3893
3894
3895
3896
3897
3898
3899
3900
3901
3902
3903
3904
3905
3906
3907
3908
3909
3910
3911
3912
3913
3914
3915
3916
3917
3918
3919
3920
3921
3922
3923
3924
3925
3926
3927
3928
3929
3930
3931
3932
3933
3934
3935
3936
3937
3938
3939
3940
3941
3942
3943
3944
3945
3946
3947
3948
3949
3950
3951
3952
3953
3954
3955
3956
3957
3958
3959
3960
3961
3962
3963
3964
3965
3966
3967
3968
3969
3970
3971
3972
3973
3974
3975
3976
3977
3978
3979
3980
3981
3982
3983
3984
3985
3986
3987
3988
3989
3990
3991
3992
3993
3994
3995
3996
3997
3998
3999
4000
4001
4002
4003
4004
4005
4006
4007
4008
4009
4010
4011
4012
4013
4014
4015
4016
4017
4018
4019
4020
4021
4022
4023
4024
4025
4026
4027
4028
4029
4030
4031
4032
4033
4034
4035
4036
4037
4038
4039
4040
4041
4042
4043
4044
4045
4046
4047
4048
4049
4050
4051
4052
4053
4054
4055
4056
4057
4058
4059
4060
4061
4062
4063
4064
4065
4066
4067
4068
4069
4070
4071
4072
4073
4074
4075
4076
4077
4078
4079
4080
4081
4082
4083
4084
4085
4086
4087
4088
4089
4090
4091
4092
4093
4094
4095
4096
4097
4098
4099
4100
4101
4102
4103
4104
4105
4106
4107
4108
4109
4110
4111
4112
4113
4114
4115
4116
4117
4118
4119
4120
4121
4122
4123
4124
4125
4126
4127
4128
4129
4130
4131
4132
4133
4134
4135
4136
4137
4138
4139
4140
4141
4142
4143
4144
4145
4146
4147
4148
4149
4150
4151
4152
4153
4154
4155
4156
4157
4158
4159
4160
4161
4162
4163
4164
4165
4166
4167
4168
4169
4170
4171
4172
4173
4174
4175
4176
4177
4178
4179
4180
4181
4182
4183
4184
4185
4186
4187
4188
4189
4190
4191
4192
4193
4194
4195
4196
4197
4198
4199
4200
4201
4202
4203
4204
4205
4206
4207
4208
4209
4210
4211
4212
4213
4214
4215
4216
4217
4218
4219
4220
4221
4222
4223
4224
4225
4226
4227
4228
4229
4230
4231
4232
4233
4234
4235
4236
4237
4238
4239
4240
4241
4242
4243
4244
4245
4246
4247
4248
4249
4250
4251
4252
4253
4254
4255
4256
4257
4258
4259
4260
4261
4262
4263
4264
4265
4266
4267
4268
4269
4270
4271
4272
4273
4274
4275
4276
4277
4278
4279
4280
4281
4282
4283
4284
4285
4286
4287
4288
4289
4290
4291
4292
4293
4294
4295
4296
4297
4298
4299
4300
4301
4302
4303
4304
4305
4306
4307
4308
4309
4310
4311
4312
4313
4314
4315
4316
4317
4318
4319
4320
4321
4322
4323
4324
4325
4326
4327
4328
4329
4330
4331
4332
4333
4334
4335
4336
4337
4338
4339
4340
4341
4342
4343
4344
4345
4346
4347
4348
4349
4350
4351
4352
4353
4354
4355
4356
4357
4358
4359
4360
4361
4362
4363
4364
4365
4366
4367
4368
4369
4370
4371
4372
4373
4374
4375
4376
4377
4378
4379
4380
4381
4382
4383
4384
4385
4386
4387
4388
4389
4390
4391
4392
4393
4394
4395
4396
4397
4398
4399
4400
4401
4402
4403
4404
4405
4406
4407
4408
4409
4410
4411
4412
4413
4414
4415
4416
4417
4418
4419
4420
4421
4422
4423
4424
4425
4426
4427
4428
4429
4430
4431
4432
4433
4434
4435
4436
4437
4438
4439
4440
4441
4442
4443
4444
4445
4446
4447
4448
4449
4450
4451
4452
4453
4454
4455
4456
4457
4458
4459
4460
4461
4462
4463
4464
4465
4466
4467
4468
4469
4470
4471
4472
4473
4474
4475
4476
4477
4478
4479
4480
4481
4482
4483
4484
4485
4486
4487
4488
4489
4490
4491
4492
4493
4494
4495
4496
4497
4498
4499
4500
4501
4502
4503
4504
4505
4506
4507
4508
4509
4510
4511
4512
4513
4514
4515
4516
4517
4518
4519
4520
4521
4522
4523
4524
4525
4526
4527
4528
4529
4530
4531
4532
4533
4534
4535
4536
4537
4538
4539
4540
4541
4542
4543
4544
4545
4546
4547
4548
4549
4550
4551
4552
4553
4554
4555
4556
4557
4558
4559
4560
4561
4562
4563
4564
4565
4566
4567
4568
4569
4570
4571
4572
4573
4574
4575
4576
4577
4578
4579
4580
4581
4582
4583
4584
4585
4586
4587
4588
4589
4590
4591
4592
4593
4594
4595
4596
4597
4598
4599
4600
4601
4602
4603
4604
4605
4606
4607
4608
4609
4610
4611
4612
4613
4614
4615
4616
4617
4618
4619
4620
4621
4622
4623
4624
4625
4626
4627
4628
4629
4630
4631
4632
4633
4634
4635
4636
4637
4638
4639
4640
4641
4642
4643
4644
4645
4646
4647
4648
4649
4650
4651
4652
4653
4654
4655
4656
4657
4658
4659
4660
4661
4662
4663
4664
4665
4666
4667
4668
4669
4670
4671
4672
4673
4674
4675
4676
4677
4678
4679
4680
4681
4682
4683
4684
4685
4686
4687
4688
4689
4690
4691
4692
4693
4694
4695
4696
4697
4698
4699
4700
4701
4702
4703
4704
4705
4706
4707
4708
4709
4710
4711
4712
4713
4714
4715
4716
4717
4718
4719
4720
4721
4722
4723
4724
4725
4726
4727
4728
4729
4730
4731
4732
4733
4734
4735
4736
4737
4738
4739
4740
4741
4742
4743
4744
4745
4746
4747
4748
4749
4750
4751
4752
4753
4754
4755
4756
4757
4758
4759
4760
4761
4762
4763
4764
4765
4766
4767
4768
4769
4770
4771
4772
4773
4774
4775
4776
4777
4778
4779
4780
4781
4782
4783
4784
4785
4786
4787
4788
4789
4790
4791
4792
4793
4794
4795
4796
4797
4798
4799
4800
4801
4802
4803
4804
4805
4806
4807
4808
4809
4810
4811
4812
4813
4814
4815
4816
4817
4818
4819
4820
4821
4822
4823
4824
4825
4826
4827
4828
4829
4830
4831
4832
4833
4834
4835
4836
4837
4838
4839
4840
4841
4842
4843
4844
4845
4846
4847
4848
4849
4850
4851
4852
4853
4854
4855
4856
4857
4858
4859
4860
4861
4862
4863
4864
4865
4866
4867
4868
4869
4870
4871
4872
4873
4874
4875
4876
4877
4878
4879
4880
4881
4882
4883
4884
4885
4886
4887
4888
4889
4890
4891
4892
4893
4894
4895
4896
4897
4898
4899
4900
4901
4902
4903
4904
4905
4906
4907
4908
4909
4910
4911
4912
4913
4914
4915
4916
4917
4918
4919
4920
4921
4922
4923
4924
4925
4926
4927
4928
4929
4930
4931
4932
4933
4934
4935
4936
4937
4938
4939
4940
4941
4942
4943
4944
4945
4946
4947
4948
4949
4950
4951
4952
4953
4954
4955
4956
4957
4958
4959
4960
4961
4962
4963
4964
4965
4966
4967
4968
4969
4970
4971
4972
4973
4974
4975
4976
4977
4978
4979
4980
4981
4982
4983
4984
4985
4986
4987
4988
4989
4990
4991
4992
4993
4994
4995
4996
4997
4998
4999
5000
5001
5002
5003
5004
5005
5006
5007
5008
5009
5010
5011
5012
5013
5014
5015
5016
5017
5018
5019
5020
5021
5022
5023
5024
5025
5026
5027
5028
5029
5030
5031
5032
5033
5034
5035
5036
5037
5038
5039
5040
5041
5042
5043
5044
5045
5046
5047
5048
5049
5050
5051
5052
5053
5054
5055
5056
5057
5058
5059
5060
5061
5062
5063
5064
5065
5066
5067
5068
5069
5070
5071
5072
5073
5074
5075
5076
5077
5078
5079
5080
5081
5082
5083
5084
5085
5086
5087
5088
5089
5090
5091
5092
5093
5094
5095
5096
5097
5098
5099
5100
5101
5102
5103
5104
5105
5106
5107
5108
5109
5110
5111
5112
5113
5114
5115
5116
5117
5118
5119
5120
5121
5122
5123
5124
5125
5126
5127
5128
5129
5130
5131
5132
5133
5134
5135
5136
5137
5138
5139
5140
5141
5142
5143
5144
5145
5146
5147
5148
5149
5150
5151
5152
5153
5154
5155
5156
5157
5158
5159
5160
5161
5162
5163
5164
5165
5166
5167
5168
5169
5170
5171
5172
5173
5174
5175
5176
5177
5178
5179
5180
5181
5182
5183
5184
5185
5186
5187
5188
5189
5190
5191
5192
5193
5194
5195
5196
5197
5198
5199
5200
5201
5202
5203
5204
5205
5206
5207
5208
5209
5210
5211
5212
5213
5214
5215
5216
5217
5218
5219
5220
5221
5222
5223
5224
5225
5226
5227
5228
5229
5230
5231
5232
5233
5234
5235
5236
5237
5238
5239
5240
5241
5242
5243
5244
5245
5246
5247
5248
5249
5250
5251
5252
5253
5254
5255
5256
5257
5258
5259
5260
5261
5262
5263
5264
5265
5266
5267
5268
5269
5270
5271
5272
5273
5274
5275
5276
5277
5278
5279
5280
5281
5282
5283
5284
5285
5286
5287
5288
5289
5290
5291
5292
5293
5294
5295
5296
5297
5298
5299
5300
5301
5302
5303
5304
5305
5306
5307
5308
5309
5310
5311
5312
5313
5314
5315
5316
5317
5318
5319
5320
5321
5322
5323
5324
5325
5326
5327
5328
5329
5330
5331
5332
5333
5334
5335
5336
5337
5338
5339
5340
5341
5342
5343
5344
5345
5346
5347
5348
5349
5350
5351
5352
5353
5354
5355
5356
5357
5358
5359
5360
5361
5362
5363
5364
5365
5366
5367
5368
5369
5370
5371
5372
5373
5374
5375
5376
5377
5378
5379
5380
5381
5382
5383
5384
5385
5386
5387
5388
5389
5390
5391
5392
5393
5394
5395
5396
5397
5398
5399
5400
5401
5402
5403
5404
5405
5406
5407
5408
5409
5410
5411
5412
5413
5414
5415
5416
5417
5418
5419
5420
5421
5422
5423
5424
5425
5426
5427
5428
5429
5430
5431
5432
5433
5434
5435
5436
5437
5438
5439
5440
5441
5442
5443
5444
5445
5446
5447
5448
5449
5450
5451
5452
5453
5454
5455
5456
5457
5458
5459
5460
5461
5462
5463
5464
5465
5466
5467
5468
5469
5470
5471
5472
5473
5474
5475
5476
5477
5478
5479
5480
5481
5482
5483
5484
5485
5486
5487
5488
5489
5490
5491
5492
5493
5494
5495
5496
5497
5498
5499
5500
5501
5502
5503
5504
5505
5506
5507
5508
5509
5510
5511
5512
5513
5514
5515
5516
5517
5518
5519
5520
5521
5522
5523
5524
5525
5526
5527
5528
5529
5530
5531
5532
5533
5534
5535
5536
5537
5538
5539
5540
5541
5542
5543
5544
5545
5546
5547
5548
5549
5550
5551
5552
5553
5554
5555
5556
5557
5558
5559
5560
5561
5562
5563
5564
5565
5566
5567
5568
5569
5570
5571
5572
5573
5574
5575
5576
5577
5578
5579
5580
5581
5582
5583
5584
5585
5586
5587
5588
5589
5590
5591
5592
5593
5594
5595
5596
5597
5598
5599
5600
5601
5602
5603
5604
5605
5606
5607
5608
5609
5610
5611
5612
5613
5614
5615
5616
5617
5618
5619
5620
5621
5622
5623
5624
5625
5626
5627
5628
5629
5630
5631
5632
5633
5634
5635
5636
5637
5638
5639
5640
5641
5642
5643
5644
5645
5646
5647
5648
5649
5650
5651
5652
5653
5654
5655
5656
5657
5658
5659
5660
5661
5662
5663
5664
5665
5666
5667
5668
5669
5670
5671
5672
5673
5674
5675
5676
5677
5678
5679
5680
5681
5682
5683
5684
5685
5686
5687
5688
5689
5690
5691
5692
5693
5694
5695
5696
5697
5698
5699
5700
5701
5702
5703
5704
5705
5706
5707
5708
5709
5710
5711
5712
5713
5714
5715
5716
5717
5718
5719
5720
5721
5722
5723
5724
5725
5726
5727
5728
5729
5730
5731
5732
5733
5734
5735
5736
5737
5738
5739
5740
5741
5742
5743
5744
5745
5746
5747
5748
5749
5750
5751
5752
5753
5754
5755
5756
5757
5758
5759
5760
5761
5762
5763
5764
5765
5766
5767
5768
5769
5770
5771
5772
5773
5774
5775
5776
5777
5778
5779
5780
5781
5782
5783
5784
5785
5786
5787
5788
5789
5790
5791
5792
5793
5794
5795
5796
5797
5798
5799
5800
5801
5802
5803
5804
5805
5806
5807
5808
5809
5810
5811
5812
5813
5814
5815
5816
5817
5818
5819
5820
5821
5822
5823
5824
5825
5826
5827
5828
5829
5830
5831
5832
5833
5834
5835
5836
5837
5838
5839
5840
5841
5842
5843
5844
5845
5846
5847
5848
5849
5850
5851
5852
5853
5854
5855
5856
5857
5858
5859
5860
5861
5862
5863
5864
5865
5866
5867
5868
5869
5870
5871
5872
5873
5874
5875
5876
5877
5878
5879
5880
5881
5882
5883
5884
5885
5886
5887
5888
5889
5890
5891
5892
5893
5894
5895
5896
5897
5898
5899
5900
5901
5902
5903
5904
5905
5906
5907
5908
5909
5910
5911
5912
5913
5914
5915
5916
5917
5918
5919
5920
5921
5922
5923
5924
5925
5926
5927
5928
5929
5930
5931
5932
5933
5934
5935
5936
5937
5938
5939
5940
5941
5942
5943
5944
5945
5946
5947
5948
5949
5950
5951
5952
5953
5954
5955
5956
5957
5958
5959
5960
5961
5962
5963
5964
5965
5966
5967
5968
5969
5970
5971
5972
5973
5974
5975
5976
5977
5978
5979
5980
5981
5982
5983
5984
5985
5986
5987
5988
5989
5990
5991
5992
5993
5994
5995
5996
5997
5998
5999
6000
6001
6002
6003
6004
6005
6006
6007
6008
6009
6010
6011
6012
6013
6014
6015
6016
6017
6018
6019
6020
6021
6022
6023
6024
6025
6026
6027
6028
6029
6030
6031
6032
6033
6034
6035
6036
6037
6038
6039
6040
6041
6042
6043
6044
6045
6046
6047
6048
6049
6050
6051
6052
6053
6054
6055
6056
6057
6058
6059
6060
6061
6062
6063
6064
6065
6066
6067
6068
6069
6070
6071
6072
6073
6074
6075
6076
6077
6078
6079
6080
6081
6082
6083
6084
6085
6086
6087
6088
6089
6090
6091
6092
6093
6094
6095
6096
6097
6098
6099
6100
6101
6102
6103
6104
6105
6106
6107
6108
6109
6110
6111
6112
6113
6114
6115
6116
6117
6118
6119
6120
6121
6122
6123
6124
6125
6126
6127
6128
6129
6130
6131
6132
6133
6134
6135
6136
6137
6138
6139
6140
6141
6142
6143
6144
6145
6146
6147
6148
6149
6150
6151
6152
6153
6154
6155
6156
6157
6158
6159
6160
6161
6162
6163
6164
6165
6166
6167
6168
6169
6170
6171
6172
6173
6174
6175
6176
6177
6178
6179
6180
6181
6182
6183
6184
6185
6186
6187
6188
6189
6190
6191
6192
6193
6194
6195
6196
6197
6198
6199
6200
6201
6202
6203
6204
6205
6206
6207
6208
6209
6210
6211
6212
6213
6214
6215
6216
6217
6218
6219
6220
6221
6222
6223
6224
6225
6226
6227
6228
6229
6230
6231
6232
6233
6234
6235
6236
6237
6238
6239
6240
6241
6242
6243
6244
6245
6246
6247
6248
6249
6250
6251
6252
6253
6254
6255
6256
6257
6258
6259
6260
6261
6262
6263
6264
6265
6266
6267
6268
6269
6270
6271
6272
6273
6274
6275
6276
6277
6278
6279
6280
6281
6282
6283
6284
6285
6286
6287
6288
6289
6290
6291
6292
6293
6294
6295
6296
6297
6298
6299
6300
6301
6302
6303
6304
6305
6306
6307
6308
6309
6310
6311
6312
6313
6314
6315
6316
6317
6318
6319
6320
6321
6322
6323
6324
6325
6326
6327
6328
6329
6330
6331
6332
6333
6334
6335
6336
6337
6338
6339
6340
6341
6342
6343
6344
6345
6346
6347
6348
6349
6350
6351
6352
6353
6354
6355
6356
6357
6358
6359
6360
6361
6362
6363
6364
6365
6366
6367
6368
6369
6370
6371
6372
6373
6374
6375
6376
6377
6378
6379
6380
6381
6382
6383
6384
6385
6386
6387
6388
6389
6390
6391
6392
6393
6394
6395
6396
6397
6398
6399
6400
6401
6402
6403
6404
6405
6406
6407
6408
6409
6410
6411
6412
6413
6414
6415
6416
6417
6418
6419
6420
6421
6422
6423
6424
6425
6426
6427
6428
6429
6430
6431
6432
6433
6434
6435
6436
6437
6438
6439
6440
6441
6442
6443
6444
6445
6446
6447
6448
6449
6450
6451
6452
6453
6454
6455
6456
6457
6458
6459
6460
6461
6462
6463
6464
6465
6466
6467
6468
6469
6470
6471
6472
6473
6474
6475
6476
6477
6478
6479
6480
6481
6482
6483
6484
6485
6486
6487
6488
6489
6490
6491
6492
6493
6494
6495
6496
6497
6498
6499
6500
6501
6502
6503
6504
6505
6506
6507
6508
6509
6510
6511
6512
6513
6514
6515
6516
6517
6518
6519
6520
6521
6522
6523
6524
6525
6526
6527
6528
6529
6530
6531
6532
6533
6534
6535
6536
6537
6538
6539
6540
6541
6542
6543
6544
6545
6546
6547
6548
6549
6550
6551
6552
6553
6554
6555
6556
6557
6558
6559
6560
6561
6562
6563
6564
6565
6566
6567
6568
6569
6570
6571
6572
6573
6574
6575
6576
6577
6578
6579
6580
6581
6582
6583
6584
6585
6586
6587
6588
6589
6590
6591
6592
6593
6594
6595
6596
6597
6598
6599
6600
6601
6602
6603
6604
6605
6606
6607
6608
6609
6610
6611
6612
6613
6614
6615
6616
6617
6618
6619
6620
6621
6622
6623
6624
6625
6626
6627
6628
6629
6630
6631
6632
6633
6634
6635
6636
6637
6638
6639
6640
6641
6642
6643
6644
6645
6646
6647
6648
6649
6650
6651
6652
6653
6654
6655
6656
6657
6658
6659
6660
6661
6662
6663
6664
6665
6666
6667
6668
6669
6670
6671
6672
6673
6674
6675
6676
6677
6678
6679
6680
6681
6682
6683
6684
6685
6686
6687
6688
6689
6690
6691
6692
6693
6694
6695
6696
6697
6698
6699
6700
6701
6702
6703
6704
6705
6706
6707
6708
6709
6710
6711
6712
6713
6714
6715
6716
6717
6718
6719
6720
6721
6722
6723
6724
6725
6726
6727
6728
6729
6730
6731
6732
6733
6734
6735
6736
6737
6738
6739
6740
6741
6742
6743
6744
6745
6746
6747
6748
6749
6750
6751
6752
6753
6754
6755
6756
6757
6758
6759
6760
6761
6762
6763
6764
6765
6766
6767
6768
6769
6770
6771
6772
6773
6774
6775
6776
6777
6778
6779
6780
6781
6782
6783
6784
6785
6786
6787
6788
6789
6790
6791
6792
6793
6794
6795
6796
6797
6798
6799
6800
6801
6802
6803
6804
6805
6806
6807
6808
6809
6810
6811
6812
6813
6814
6815
6816
6817
6818
6819
6820
6821
6822
6823
6824
6825
6826
6827
6828
6829
6830
6831
6832
6833
6834
6835
6836
6837
6838
6839
6840
6841
6842
6843
6844
6845
6846
6847
6848
6849
6850
6851
6852
6853
6854
6855
6856
6857
6858
6859
6860
6861
6862
6863
6864
6865
6866
6867
6868
6869
6870
6871
6872
6873
6874
6875
6876
6877
6878
6879
6880
6881
6882
6883
6884
6885
6886
6887
6888
6889
6890
6891
6892
6893
6894
6895
6896
6897
6898
6899
6900
6901
6902
6903
6904
6905
6906
6907
6908
6909
6910
6911
6912
6913
6914
6915
6916
6917
6918
6919
6920
6921
6922
6923
6924
6925
6926
6927
6928
6929
6930
6931
6932
6933
6934
6935
6936
6937
6938
6939
6940
6941
6942
6943
6944
6945
6946
6947
6948
6949
6950
6951
6952
6953
6954
6955
6956
6957
6958
6959
6960
6961
6962
6963
6964
6965
6966
6967
6968
6969
6970
6971
6972
6973
6974
6975
6976
6977
6978
6979
6980
6981
6982
6983
6984
6985
6986
6987
6988
6989
6990
6991
6992
6993
6994
6995
6996
6997
6998
6999
7000
7001
7002
7003
7004
7005
7006
7007
7008
7009
7010
7011
7012
7013
7014
7015
7016
7017
7018
7019
7020
7021
7022
7023
7024
7025
7026
7027
7028
7029
7030
7031
7032
7033
7034
7035
7036
7037
7038
7039
7040
7041
7042
7043
7044
7045
7046
7047
7048
7049
7050
7051
7052
7053
7054
7055
7056
7057
7058
7059
7060
7061
7062
7063
7064
7065
7066
7067
7068
7069
7070
7071
7072
7073
7074
7075
7076
7077
7078
7079
7080
7081
7082
7083
7084
7085
7086
7087
7088
7089
7090
7091
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * INET                An implementation of the TCP/IP protocol suite for the LINUX
 *                operating system.  INET is implemented using the  BSD Socket
 *                interface as the means of communication with the user level.
 *
 *                Routing netlink socket interface: protocol independent part.
 *
 * Authors:        Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
 *
 *        Fixes:
 *        Vitaly E. Lavrov                RTA_OK arithmetic was wrong.
 */

#include <linux/bitops.h>
#include <linux/errno.h>
#include <linux/module.h>
#include <linux/types.h>
#include <linux/socket.h>
#include <linux/kernel.h>
#include <linux/timer.h>
#include <linux/string.h>
#include <linux/sockios.h>
#include <linux/net.h>
#include <linux/fcntl.h>
#include <linux/mm.h>
#include <linux/slab.h>
#include <linux/interrupt.h>
#include <linux/capability.h>
#include <linux/skbuff.h>
#include <linux/init.h>
#include <linux/security.h>
#include <linux/mutex.h>
#include <linux/if_addr.h>
#include <linux/if_bridge.h>
#include <linux/if_vlan.h>
#include <linux/pci.h>
#include <linux/etherdevice.h>
#include <linux/bpf.h>

#include <linux/uaccess.h>

#include <linux/inet.h>
#include <linux/netdevice.h>
#include <net/ip.h>
#include <net/protocol.h>
#include <net/arp.h>
#include <net/route.h>
#include <net/udp.h>
#include <net/tcp.h>
#include <net/sock.h>
#include <net/pkt_sched.h>
#include <net/fib_rules.h>
#include <net/rtnetlink.h>
#include <net/net_namespace.h>
#include <net/netdev_lock.h>
#include <net/devlink.h>
#if IS_ENABLED(CONFIG_IPV6)
#include <net/addrconf.h>
#endif
#include <linux/dpll.h>

#include "dev.h"

#define RTNL_MAX_TYPE                50
#define RTNL_SLAVE_MAX_TYPE        44

struct rtnl_link {
        rtnl_doit_func                doit;
        rtnl_dumpit_func        dumpit;
        struct module                *owner;
        unsigned int                flags;
        struct rcu_head                rcu;
};

static DEFINE_MUTEX(rtnl_mutex);

void rtnl_lock(void)
{
        mutex_lock(&rtnl_mutex);
}
EXPORT_SYMBOL(rtnl_lock);

int rtnl_lock_interruptible(void)
{
        return mutex_lock_interruptible(&rtnl_mutex);
}

int rtnl_lock_killable(void)
{
        return mutex_lock_killable(&rtnl_mutex);
}

static struct sk_buff *defer_kfree_skb_list;
void rtnl_kfree_skbs(struct sk_buff *head, struct sk_buff *tail)
{
        if (head && tail) {
                tail->next = defer_kfree_skb_list;
                defer_kfree_skb_list = head;
        }
}
EXPORT_SYMBOL(rtnl_kfree_skbs);

void __rtnl_unlock(void)
{
        struct sk_buff *head = defer_kfree_skb_list;

        defer_kfree_skb_list = NULL;

        /* Ensure that we didn't actually add any TODO item when __rtnl_unlock()
         * is used. In some places, e.g. in cfg80211, we have code that will do
         * something like
         *   rtnl_lock()
         *   wiphy_lock()
         *   ...
         *   rtnl_unlock()
         *
         * and because netdev_run_todo() acquires the RTNL for items on the list
         * we could cause a situation such as this:
         * Thread 1                        Thread 2
         *                                  rtnl_lock()
         *                                  unregister_netdevice()
         *                                  __rtnl_unlock()
         * rtnl_lock()
         * wiphy_lock()
         * rtnl_unlock()
         *   netdev_run_todo()
         *     __rtnl_unlock()
         *
         *     // list not empty now
         *     // because of thread 2
         *                                  rtnl_lock()
         *     while (!list_empty(...))
         *       rtnl_lock()
         *                                  wiphy_lock()
         * **** DEADLOCK ****
         *
         * However, usage of __rtnl_unlock() is rare, and so we can ensure that
         * it's not used in cases where something is added to do the list.
         */
        WARN_ON(!list_empty(&net_todo_list));

        mutex_unlock(&rtnl_mutex);

        while (head) {
                struct sk_buff *next = head->next;

                kfree_skb(head);
                cond_resched();
                head = next;
        }
}

void rtnl_unlock(void)
{
        /* This fellow will unlock it for us. */
        netdev_run_todo();
}
EXPORT_SYMBOL(rtnl_unlock);

int rtnl_trylock(void)
{
        return mutex_trylock(&rtnl_mutex);
}
EXPORT_SYMBOL(rtnl_trylock);

int rtnl_is_locked(void)
{
        return mutex_is_locked(&rtnl_mutex);
}
EXPORT_SYMBOL(rtnl_is_locked);

bool refcount_dec_and_rtnl_lock(refcount_t *r)
{
        return refcount_dec_and_mutex_lock(r, &rtnl_mutex);
}
EXPORT_SYMBOL(refcount_dec_and_rtnl_lock);

#ifdef CONFIG_PROVE_LOCKING
bool lockdep_rtnl_is_held(void)
{
        return lockdep_is_held(&rtnl_mutex);
}
EXPORT_SYMBOL(lockdep_rtnl_is_held);
#endif /* #ifdef CONFIG_PROVE_LOCKING */

#ifdef CONFIG_DEBUG_NET_SMALL_RTNL
void __rtnl_net_lock(struct net *net)
{
        ASSERT_RTNL();

        mutex_lock(&net->rtnl_mutex);
}
EXPORT_SYMBOL(__rtnl_net_lock);

void __rtnl_net_unlock(struct net *net)
{
        ASSERT_RTNL();

        mutex_unlock(&net->rtnl_mutex);
}
EXPORT_SYMBOL(__rtnl_net_unlock);

void rtnl_net_lock(struct net *net)
{
        rtnl_lock();
        __rtnl_net_lock(net);
}
EXPORT_SYMBOL(rtnl_net_lock);

void rtnl_net_unlock(struct net *net)
{
        __rtnl_net_unlock(net);
        rtnl_unlock();
}
EXPORT_SYMBOL(rtnl_net_unlock);

int rtnl_net_trylock(struct net *net)
{
        int ret = rtnl_trylock();

        if (ret)
                __rtnl_net_lock(net);

        return ret;
}
EXPORT_SYMBOL(rtnl_net_trylock);

int rtnl_net_lock_killable(struct net *net)
{
        int ret = rtnl_lock_killable();

        if (!ret)
                __rtnl_net_lock(net);

        return ret;
}

static int rtnl_net_cmp_locks(const struct net *net_a, const struct net *net_b)
{
        if (net_eq(net_a, net_b))
                return 0;

        /* always init_net first */
        if (net_eq(net_a, &init_net))
                return -1;

        if (net_eq(net_b, &init_net))
                return 1;

        /* otherwise lock in ascending order */
        return net_a < net_b ? -1 : 1;
}

int rtnl_net_lock_cmp_fn(const struct lockdep_map *a, const struct lockdep_map *b)
{
        const struct net *net_a, *net_b;

        net_a = container_of(a, struct net, rtnl_mutex.dep_map);
        net_b = container_of(b, struct net, rtnl_mutex.dep_map);

        return rtnl_net_cmp_locks(net_a, net_b);
}

bool rtnl_net_is_locked(struct net *net)
{
        return rtnl_is_locked() && mutex_is_locked(&net->rtnl_mutex);
}
EXPORT_SYMBOL(rtnl_net_is_locked);

bool lockdep_rtnl_net_is_held(struct net *net)
{
        return lockdep_rtnl_is_held() && lockdep_is_held(&net->rtnl_mutex);
}
EXPORT_SYMBOL(lockdep_rtnl_net_is_held);
#else
static int rtnl_net_cmp_locks(const struct net *net_a, const struct net *net_b)
{
        /* No need to swap */
        return -1;
}
#endif

struct rtnl_nets {
        /* ->newlink() needs to freeze 3 netns at most;
         * 2 for the new device, 1 for its peer.
         */
        struct net *net[3];
        unsigned char len;
};

static void rtnl_nets_init(struct rtnl_nets *rtnl_nets)
{
        memset(rtnl_nets, 0, sizeof(*rtnl_nets));
}

static void rtnl_nets_destroy(struct rtnl_nets *rtnl_nets)
{
        int i;

        for (i = 0; i < rtnl_nets->len; i++) {
                put_net(rtnl_nets->net[i]);
                rtnl_nets->net[i] = NULL;
        }

        rtnl_nets->len = 0;
}

/**
 * rtnl_nets_add - Add netns to be locked before ->newlink().
 *
 * @rtnl_nets: rtnl_nets pointer passed to ->get_peer_net().
 * @net: netns pointer with an extra refcnt held.
 *
 * The extra refcnt is released in rtnl_nets_destroy().
 */
static void rtnl_nets_add(struct rtnl_nets *rtnl_nets, struct net *net)
{
        int i;

        DEBUG_NET_WARN_ON_ONCE(rtnl_nets->len == ARRAY_SIZE(rtnl_nets->net));

        for (i = 0; i < rtnl_nets->len; i++) {
                switch (rtnl_net_cmp_locks(rtnl_nets->net[i], net)) {
                case 0:
                        put_net(net);
                        return;
                case 1:
                        swap(rtnl_nets->net[i], net);
                }
        }

        rtnl_nets->net[i] = net;
        rtnl_nets->len++;
}

static void rtnl_nets_lock(struct rtnl_nets *rtnl_nets)
{
        int i;

        rtnl_lock();

        for (i = 0; i < rtnl_nets->len; i++)
                __rtnl_net_lock(rtnl_nets->net[i]);
}

static void rtnl_nets_unlock(struct rtnl_nets *rtnl_nets)
{
        int i;

        for (i = 0; i < rtnl_nets->len; i++)
                __rtnl_net_unlock(rtnl_nets->net[i]);

        rtnl_unlock();
}

static struct rtnl_link __rcu *__rcu *rtnl_msg_handlers[RTNL_FAMILY_MAX + 1];

static inline int rtm_msgindex(int msgtype)
{
        int msgindex = msgtype - RTM_BASE;

        /*
         * msgindex < 0 implies someone tried to register a netlink
         * control code. msgindex >= RTM_NR_MSGTYPES may indicate that
         * the message type has not been added to linux/rtnetlink.h
         */
        BUG_ON(msgindex < 0 || msgindex >= RTM_NR_MSGTYPES);

        return msgindex;
}

static struct rtnl_link *rtnl_get_link(int protocol, int msgtype)
{
        struct rtnl_link __rcu **tab;

        if (protocol >= ARRAY_SIZE(rtnl_msg_handlers))
                protocol = PF_UNSPEC;

        tab = rcu_dereference_rtnl(rtnl_msg_handlers[protocol]);
        if (!tab)
                tab = rcu_dereference_rtnl(rtnl_msg_handlers[PF_UNSPEC]);

        return rcu_dereference_rtnl(tab[msgtype]);
}

static int rtnl_register_internal(struct module *owner,
                                  int protocol, int msgtype,
                                  rtnl_doit_func doit, rtnl_dumpit_func dumpit,
                                  unsigned int flags)
{
        struct rtnl_link *link, *old;
        struct rtnl_link __rcu **tab;
        int msgindex;
        int ret = -ENOBUFS;

        BUG_ON(protocol < 0 || protocol > RTNL_FAMILY_MAX);
        msgindex = rtm_msgindex(msgtype);

        rtnl_lock();
        tab = rtnl_dereference(rtnl_msg_handlers[protocol]);
        if (tab == NULL) {
                tab = kcalloc(RTM_NR_MSGTYPES, sizeof(void *), GFP_KERNEL);
                if (!tab)
                        goto unlock;

                /* ensures we see the 0 stores */
                rcu_assign_pointer(rtnl_msg_handlers[protocol], tab);
        }

        old = rtnl_dereference(tab[msgindex]);
        if (old) {
                link = kmemdup(old, sizeof(*old), GFP_KERNEL);
                if (!link)
                        goto unlock;
        } else {
                link = kzalloc(sizeof(*link), GFP_KERNEL);
                if (!link)
                        goto unlock;
        }

        WARN_ON(link->owner && link->owner != owner);
        link->owner = owner;

        WARN_ON(doit && link->doit && link->doit != doit);
        if (doit)
                link->doit = doit;
        WARN_ON(dumpit && link->dumpit && link->dumpit != dumpit);
        if (dumpit)
                link->dumpit = dumpit;

        WARN_ON(rtnl_msgtype_kind(msgtype) != RTNL_KIND_DEL &&
                (flags & RTNL_FLAG_BULK_DEL_SUPPORTED));
        link->flags |= flags;

        /* publish protocol:msgtype */
        rcu_assign_pointer(tab[msgindex], link);
        ret = 0;
        if (old)
                kfree_rcu(old, rcu);
unlock:
        rtnl_unlock();
        return ret;
}

/**
 * rtnl_unregister - Unregister a rtnetlink message type
 * @protocol: Protocol family or PF_UNSPEC
 * @msgtype: rtnetlink message type
 *
 * Returns 0 on success or a negative error code.
 */
static int rtnl_unregister(int protocol, int msgtype)
{
        struct rtnl_link __rcu **tab;
        struct rtnl_link *link;
        int msgindex;

        BUG_ON(protocol < 0 || protocol > RTNL_FAMILY_MAX);
        msgindex = rtm_msgindex(msgtype);

        rtnl_lock();
        tab = rtnl_dereference(rtnl_msg_handlers[protocol]);
        if (!tab) {
                rtnl_unlock();
                return -ENOENT;
        }

        link = rcu_replace_pointer_rtnl(tab[msgindex], NULL);
        rtnl_unlock();

        kfree_rcu(link, rcu);

        return 0;
}

/**
 * rtnl_unregister_all - Unregister all rtnetlink message type of a protocol
 * @protocol : Protocol family or PF_UNSPEC
 *
 * Identical to calling rtnl_unregster() for all registered message types
 * of a certain protocol family.
 */
void rtnl_unregister_all(int protocol)
{
        struct rtnl_link __rcu **tab;
        struct rtnl_link *link;
        int msgindex;

        BUG_ON(protocol < 0 || protocol > RTNL_FAMILY_MAX);

        rtnl_lock();
        tab = rcu_replace_pointer_rtnl(rtnl_msg_handlers[protocol], NULL);
        if (!tab) {
                rtnl_unlock();
                return;
        }
        for (msgindex = 0; msgindex < RTM_NR_MSGTYPES; msgindex++) {
                link = rcu_replace_pointer_rtnl(tab[msgindex], NULL);
                kfree_rcu(link, rcu);
        }
        rtnl_unlock();

        synchronize_net();

        kfree(tab);
}
EXPORT_SYMBOL_GPL(rtnl_unregister_all);

/**
 * __rtnl_register_many - Register rtnetlink message types
 * @handlers: Array of struct rtnl_msg_handlers
 * @n: The length of @handlers
 *
 * Registers the specified function pointers (at least one of them has
 * to be non-NULL) to be called whenever a request message for the
 * specified protocol family and message type is received.
 *
 * The special protocol family PF_UNSPEC may be used to define fallback
 * function pointers for the case when no entry for the specific protocol
 * family exists.
 *
 * When one element of @handlers fails to register,
 * 1) built-in: panics.
 * 2) modules : the previous successful registrations are unwinded
 *              and an error is returned.
 *
 * Use rtnl_register_many().
 */
int __rtnl_register_many(const struct rtnl_msg_handler *handlers, int n)
{
        const struct rtnl_msg_handler *handler;
        int i, err;

        for (i = 0, handler = handlers; i < n; i++, handler++) {
                err = rtnl_register_internal(handler->owner, handler->protocol,
                                             handler->msgtype, handler->doit,
                                             handler->dumpit, handler->flags);
                if (err) {
                        if (!handler->owner)
                                panic("Unable to register rtnetlink message "
                                      "handlers, %pS\n", handlers);

                        __rtnl_unregister_many(handlers, i);
                        break;
                }
        }

        return err;
}
EXPORT_SYMBOL_GPL(__rtnl_register_many);

void __rtnl_unregister_many(const struct rtnl_msg_handler *handlers, int n)
{
        const struct rtnl_msg_handler *handler;
        int i;

        for (i = n - 1, handler = handlers + n - 1; i >= 0; i--, handler--)
                rtnl_unregister(handler->protocol, handler->msgtype);
}
EXPORT_SYMBOL_GPL(__rtnl_unregister_many);

static DEFINE_MUTEX(link_ops_mutex);
static LIST_HEAD(link_ops);

static struct rtnl_link_ops *rtnl_link_ops_get(const char *kind, int *srcu_index)
{
        struct rtnl_link_ops *ops;

        rcu_read_lock();

        list_for_each_entry_rcu(ops, &link_ops, list) {
                if (!strcmp(ops->kind, kind)) {
                        *srcu_index = srcu_read_lock(&ops->srcu);
                        goto unlock;
                }
        }

        ops = NULL;
unlock:
        rcu_read_unlock();

        return ops;
}

static void rtnl_link_ops_put(struct rtnl_link_ops *ops, int srcu_index)
{
        srcu_read_unlock(&ops->srcu, srcu_index);
}

/**
 * rtnl_link_register - Register rtnl_link_ops with rtnetlink.
 * @ops: struct rtnl_link_ops * to register
 *
 * Returns 0 on success or a negative error code.
 */
int rtnl_link_register(struct rtnl_link_ops *ops)
{
        struct rtnl_link_ops *tmp;
        int err;

        /* Sanity-check max sizes to avoid stack buffer overflow. */
        if (WARN_ON(ops->maxtype > RTNL_MAX_TYPE ||
                    ops->slave_maxtype > RTNL_SLAVE_MAX_TYPE))
                return -EINVAL;

        /* The check for alloc/setup is here because if ops
         * does not have that filled up, it is not possible
         * to use the ops for creating device. So do not
         * fill up dellink as well. That disables rtnl_dellink.
         */
        if ((ops->alloc || ops->setup) && !ops->dellink)
                ops->dellink = unregister_netdevice_queue;

        err = init_srcu_struct(&ops->srcu);
        if (err)
                return err;

        mutex_lock(&link_ops_mutex);

        list_for_each_entry(tmp, &link_ops, list) {
                if (!strcmp(ops->kind, tmp->kind)) {
                        err = -EEXIST;
                        goto unlock;
                }
        }

        list_add_tail_rcu(&ops->list, &link_ops);
unlock:
        mutex_unlock(&link_ops_mutex);

        return err;
}
EXPORT_SYMBOL_GPL(rtnl_link_register);

static void __rtnl_kill_links(struct net *net, struct rtnl_link_ops *ops)
{
        struct net_device *dev;
        LIST_HEAD(list_kill);

        for_each_netdev(net, dev) {
                if (dev->rtnl_link_ops == ops)
                        ops->dellink(dev, &list_kill);
        }
        unregister_netdevice_many(&list_kill);
}

/* Return with the rtnl_lock held when there are no network
 * devices unregistering in any network namespace.
 */
static void rtnl_lock_unregistering_all(void)
{
        DEFINE_WAIT_FUNC(wait, woken_wake_function);

        add_wait_queue(&netdev_unregistering_wq, &wait);
        for (;;) {
                rtnl_lock();
                /* We held write locked pernet_ops_rwsem, and parallel
                 * setup_net() and cleanup_net() are not possible.
                 */
                if (!atomic_read(&dev_unreg_count))
                        break;
                __rtnl_unlock();

                wait_woken(&wait, TASK_UNINTERRUPTIBLE, MAX_SCHEDULE_TIMEOUT);
        }
        remove_wait_queue(&netdev_unregistering_wq, &wait);
}

/**
 * rtnl_link_unregister - Unregister rtnl_link_ops from rtnetlink.
 * @ops: struct rtnl_link_ops * to unregister
 */
void rtnl_link_unregister(struct rtnl_link_ops *ops)
{
        struct net *net;

        mutex_lock(&link_ops_mutex);
        list_del_rcu(&ops->list);
        mutex_unlock(&link_ops_mutex);

        synchronize_srcu(&ops->srcu);
        cleanup_srcu_struct(&ops->srcu);

        /* Close the race with setup_net() and cleanup_net() */
        down_write(&pernet_ops_rwsem);
        rtnl_lock_unregistering_all();

        for_each_net(net)
                __rtnl_kill_links(net, ops);

        rtnl_unlock();
        up_write(&pernet_ops_rwsem);
}
EXPORT_SYMBOL_GPL(rtnl_link_unregister);

static size_t rtnl_link_get_slave_info_data_size(const struct net_device *dev)
{
        struct net_device *master_dev;
        const struct rtnl_link_ops *ops;
        size_t size = 0;

        rcu_read_lock();

        master_dev = netdev_master_upper_dev_get_rcu((struct net_device *)dev);
        if (!master_dev)
                goto out;

        ops = master_dev->rtnl_link_ops;
        if (!ops || !ops->get_slave_size)
                goto out;
        /* IFLA_INFO_SLAVE_DATA + nested data */
        size = nla_total_size(sizeof(struct nlattr)) +
               ops->get_slave_size(master_dev, dev);

out:
        rcu_read_unlock();
        return size;
}

static size_t rtnl_link_get_size(const struct net_device *dev)
{
        const struct rtnl_link_ops *ops = dev->rtnl_link_ops;
        size_t size;

        if (!ops)
                return 0;

        size = nla_total_size(sizeof(struct nlattr)) + /* IFLA_LINKINFO */
               nla_total_size(strlen(ops->kind) + 1);  /* IFLA_INFO_KIND */

        if (ops->get_size)
                /* IFLA_INFO_DATA + nested data */
                size += nla_total_size(sizeof(struct nlattr)) +
                        ops->get_size(dev);

        if (ops->get_xstats_size)
                /* IFLA_INFO_XSTATS */
                size += nla_total_size(ops->get_xstats_size(dev));

        size += rtnl_link_get_slave_info_data_size(dev);

        return size;
}

static LIST_HEAD(rtnl_af_ops);

static struct rtnl_af_ops *rtnl_af_lookup(const int family, int *srcu_index)
{
        struct rtnl_af_ops *ops;

        ASSERT_RTNL();

        rcu_read_lock();

        list_for_each_entry_rcu(ops, &rtnl_af_ops, list) {
                if (ops->family == family) {
                        *srcu_index = srcu_read_lock(&ops->srcu);
                        goto unlock;
                }
        }

        ops = NULL;
unlock:
        rcu_read_unlock();

        return ops;
}

static void rtnl_af_put(struct rtnl_af_ops *ops, int srcu_index)
{
        srcu_read_unlock(&ops->srcu, srcu_index);
}

/**
 * rtnl_af_register - Register rtnl_af_ops with rtnetlink.
 * @ops: struct rtnl_af_ops * to register
 *
 * Return: 0 on success or a negative error code.
 */
int rtnl_af_register(struct rtnl_af_ops *ops)
{
        int err = init_srcu_struct(&ops->srcu);

        if (err)
                return err;

        rtnl_lock();
        list_add_tail_rcu(&ops->list, &rtnl_af_ops);
        rtnl_unlock();

        return 0;
}
EXPORT_SYMBOL_GPL(rtnl_af_register);

/**
 * rtnl_af_unregister - Unregister rtnl_af_ops from rtnetlink.
 * @ops: struct rtnl_af_ops * to unregister
 */
void rtnl_af_unregister(struct rtnl_af_ops *ops)
{
        rtnl_lock();
        list_del_rcu(&ops->list);
        rtnl_unlock();

        synchronize_rcu();
        synchronize_srcu(&ops->srcu);
        cleanup_srcu_struct(&ops->srcu);
}
EXPORT_SYMBOL_GPL(rtnl_af_unregister);

static size_t rtnl_link_get_af_size(const struct net_device *dev,
                                    u32 ext_filter_mask)
{
        struct rtnl_af_ops *af_ops;
        size_t size;

        /* IFLA_AF_SPEC */
        size = nla_total_size(sizeof(struct nlattr));

        rcu_read_lock();
        list_for_each_entry_rcu(af_ops, &rtnl_af_ops, list) {
                if (af_ops->get_link_af_size) {
                        /* AF_* + nested data */
                        size += nla_total_size(sizeof(struct nlattr)) +
                                af_ops->get_link_af_size(dev, ext_filter_mask);
                }
        }
        rcu_read_unlock();

        return size;
}

static bool rtnl_have_link_slave_info(const struct net_device *dev)
{
        struct net_device *master_dev;
        bool ret = false;

        rcu_read_lock();

        master_dev = netdev_master_upper_dev_get_rcu((struct net_device *)dev);
        if (master_dev && master_dev->rtnl_link_ops)
                ret = true;
        rcu_read_unlock();
        return ret;
}

static int rtnl_link_slave_info_fill(struct sk_buff *skb,
                                     const struct net_device *dev)
{
        struct net_device *master_dev;
        const struct rtnl_link_ops *ops;
        struct nlattr *slave_data;
        int err;

        master_dev = netdev_master_upper_dev_get((struct net_device *) dev);
        if (!master_dev)
                return 0;
        ops = master_dev->rtnl_link_ops;
        if (!ops)
                return 0;
        if (nla_put_string(skb, IFLA_INFO_SLAVE_KIND, ops->kind) < 0)
                return -EMSGSIZE;
        if (ops->fill_slave_info) {
                slave_data = nla_nest_start_noflag(skb, IFLA_INFO_SLAVE_DATA);
                if (!slave_data)
                        return -EMSGSIZE;
                err = ops->fill_slave_info(skb, master_dev, dev);
                if (err < 0)
                        goto err_cancel_slave_data;
                nla_nest_end(skb, slave_data);
        }
        return 0;

err_cancel_slave_data:
        nla_nest_cancel(skb, slave_data);
        return err;
}

static int rtnl_link_info_fill(struct sk_buff *skb,
                               const struct net_device *dev)
{
        const struct rtnl_link_ops *ops = dev->rtnl_link_ops;
        struct nlattr *data;
        int err;

        if (!ops)
                return 0;
        if (nla_put_string(skb, IFLA_INFO_KIND, ops->kind) < 0)
                return -EMSGSIZE;
        if (ops->fill_xstats) {
                err = ops->fill_xstats(skb, dev);
                if (err < 0)
                        return err;
        }
        if (ops->fill_info) {
                data = nla_nest_start_noflag(skb, IFLA_INFO_DATA);
                if (data == NULL)
                        return -EMSGSIZE;
                err = ops->fill_info(skb, dev);
                if (err < 0)
                        goto err_cancel_data;
                nla_nest_end(skb, data);
        }
        return 0;

err_cancel_data:
        nla_nest_cancel(skb, data);
        return err;
}

static int rtnl_link_fill(struct sk_buff *skb, const struct net_device *dev)
{
        struct nlattr *linkinfo;
        int err = -EMSGSIZE;

        linkinfo = nla_nest_start_noflag(skb, IFLA_LINKINFO);
        if (linkinfo == NULL)
                goto out;

        err = rtnl_link_info_fill(skb, dev);
        if (err < 0)
                goto err_cancel_link;

        err = rtnl_link_slave_info_fill(skb, dev);
        if (err < 0)
                goto err_cancel_link;

        nla_nest_end(skb, linkinfo);
        return 0;

err_cancel_link:
        nla_nest_cancel(skb, linkinfo);
out:
        return err;
}

int rtnetlink_send(struct sk_buff *skb, struct net *net, u32 pid, unsigned int group, int echo)
{
        struct sock *rtnl = net->rtnl;

        return nlmsg_notify(rtnl, skb, pid, group, echo, GFP_KERNEL);
}

int rtnl_unicast(struct sk_buff *skb, struct net *net, u32 pid)
{
        struct sock *rtnl = net->rtnl;

        return nlmsg_unicast(rtnl, skb, pid);
}
EXPORT_SYMBOL(rtnl_unicast);

void rtnl_notify(struct sk_buff *skb, struct net *net, u32 pid, u32 group,
                 const struct nlmsghdr *nlh, gfp_t flags)
{
        struct sock *rtnl = net->rtnl;

        nlmsg_notify(rtnl, skb, pid, group, nlmsg_report(nlh), flags);
}
EXPORT_SYMBOL(rtnl_notify);

void rtnl_set_sk_err(struct net *net, u32 group, int error)
{
        struct sock *rtnl = net->rtnl;

        netlink_set_err(rtnl, 0, group, error);
}
EXPORT_SYMBOL(rtnl_set_sk_err);

int rtnetlink_put_metrics(struct sk_buff *skb, u32 *metrics)
{
        struct nlattr *mx;
        int i, valid = 0;

        /* nothing is dumped for dst_default_metrics, so just skip the loop */
        if (metrics == dst_default_metrics.metrics)
                return 0;

        mx = nla_nest_start_noflag(skb, RTA_METRICS);
        if (mx == NULL)
                return -ENOBUFS;

        for (i = 0; i < RTAX_MAX; i++) {
                if (metrics[i]) {
                        if (i == RTAX_CC_ALGO - 1) {
                                char tmp[TCP_CA_NAME_MAX], *name;

                                name = tcp_ca_get_name_by_key(metrics[i], tmp);
                                if (!name)
                                        continue;
                                if (nla_put_string(skb, i + 1, name))
                                        goto nla_put_failure;
                        } else if (i == RTAX_FEATURES - 1) {
                                u32 user_features = metrics[i] & RTAX_FEATURE_MASK;

                                if (!user_features)
                                        continue;
                                BUILD_BUG_ON(RTAX_FEATURE_MASK & DST_FEATURE_MASK);
                                if (nla_put_u32(skb, i + 1, user_features))
                                        goto nla_put_failure;
                        } else {
                                if (nla_put_u32(skb, i + 1, metrics[i]))
                                        goto nla_put_failure;
                        }
                        valid++;
                }
        }

        if (!valid) {
                nla_nest_cancel(skb, mx);
                return 0;
        }

        return nla_nest_end(skb, mx);

nla_put_failure:
        nla_nest_cancel(skb, mx);
        return -EMSGSIZE;
}
EXPORT_SYMBOL(rtnetlink_put_metrics);

int rtnl_put_cacheinfo(struct sk_buff *skb, struct dst_entry *dst, u32 id,
                       long expires, u32 error)
{
        struct rta_cacheinfo ci = {
                .rta_error = error,
                .rta_id =  id,
        };
        unsigned long delta;

        if (dst) {
                delta = jiffies - READ_ONCE(dst->lastuse);
                ci.rta_lastuse = jiffies_delta_to_clock_t(delta);
                ci.rta_used = dst->__use;
                ci.rta_clntref = rcuref_read(&dst->__rcuref);
        }
        if (expires) {
                unsigned long clock;

                clock = jiffies_to_clock_t(abs(expires));
                clock = min_t(unsigned long, clock, INT_MAX);
                ci.rta_expires = (expires > 0) ? clock : -clock;
        }
        return nla_put(skb, RTA_CACHEINFO, sizeof(ci), &ci);
}
EXPORT_SYMBOL_GPL(rtnl_put_cacheinfo);

void netif_set_operstate(struct net_device *dev, int newstate)
{
        unsigned int old = READ_ONCE(dev->operstate);

        do {
                if (old == newstate)
                        return;
        } while (!try_cmpxchg(&dev->operstate, &old, newstate));

        netif_state_change(dev);
}
EXPORT_SYMBOL(netif_set_operstate);

static void set_operstate(struct net_device *dev, unsigned char transition)
{
        unsigned char operstate = READ_ONCE(dev->operstate);

        switch (transition) {
        case IF_OPER_UP:
                if ((operstate == IF_OPER_DORMANT ||
                     operstate == IF_OPER_TESTING ||
                     operstate == IF_OPER_UNKNOWN) &&
                    !netif_dormant(dev) && !netif_testing(dev))
                        operstate = IF_OPER_UP;
                break;

        case IF_OPER_TESTING:
                if (netif_oper_up(dev))
                        operstate = IF_OPER_TESTING;
                break;

        case IF_OPER_DORMANT:
                if (netif_oper_up(dev))
                        operstate = IF_OPER_DORMANT;
                break;
        }

        netif_set_operstate(dev, operstate);
}

static unsigned int rtnl_dev_get_flags(const struct net_device *dev)
{
        return (dev->flags & ~(IFF_PROMISC | IFF_ALLMULTI)) |
               (dev->gflags & (IFF_PROMISC | IFF_ALLMULTI));
}

static unsigned int rtnl_dev_combine_flags(const struct net_device *dev,
                                           const struct ifinfomsg *ifm)
{
        unsigned int flags = ifm->ifi_flags;

        /* bugwards compatibility: ifi_change == 0 is treated as ~0 */
        if (ifm->ifi_change)
                flags = (flags & ifm->ifi_change) |
                        (rtnl_dev_get_flags(dev) & ~ifm->ifi_change);

        return flags;
}

static void copy_rtnl_link_stats(struct rtnl_link_stats *a,
                                 const struct rtnl_link_stats64 *b)
{
        a->rx_packets = b->rx_packets;
        a->tx_packets = b->tx_packets;
        a->rx_bytes = b->rx_bytes;
        a->tx_bytes = b->tx_bytes;
        a->rx_errors = b->rx_errors;
        a->tx_errors = b->tx_errors;
        a->rx_dropped = b->rx_dropped;
        a->tx_dropped = b->tx_dropped;

        a->multicast = b->multicast;
        a->collisions = b->collisions;

        a->rx_length_errors = b->rx_length_errors;
        a->rx_over_errors = b->rx_over_errors;
        a->rx_crc_errors = b->rx_crc_errors;
        a->rx_frame_errors = b->rx_frame_errors;
        a->rx_fifo_errors = b->rx_fifo_errors;
        a->rx_missed_errors = b->rx_missed_errors;

        a->tx_aborted_errors = b->tx_aborted_errors;
        a->tx_carrier_errors = b->tx_carrier_errors;
        a->tx_fifo_errors = b->tx_fifo_errors;
        a->tx_heartbeat_errors = b->tx_heartbeat_errors;
        a->tx_window_errors = b->tx_window_errors;

        a->rx_compressed = b->rx_compressed;
        a->tx_compressed = b->tx_compressed;

        a->rx_nohandler = b->rx_nohandler;
}

/* All VF info */
static inline int rtnl_vfinfo_size(const struct net_device *dev,
                                   u32 ext_filter_mask)
{
        if (dev->dev.parent && (ext_filter_mask & RTEXT_FILTER_VF)) {
                int num_vfs = dev_num_vf(dev->dev.parent);
                size_t size = nla_total_size(0);
                size += num_vfs *
                        (nla_total_size(0) +
                         nla_total_size(sizeof(struct ifla_vf_mac)) +
                         nla_total_size(sizeof(struct ifla_vf_broadcast)) +
                         nla_total_size(sizeof(struct ifla_vf_vlan)) +
                         nla_total_size(0) + /* nest IFLA_VF_VLAN_LIST */
                         nla_total_size(MAX_VLAN_LIST_LEN *
                                        sizeof(struct ifla_vf_vlan_info)) +
                         nla_total_size(sizeof(struct ifla_vf_spoofchk)) +
                         nla_total_size(sizeof(struct ifla_vf_tx_rate)) +
                         nla_total_size(sizeof(struct ifla_vf_rate)) +
                         nla_total_size(sizeof(struct ifla_vf_link_state)) +
                         nla_total_size(sizeof(struct ifla_vf_rss_query_en)) +
                         nla_total_size(sizeof(struct ifla_vf_trust)));
                if (~ext_filter_mask & RTEXT_FILTER_SKIP_STATS) {
                        size += num_vfs *
                                (nla_total_size(0) + /* nest IFLA_VF_STATS */
                                 /* IFLA_VF_STATS_RX_PACKETS */
                                 nla_total_size_64bit(sizeof(__u64)) +
                                 /* IFLA_VF_STATS_TX_PACKETS */
                                 nla_total_size_64bit(sizeof(__u64)) +
                                 /* IFLA_VF_STATS_RX_BYTES */
                                 nla_total_size_64bit(sizeof(__u64)) +
                                 /* IFLA_VF_STATS_TX_BYTES */
                                 nla_total_size_64bit(sizeof(__u64)) +
                                 /* IFLA_VF_STATS_BROADCAST */
                                 nla_total_size_64bit(sizeof(__u64)) +
                                 /* IFLA_VF_STATS_MULTICAST */
                                 nla_total_size_64bit(sizeof(__u64)) +
                                 /* IFLA_VF_STATS_RX_DROPPED */
                                 nla_total_size_64bit(sizeof(__u64)) +
                                 /* IFLA_VF_STATS_TX_DROPPED */
                                 nla_total_size_64bit(sizeof(__u64)));
                }
                if (dev->netdev_ops->ndo_get_vf_guid)
                        size += num_vfs * 2 *
                                nla_total_size(sizeof(struct ifla_vf_guid));
                return size;
        } else
                return 0;
}

static size_t rtnl_port_size(const struct net_device *dev,
                             u32 ext_filter_mask)
{
        size_t port_size = nla_total_size(4)                /* PORT_VF */
                + nla_total_size(PORT_PROFILE_MAX)        /* PORT_PROFILE */
                + nla_total_size(PORT_UUID_MAX)                /* PORT_INSTANCE_UUID */
                + nla_total_size(PORT_UUID_MAX)                /* PORT_HOST_UUID */
                + nla_total_size(1)                        /* PROT_VDP_REQUEST */
                + nla_total_size(2);                        /* PORT_VDP_RESPONSE */
        size_t vf_ports_size = nla_total_size(sizeof(struct nlattr));
        size_t vf_port_size = nla_total_size(sizeof(struct nlattr))
                + port_size;
        size_t port_self_size = nla_total_size(sizeof(struct nlattr))
                + port_size;

        if (!dev->netdev_ops->ndo_get_vf_port || !dev->dev.parent ||
            !(ext_filter_mask & RTEXT_FILTER_VF))
                return 0;
        if (dev_num_vf(dev->dev.parent))
                return port_self_size + vf_ports_size +
                        vf_port_size * dev_num_vf(dev->dev.parent);
        else
                return port_self_size;
}

static size_t rtnl_xdp_size(void)
{
        size_t xdp_size = nla_total_size(0) +        /* nest IFLA_XDP */
                          nla_total_size(1) +        /* XDP_ATTACHED */
                          nla_total_size(4) +        /* XDP_PROG_ID (or 1st mode) */
                          nla_total_size(4);        /* XDP_<mode>_PROG_ID */

        return xdp_size;
}

static size_t rtnl_prop_list_size(const struct net_device *dev)
{
        struct netdev_name_node *name_node;
        unsigned int cnt = 0;

        rcu_read_lock();
        list_for_each_entry_rcu(name_node, &dev->name_node->list, list)
                cnt++;
        rcu_read_unlock();

        if (!cnt)
                return 0;

        return nla_total_size(0) + cnt * nla_total_size(ALTIFNAMSIZ);
}

static size_t rtnl_proto_down_size(const struct net_device *dev)
{
        size_t size = nla_total_size(1);

        /* Assume dev->proto_down_reason is not zero. */
        size += nla_total_size(0) + nla_total_size(4);

        return size;
}

static size_t rtnl_devlink_port_size(const struct net_device *dev)
{
        size_t size = nla_total_size(0); /* nest IFLA_DEVLINK_PORT */

        if (dev->devlink_port)
                size += devlink_nl_port_handle_size(dev->devlink_port);

        return size;
}

static size_t rtnl_dpll_pin_size(const struct net_device *dev)
{
        size_t size = nla_total_size(0); /* nest IFLA_DPLL_PIN */

        size += dpll_netdev_pin_handle_size(dev);

        return size;
}

static noinline size_t if_nlmsg_size(const struct net_device *dev,
                                     u32 ext_filter_mask)
{
        return NLMSG_ALIGN(sizeof(struct ifinfomsg))
               + nla_total_size(IFNAMSIZ) /* IFLA_IFNAME */
               + nla_total_size(IFALIASZ) /* IFLA_IFALIAS */
               + nla_total_size(IFNAMSIZ) /* IFLA_QDISC */
               + nla_total_size_64bit(sizeof(struct rtnl_link_ifmap))
               + nla_total_size(sizeof(struct rtnl_link_stats))
               + nla_total_size_64bit(sizeof(struct rtnl_link_stats64))
               + nla_total_size(MAX_ADDR_LEN) /* IFLA_ADDRESS */
               + nla_total_size(MAX_ADDR_LEN) /* IFLA_BROADCAST */
               + nla_total_size(4) /* IFLA_TXQLEN */
               + nla_total_size(4) /* IFLA_WEIGHT */
               + nla_total_size(4) /* IFLA_MTU */
               + nla_total_size(4) /* IFLA_LINK */
               + nla_total_size(4) /* IFLA_MASTER */
               + nla_total_size(1) /* IFLA_CARRIER */
               + nla_total_size(4) /* IFLA_PROMISCUITY */
               + nla_total_size(4) /* IFLA_ALLMULTI */
               + nla_total_size(4) /* IFLA_NUM_TX_QUEUES */
               + nla_total_size(4) /* IFLA_NUM_RX_QUEUES */
               + nla_total_size(4) /* IFLA_GSO_MAX_SEGS */
               + nla_total_size(4) /* IFLA_GSO_MAX_SIZE */
               + nla_total_size(4) /* IFLA_GRO_MAX_SIZE */
               + nla_total_size(4) /* IFLA_GSO_IPV4_MAX_SIZE */
               + nla_total_size(4) /* IFLA_GRO_IPV4_MAX_SIZE */
               + nla_total_size(4) /* IFLA_TSO_MAX_SIZE */
               + nla_total_size(4) /* IFLA_TSO_MAX_SEGS */
               + nla_total_size(1) /* IFLA_OPERSTATE */
               + nla_total_size(1) /* IFLA_LINKMODE */
               + nla_total_size(1) /* IFLA_NETNS_IMMUTABLE */
               + nla_total_size(4) /* IFLA_CARRIER_CHANGES */
               + nla_total_size(4) /* IFLA_LINK_NETNSID */
               + nla_total_size(4) /* IFLA_GROUP */
               + nla_total_size(ext_filter_mask
                                & RTEXT_FILTER_VF ? 4 : 0) /* IFLA_NUM_VF */
               + rtnl_vfinfo_size(dev, ext_filter_mask) /* IFLA_VFINFO_LIST */
               + rtnl_port_size(dev, ext_filter_mask) /* IFLA_VF_PORTS + IFLA_PORT_SELF */
               + rtnl_link_get_size(dev) /* IFLA_LINKINFO */
               + rtnl_link_get_af_size(dev, ext_filter_mask) /* IFLA_AF_SPEC */
               + nla_total_size(MAX_PHYS_ITEM_ID_LEN) /* IFLA_PHYS_PORT_ID */
               + nla_total_size(MAX_PHYS_ITEM_ID_LEN) /* IFLA_PHYS_SWITCH_ID */
               + nla_total_size(IFNAMSIZ) /* IFLA_PHYS_PORT_NAME */
               + rtnl_xdp_size() /* IFLA_XDP */
               + nla_total_size(4)  /* IFLA_EVENT */
               + nla_total_size(4)  /* IFLA_NEW_NETNSID */
               + nla_total_size(4)  /* IFLA_NEW_IFINDEX */
               + rtnl_proto_down_size(dev)  /* proto down */
               + nla_total_size(4)  /* IFLA_TARGET_NETNSID */
               + nla_total_size(4)  /* IFLA_CARRIER_UP_COUNT */
               + nla_total_size(4)  /* IFLA_CARRIER_DOWN_COUNT */
               + nla_total_size(4)  /* IFLA_MIN_MTU */
               + nla_total_size(4)  /* IFLA_MAX_MTU */
               + rtnl_prop_list_size(dev)
               + nla_total_size(MAX_ADDR_LEN) /* IFLA_PERM_ADDRESS */
               + rtnl_devlink_port_size(dev)
               + rtnl_dpll_pin_size(dev)
               + nla_total_size(8)  /* IFLA_MAX_PACING_OFFLOAD_HORIZON */
               + 0;
}

static int rtnl_vf_ports_fill(struct sk_buff *skb, struct net_device *dev)
{
        struct nlattr *vf_ports;
        struct nlattr *vf_port;
        int vf;
        int err;

        vf_ports = nla_nest_start_noflag(skb, IFLA_VF_PORTS);
        if (!vf_ports)
                return -EMSGSIZE;

        for (vf = 0; vf < dev_num_vf(dev->dev.parent); vf++) {
                vf_port = nla_nest_start_noflag(skb, IFLA_VF_PORT);
                if (!vf_port)
                        goto nla_put_failure;
                if (nla_put_u32(skb, IFLA_PORT_VF, vf))
                        goto nla_put_failure;
                err = dev->netdev_ops->ndo_get_vf_port(dev, vf, skb);
                if (err == -EMSGSIZE)
                        goto nla_put_failure;
                if (err) {
                        nla_nest_cancel(skb, vf_port);
                        continue;
                }
                nla_nest_end(skb, vf_port);
        }

        nla_nest_end(skb, vf_ports);

        return 0;

nla_put_failure:
        nla_nest_cancel(skb, vf_ports);
        return -EMSGSIZE;
}

static int rtnl_port_self_fill(struct sk_buff *skb, struct net_device *dev)
{
        struct nlattr *port_self;
        int err;

        port_self = nla_nest_start_noflag(skb, IFLA_PORT_SELF);
        if (!port_self)
                return -EMSGSIZE;

        err = dev->netdev_ops->ndo_get_vf_port(dev, PORT_SELF_VF, skb);
        if (err) {
                nla_nest_cancel(skb, port_self);
                return (err == -EMSGSIZE) ? err : 0;
        }

        nla_nest_end(skb, port_self);

        return 0;
}

static int rtnl_port_fill(struct sk_buff *skb, struct net_device *dev,
                          u32 ext_filter_mask)
{
        int err;

        if (!dev->netdev_ops->ndo_get_vf_port || !dev->dev.parent ||
            !(ext_filter_mask & RTEXT_FILTER_VF))
                return 0;

        err = rtnl_port_self_fill(skb, dev);
        if (err)
                return err;

        if (dev_num_vf(dev->dev.parent)) {
                err = rtnl_vf_ports_fill(skb, dev);
                if (err)
                        return err;
        }

        return 0;
}

static int rtnl_phys_port_id_fill(struct sk_buff *skb, struct net_device *dev)
{
        int err;
        struct netdev_phys_item_id ppid;

        err = dev_get_phys_port_id(dev, &ppid);
        if (err) {
                if (err == -EOPNOTSUPP)
                        return 0;
                return err;
        }

        if (nla_put(skb, IFLA_PHYS_PORT_ID, ppid.id_len, ppid.id))
                return -EMSGSIZE;

        return 0;
}

static int rtnl_phys_port_name_fill(struct sk_buff *skb, struct net_device *dev)
{
        char name[IFNAMSIZ];
        int err;

        err = dev_get_phys_port_name(dev, name, sizeof(name));
        if (err) {
                if (err == -EOPNOTSUPP)
                        return 0;
                return err;
        }

        if (nla_put_string(skb, IFLA_PHYS_PORT_NAME, name))
                return -EMSGSIZE;

        return 0;
}

static int rtnl_phys_switch_id_fill(struct sk_buff *skb, struct net_device *dev)
{
        struct netdev_phys_item_id ppid = { };
        int err;

        err = dev_get_port_parent_id(dev, &ppid, false);
        if (err) {
                if (err == -EOPNOTSUPP)
                        return 0;
                return err;
        }

        if (nla_put(skb, IFLA_PHYS_SWITCH_ID, ppid.id_len, ppid.id))
                return -EMSGSIZE;

        return 0;
}

static noinline_for_stack int rtnl_fill_stats(struct sk_buff *skb,
                                              struct net_device *dev)
{
        struct rtnl_link_stats64 *sp;
        struct nlattr *attr;

        attr = nla_reserve_64bit(skb, IFLA_STATS64,
                                 sizeof(struct rtnl_link_stats64), IFLA_PAD);
        if (!attr)
                return -EMSGSIZE;

        sp = nla_data(attr);
        dev_get_stats(dev, sp);

        attr = nla_reserve(skb, IFLA_STATS,
                           sizeof(struct rtnl_link_stats));
        if (!attr)
                return -EMSGSIZE;

        copy_rtnl_link_stats(nla_data(attr), sp);

        return 0;
}

static noinline_for_stack int rtnl_fill_vfinfo(struct sk_buff *skb,
                                               struct net_device *dev,
                                               int vfs_num,
                                               u32 ext_filter_mask)
{
        struct ifla_vf_rss_query_en vf_rss_query_en;
        struct nlattr *vf, *vfstats, *vfvlanlist;
        struct ifla_vf_link_state vf_linkstate;
        struct ifla_vf_vlan_info vf_vlan_info;
        struct ifla_vf_spoofchk vf_spoofchk;
        struct ifla_vf_tx_rate vf_tx_rate;
        struct ifla_vf_stats vf_stats;
        struct ifla_vf_trust vf_trust;
        struct ifla_vf_vlan vf_vlan;
        struct ifla_vf_rate vf_rate;
        struct ifla_vf_mac vf_mac;
        struct ifla_vf_broadcast vf_broadcast;
        struct ifla_vf_info ivi;
        struct ifla_vf_guid node_guid;
        struct ifla_vf_guid port_guid;

        memset(&ivi, 0, sizeof(ivi));

        /* Not all SR-IOV capable drivers support the
         * spoofcheck and "RSS query enable" query.  Preset to
         * -1 so the user space tool can detect that the driver
         * didn't report anything.
         */
        ivi.spoofchk = -1;
        ivi.rss_query_en = -1;
        ivi.trusted = -1;
        /* The default value for VF link state is "auto"
         * IFLA_VF_LINK_STATE_AUTO which equals zero
         */
        ivi.linkstate = 0;
        /* VLAN Protocol by default is 802.1Q */
        ivi.vlan_proto = htons(ETH_P_8021Q);
        if (dev->netdev_ops->ndo_get_vf_config(dev, vfs_num, &ivi))
                return 0;

        memset(&vf_vlan_info, 0, sizeof(vf_vlan_info));
        memset(&node_guid, 0, sizeof(node_guid));
        memset(&port_guid, 0, sizeof(port_guid));

        vf_mac.vf =
                vf_vlan.vf =
                vf_vlan_info.vf =
                vf_rate.vf =
                vf_tx_rate.vf =
                vf_spoofchk.vf =
                vf_linkstate.vf =
                vf_rss_query_en.vf =
                vf_trust.vf =
                node_guid.vf =
                port_guid.vf = ivi.vf;

        memcpy(vf_mac.mac, ivi.mac, sizeof(ivi.mac));
        memcpy(vf_broadcast.broadcast, dev->broadcast, dev->addr_len);
        vf_vlan.vlan = ivi.vlan;
        vf_vlan.qos = ivi.qos;
        vf_vlan_info.vlan = ivi.vlan;
        vf_vlan_info.qos = ivi.qos;
        vf_vlan_info.vlan_proto = ivi.vlan_proto;
        vf_tx_rate.rate = ivi.max_tx_rate;
        vf_rate.min_tx_rate = ivi.min_tx_rate;
        vf_rate.max_tx_rate = ivi.max_tx_rate;
        vf_spoofchk.setting = ivi.spoofchk;
        vf_linkstate.link_state = ivi.linkstate;
        vf_rss_query_en.setting = ivi.rss_query_en;
        vf_trust.setting = ivi.trusted;
        vf = nla_nest_start_noflag(skb, IFLA_VF_INFO);
        if (!vf)
                return -EMSGSIZE;
        if (nla_put(skb, IFLA_VF_MAC, sizeof(vf_mac), &vf_mac) ||
            nla_put(skb, IFLA_VF_BROADCAST, sizeof(vf_broadcast), &vf_broadcast) ||
            nla_put(skb, IFLA_VF_VLAN, sizeof(vf_vlan), &vf_vlan) ||
            nla_put(skb, IFLA_VF_RATE, sizeof(vf_rate),
                    &vf_rate) ||
            nla_put(skb, IFLA_VF_TX_RATE, sizeof(vf_tx_rate),
                    &vf_tx_rate) ||
            nla_put(skb, IFLA_VF_SPOOFCHK, sizeof(vf_spoofchk),
                    &vf_spoofchk) ||
            nla_put(skb, IFLA_VF_LINK_STATE, sizeof(vf_linkstate),
                    &vf_linkstate) ||
            nla_put(skb, IFLA_VF_RSS_QUERY_EN,
                    sizeof(vf_rss_query_en),
                    &vf_rss_query_en) ||
            nla_put(skb, IFLA_VF_TRUST,
                    sizeof(vf_trust), &vf_trust))
                goto nla_put_vf_failure;

        if (dev->netdev_ops->ndo_get_vf_guid &&
            !dev->netdev_ops->ndo_get_vf_guid(dev, vfs_num, &node_guid,
                                              &port_guid)) {
                if (nla_put(skb, IFLA_VF_IB_NODE_GUID, sizeof(node_guid),
                            &node_guid) ||
                    nla_put(skb, IFLA_VF_IB_PORT_GUID, sizeof(port_guid),
                            &port_guid))
                        goto nla_put_vf_failure;
        }
        vfvlanlist = nla_nest_start_noflag(skb, IFLA_VF_VLAN_LIST);
        if (!vfvlanlist)
                goto nla_put_vf_failure;
        if (nla_put(skb, IFLA_VF_VLAN_INFO, sizeof(vf_vlan_info),
                    &vf_vlan_info)) {
                nla_nest_cancel(skb, vfvlanlist);
                goto nla_put_vf_failure;
        }
        nla_nest_end(skb, vfvlanlist);
        if (~ext_filter_mask & RTEXT_FILTER_SKIP_STATS) {
                memset(&vf_stats, 0, sizeof(vf_stats));
                if (dev->netdev_ops->ndo_get_vf_stats)
                        dev->netdev_ops->ndo_get_vf_stats(dev, vfs_num,
                                                          &vf_stats);
                vfstats = nla_nest_start_noflag(skb, IFLA_VF_STATS);
                if (!vfstats)
                        goto nla_put_vf_failure;
                if (nla_put_u64_64bit(skb, IFLA_VF_STATS_RX_PACKETS,
                                      vf_stats.rx_packets, IFLA_VF_STATS_PAD) ||
                    nla_put_u64_64bit(skb, IFLA_VF_STATS_TX_PACKETS,
                                      vf_stats.tx_packets, IFLA_VF_STATS_PAD) ||
                    nla_put_u64_64bit(skb, IFLA_VF_STATS_RX_BYTES,
                                      vf_stats.rx_bytes, IFLA_VF_STATS_PAD) ||
                    nla_put_u64_64bit(skb, IFLA_VF_STATS_TX_BYTES,
                                      vf_stats.tx_bytes, IFLA_VF_STATS_PAD) ||
                    nla_put_u64_64bit(skb, IFLA_VF_STATS_BROADCAST,
                                      vf_stats.broadcast, IFLA_VF_STATS_PAD) ||
                    nla_put_u64_64bit(skb, IFLA_VF_STATS_MULTICAST,
                                      vf_stats.multicast, IFLA_VF_STATS_PAD) ||
                    nla_put_u64_64bit(skb, IFLA_VF_STATS_RX_DROPPED,
                                      vf_stats.rx_dropped, IFLA_VF_STATS_PAD) ||
                    nla_put_u64_64bit(skb, IFLA_VF_STATS_TX_DROPPED,
                                      vf_stats.tx_dropped, IFLA_VF_STATS_PAD)) {
                        nla_nest_cancel(skb, vfstats);
                        goto nla_put_vf_failure;
                }
                nla_nest_end(skb, vfstats);
        }
        nla_nest_end(skb, vf);
        return 0;

nla_put_vf_failure:
        nla_nest_cancel(skb, vf);
        return -EMSGSIZE;
}

static noinline_for_stack int rtnl_fill_vf(struct sk_buff *skb,
                                           struct net_device *dev,
                                           u32 ext_filter_mask)
{
        struct nlattr *vfinfo;
        int i, num_vfs;

        if (!dev->dev.parent || ((ext_filter_mask & RTEXT_FILTER_VF) == 0))
                return 0;

        num_vfs = dev_num_vf(dev->dev.parent);
        if (nla_put_u32(skb, IFLA_NUM_VF, num_vfs))
                return -EMSGSIZE;

        if (!dev->netdev_ops->ndo_get_vf_config)
                return 0;

        vfinfo = nla_nest_start_noflag(skb, IFLA_VFINFO_LIST);
        if (!vfinfo)
                return -EMSGSIZE;

        for (i = 0; i < num_vfs; i++) {
                if (rtnl_fill_vfinfo(skb, dev, i, ext_filter_mask)) {
                        nla_nest_cancel(skb, vfinfo);
                        return -EMSGSIZE;
                }
        }

        nla_nest_end(skb, vfinfo);
        return 0;
}

static int rtnl_fill_link_ifmap(struct sk_buff *skb,
                                const struct net_device *dev)
{
        struct rtnl_link_ifmap map;

        memset(&map, 0, sizeof(map));
        map.mem_start = READ_ONCE(dev->mem_start);
        map.mem_end   = READ_ONCE(dev->mem_end);
        map.base_addr = READ_ONCE(dev->base_addr);
        map.irq       = READ_ONCE(dev->irq);
        map.dma       = READ_ONCE(dev->dma);
        map.port      = READ_ONCE(dev->if_port);

        if (nla_put_64bit(skb, IFLA_MAP, sizeof(map), &map, IFLA_PAD))
                return -EMSGSIZE;

        return 0;
}

static u32 rtnl_xdp_prog_skb(struct net_device *dev)
{
        const struct bpf_prog *generic_xdp_prog;
        u32 res = 0;

        rcu_read_lock();
        generic_xdp_prog = rcu_dereference(dev->xdp_prog);
        if (generic_xdp_prog)
                res = generic_xdp_prog->aux->id;
        rcu_read_unlock();

        return res;
}

static u32 rtnl_xdp_prog_drv(struct net_device *dev)
{
        return dev_xdp_prog_id(dev, XDP_MODE_DRV);
}

static u32 rtnl_xdp_prog_hw(struct net_device *dev)
{
        return dev_xdp_prog_id(dev, XDP_MODE_HW);
}

static int rtnl_xdp_report_one(struct sk_buff *skb, struct net_device *dev,
                               u32 *prog_id, u8 *mode, u8 tgt_mode, u32 attr,
                               u32 (*get_prog_id)(struct net_device *dev))
{
        u32 curr_id;
        int err;

        curr_id = get_prog_id(dev);
        if (!curr_id)
                return 0;

        *prog_id = curr_id;
        err = nla_put_u32(skb, attr, curr_id);
        if (err)
                return err;

        if (*mode != XDP_ATTACHED_NONE)
                *mode = XDP_ATTACHED_MULTI;
        else
                *mode = tgt_mode;

        return 0;
}

static int rtnl_xdp_fill(struct sk_buff *skb, struct net_device *dev)
{
        struct nlattr *xdp;
        u32 prog_id;
        int err;
        u8 mode;

        xdp = nla_nest_start_noflag(skb, IFLA_XDP);
        if (!xdp)
                return -EMSGSIZE;

        prog_id = 0;
        mode = XDP_ATTACHED_NONE;
        err = rtnl_xdp_report_one(skb, dev, &prog_id, &mode, XDP_ATTACHED_SKB,
                                  IFLA_XDP_SKB_PROG_ID, rtnl_xdp_prog_skb);
        if (err)
                goto err_cancel;
        err = rtnl_xdp_report_one(skb, dev, &prog_id, &mode, XDP_ATTACHED_DRV,
                                  IFLA_XDP_DRV_PROG_ID, rtnl_xdp_prog_drv);
        if (err)
                goto err_cancel;
        err = rtnl_xdp_report_one(skb, dev, &prog_id, &mode, XDP_ATTACHED_HW,
                                  IFLA_XDP_HW_PROG_ID, rtnl_xdp_prog_hw);
        if (err)
                goto err_cancel;

        err = nla_put_u8(skb, IFLA_XDP_ATTACHED, mode);
        if (err)
                goto err_cancel;

        if (prog_id && mode != XDP_ATTACHED_MULTI) {
                err = nla_put_u32(skb, IFLA_XDP_PROG_ID, prog_id);
                if (err)
                        goto err_cancel;
        }

        nla_nest_end(skb, xdp);
        return 0;

err_cancel:
        nla_nest_cancel(skb, xdp);
        return err;
}

static u32 rtnl_get_event(unsigned long event)
{
        u32 rtnl_event_type = IFLA_EVENT_NONE;

        switch (event) {
        case NETDEV_REBOOT:
                rtnl_event_type = IFLA_EVENT_REBOOT;
                break;
        case NETDEV_FEAT_CHANGE:
                rtnl_event_type = IFLA_EVENT_FEATURES;
                break;
        case NETDEV_BONDING_FAILOVER:
                rtnl_event_type = IFLA_EVENT_BONDING_FAILOVER;
                break;
        case NETDEV_NOTIFY_PEERS:
                rtnl_event_type = IFLA_EVENT_NOTIFY_PEERS;
                break;
        case NETDEV_RESEND_IGMP:
                rtnl_event_type = IFLA_EVENT_IGMP_RESEND;
                break;
        case NETDEV_CHANGEINFODATA:
                rtnl_event_type = IFLA_EVENT_BONDING_OPTIONS;
                break;
        default:
                break;
        }

        return rtnl_event_type;
}

static int put_master_ifindex(struct sk_buff *skb, struct net_device *dev)
{
        const struct net_device *upper_dev;
        int ret = 0;

        rcu_read_lock();

        upper_dev = netdev_master_upper_dev_get_rcu(dev);
        if (upper_dev)
                ret = nla_put_u32(skb, IFLA_MASTER,
                                  READ_ONCE(upper_dev->ifindex));

        rcu_read_unlock();
        return ret;
}

static int nla_put_iflink(struct sk_buff *skb, const struct net_device *dev,
                          bool force)
{
        int iflink = dev_get_iflink(dev);

        if (force || READ_ONCE(dev->ifindex) != iflink)
                return nla_put_u32(skb, IFLA_LINK, iflink);

        return 0;
}

static noinline_for_stack int nla_put_ifalias(struct sk_buff *skb,
                                              struct net_device *dev)
{
        char buf[IFALIASZ];
        int ret;

        ret = dev_get_alias(dev, buf, sizeof(buf));
        return ret > 0 ? nla_put_string(skb, IFLA_IFALIAS, buf) : 0;
}

static int rtnl_fill_link_netnsid(struct sk_buff *skb,
                                  const struct net_device *dev,
                                  struct net *src_net, gfp_t gfp)
{
        bool put_iflink = false;

        if (dev->rtnl_link_ops && dev->rtnl_link_ops->get_link_net) {
                struct net *link_net = dev->rtnl_link_ops->get_link_net(dev);

                if (!net_eq(dev_net(dev), link_net)) {
                        int id = peernet2id_alloc(src_net, link_net, gfp);

                        if (nla_put_s32(skb, IFLA_LINK_NETNSID, id))
                                return -EMSGSIZE;

                        put_iflink = true;
                }
        }

        return nla_put_iflink(skb, dev, put_iflink);
}

static int rtnl_fill_link_af(struct sk_buff *skb,
                             const struct net_device *dev,
                             u32 ext_filter_mask)
{
        const struct rtnl_af_ops *af_ops;
        struct nlattr *af_spec;

        af_spec = nla_nest_start_noflag(skb, IFLA_AF_SPEC);
        if (!af_spec)
                return -EMSGSIZE;

        list_for_each_entry_rcu(af_ops, &rtnl_af_ops, list) {
                struct nlattr *af;
                int err;

                if (!af_ops->fill_link_af)
                        continue;

                af = nla_nest_start_noflag(skb, af_ops->family);
                if (!af)
                        return -EMSGSIZE;

                err = af_ops->fill_link_af(skb, dev, ext_filter_mask);
                /*
                 * Caller may return ENODATA to indicate that there
                 * was no data to be dumped. This is not an error, it
                 * means we should trim the attribute header and
                 * continue.
                 */
                if (err == -ENODATA)
                        nla_nest_cancel(skb, af);
                else if (err < 0)
                        return -EMSGSIZE;

                nla_nest_end(skb, af);
        }

        nla_nest_end(skb, af_spec);
        return 0;
}

static int rtnl_fill_alt_ifnames(struct sk_buff *skb,
                                 const struct net_device *dev)
{
        struct netdev_name_node *name_node;
        int count = 0;

        list_for_each_entry_rcu(name_node, &dev->name_node->list, list) {
                if (nla_put_string(skb, IFLA_ALT_IFNAME, name_node->name))
                        return -EMSGSIZE;
                count++;
        }
        return count;
}

/* RCU protected. */
static int rtnl_fill_prop_list(struct sk_buff *skb,
                               const struct net_device *dev)
{
        struct nlattr *prop_list;
        int ret;

        prop_list = nla_nest_start(skb, IFLA_PROP_LIST);
        if (!prop_list)
                return -EMSGSIZE;

        ret = rtnl_fill_alt_ifnames(skb, dev);
        if (ret <= 0)
                goto nest_cancel;

        nla_nest_end(skb, prop_list);
        return 0;

nest_cancel:
        nla_nest_cancel(skb, prop_list);
        return ret;
}

static int rtnl_fill_proto_down(struct sk_buff *skb,
                                const struct net_device *dev)
{
        struct nlattr *pr;
        u32 preason;

        if (nla_put_u8(skb, IFLA_PROTO_DOWN, READ_ONCE(dev->proto_down)))
                goto nla_put_failure;

        preason = READ_ONCE(dev->proto_down_reason);
        if (!preason)
                return 0;

        pr = nla_nest_start(skb, IFLA_PROTO_DOWN_REASON);
        if (!pr)
                return -EMSGSIZE;

        if (nla_put_u32(skb, IFLA_PROTO_DOWN_REASON_VALUE, preason)) {
                nla_nest_cancel(skb, pr);
                goto nla_put_failure;
        }

        nla_nest_end(skb, pr);
        return 0;

nla_put_failure:
        return -EMSGSIZE;
}

static int rtnl_fill_devlink_port(struct sk_buff *skb,
                                  const struct net_device *dev)
{
        struct nlattr *devlink_port_nest;
        int ret;

        devlink_port_nest = nla_nest_start(skb, IFLA_DEVLINK_PORT);
        if (!devlink_port_nest)
                return -EMSGSIZE;

        if (dev->devlink_port) {
                ret = devlink_nl_port_handle_fill(skb, dev->devlink_port);
                if (ret < 0)
                        goto nest_cancel;
        }

        nla_nest_end(skb, devlink_port_nest);
        return 0;

nest_cancel:
        nla_nest_cancel(skb, devlink_port_nest);
        return ret;
}

static int rtnl_fill_dpll_pin(struct sk_buff *skb,
                              const struct net_device *dev)
{
        struct nlattr *dpll_pin_nest;
        int ret;

        dpll_pin_nest = nla_nest_start(skb, IFLA_DPLL_PIN);
        if (!dpll_pin_nest)
                return -EMSGSIZE;

        ret = dpll_netdev_add_pin_handle(skb, dev);
        if (ret < 0)
                goto nest_cancel;

        nla_nest_end(skb, dpll_pin_nest);
        return 0;

nest_cancel:
        nla_nest_cancel(skb, dpll_pin_nest);
        return ret;
}

static int rtnl_fill_ifinfo(struct sk_buff *skb,
                            struct net_device *dev, struct net *src_net,
                            int type, u32 pid, u32 seq, u32 change,
                            unsigned int flags, u32 ext_filter_mask,
                            u32 event, int *new_nsid, int new_ifindex,
                            int tgt_netnsid, gfp_t gfp)
{
        char devname[IFNAMSIZ];
        struct ifinfomsg *ifm;
        struct nlmsghdr *nlh;
        struct Qdisc *qdisc;

        ASSERT_RTNL();
        nlh = nlmsg_put(skb, pid, seq, type, sizeof(*ifm), flags);
        if (nlh == NULL)
                return -EMSGSIZE;

        ifm = nlmsg_data(nlh);
        ifm->ifi_family = AF_UNSPEC;
        ifm->__ifi_pad = 0;
        ifm->ifi_type = READ_ONCE(dev->type);
        ifm->ifi_index = READ_ONCE(dev->ifindex);
        ifm->ifi_flags = dev_get_flags(dev);
        ifm->ifi_change = change;

        if (tgt_netnsid >= 0 && nla_put_s32(skb, IFLA_TARGET_NETNSID, tgt_netnsid))
                goto nla_put_failure;

        netdev_copy_name(dev, devname);
        if (nla_put_string(skb, IFLA_IFNAME, devname))
                goto nla_put_failure;

        if (nla_put_u32(skb, IFLA_TXQLEN, READ_ONCE(dev->tx_queue_len)) ||
            nla_put_u8(skb, IFLA_OPERSTATE,
                       netif_running(dev) ? READ_ONCE(dev->operstate) :
                                            IF_OPER_DOWN) ||
            nla_put_u8(skb, IFLA_LINKMODE, READ_ONCE(dev->link_mode)) ||
            nla_put_u8(skb, IFLA_NETNS_IMMUTABLE, dev->netns_immutable) ||
            nla_put_u32(skb, IFLA_MTU, READ_ONCE(dev->mtu)) ||
            nla_put_u32(skb, IFLA_MIN_MTU, READ_ONCE(dev->min_mtu)) ||
            nla_put_u32(skb, IFLA_MAX_MTU, READ_ONCE(dev->max_mtu)) ||
            nla_put_u32(skb, IFLA_GROUP, READ_ONCE(dev->group)) ||
            nla_put_u32(skb, IFLA_PROMISCUITY, READ_ONCE(dev->promiscuity)) ||
            nla_put_u32(skb, IFLA_ALLMULTI, READ_ONCE(dev->allmulti)) ||
            nla_put_u32(skb, IFLA_NUM_TX_QUEUES,
                        READ_ONCE(dev->num_tx_queues)) ||
            nla_put_u32(skb, IFLA_GSO_MAX_SEGS,
                        READ_ONCE(dev->gso_max_segs)) ||
            nla_put_u32(skb, IFLA_GSO_MAX_SIZE,
                        READ_ONCE(dev->gso_max_size)) ||
            nla_put_u32(skb, IFLA_GRO_MAX_SIZE,
                        READ_ONCE(dev->gro_max_size)) ||
            nla_put_u32(skb, IFLA_GSO_IPV4_MAX_SIZE,
                        READ_ONCE(dev->gso_ipv4_max_size)) ||
            nla_put_u32(skb, IFLA_GRO_IPV4_MAX_SIZE,
                        READ_ONCE(dev->gro_ipv4_max_size)) ||
            nla_put_u32(skb, IFLA_TSO_MAX_SIZE,
                        READ_ONCE(dev->tso_max_size)) ||
            nla_put_u32(skb, IFLA_TSO_MAX_SEGS,
                        READ_ONCE(dev->tso_max_segs)) ||
            nla_put_uint(skb, IFLA_MAX_PACING_OFFLOAD_HORIZON,
                         READ_ONCE(dev->max_pacing_offload_horizon)) ||
#ifdef CONFIG_RPS
            nla_put_u32(skb, IFLA_NUM_RX_QUEUES,
                        READ_ONCE(dev->num_rx_queues)) ||
#endif
            put_master_ifindex(skb, dev) ||
            nla_put_u8(skb, IFLA_CARRIER, netif_carrier_ok(dev)) ||
            nla_put_ifalias(skb, dev) ||
            nla_put_u32(skb, IFLA_CARRIER_CHANGES,
                        atomic_read(&dev->carrier_up_count) +
                        atomic_read(&dev->carrier_down_count)) ||
            nla_put_u32(skb, IFLA_CARRIER_UP_COUNT,
                        atomic_read(&dev->carrier_up_count)) ||
            nla_put_u32(skb, IFLA_CARRIER_DOWN_COUNT,
                        atomic_read(&dev->carrier_down_count)))
                goto nla_put_failure;

        if (rtnl_fill_proto_down(skb, dev))
                goto nla_put_failure;

        if (event != IFLA_EVENT_NONE) {
                if (nla_put_u32(skb, IFLA_EVENT, event))
                        goto nla_put_failure;
        }

        if (dev->addr_len) {
                if (nla_put(skb, IFLA_ADDRESS, dev->addr_len, dev->dev_addr) ||
                    nla_put(skb, IFLA_BROADCAST, dev->addr_len, dev->broadcast))
                        goto nla_put_failure;
        }

        if (rtnl_phys_port_id_fill(skb, dev))
                goto nla_put_failure;

        if (rtnl_phys_port_name_fill(skb, dev))
                goto nla_put_failure;

        if (rtnl_phys_switch_id_fill(skb, dev))
                goto nla_put_failure;

        if (rtnl_fill_stats(skb, dev))
                goto nla_put_failure;

        if (rtnl_fill_vf(skb, dev, ext_filter_mask))
                goto nla_put_failure;

        if (rtnl_port_fill(skb, dev, ext_filter_mask))
                goto nla_put_failure;

        if (rtnl_xdp_fill(skb, dev))
                goto nla_put_failure;

        if (dev->rtnl_link_ops || rtnl_have_link_slave_info(dev)) {
                if (rtnl_link_fill(skb, dev) < 0)
                        goto nla_put_failure;
        }

        if (new_nsid &&
            nla_put_s32(skb, IFLA_NEW_NETNSID, *new_nsid) < 0)
                goto nla_put_failure;
        if (new_ifindex &&
            nla_put_s32(skb, IFLA_NEW_IFINDEX, new_ifindex) < 0)
                goto nla_put_failure;

        if (memchr_inv(dev->perm_addr, '\0', dev->addr_len) &&
            nla_put(skb, IFLA_PERM_ADDRESS, dev->addr_len, dev->perm_addr))
                goto nla_put_failure;

        rcu_read_lock();
        if (rtnl_fill_link_netnsid(skb, dev, src_net, GFP_ATOMIC))
                goto nla_put_failure_rcu;
        qdisc = rcu_dereference(dev->qdisc);
        if (qdisc && nla_put_string(skb, IFLA_QDISC, qdisc->ops->id))
                goto nla_put_failure_rcu;
        if (rtnl_fill_link_af(skb, dev, ext_filter_mask))
                goto nla_put_failure_rcu;
        if (rtnl_fill_link_ifmap(skb, dev))
                goto nla_put_failure_rcu;
        if (rtnl_fill_prop_list(skb, dev))
                goto nla_put_failure_rcu;
        rcu_read_unlock();

        if (dev->dev.parent &&
            nla_put_string(skb, IFLA_PARENT_DEV_NAME,
                           dev_name(dev->dev.parent)))
                goto nla_put_failure;

        if (dev->dev.parent && dev->dev.parent->bus &&
            nla_put_string(skb, IFLA_PARENT_DEV_BUS_NAME,
                           dev->dev.parent->bus->name))
                goto nla_put_failure;

        if (rtnl_fill_devlink_port(skb, dev))
                goto nla_put_failure;

        if (rtnl_fill_dpll_pin(skb, dev))
                goto nla_put_failure;

        nlmsg_end(skb, nlh);
        return 0;

nla_put_failure_rcu:
        rcu_read_unlock();
nla_put_failure:
        nlmsg_cancel(skb, nlh);
        return -EMSGSIZE;
}

static const struct nla_policy ifla_policy[IFLA_MAX+1] = {
        [IFLA_UNSPEC]                = { .strict_start_type = IFLA_DPLL_PIN },
        [IFLA_IFNAME]                = { .type = NLA_STRING, .len = IFNAMSIZ-1 },
        [IFLA_ADDRESS]                = { .type = NLA_BINARY, .len = MAX_ADDR_LEN },
        [IFLA_BROADCAST]        = { .type = NLA_BINARY, .len = MAX_ADDR_LEN },
        [IFLA_MAP]                = { .len = sizeof(struct rtnl_link_ifmap) },
        [IFLA_MTU]                = { .type = NLA_U32 },
        [IFLA_LINK]                = { .type = NLA_U32 },
        [IFLA_MASTER]                = { .type = NLA_U32 },
        [IFLA_CARRIER]                = { .type = NLA_U8 },
        [IFLA_TXQLEN]                = { .type = NLA_U32 },
        [IFLA_WEIGHT]                = { .type = NLA_U32 },
        [IFLA_OPERSTATE]        = { .type = NLA_U8 },
        [IFLA_LINKMODE]                = { .type = NLA_U8 },
        [IFLA_LINKINFO]                = { .type = NLA_NESTED },
        [IFLA_NET_NS_PID]        = { .type = NLA_U32 },
        [IFLA_NET_NS_FD]        = { .type = NLA_U32 },
        /* IFLA_IFALIAS is a string, but policy is set to NLA_BINARY to
         * allow 0-length string (needed to remove an alias).
         */
        [IFLA_IFALIAS]                = { .type = NLA_BINARY, .len = IFALIASZ - 1 },
        [IFLA_VFINFO_LIST]        = {. type = NLA_NESTED },
        [IFLA_VF_PORTS]                = { .type = NLA_NESTED },
        [IFLA_PORT_SELF]        = { .type = NLA_NESTED },
        [IFLA_AF_SPEC]                = { .type = NLA_NESTED },
        [IFLA_EXT_MASK]                = { .type = NLA_U32 },
        [IFLA_PROMISCUITY]        = { .type = NLA_U32 },
        [IFLA_NUM_TX_QUEUES]        = { .type = NLA_U32 },
        [IFLA_NUM_RX_QUEUES]        = { .type = NLA_U32 },
        [IFLA_GSO_MAX_SEGS]        = { .type = NLA_U32 },
        [IFLA_GSO_MAX_SIZE]        = NLA_POLICY_MIN(NLA_U32, MAX_TCP_HEADER + 1),
        [IFLA_PHYS_PORT_ID]        = { .type = NLA_BINARY, .len = MAX_PHYS_ITEM_ID_LEN },
        [IFLA_CARRIER_CHANGES]        = { .type = NLA_U32 },  /* ignored */
        [IFLA_PHYS_SWITCH_ID]        = { .type = NLA_BINARY, .len = MAX_PHYS_ITEM_ID_LEN },
        [IFLA_LINK_NETNSID]        = { .type = NLA_S32 },
        [IFLA_PROTO_DOWN]        = { .type = NLA_U8 },
        [IFLA_XDP]                = { .type = NLA_NESTED },
        [IFLA_EVENT]                = { .type = NLA_U32 },
        [IFLA_GROUP]                = { .type = NLA_U32 },
        [IFLA_TARGET_NETNSID]        = { .type = NLA_S32 },
        [IFLA_CARRIER_UP_COUNT]        = { .type = NLA_U32 },
        [IFLA_CARRIER_DOWN_COUNT] = { .type = NLA_U32 },
        [IFLA_MIN_MTU]                = { .type = NLA_U32 },
        [IFLA_MAX_MTU]                = { .type = NLA_U32 },
        [IFLA_PROP_LIST]        = { .type = NLA_NESTED },
        [IFLA_ALT_IFNAME]        = { .type = NLA_STRING,
                                    .len = ALTIFNAMSIZ - 1 },
        [IFLA_PERM_ADDRESS]        = { .type = NLA_REJECT },
        [IFLA_PROTO_DOWN_REASON] = { .type = NLA_NESTED },
        [IFLA_NEW_IFINDEX]        = NLA_POLICY_MIN(NLA_S32, 1),
        [IFLA_PARENT_DEV_NAME]        = { .type = NLA_NUL_STRING },
        [IFLA_GRO_MAX_SIZE]        = { .type = NLA_U32 },
        [IFLA_TSO_MAX_SIZE]        = { .type = NLA_REJECT },
        [IFLA_TSO_MAX_SEGS]        = { .type = NLA_REJECT },
        [IFLA_ALLMULTI]                = { .type = NLA_REJECT },
        [IFLA_GSO_IPV4_MAX_SIZE]        = NLA_POLICY_MIN(NLA_U32, MAX_TCP_HEADER + 1),
        [IFLA_GRO_IPV4_MAX_SIZE]        = { .type = NLA_U32 },
        [IFLA_NETNS_IMMUTABLE]        = { .type = NLA_REJECT },
};

static const struct nla_policy ifla_info_policy[IFLA_INFO_MAX+1] = {
        [IFLA_INFO_KIND]        = { .type = NLA_STRING },
        [IFLA_INFO_DATA]        = { .type = NLA_NESTED },
        [IFLA_INFO_SLAVE_KIND]        = { .type = NLA_STRING },
        [IFLA_INFO_SLAVE_DATA]        = { .type = NLA_NESTED },
};

static const struct nla_policy ifla_vf_policy[IFLA_VF_MAX+1] = {
        [IFLA_VF_MAC]                = { .len = sizeof(struct ifla_vf_mac) },
        [IFLA_VF_BROADCAST]        = { .type = NLA_REJECT },
        [IFLA_VF_VLAN]                = { .len = sizeof(struct ifla_vf_vlan) },
        [IFLA_VF_VLAN_LIST]     = { .type = NLA_NESTED },
        [IFLA_VF_TX_RATE]        = { .len = sizeof(struct ifla_vf_tx_rate) },
        [IFLA_VF_SPOOFCHK]        = { .len = sizeof(struct ifla_vf_spoofchk) },
        [IFLA_VF_RATE]                = { .len = sizeof(struct ifla_vf_rate) },
        [IFLA_VF_LINK_STATE]        = { .len = sizeof(struct ifla_vf_link_state) },
        [IFLA_VF_RSS_QUERY_EN]        = { .len = sizeof(struct ifla_vf_rss_query_en) },
        [IFLA_VF_STATS]                = { .type = NLA_NESTED },
        [IFLA_VF_TRUST]                = { .len = sizeof(struct ifla_vf_trust) },
        [IFLA_VF_IB_NODE_GUID]        = { .len = sizeof(struct ifla_vf_guid) },
        [IFLA_VF_IB_PORT_GUID]        = { .len = sizeof(struct ifla_vf_guid) },
};

static const struct nla_policy ifla_port_policy[IFLA_PORT_MAX+1] = {
        [IFLA_PORT_VF]                = { .type = NLA_U32 },
        [IFLA_PORT_PROFILE]        = { .type = NLA_STRING,
                                    .len = PORT_PROFILE_MAX },
        [IFLA_PORT_INSTANCE_UUID] = { .type = NLA_BINARY,
                                      .len = PORT_UUID_MAX },
        [IFLA_PORT_HOST_UUID]        = { .type = NLA_STRING,
                                    .len = PORT_UUID_MAX },
        [IFLA_PORT_REQUEST]        = { .type = NLA_U8, },
        [IFLA_PORT_RESPONSE]        = { .type = NLA_U16, },

        /* Unused, but we need to keep it here since user space could
         * fill it. It's also broken with regard to NLA_BINARY use in
         * combination with structs.
         */
        [IFLA_PORT_VSI_TYPE]        = { .type = NLA_BINARY,
                                    .len = sizeof(struct ifla_port_vsi) },
};

static const struct nla_policy ifla_xdp_policy[IFLA_XDP_MAX + 1] = {
        [IFLA_XDP_UNSPEC]        = { .strict_start_type = IFLA_XDP_EXPECTED_FD },
        [IFLA_XDP_FD]                = { .type = NLA_S32 },
        [IFLA_XDP_EXPECTED_FD]        = { .type = NLA_S32 },
        [IFLA_XDP_ATTACHED]        = { .type = NLA_U8 },
        [IFLA_XDP_FLAGS]        = { .type = NLA_U32 },
        [IFLA_XDP_PROG_ID]        = { .type = NLA_U32 },
};

static struct rtnl_link_ops *linkinfo_to_kind_ops(const struct nlattr *nla,
                                                  int *ops_srcu_index)
{
        struct nlattr *linfo[IFLA_INFO_MAX + 1];
        struct rtnl_link_ops *ops = NULL;

        if (nla_parse_nested_deprecated(linfo, IFLA_INFO_MAX, nla, ifla_info_policy, NULL) < 0)
                return NULL;

        if (linfo[IFLA_INFO_KIND]) {
                char kind[MODULE_NAME_LEN];

                nla_strscpy(kind, linfo[IFLA_INFO_KIND], sizeof(kind));
                ops = rtnl_link_ops_get(kind, ops_srcu_index);
        }

        return ops;
}

static bool link_master_filtered(struct net_device *dev, int master_idx)
{
        struct net_device *master;

        if (!master_idx)
                return false;

        master = netdev_master_upper_dev_get(dev);

        /* 0 is already used to denote IFLA_MASTER wasn't passed, therefore need
         * another invalid value for ifindex to denote "no master".
         */
        if (master_idx == -1)
                return !!master;

        if (!master || master->ifindex != master_idx)
                return true;

        return false;
}

static bool link_kind_filtered(const struct net_device *dev,
                               const struct rtnl_link_ops *kind_ops)
{
        if (kind_ops && dev->rtnl_link_ops != kind_ops)
                return true;

        return false;
}

static bool link_dump_filtered(struct net_device *dev,
                               int master_idx,
                               const struct rtnl_link_ops *kind_ops)
{
        if (link_master_filtered(dev, master_idx) ||
            link_kind_filtered(dev, kind_ops))
                return true;

        return false;
}

/**
 * rtnl_get_net_ns_capable - Get netns if sufficiently privileged.
 * @sk: netlink socket
 * @netnsid: network namespace identifier
 *
 * Returns the network namespace identified by netnsid on success or an error
 * pointer on failure.
 */
struct net *rtnl_get_net_ns_capable(struct sock *sk, int netnsid)
{
        struct net *net;

        net = get_net_ns_by_id(sock_net(sk), netnsid);
        if (!net)
                return ERR_PTR(-EINVAL);

        /* For now, the caller is required to have CAP_NET_ADMIN in
         * the user namespace owning the target net ns.
         */
        if (!sk_ns_capable(sk, net->user_ns, CAP_NET_ADMIN)) {
                put_net(net);
                return ERR_PTR(-EACCES);
        }
        return net;
}
EXPORT_SYMBOL_GPL(rtnl_get_net_ns_capable);

static int rtnl_valid_dump_ifinfo_req(const struct nlmsghdr *nlh,
                                      bool strict_check, struct nlattr **tb,
                                      struct netlink_ext_ack *extack)
{
        int hdrlen;

        if (strict_check) {
                struct ifinfomsg *ifm;

                ifm = nlmsg_payload(nlh, sizeof(*ifm));
                if (!ifm) {
                        NL_SET_ERR_MSG(extack, "Invalid header for link dump");
                        return -EINVAL;
                }

                if (ifm->__ifi_pad || ifm->ifi_type || ifm->ifi_flags ||
                    ifm->ifi_change) {
                        NL_SET_ERR_MSG(extack, "Invalid values in header for link dump request");
                        return -EINVAL;
                }
                if (ifm->ifi_index) {
                        NL_SET_ERR_MSG(extack, "Filter by device index not supported for link dumps");
                        return -EINVAL;
                }

                return nlmsg_parse_deprecated_strict(nlh, sizeof(*ifm), tb,
                                                     IFLA_MAX, ifla_policy,
                                                     extack);
        }

        /* A hack to preserve kernel<->userspace interface.
         * The correct header is ifinfomsg. It is consistent with rtnl_getlink.
         * However, before Linux v3.9 the code here assumed rtgenmsg and that's
         * what iproute2 < v3.9.0 used.
         * We can detect the old iproute2. Even including the IFLA_EXT_MASK
         * attribute, its netlink message is shorter than struct ifinfomsg.
         */
        hdrlen = nlmsg_len(nlh) < sizeof(struct ifinfomsg) ?
                 sizeof(struct rtgenmsg) : sizeof(struct ifinfomsg);

        return nlmsg_parse_deprecated(nlh, hdrlen, tb, IFLA_MAX, ifla_policy,
                                      extack);
}

static int rtnl_dump_ifinfo(struct sk_buff *skb, struct netlink_callback *cb)
{
        struct netlink_ext_ack *extack = cb->extack;
        struct rtnl_link_ops *kind_ops = NULL;
        const struct nlmsghdr *nlh = cb->nlh;
        struct net *net = sock_net(skb->sk);
        unsigned int flags = NLM_F_MULTI;
        struct nlattr *tb[IFLA_MAX+1];
        struct {
                unsigned long ifindex;
        } *ctx = (void *)cb->ctx;
        struct net *tgt_net = net;
        u32 ext_filter_mask = 0;
        struct net_device *dev;
        int ops_srcu_index;
        int master_idx = 0;
        int netnsid = -1;
        int err, i;

        err = rtnl_valid_dump_ifinfo_req(nlh, cb->strict_check, tb, extack);
        if (err < 0) {
                if (cb->strict_check)
                        return err;

                goto walk_entries;
        }

        for (i = 0; i <= IFLA_MAX; ++i) {
                if (!tb[i])
                        continue;

                /* new attributes should only be added with strict checking */
                switch (i) {
                case IFLA_TARGET_NETNSID:
                        netnsid = nla_get_s32(tb[i]);
                        tgt_net = rtnl_get_net_ns_capable(skb->sk, netnsid);
                        if (IS_ERR(tgt_net)) {
                                NL_SET_ERR_MSG(extack, "Invalid target network namespace id");
                                err = PTR_ERR(tgt_net);
                                netnsid = -1;
                                goto out;
                        }
                        break;
                case IFLA_EXT_MASK:
                        ext_filter_mask = nla_get_u32(tb[i]);
                        break;
                case IFLA_MASTER:
                        master_idx = nla_get_u32(tb[i]);
                        break;
                case IFLA_LINKINFO:
                        kind_ops = linkinfo_to_kind_ops(tb[i], &ops_srcu_index);
                        break;
                default:
                        if (cb->strict_check) {
                                NL_SET_ERR_MSG(extack, "Unsupported attribute in link dump request");
                                err = -EINVAL;
                                goto out;
                        }
                }
        }

        if (master_idx || kind_ops)
                flags |= NLM_F_DUMP_FILTERED;

walk_entries:
        err = 0;
        for_each_netdev_dump(tgt_net, dev, ctx->ifindex) {
                if (link_dump_filtered(dev, master_idx, kind_ops))
                        continue;
                err = rtnl_fill_ifinfo(skb, dev, net, RTM_NEWLINK,
                                       NETLINK_CB(cb->skb).portid,
                                       nlh->nlmsg_seq, 0, flags,
                                       ext_filter_mask, 0, NULL, 0,
                                       netnsid, GFP_KERNEL);
                if (err < 0)
                        break;
        }


        cb->seq = tgt_net->dev_base_seq;
        nl_dump_check_consistent(cb, nlmsg_hdr(skb));

out:

        if (kind_ops)
                rtnl_link_ops_put(kind_ops, ops_srcu_index);
        if (netnsid >= 0)
                put_net(tgt_net);

        return err;
}

int rtnl_nla_parse_ifinfomsg(struct nlattr **tb, const struct nlattr *nla_peer,
                             struct netlink_ext_ack *exterr)
{
        const struct ifinfomsg *ifmp;
        const struct nlattr *attrs;
        size_t len;

        ifmp = nla_data(nla_peer);
        attrs = nla_data(nla_peer) + sizeof(struct ifinfomsg);
        len = nla_len(nla_peer) - sizeof(struct ifinfomsg);

        if (ifmp->ifi_index < 0) {
                NL_SET_ERR_MSG_ATTR(exterr, nla_peer,
                                    "ifindex can't be negative");
                return -EINVAL;
        }

        return nla_parse_deprecated(tb, IFLA_MAX, attrs, len, ifla_policy,
                                    exterr);
}
EXPORT_SYMBOL(rtnl_nla_parse_ifinfomsg);

static struct net *rtnl_link_get_net_ifla(struct nlattr *tb[])
{
        struct net *net = NULL;

        /* Examine the link attributes and figure out which
         * network namespace we are talking about.
         */
        if (tb[IFLA_NET_NS_PID])
                net = get_net_ns_by_pid(nla_get_u32(tb[IFLA_NET_NS_PID]));
        else if (tb[IFLA_NET_NS_FD])
                net = get_net_ns_by_fd(nla_get_u32(tb[IFLA_NET_NS_FD]));

        return net;
}

struct net *rtnl_link_get_net(struct net *src_net, struct nlattr *tb[])
{
        struct net *net = rtnl_link_get_net_ifla(tb);

        if (!net)
                net = get_net(src_net);

        return net;
}
EXPORT_SYMBOL(rtnl_link_get_net);

/* Figure out which network namespace we are talking about by
 * examining the link attributes in the following order:
 *
 * 1. IFLA_NET_NS_PID
 * 2. IFLA_NET_NS_FD
 * 3. IFLA_TARGET_NETNSID
 */
static struct net *rtnl_link_get_net_by_nlattr(struct net *src_net,
                                               struct nlattr *tb[])
{
        struct net *net;

        if (tb[IFLA_NET_NS_PID] || tb[IFLA_NET_NS_FD])
                return rtnl_link_get_net(src_net, tb);

        if (!tb[IFLA_TARGET_NETNSID])
                return get_net(src_net);

        net = get_net_ns_by_id(src_net, nla_get_u32(tb[IFLA_TARGET_NETNSID]));
        if (!net)
                return ERR_PTR(-EINVAL);

        return net;
}

static struct net *rtnl_link_get_net_capable(const struct sk_buff *skb,
                                             struct net *src_net,
                                             struct nlattr *tb[], int cap)
{
        struct net *net;

        net = rtnl_link_get_net_by_nlattr(src_net, tb);
        if (IS_ERR(net))
                return net;

        if (!netlink_ns_capable(skb, net->user_ns, cap)) {
                put_net(net);
                return ERR_PTR(-EPERM);
        }

        return net;
}

/* Verify that rtnetlink requests do not pass additional properties
 * potentially referring to different network namespaces.
 */
static int rtnl_ensure_unique_netns(struct nlattr *tb[],
                                    struct netlink_ext_ack *extack,
                                    bool netns_id_only)
{

        if (netns_id_only) {
                if (!tb[IFLA_NET_NS_PID] && !tb[IFLA_NET_NS_FD])
                        return 0;

                NL_SET_ERR_MSG(extack, "specified netns attribute not supported");
                return -EOPNOTSUPP;
        }

        if (tb[IFLA_TARGET_NETNSID] && (tb[IFLA_NET_NS_PID] || tb[IFLA_NET_NS_FD]))
                goto invalid_attr;

        if (tb[IFLA_NET_NS_PID] && (tb[IFLA_TARGET_NETNSID] || tb[IFLA_NET_NS_FD]))
                goto invalid_attr;

        if (tb[IFLA_NET_NS_FD] && (tb[IFLA_TARGET_NETNSID] || tb[IFLA_NET_NS_PID]))
                goto invalid_attr;

        return 0;

invalid_attr:
        NL_SET_ERR_MSG(extack, "multiple netns identifying attributes specified");
        return -EINVAL;
}

static        int rtnl_set_vf_rate(struct net_device *dev, int vf, int min_tx_rate,
                             int max_tx_rate)
{
        const struct net_device_ops *ops = dev->netdev_ops;

        if (!ops->ndo_set_vf_rate)
                return -EOPNOTSUPP;
        if (max_tx_rate && max_tx_rate < min_tx_rate)
                return -EINVAL;

        return ops->ndo_set_vf_rate(dev, vf, min_tx_rate, max_tx_rate);
}

static int validate_linkmsg(struct net_device *dev, struct nlattr *tb[],
                            struct netlink_ext_ack *extack)
{
        if (tb[IFLA_ADDRESS] &&
            nla_len(tb[IFLA_ADDRESS]) < dev->addr_len)
                return -EINVAL;

        if (tb[IFLA_BROADCAST] &&
            nla_len(tb[IFLA_BROADCAST]) < dev->addr_len)
                return -EINVAL;

        if (tb[IFLA_GSO_MAX_SIZE] &&
            nla_get_u32(tb[IFLA_GSO_MAX_SIZE]) > dev->tso_max_size) {
                NL_SET_ERR_MSG(extack, "too big gso_max_size");
                return -EINVAL;
        }

        if (tb[IFLA_GSO_MAX_SEGS] &&
            (nla_get_u32(tb[IFLA_GSO_MAX_SEGS]) > GSO_MAX_SEGS ||
             nla_get_u32(tb[IFLA_GSO_MAX_SEGS]) > dev->tso_max_segs)) {
                NL_SET_ERR_MSG(extack, "too big gso_max_segs");
                return -EINVAL;
        }

        if (tb[IFLA_GRO_MAX_SIZE] &&
            nla_get_u32(tb[IFLA_GRO_MAX_SIZE]) > GRO_MAX_SIZE) {
                NL_SET_ERR_MSG(extack, "too big gro_max_size");
                return -EINVAL;
        }

        if (tb[IFLA_GSO_IPV4_MAX_SIZE] &&
            nla_get_u32(tb[IFLA_GSO_IPV4_MAX_SIZE]) > dev->tso_max_size) {
                NL_SET_ERR_MSG(extack, "too big gso_ipv4_max_size");
                return -EINVAL;
        }

        if (tb[IFLA_GRO_IPV4_MAX_SIZE] &&
            nla_get_u32(tb[IFLA_GRO_IPV4_MAX_SIZE]) > GRO_MAX_SIZE) {
                NL_SET_ERR_MSG(extack, "too big gro_ipv4_max_size");
                return -EINVAL;
        }

        if (tb[IFLA_AF_SPEC]) {
                struct nlattr *af;
                int rem, err;

                nla_for_each_nested(af, tb[IFLA_AF_SPEC], rem) {
                        struct rtnl_af_ops *af_ops;
                        int af_ops_srcu_index;

                        af_ops = rtnl_af_lookup(nla_type(af), &af_ops_srcu_index);
                        if (!af_ops)
                                return -EAFNOSUPPORT;

                        if (!af_ops->set_link_af)
                                err = -EOPNOTSUPP;
                        else if (af_ops->validate_link_af)
                                err = af_ops->validate_link_af(dev, af, extack);
                        else
                                err = 0;

                        rtnl_af_put(af_ops, af_ops_srcu_index);

                        if (err < 0)
                                return err;
                }
        }

        return 0;
}

static int handle_infiniband_guid(struct net_device *dev, struct ifla_vf_guid *ivt,
                                  int guid_type)
{
        const struct net_device_ops *ops = dev->netdev_ops;

        return ops->ndo_set_vf_guid(dev, ivt->vf, ivt->guid, guid_type);
}

static int handle_vf_guid(struct net_device *dev, struct ifla_vf_guid *ivt, int guid_type)
{
        if (dev->type != ARPHRD_INFINIBAND)
                return -EOPNOTSUPP;

        return handle_infiniband_guid(dev, ivt, guid_type);
}

static int do_setvfinfo(struct net_device *dev, struct nlattr **tb)
{
        const struct net_device_ops *ops = dev->netdev_ops;
        int err = -EINVAL;

        if (tb[IFLA_VF_MAC]) {
                struct ifla_vf_mac *ivm = nla_data(tb[IFLA_VF_MAC]);

                if (ivm->vf >= INT_MAX)
                        return -EINVAL;
                err = -EOPNOTSUPP;
                if (ops->ndo_set_vf_mac)
                        err = ops->ndo_set_vf_mac(dev, ivm->vf,
                                                  ivm->mac);
                if (err < 0)
                        return err;
        }

        if (tb[IFLA_VF_VLAN]) {
                struct ifla_vf_vlan *ivv = nla_data(tb[IFLA_VF_VLAN]);

                if (ivv->vf >= INT_MAX)
                        return -EINVAL;
                err = -EOPNOTSUPP;
                if (ops->ndo_set_vf_vlan)
                        err = ops->ndo_set_vf_vlan(dev, ivv->vf, ivv->vlan,
                                                   ivv->qos,
                                                   htons(ETH_P_8021Q));
                if (err < 0)
                        return err;
        }

        if (tb[IFLA_VF_VLAN_LIST]) {
                struct ifla_vf_vlan_info *ivvl[MAX_VLAN_LIST_LEN];
                struct nlattr *attr;
                int rem, len = 0;

                err = -EOPNOTSUPP;
                if (!ops->ndo_set_vf_vlan)
                        return err;

                nla_for_each_nested(attr, tb[IFLA_VF_VLAN_LIST], rem) {
                        if (nla_type(attr) != IFLA_VF_VLAN_INFO ||
                            nla_len(attr) < sizeof(struct ifla_vf_vlan_info)) {
                                return -EINVAL;
                        }
                        if (len >= MAX_VLAN_LIST_LEN)
                                return -EOPNOTSUPP;
                        ivvl[len] = nla_data(attr);

                        len++;
                }
                if (len == 0)
                        return -EINVAL;

                if (ivvl[0]->vf >= INT_MAX)
                        return -EINVAL;
                err = ops->ndo_set_vf_vlan(dev, ivvl[0]->vf, ivvl[0]->vlan,
                                           ivvl[0]->qos, ivvl[0]->vlan_proto);
                if (err < 0)
                        return err;
        }

        if (tb[IFLA_VF_TX_RATE]) {
                struct ifla_vf_tx_rate *ivt = nla_data(tb[IFLA_VF_TX_RATE]);
                struct ifla_vf_info ivf;

                if (ivt->vf >= INT_MAX)
                        return -EINVAL;
                err = -EOPNOTSUPP;
                if (ops->ndo_get_vf_config)
                        err = ops->ndo_get_vf_config(dev, ivt->vf, &ivf);
                if (err < 0)
                        return err;

                err = rtnl_set_vf_rate(dev, ivt->vf,
                                       ivf.min_tx_rate, ivt->rate);
                if (err < 0)
                        return err;
        }

        if (tb[IFLA_VF_RATE]) {
                struct ifla_vf_rate *ivt = nla_data(tb[IFLA_VF_RATE]);

                if (ivt->vf >= INT_MAX)
                        return -EINVAL;

                err = rtnl_set_vf_rate(dev, ivt->vf,
                                       ivt->min_tx_rate, ivt->max_tx_rate);
                if (err < 0)
                        return err;
        }

        if (tb[IFLA_VF_SPOOFCHK]) {
                struct ifla_vf_spoofchk *ivs = nla_data(tb[IFLA_VF_SPOOFCHK]);

                if (ivs->vf >= INT_MAX)
                        return -EINVAL;
                err = -EOPNOTSUPP;
                if (ops->ndo_set_vf_spoofchk)
                        err = ops->ndo_set_vf_spoofchk(dev, ivs->vf,
                                                       ivs->setting);
                if (err < 0)
                        return err;
        }

        if (tb[IFLA_VF_LINK_STATE]) {
                struct ifla_vf_link_state *ivl = nla_data(tb[IFLA_VF_LINK_STATE]);

                if (ivl->vf >= INT_MAX)
                        return -EINVAL;
                err = -EOPNOTSUPP;
                if (ops->ndo_set_vf_link_state)
                        err = ops->ndo_set_vf_link_state(dev, ivl->vf,
                                                         ivl->link_state);
                if (err < 0)
                        return err;
        }

        if (tb[IFLA_VF_RSS_QUERY_EN]) {
                struct ifla_vf_rss_query_en *ivrssq_en;

                err = -EOPNOTSUPP;
                ivrssq_en = nla_data(tb[IFLA_VF_RSS_QUERY_EN]);
                if (ivrssq_en->vf >= INT_MAX)
                        return -EINVAL;
                if (ops->ndo_set_vf_rss_query_en)
                        err = ops->ndo_set_vf_rss_query_en(dev, ivrssq_en->vf,
                                                           ivrssq_en->setting);
                if (err < 0)
                        return err;
        }

        if (tb[IFLA_VF_TRUST]) {
                struct ifla_vf_trust *ivt = nla_data(tb[IFLA_VF_TRUST]);

                if (ivt->vf >= INT_MAX)
                        return -EINVAL;
                err = -EOPNOTSUPP;
                if (ops->ndo_set_vf_trust)
                        err = ops->ndo_set_vf_trust(dev, ivt->vf, ivt->setting);
                if (err < 0)
                        return err;
        }

        if (tb[IFLA_VF_IB_NODE_GUID]) {
                struct ifla_vf_guid *ivt = nla_data(tb[IFLA_VF_IB_NODE_GUID]);

                if (ivt->vf >= INT_MAX)
                        return -EINVAL;
                if (!ops->ndo_set_vf_guid)
                        return -EOPNOTSUPP;
                return handle_vf_guid(dev, ivt, IFLA_VF_IB_NODE_GUID);
        }

        if (tb[IFLA_VF_IB_PORT_GUID]) {
                struct ifla_vf_guid *ivt = nla_data(tb[IFLA_VF_IB_PORT_GUID]);

                if (ivt->vf >= INT_MAX)
                        return -EINVAL;
                if (!ops->ndo_set_vf_guid)
                        return -EOPNOTSUPP;

                return handle_vf_guid(dev, ivt, IFLA_VF_IB_PORT_GUID);
        }

        return err;
}

static int do_set_master(struct net_device *dev, int ifindex,
                         struct netlink_ext_ack *extack)
{
        struct net_device *upper_dev = netdev_master_upper_dev_get(dev);
        const struct net_device_ops *ops;
        int err;

        /* Release the lower lock, the upper is responsible for locking
         * the lower if needed. None of the existing upper devices
         * use netdev instance lock, so don't grab it.
         */

        if (upper_dev) {
                if (upper_dev->ifindex == ifindex)
                        return 0;
                ops = upper_dev->netdev_ops;
                if (ops->ndo_del_slave) {
                        netdev_unlock_ops(dev);
                        err = ops->ndo_del_slave(upper_dev, dev);
                        netdev_lock_ops(dev);
                        if (err)
                                return err;
                } else {
                        return -EOPNOTSUPP;
                }
        }

        if (ifindex) {
                upper_dev = __dev_get_by_index(dev_net(dev), ifindex);
                if (!upper_dev)
                        return -EINVAL;
                ops = upper_dev->netdev_ops;
                if (ops->ndo_add_slave) {
                        netdev_unlock_ops(dev);
                        err = ops->ndo_add_slave(upper_dev, dev, extack);
                        netdev_lock_ops(dev);
                        if (err)
                                return err;
                } else {
                        return -EOPNOTSUPP;
                }
        }
        return 0;
}

static const struct nla_policy ifla_proto_down_reason_policy[IFLA_PROTO_DOWN_REASON_VALUE + 1] = {
        [IFLA_PROTO_DOWN_REASON_MASK]        = { .type = NLA_U32 },
        [IFLA_PROTO_DOWN_REASON_VALUE]        = { .type = NLA_U32 },
};

static int do_set_proto_down(struct net_device *dev,
                             struct nlattr *nl_proto_down,
                             struct nlattr *nl_proto_down_reason,
                             struct netlink_ext_ack *extack)
{
        struct nlattr *pdreason[IFLA_PROTO_DOWN_REASON_MAX + 1];
        unsigned long mask = 0;
        u32 value;
        bool proto_down;
        int err;

        if (!dev->change_proto_down) {
                NL_SET_ERR_MSG(extack,  "Protodown not supported by device");
                return -EOPNOTSUPP;
        }

        if (nl_proto_down_reason) {
                err = nla_parse_nested_deprecated(pdreason,
                                                  IFLA_PROTO_DOWN_REASON_MAX,
                                                  nl_proto_down_reason,
                                                  ifla_proto_down_reason_policy,
                                                  NULL);
                if (err < 0)
                        return err;

                if (!pdreason[IFLA_PROTO_DOWN_REASON_VALUE]) {
                        NL_SET_ERR_MSG(extack, "Invalid protodown reason value");
                        return -EINVAL;
                }

                value = nla_get_u32(pdreason[IFLA_PROTO_DOWN_REASON_VALUE]);

                if (pdreason[IFLA_PROTO_DOWN_REASON_MASK])
                        mask = nla_get_u32(pdreason[IFLA_PROTO_DOWN_REASON_MASK]);

                netdev_change_proto_down_reason_locked(dev, mask, value);
        }

        if (nl_proto_down) {
                proto_down = nla_get_u8(nl_proto_down);

                /* Don't turn off protodown if there are active reasons */
                if (!proto_down && dev->proto_down_reason) {
                        NL_SET_ERR_MSG(extack, "Cannot clear protodown, active reasons");
                        return -EBUSY;
                }
                err = netif_change_proto_down(dev, proto_down);
                if (err)
                        return err;
        }

        return 0;
}

#define DO_SETLINK_MODIFIED        0x01
/* notify flag means notify + modified. */
#define DO_SETLINK_NOTIFY        0x03
static int do_setlink(const struct sk_buff *skb, struct net_device *dev,
                      struct net *tgt_net, struct ifinfomsg *ifm,
                      struct netlink_ext_ack *extack,
                      struct nlattr **tb, int status)
{
        const struct net_device_ops *ops = dev->netdev_ops;
        char ifname[IFNAMSIZ];
        int err;

        err = validate_linkmsg(dev, tb, extack);
        if (err < 0)
                return err;

        if (tb[IFLA_IFNAME])
                nla_strscpy(ifname, tb[IFLA_IFNAME], IFNAMSIZ);
        else
                ifname[0] = '\0';

        if (!net_eq(tgt_net, dev_net(dev))) {
                const char *pat = ifname[0] ? ifname : NULL;
                int new_ifindex;

                new_ifindex = nla_get_s32_default(tb[IFLA_NEW_IFINDEX], 0);

                err = __dev_change_net_namespace(dev, tgt_net, pat,
                                                 new_ifindex, extack);
                if (err)
                        return err;

                status |= DO_SETLINK_MODIFIED;
        }

        netdev_lock_ops(dev);

        if (tb[IFLA_MAP]) {
                struct rtnl_link_ifmap *u_map;
                struct ifmap k_map;

                if (!ops->ndo_set_config) {
                        err = -EOPNOTSUPP;
                        goto errout;
                }

                if (!netif_device_present(dev)) {
                        err = -ENODEV;
                        goto errout;
                }

                u_map = nla_data(tb[IFLA_MAP]);
                k_map.mem_start = (unsigned long) u_map->mem_start;
                k_map.mem_end = (unsigned long) u_map->mem_end;
                k_map.base_addr = (unsigned short) u_map->base_addr;
                k_map.irq = (unsigned char) u_map->irq;
                k_map.dma = (unsigned char) u_map->dma;
                k_map.port = (unsigned char) u_map->port;

                err = ops->ndo_set_config(dev, &k_map);
                if (err < 0)
                        goto errout;

                status |= DO_SETLINK_NOTIFY;
        }

        if (tb[IFLA_ADDRESS]) {
                struct sockaddr_storage ss = { };

                netdev_unlock_ops(dev);

                /* dev_addr_sem is an outer lock, enforce proper ordering */
                down_write(&dev_addr_sem);
                netdev_lock_ops(dev);

                ss.ss_family = dev->type;
                memcpy(ss.__data, nla_data(tb[IFLA_ADDRESS]), dev->addr_len);
                err = netif_set_mac_address(dev, &ss, extack);
                if (err) {
                        up_write(&dev_addr_sem);
                        goto errout;
                }
                status |= DO_SETLINK_MODIFIED;

                up_write(&dev_addr_sem);
        }

        if (tb[IFLA_MTU]) {
                err = netif_set_mtu_ext(dev, nla_get_u32(tb[IFLA_MTU]), extack);
                if (err < 0)
                        goto errout;
                status |= DO_SETLINK_MODIFIED;
        }

        if (tb[IFLA_GROUP]) {
                netif_set_group(dev, nla_get_u32(tb[IFLA_GROUP]));
                status |= DO_SETLINK_NOTIFY;
        }

        /*
         * Interface selected by interface index but interface
         * name provided implies that a name change has been
         * requested.
         */
        if (ifm->ifi_index > 0 && ifname[0]) {
                err = netif_change_name(dev, ifname);
                if (err < 0)
                        goto errout;
                status |= DO_SETLINK_MODIFIED;
        }

        if (tb[IFLA_IFALIAS]) {
                err = netif_set_alias(dev, nla_data(tb[IFLA_IFALIAS]),
                                      nla_len(tb[IFLA_IFALIAS]));
                if (err < 0)
                        goto errout;
                status |= DO_SETLINK_NOTIFY;
        }

        if (tb[IFLA_BROADCAST]) {
                nla_memcpy(dev->broadcast, tb[IFLA_BROADCAST], dev->addr_len);
                call_netdevice_notifiers(NETDEV_CHANGEADDR, dev);
        }

        if (ifm->ifi_flags || ifm->ifi_change) {
                err = netif_change_flags(dev, rtnl_dev_combine_flags(dev, ifm),
                                         extack);
                if (err < 0)
                        goto errout;
        }

        if (tb[IFLA_MASTER]) {
                err = do_set_master(dev, nla_get_u32(tb[IFLA_MASTER]), extack);
                if (err)
                        goto errout;
                status |= DO_SETLINK_MODIFIED;
        }

        if (tb[IFLA_CARRIER]) {
                err = netif_change_carrier(dev, nla_get_u8(tb[IFLA_CARRIER]));
                if (err)
                        goto errout;
                status |= DO_SETLINK_MODIFIED;
        }

        if (tb[IFLA_TXQLEN]) {
                unsigned int value = nla_get_u32(tb[IFLA_TXQLEN]);

                err = netif_change_tx_queue_len(dev, value);
                if (err)
                        goto errout;
                status |= DO_SETLINK_MODIFIED;
        }

        if (tb[IFLA_GSO_MAX_SIZE]) {
                u32 max_size = nla_get_u32(tb[IFLA_GSO_MAX_SIZE]);

                if (dev->gso_max_size ^ max_size) {
                        netif_set_gso_max_size(dev, max_size);
                        status |= DO_SETLINK_MODIFIED;
                }
        }

        if (tb[IFLA_GSO_MAX_SEGS]) {
                u32 max_segs = nla_get_u32(tb[IFLA_GSO_MAX_SEGS]);

                if (dev->gso_max_segs ^ max_segs) {
                        netif_set_gso_max_segs(dev, max_segs);
                        status |= DO_SETLINK_MODIFIED;
                }
        }

        if (tb[IFLA_GRO_MAX_SIZE]) {
                u32 gro_max_size = nla_get_u32(tb[IFLA_GRO_MAX_SIZE]);

                if (dev->gro_max_size ^ gro_max_size) {
                        netif_set_gro_max_size(dev, gro_max_size);
                        status |= DO_SETLINK_MODIFIED;
                }
        }

        if (tb[IFLA_GSO_IPV4_MAX_SIZE]) {
                u32 max_size = nla_get_u32(tb[IFLA_GSO_IPV4_MAX_SIZE]);

                if (dev->gso_ipv4_max_size ^ max_size) {
                        netif_set_gso_ipv4_max_size(dev, max_size);
                        status |= DO_SETLINK_MODIFIED;
                }
        }

        if (tb[IFLA_GRO_IPV4_MAX_SIZE]) {
                u32 gro_max_size = nla_get_u32(tb[IFLA_GRO_IPV4_MAX_SIZE]);

                if (dev->gro_ipv4_max_size ^ gro_max_size) {
                        netif_set_gro_ipv4_max_size(dev, gro_max_size);
                        status |= DO_SETLINK_MODIFIED;
                }
        }

        if (tb[IFLA_OPERSTATE])
                set_operstate(dev, nla_get_u8(tb[IFLA_OPERSTATE]));

        if (tb[IFLA_LINKMODE]) {
                unsigned char value = nla_get_u8(tb[IFLA_LINKMODE]);

                if (dev->link_mode ^ value)
                        status |= DO_SETLINK_NOTIFY;
                WRITE_ONCE(dev->link_mode, value);
        }

        if (tb[IFLA_VFINFO_LIST]) {
                struct nlattr *vfinfo[IFLA_VF_MAX + 1];
                struct nlattr *attr;
                int rem;

                nla_for_each_nested(attr, tb[IFLA_VFINFO_LIST], rem) {
                        if (nla_type(attr) != IFLA_VF_INFO ||
                            nla_len(attr) < NLA_HDRLEN) {
                                err = -EINVAL;
                                goto errout;
                        }
                        err = nla_parse_nested_deprecated(vfinfo, IFLA_VF_MAX,
                                                          attr,
                                                          ifla_vf_policy,
                                                          NULL);
                        if (err < 0)
                                goto errout;
                        err = do_setvfinfo(dev, vfinfo);
                        if (err < 0)
                                goto errout;
                        status |= DO_SETLINK_NOTIFY;
                }
        }
        err = 0;

        if (tb[IFLA_VF_PORTS]) {
                struct nlattr *port[IFLA_PORT_MAX+1];
                struct nlattr *attr;
                int vf;
                int rem;

                err = -EOPNOTSUPP;
                if (!ops->ndo_set_vf_port)
                        goto errout;

                nla_for_each_nested(attr, tb[IFLA_VF_PORTS], rem) {
                        if (nla_type(attr) != IFLA_VF_PORT ||
                            nla_len(attr) < NLA_HDRLEN) {
                                err = -EINVAL;
                                goto errout;
                        }
                        err = nla_parse_nested_deprecated(port, IFLA_PORT_MAX,
                                                          attr,
                                                          ifla_port_policy,
                                                          NULL);
                        if (err < 0)
                                goto errout;
                        if (!port[IFLA_PORT_VF]) {
                                err = -EOPNOTSUPP;
                                goto errout;
                        }
                        vf = nla_get_u32(port[IFLA_PORT_VF]);
                        err = ops->ndo_set_vf_port(dev, vf, port);
                        if (err < 0)
                                goto errout;
                        status |= DO_SETLINK_NOTIFY;
                }
        }
        err = 0;

        if (tb[IFLA_PORT_SELF]) {
                struct nlattr *port[IFLA_PORT_MAX+1];

                err = nla_parse_nested_deprecated(port, IFLA_PORT_MAX,
                                                  tb[IFLA_PORT_SELF],
                                                  ifla_port_policy, NULL);
                if (err < 0)
                        goto errout;

                err = -EOPNOTSUPP;
                if (ops->ndo_set_vf_port)
                        err = ops->ndo_set_vf_port(dev, PORT_SELF_VF, port);
                if (err < 0)
                        goto errout;
                status |= DO_SETLINK_NOTIFY;
        }

        if (tb[IFLA_AF_SPEC]) {
                struct nlattr *af;
                int rem;

                nla_for_each_nested(af, tb[IFLA_AF_SPEC], rem) {
                        struct rtnl_af_ops *af_ops;
                        int af_ops_srcu_index;

                        af_ops = rtnl_af_lookup(nla_type(af), &af_ops_srcu_index);
                        if (!af_ops) {
                                err = -EAFNOSUPPORT;
                                goto errout;
                        }

                        err = af_ops->set_link_af(dev, af, extack);
                        rtnl_af_put(af_ops, af_ops_srcu_index);

                        if (err < 0)
                                goto errout;

                        status |= DO_SETLINK_NOTIFY;
                }
        }
        err = 0;

        if (tb[IFLA_PROTO_DOWN] || tb[IFLA_PROTO_DOWN_REASON]) {
                err = do_set_proto_down(dev, tb[IFLA_PROTO_DOWN],
                                        tb[IFLA_PROTO_DOWN_REASON], extack);
                if (err)
                        goto errout;
                status |= DO_SETLINK_NOTIFY;
        }

        if (tb[IFLA_XDP]) {
                struct nlattr *xdp[IFLA_XDP_MAX + 1];
                u32 xdp_flags = 0;

                err = nla_parse_nested_deprecated(xdp, IFLA_XDP_MAX,
                                                  tb[IFLA_XDP],
                                                  ifla_xdp_policy, NULL);
                if (err < 0)
                        goto errout;

                if (xdp[IFLA_XDP_ATTACHED] || xdp[IFLA_XDP_PROG_ID]) {
                        err = -EINVAL;
                        goto errout;
                }

                if (xdp[IFLA_XDP_FLAGS]) {
                        xdp_flags = nla_get_u32(xdp[IFLA_XDP_FLAGS]);
                        if (xdp_flags & ~XDP_FLAGS_MASK) {
                                err = -EINVAL;
                                goto errout;
                        }
                        if (hweight32(xdp_flags & XDP_FLAGS_MODES) > 1) {
                                err = -EINVAL;
                                goto errout;
                        }
                }

                if (xdp[IFLA_XDP_FD]) {
                        int expected_fd = -1;

                        if (xdp_flags & XDP_FLAGS_REPLACE) {
                                if (!xdp[IFLA_XDP_EXPECTED_FD]) {
                                        err = -EINVAL;
                                        goto errout;
                                }
                                expected_fd =
                                        nla_get_s32(xdp[IFLA_XDP_EXPECTED_FD]);
                        }

                        err = dev_change_xdp_fd(dev, extack,
                                                nla_get_s32(xdp[IFLA_XDP_FD]),
                                                expected_fd,
                                                xdp_flags);
                        if (err)
                                goto errout;
                        status |= DO_SETLINK_NOTIFY;
                }
        }

errout:
        if (status & DO_SETLINK_MODIFIED) {
                if ((status & DO_SETLINK_NOTIFY) == DO_SETLINK_NOTIFY)
                        netif_state_change(dev);

                if (err < 0)
                        net_warn_ratelimited("A link change request failed with some changes committed already. Interface %s may have been left with an inconsistent configuration, please check.\n",
                                             dev->name);
        }

        netdev_unlock_ops(dev);

        return err;
}

static struct net_device *rtnl_dev_get(struct net *net,
                                       struct nlattr *tb[])
{
        char ifname[ALTIFNAMSIZ];

        if (tb[IFLA_IFNAME])
                nla_strscpy(ifname, tb[IFLA_IFNAME], IFNAMSIZ);
        else if (tb[IFLA_ALT_IFNAME])
                nla_strscpy(ifname, tb[IFLA_ALT_IFNAME], ALTIFNAMSIZ);
        else
                return NULL;

        return __dev_get_by_name(net, ifname);
}

static int rtnl_setlink(struct sk_buff *skb, struct nlmsghdr *nlh,
                        struct netlink_ext_ack *extack)
{
        struct ifinfomsg *ifm = nlmsg_data(nlh);
        struct net *net = sock_net(skb->sk);
        struct nlattr *tb[IFLA_MAX+1];
        struct net_device *dev = NULL;
        struct rtnl_nets rtnl_nets;
        struct net *tgt_net;
        int err;

        err = nlmsg_parse_deprecated(nlh, sizeof(*ifm), tb, IFLA_MAX,
                                     ifla_policy, extack);
        if (err < 0)
                goto errout;

        err = rtnl_ensure_unique_netns(tb, extack, false);
        if (err < 0)
                goto errout;

        tgt_net = rtnl_link_get_net_capable(skb, net, tb, CAP_NET_ADMIN);
        if (IS_ERR(tgt_net)) {
                err = PTR_ERR(tgt_net);
                goto errout;
        }

        rtnl_nets_init(&rtnl_nets);
        rtnl_nets_add(&rtnl_nets, get_net(net));
        rtnl_nets_add(&rtnl_nets, tgt_net);

        rtnl_nets_lock(&rtnl_nets);

        if (ifm->ifi_index > 0)
                dev = __dev_get_by_index(net, ifm->ifi_index);
        else if (tb[IFLA_IFNAME] || tb[IFLA_ALT_IFNAME])
                dev = rtnl_dev_get(net, tb);
        else
                err = -EINVAL;

        if (dev)
                err = do_setlink(skb, dev, tgt_net, ifm, extack, tb, 0);
        else if (!err)
                err = -ENODEV;

        rtnl_nets_unlock(&rtnl_nets);
        rtnl_nets_destroy(&rtnl_nets);
errout:
        return err;
}

static int rtnl_group_dellink(const struct net *net, int group)
{
        struct net_device *dev, *aux;
        LIST_HEAD(list_kill);
        bool found = false;

        if (!group)
                return -EPERM;

        for_each_netdev(net, dev) {
                if (dev->group == group) {
                        const struct rtnl_link_ops *ops;

                        found = true;
                        ops = dev->rtnl_link_ops;
                        if (!ops || !ops->dellink)
                                return -EOPNOTSUPP;
                }
        }

        if (!found)
                return -ENODEV;

        for_each_netdev_safe(net, dev, aux) {
                if (dev->group == group) {
                        const struct rtnl_link_ops *ops;

                        ops = dev->rtnl_link_ops;
                        ops->dellink(dev, &list_kill);
                }
        }
        unregister_netdevice_many(&list_kill);

        return 0;
}

int rtnl_delete_link(struct net_device *dev, u32 portid, const struct nlmsghdr *nlh)
{
        const struct rtnl_link_ops *ops;
        LIST_HEAD(list_kill);

        ops = dev->rtnl_link_ops;
        if (!ops || !ops->dellink)
                return -EOPNOTSUPP;

        ops->dellink(dev, &list_kill);
        unregister_netdevice_many_notify(&list_kill, portid, nlh);

        return 0;
}
EXPORT_SYMBOL_GPL(rtnl_delete_link);

static int rtnl_dellink(struct sk_buff *skb, struct nlmsghdr *nlh,
                        struct netlink_ext_ack *extack)
{
        struct ifinfomsg *ifm = nlmsg_data(nlh);
        struct net *net = sock_net(skb->sk);
        u32 portid = NETLINK_CB(skb).portid;
        struct nlattr *tb[IFLA_MAX+1];
        struct net_device *dev = NULL;
        struct net *tgt_net = net;
        int netnsid = -1;
        int err;

        err = nlmsg_parse_deprecated(nlh, sizeof(*ifm), tb, IFLA_MAX,
                                     ifla_policy, extack);
        if (err < 0)
                return err;

        err = rtnl_ensure_unique_netns(tb, extack, true);
        if (err < 0)
                return err;

        if (tb[IFLA_TARGET_NETNSID]) {
                netnsid = nla_get_s32(tb[IFLA_TARGET_NETNSID]);
                tgt_net = rtnl_get_net_ns_capable(NETLINK_CB(skb).sk, netnsid);
                if (IS_ERR(tgt_net))
                        return PTR_ERR(tgt_net);
        }

        rtnl_net_lock(tgt_net);

        if (ifm->ifi_index > 0)
                dev = __dev_get_by_index(tgt_net, ifm->ifi_index);
        else if (tb[IFLA_IFNAME] || tb[IFLA_ALT_IFNAME])
                dev = rtnl_dev_get(tgt_net, tb);

        if (dev)
                err = rtnl_delete_link(dev, portid, nlh);
        else if (ifm->ifi_index > 0 || tb[IFLA_IFNAME] || tb[IFLA_ALT_IFNAME])
                err = -ENODEV;
        else if (tb[IFLA_GROUP])
                err = rtnl_group_dellink(tgt_net, nla_get_u32(tb[IFLA_GROUP]));
        else
                err = -EINVAL;

        rtnl_net_unlock(tgt_net);

        if (netnsid >= 0)
                put_net(tgt_net);

        return err;
}

int rtnl_configure_link(struct net_device *dev, const struct ifinfomsg *ifm,
                        u32 portid, const struct nlmsghdr *nlh)
{
        unsigned int old_flags, changed;
        int err;

        old_flags = dev->flags;
        if (ifm && (ifm->ifi_flags || ifm->ifi_change)) {
                err = __dev_change_flags(dev, rtnl_dev_combine_flags(dev, ifm),
                                         NULL);
                if (err < 0)
                        return err;
        }

        changed = old_flags ^ dev->flags;
        if (dev->rtnl_link_initializing) {
                dev->rtnl_link_initializing = false;
                changed = ~0U;
        }

        __dev_notify_flags(dev, old_flags, changed, portid, nlh);
        return 0;
}
EXPORT_SYMBOL(rtnl_configure_link);

struct net_device *rtnl_create_link(struct net *net, const char *ifname,
                                    unsigned char name_assign_type,
                                    const struct rtnl_link_ops *ops,
                                    struct nlattr *tb[],
                                    struct netlink_ext_ack *extack)
{
        struct net_device *dev;
        unsigned int num_tx_queues = 1;
        unsigned int num_rx_queues = 1;
        int err;

        if (tb[IFLA_NUM_TX_QUEUES])
                num_tx_queues = nla_get_u32(tb[IFLA_NUM_TX_QUEUES]);
        else if (ops->get_num_tx_queues)
                num_tx_queues = ops->get_num_tx_queues();

        if (tb[IFLA_NUM_RX_QUEUES])
                num_rx_queues = nla_get_u32(tb[IFLA_NUM_RX_QUEUES]);
        else if (ops->get_num_rx_queues)
                num_rx_queues = ops->get_num_rx_queues();

        if (num_tx_queues < 1 || num_tx_queues > 4096) {
                NL_SET_ERR_MSG(extack, "Invalid number of transmit queues");
                return ERR_PTR(-EINVAL);
        }

        if (num_rx_queues < 1 || num_rx_queues > 4096) {
                NL_SET_ERR_MSG(extack, "Invalid number of receive queues");
                return ERR_PTR(-EINVAL);
        }

        if (ops->alloc) {
                dev = ops->alloc(tb, ifname, name_assign_type,
                                 num_tx_queues, num_rx_queues);
                if (IS_ERR(dev))
                        return dev;
        } else {
                dev = alloc_netdev_mqs(ops->priv_size, ifname,
                                       name_assign_type, ops->setup,
                                       num_tx_queues, num_rx_queues);
        }

        if (!dev)
                return ERR_PTR(-ENOMEM);

        err = validate_linkmsg(dev, tb, extack);
        if (err < 0) {
                free_netdev(dev);
                return ERR_PTR(err);
        }

        dev_net_set(dev, net);
        dev->rtnl_link_ops = ops;
        dev->rtnl_link_initializing = true;

        if (tb[IFLA_MTU]) {
                u32 mtu = nla_get_u32(tb[IFLA_MTU]);

                err = dev_validate_mtu(dev, mtu, extack);
                if (err) {
                        free_netdev(dev);
                        return ERR_PTR(err);
                }
                dev->mtu = mtu;
        }
        if (tb[IFLA_ADDRESS]) {
                __dev_addr_set(dev, nla_data(tb[IFLA_ADDRESS]),
                               nla_len(tb[IFLA_ADDRESS]));
                dev->addr_assign_type = NET_ADDR_SET;
        }
        if (tb[IFLA_BROADCAST])
                memcpy(dev->broadcast, nla_data(tb[IFLA_BROADCAST]),
                                nla_len(tb[IFLA_BROADCAST]));
        if (tb[IFLA_TXQLEN])
                dev->tx_queue_len = nla_get_u32(tb[IFLA_TXQLEN]);
        if (tb[IFLA_OPERSTATE])
                set_operstate(dev, nla_get_u8(tb[IFLA_OPERSTATE]));
        if (tb[IFLA_LINKMODE])
                dev->link_mode = nla_get_u8(tb[IFLA_LINKMODE]);
        if (tb[IFLA_GROUP])
                netif_set_group(dev, nla_get_u32(tb[IFLA_GROUP]));
        if (tb[IFLA_GSO_MAX_SIZE])
                netif_set_gso_max_size(dev, nla_get_u32(tb[IFLA_GSO_MAX_SIZE]));
        if (tb[IFLA_GSO_MAX_SEGS])
                netif_set_gso_max_segs(dev, nla_get_u32(tb[IFLA_GSO_MAX_SEGS]));
        if (tb[IFLA_GRO_MAX_SIZE])
                netif_set_gro_max_size(dev, nla_get_u32(tb[IFLA_GRO_MAX_SIZE]));
        if (tb[IFLA_GSO_IPV4_MAX_SIZE])
                netif_set_gso_ipv4_max_size(dev, nla_get_u32(tb[IFLA_GSO_IPV4_MAX_SIZE]));
        if (tb[IFLA_GRO_IPV4_MAX_SIZE])
                netif_set_gro_ipv4_max_size(dev, nla_get_u32(tb[IFLA_GRO_IPV4_MAX_SIZE]));

        return dev;
}
EXPORT_SYMBOL(rtnl_create_link);

struct rtnl_newlink_tbs {
        struct nlattr *tb[IFLA_MAX + 1];
        struct nlattr *linkinfo[IFLA_INFO_MAX + 1];
        struct nlattr *attr[RTNL_MAX_TYPE + 1];
        struct nlattr *slave_attr[RTNL_SLAVE_MAX_TYPE + 1];
};

static int rtnl_changelink(const struct sk_buff *skb, struct nlmsghdr *nlh,
                           const struct rtnl_link_ops *ops,
                           struct net_device *dev, struct net *tgt_net,
                           struct rtnl_newlink_tbs *tbs,
                           struct nlattr **data,
                           struct netlink_ext_ack *extack)
{
        struct nlattr ** const linkinfo = tbs->linkinfo;
        struct nlattr ** const tb = tbs->tb;
        int status = 0;
        int err;

        if (nlh->nlmsg_flags & NLM_F_EXCL)
                return -EEXIST;

        if (nlh->nlmsg_flags & NLM_F_REPLACE)
                return -EOPNOTSUPP;

        if (linkinfo[IFLA_INFO_DATA]) {
                if (!ops || ops != dev->rtnl_link_ops || !ops->changelink)
                        return -EOPNOTSUPP;

                err = ops->changelink(dev, tb, data, extack);
                if (err < 0)
                        return err;

                status |= DO_SETLINK_NOTIFY;
        }

        if (linkinfo[IFLA_INFO_SLAVE_DATA]) {
                const struct rtnl_link_ops *m_ops = NULL;
                struct nlattr **slave_data = NULL;
                struct net_device *master_dev;

                master_dev = netdev_master_upper_dev_get(dev);
                if (master_dev)
                        m_ops = master_dev->rtnl_link_ops;

                if (!m_ops || !m_ops->slave_changelink)
                        return -EOPNOTSUPP;

                if (m_ops->slave_maxtype > RTNL_SLAVE_MAX_TYPE)
                        return -EINVAL;

                if (m_ops->slave_maxtype) {
                        err = nla_parse_nested_deprecated(tbs->slave_attr,
                                                          m_ops->slave_maxtype,
                                                          linkinfo[IFLA_INFO_SLAVE_DATA],
                                                          m_ops->slave_policy, extack);
                        if (err < 0)
                                return err;

                        slave_data = tbs->slave_attr;
                }

                err = m_ops->slave_changelink(master_dev, dev, tb, slave_data, extack);
                if (err < 0)
                        return err;

                status |= DO_SETLINK_NOTIFY;
        }

        return do_setlink(skb, dev, tgt_net, nlmsg_data(nlh), extack, tb, status);
}

static int rtnl_group_changelink(const struct sk_buff *skb,
                                 struct net *net, struct net *tgt_net,
                                 int group, struct ifinfomsg *ifm,
                                 struct netlink_ext_ack *extack,
                                 struct nlattr **tb)
{
        struct net_device *dev, *aux;
        int err;

        for_each_netdev_safe(net, dev, aux) {
                if (dev->group == group) {
                        err = do_setlink(skb, dev, tgt_net, ifm, extack, tb, 0);
                        if (err < 0)
                                return err;
                }
        }

        return 0;
}

static int rtnl_newlink_create(struct sk_buff *skb, struct ifinfomsg *ifm,
                               const struct rtnl_link_ops *ops,
                               struct net *tgt_net, struct net *link_net,
                               struct net *peer_net,
                               const struct nlmsghdr *nlh,
                               struct nlattr **tb, struct nlattr **data,
                               struct netlink_ext_ack *extack)
{
        unsigned char name_assign_type = NET_NAME_USER;
        struct rtnl_newlink_params params = {
                .src_net = sock_net(skb->sk),
                .link_net = link_net,
                .peer_net = peer_net,
                .tb = tb,
                .data = data,
        };
        u32 portid = NETLINK_CB(skb).portid;
        struct net_device *dev;
        char ifname[IFNAMSIZ];
        int err;

        if (!ops->alloc && !ops->setup)
                return -EOPNOTSUPP;

        if (tb[IFLA_IFNAME]) {
                nla_strscpy(ifname, tb[IFLA_IFNAME], IFNAMSIZ);
        } else {
                snprintf(ifname, IFNAMSIZ, "%s%%d", ops->kind);
                name_assign_type = NET_NAME_ENUM;
        }

        dev = rtnl_create_link(tgt_net, ifname, name_assign_type, ops, tb,
                               extack);
        if (IS_ERR(dev)) {
                err = PTR_ERR(dev);
                goto out;
        }

        dev->ifindex = ifm->ifi_index;

        if (ops->newlink)
                err = ops->newlink(dev, &params, extack);
        else
                err = register_netdevice(dev);
        if (err < 0) {
                free_netdev(dev);
                goto out;
        }

        netdev_lock_ops(dev);

        err = rtnl_configure_link(dev, ifm, portid, nlh);
        if (err < 0)
                goto out_unregister;
        if (tb[IFLA_MASTER]) {
                err = do_set_master(dev, nla_get_u32(tb[IFLA_MASTER]), extack);
                if (err)
                        goto out_unregister;
        }

        netdev_unlock_ops(dev);
out:
        return err;
out_unregister:
        netdev_unlock_ops(dev);
        if (ops->newlink) {
                LIST_HEAD(list_kill);

                ops->dellink(dev, &list_kill);
                unregister_netdevice_many(&list_kill);
        } else {
                unregister_netdevice(dev);
        }
        goto out;
}

static struct net *rtnl_get_peer_net(const struct rtnl_link_ops *ops,
                                     struct nlattr *tbp[],
                                     struct nlattr *data[],
                                     struct netlink_ext_ack *extack)
{
        struct nlattr *tb[IFLA_MAX + 1];
        int err;

        if (!data || !data[ops->peer_type])
                return rtnl_link_get_net_ifla(tbp);

        err = rtnl_nla_parse_ifinfomsg(tb, data[ops->peer_type], extack);
        if (err < 0)
                return ERR_PTR(err);

        if (ops->validate) {
                err = ops->validate(tb, NULL, extack);
                if (err < 0)
                        return ERR_PTR(err);
        }

        return rtnl_link_get_net_ifla(tb);
}

static int __rtnl_newlink(struct sk_buff *skb, struct nlmsghdr *nlh,
                          const struct rtnl_link_ops *ops,
                          struct net *tgt_net, struct net *link_net,
                          struct net *peer_net,
                          struct rtnl_newlink_tbs *tbs,
                          struct nlattr **data,
                          struct netlink_ext_ack *extack)
{
        struct nlattr ** const tb = tbs->tb;
        struct net *net = sock_net(skb->sk);
        struct net *device_net;
        struct net_device *dev;
        struct ifinfomsg *ifm;
        bool link_specified;

        /* When creating, lookup for existing device in target net namespace */
        device_net = (nlh->nlmsg_flags & NLM_F_CREATE) &&
                     (nlh->nlmsg_flags & NLM_F_EXCL) ?
                     tgt_net : net;

        ifm = nlmsg_data(nlh);
        if (ifm->ifi_index > 0) {
                link_specified = true;
                dev = __dev_get_by_index(device_net, ifm->ifi_index);
        } else if (ifm->ifi_index < 0) {
                NL_SET_ERR_MSG(extack, "ifindex can't be negative");
                return -EINVAL;
        } else if (tb[IFLA_IFNAME] || tb[IFLA_ALT_IFNAME]) {
                link_specified = true;
                dev = rtnl_dev_get(device_net, tb);
        } else {
                link_specified = false;
                dev = NULL;
        }

        if (dev)
                return rtnl_changelink(skb, nlh, ops, dev, tgt_net, tbs, data, extack);

        if (!(nlh->nlmsg_flags & NLM_F_CREATE)) {
                /* No dev found and NLM_F_CREATE not set. Requested dev does not exist,
                 * or it's for a group
                */
                if (link_specified || !tb[IFLA_GROUP])
                        return -ENODEV;

                return rtnl_group_changelink(skb, net, tgt_net,
                                             nla_get_u32(tb[IFLA_GROUP]),
                                             ifm, extack, tb);
        }

        if (tb[IFLA_MAP] || tb[IFLA_PROTINFO])
                return -EOPNOTSUPP;

        if (!ops) {
                NL_SET_ERR_MSG(extack, "Unknown device type");
                return -EOPNOTSUPP;
        }

        return rtnl_newlink_create(skb, ifm, ops, tgt_net, link_net, peer_net, nlh,
                                   tb, data, extack);
}

static int rtnl_newlink(struct sk_buff *skb, struct nlmsghdr *nlh,
                        struct netlink_ext_ack *extack)
{
        struct net *tgt_net, *link_net = NULL, *peer_net = NULL;
        struct nlattr **tb, **linkinfo, **data = NULL;
        struct rtnl_link_ops *ops = NULL;
        struct rtnl_newlink_tbs *tbs;
        struct rtnl_nets rtnl_nets;
        int ops_srcu_index;
        int ret;

        tbs = kmalloc(sizeof(*tbs), GFP_KERNEL);
        if (!tbs)
                return -ENOMEM;

        tb = tbs->tb;
        ret = nlmsg_parse_deprecated(nlh, sizeof(struct ifinfomsg), tb,
                                     IFLA_MAX, ifla_policy, extack);
        if (ret < 0)
                goto free;

        ret = rtnl_ensure_unique_netns(tb, extack, false);
        if (ret < 0)
                goto free;

        linkinfo = tbs->linkinfo;
        if (tb[IFLA_LINKINFO]) {
                ret = nla_parse_nested_deprecated(linkinfo, IFLA_INFO_MAX,
                                                  tb[IFLA_LINKINFO],
                                                  ifla_info_policy, NULL);
                if (ret < 0)
                        goto free;
        } else {
                memset(linkinfo, 0, sizeof(tbs->linkinfo));
        }

        if (linkinfo[IFLA_INFO_KIND]) {
                char kind[MODULE_NAME_LEN];

                nla_strscpy(kind, linkinfo[IFLA_INFO_KIND], sizeof(kind));
                ops = rtnl_link_ops_get(kind, &ops_srcu_index);
#ifdef CONFIG_MODULES
                if (!ops) {
                        request_module("rtnl-link-%s", kind);
                        ops = rtnl_link_ops_get(kind, &ops_srcu_index);
                }
#endif
        }

        rtnl_nets_init(&rtnl_nets);

        if (ops) {
                if (ops->maxtype > RTNL_MAX_TYPE) {
                        ret = -EINVAL;
                        goto put_ops;
                }

                if (ops->maxtype && linkinfo[IFLA_INFO_DATA]) {
                        ret = nla_parse_nested_deprecated(tbs->attr, ops->maxtype,
                                                          linkinfo[IFLA_INFO_DATA],
                                                          ops->policy, extack);
                        if (ret < 0)
                                goto put_ops;

                        data = tbs->attr;
                }

                if (ops->validate) {
                        ret = ops->validate(tb, data, extack);
                        if (ret < 0)
                                goto put_ops;
                }

                if (ops->peer_type) {
                        peer_net = rtnl_get_peer_net(ops, tb, data, extack);
                        if (IS_ERR(peer_net)) {
                                ret = PTR_ERR(peer_net);
                                goto put_ops;
                        }
                        if (peer_net)
                                rtnl_nets_add(&rtnl_nets, peer_net);
                }
        }

        tgt_net = rtnl_link_get_net_capable(skb, sock_net(skb->sk), tb, CAP_NET_ADMIN);
        if (IS_ERR(tgt_net)) {
                ret = PTR_ERR(tgt_net);
                goto put_net;
        }

        rtnl_nets_add(&rtnl_nets, tgt_net);

        if (tb[IFLA_LINK_NETNSID]) {
                int id = nla_get_s32(tb[IFLA_LINK_NETNSID]);

                link_net = get_net_ns_by_id(tgt_net, id);
                if (!link_net) {
                        NL_SET_ERR_MSG(extack, "Unknown network namespace id");
                        ret =  -EINVAL;
                        goto put_net;
                }

                rtnl_nets_add(&rtnl_nets, link_net);

                if (!netlink_ns_capable(skb, link_net->user_ns, CAP_NET_ADMIN)) {
                        ret = -EPERM;
                        goto put_net;
                }
        }

        rtnl_nets_lock(&rtnl_nets);
        ret = __rtnl_newlink(skb, nlh, ops, tgt_net, link_net, peer_net, tbs, data, extack);
        rtnl_nets_unlock(&rtnl_nets);

put_net:
        rtnl_nets_destroy(&rtnl_nets);
put_ops:
        if (ops)
                rtnl_link_ops_put(ops, ops_srcu_index);
free:
        kfree(tbs);
        return ret;
}

static int rtnl_valid_getlink_req(struct sk_buff *skb,
                                  const struct nlmsghdr *nlh,
                                  struct nlattr **tb,
                                  struct netlink_ext_ack *extack)
{
        struct ifinfomsg *ifm;
        int i, err;

        ifm = nlmsg_payload(nlh, sizeof(*ifm));
        if (!ifm) {
                NL_SET_ERR_MSG(extack, "Invalid header for get link");
                return -EINVAL;
        }

        if (!netlink_strict_get_check(skb))
                return nlmsg_parse_deprecated(nlh, sizeof(*ifm), tb, IFLA_MAX,
                                              ifla_policy, extack);

        if (ifm->__ifi_pad || ifm->ifi_type || ifm->ifi_flags ||
            ifm->ifi_change) {
                NL_SET_ERR_MSG(extack, "Invalid values in header for get link request");
                return -EINVAL;
        }

        err = nlmsg_parse_deprecated_strict(nlh, sizeof(*ifm), tb, IFLA_MAX,
                                            ifla_policy, extack);
        if (err)
                return err;

        for (i = 0; i <= IFLA_MAX; i++) {
                if (!tb[i])
                        continue;

                switch (i) {
                case IFLA_IFNAME:
                case IFLA_ALT_IFNAME:
                case IFLA_EXT_MASK:
                case IFLA_TARGET_NETNSID:
                        break;
                default:
                        NL_SET_ERR_MSG(extack, "Unsupported attribute in get link request");
                        return -EINVAL;
                }
        }

        return 0;
}

static int rtnl_getlink(struct sk_buff *skb, struct nlmsghdr *nlh,
                        struct netlink_ext_ack *extack)
{
        struct net *net = sock_net(skb->sk);
        struct net *tgt_net = net;
        struct ifinfomsg *ifm;
        struct nlattr *tb[IFLA_MAX+1];
        struct net_device *dev = NULL;
        struct sk_buff *nskb;
        int netnsid = -1;
        int err;
        u32 ext_filter_mask = 0;

        err = rtnl_valid_getlink_req(skb, nlh, tb, extack);
        if (err < 0)
                return err;

        err = rtnl_ensure_unique_netns(tb, extack, true);
        if (err < 0)
                return err;

        if (tb[IFLA_TARGET_NETNSID]) {
                netnsid = nla_get_s32(tb[IFLA_TARGET_NETNSID]);
                tgt_net = rtnl_get_net_ns_capable(NETLINK_CB(skb).sk, netnsid);
                if (IS_ERR(tgt_net))
                        return PTR_ERR(tgt_net);
        }

        if (tb[IFLA_EXT_MASK])
                ext_filter_mask = nla_get_u32(tb[IFLA_EXT_MASK]);

        err = -EINVAL;
        ifm = nlmsg_data(nlh);
        if (ifm->ifi_index > 0)
                dev = __dev_get_by_index(tgt_net, ifm->ifi_index);
        else if (tb[IFLA_IFNAME] || tb[IFLA_ALT_IFNAME])
                dev = rtnl_dev_get(tgt_net, tb);
        else
                goto out;

        err = -ENODEV;
        if (dev == NULL)
                goto out;

        err = -ENOBUFS;
        nskb = nlmsg_new_large(if_nlmsg_size(dev, ext_filter_mask));
        if (nskb == NULL)
                goto out;

        /* Synchronize the carrier state so we don't report a state
         * that we're not actually going to honour immediately; if
         * the driver just did a carrier off->on transition, we can
         * only TX if link watch work has run, but without this we'd
         * already report carrier on, even if it doesn't work yet.
         */
        linkwatch_sync_dev(dev);

        err = rtnl_fill_ifinfo(nskb, dev, net,
                               RTM_NEWLINK, NETLINK_CB(skb).portid,
                               nlh->nlmsg_seq, 0, 0, ext_filter_mask,
                               0, NULL, 0, netnsid, GFP_KERNEL);
        if (err < 0) {
                /* -EMSGSIZE implies BUG in if_nlmsg_size */
                WARN_ON(err == -EMSGSIZE);
                kfree_skb(nskb);
        } else
                err = rtnl_unicast(nskb, net, NETLINK_CB(skb).portid);
out:
        if (netnsid >= 0)
                put_net(tgt_net);

        return err;
}

static int rtnl_alt_ifname(int cmd, struct net_device *dev, struct nlattr *attr,
                           bool *changed, struct netlink_ext_ack *extack)
{
        char *alt_ifname;
        size_t size;
        int err;

        err = nla_validate(attr, attr->nla_len, IFLA_MAX, ifla_policy, extack);
        if (err)
                return err;

        if (cmd == RTM_NEWLINKPROP) {
                size = rtnl_prop_list_size(dev);
                size += nla_total_size(ALTIFNAMSIZ);
                if (size >= U16_MAX) {
                        NL_SET_ERR_MSG(extack,
                                       "effective property list too long");
                        return -EINVAL;
                }
        }

        alt_ifname = nla_strdup(attr, GFP_KERNEL_ACCOUNT);
        if (!alt_ifname)
                return -ENOMEM;

        if (cmd == RTM_NEWLINKPROP) {
                err = netdev_name_node_alt_create(dev, alt_ifname);
                if (!err)
                        alt_ifname = NULL;
        } else if (cmd == RTM_DELLINKPROP) {
                err = netdev_name_node_alt_destroy(dev, alt_ifname);
        } else {
                WARN_ON_ONCE(1);
                err = -EINVAL;
        }

        kfree(alt_ifname);
        if (!err)
                *changed = true;
        return err;
}

static int rtnl_linkprop(int cmd, struct sk_buff *skb, struct nlmsghdr *nlh,
                         struct netlink_ext_ack *extack)
{
        struct net *net = sock_net(skb->sk);
        struct nlattr *tb[IFLA_MAX + 1];
        struct net_device *dev;
        struct ifinfomsg *ifm;
        bool changed = false;
        struct nlattr *attr;
        int err, rem;

        err = nlmsg_parse(nlh, sizeof(*ifm), tb, IFLA_MAX, ifla_policy, extack);
        if (err)
                return err;

        err = rtnl_ensure_unique_netns(tb, extack, true);
        if (err)
                return err;

        ifm = nlmsg_data(nlh);
        if (ifm->ifi_index > 0)
                dev = __dev_get_by_index(net, ifm->ifi_index);
        else if (tb[IFLA_IFNAME] || tb[IFLA_ALT_IFNAME])
                dev = rtnl_dev_get(net, tb);
        else
                return -EINVAL;

        if (!dev)
                return -ENODEV;

        if (!tb[IFLA_PROP_LIST])
                return 0;

        nla_for_each_nested(attr, tb[IFLA_PROP_LIST], rem) {
                switch (nla_type(attr)) {
                case IFLA_ALT_IFNAME:
                        err = rtnl_alt_ifname(cmd, dev, attr, &changed, extack);
                        if (err)
                                return err;
                        break;
                }
        }

        if (changed)
                netdev_state_change(dev);
        return 0;
}

static int rtnl_newlinkprop(struct sk_buff *skb, struct nlmsghdr *nlh,
                            struct netlink_ext_ack *extack)
{
        return rtnl_linkprop(RTM_NEWLINKPROP, skb, nlh, extack);
}

static int rtnl_dellinkprop(struct sk_buff *skb, struct nlmsghdr *nlh,
                            struct netlink_ext_ack *extack)
{
        return rtnl_linkprop(RTM_DELLINKPROP, skb, nlh, extack);
}

static noinline_for_stack u32 rtnl_calcit(struct sk_buff *skb,
                                          struct nlmsghdr *nlh)
{
        struct net *net = sock_net(skb->sk);
        size_t min_ifinfo_dump_size = 0;
        u32 ext_filter_mask = 0;
        struct net_device *dev;
        struct nlattr *nla;
        int hdrlen, rem;

        /* Same kernel<->userspace interface hack as in rtnl_dump_ifinfo. */
        hdrlen = nlmsg_len(nlh) < sizeof(struct ifinfomsg) ?
                 sizeof(struct rtgenmsg) : sizeof(struct ifinfomsg);

        if (nlh->nlmsg_len < nlmsg_msg_size(hdrlen))
                return NLMSG_GOODSIZE;

        nla_for_each_attr_type(nla, IFLA_EXT_MASK,
                               nlmsg_attrdata(nlh, hdrlen),
                               nlmsg_attrlen(nlh, hdrlen), rem) {
                if (nla_len(nla) == sizeof(u32))
                        ext_filter_mask = nla_get_u32(nla);
        }

        if (!ext_filter_mask)
                return NLMSG_GOODSIZE;
        /*
         * traverse the list of net devices and compute the minimum
         * buffer size based upon the filter mask.
         */
        rcu_read_lock();
        for_each_netdev_rcu(net, dev) {
                min_ifinfo_dump_size = max(min_ifinfo_dump_size,
                                           if_nlmsg_size(dev, ext_filter_mask));
        }
        rcu_read_unlock();

        return nlmsg_total_size(min_ifinfo_dump_size);
}

static int rtnl_dump_all(struct sk_buff *skb, struct netlink_callback *cb)
{
        int idx;
        int s_idx = cb->family;
        int type = cb->nlh->nlmsg_type - RTM_BASE;
        int ret = 0;

        if (s_idx == 0)
                s_idx = 1;

        for (idx = 1; idx <= RTNL_FAMILY_MAX; idx++) {
                struct rtnl_link __rcu **tab;
                struct rtnl_link *link;
                rtnl_dumpit_func dumpit;

                if (idx < s_idx || idx == PF_PACKET)
                        continue;

                if (type < 0 || type >= RTM_NR_MSGTYPES)
                        continue;

                tab = rcu_dereference_rtnl(rtnl_msg_handlers[idx]);
                if (!tab)
                        continue;

                link = rcu_dereference_rtnl(tab[type]);
                if (!link)
                        continue;

                dumpit = link->dumpit;
                if (!dumpit)
                        continue;

                if (idx > s_idx) {
                        memset(&cb->args[0], 0, sizeof(cb->args));
                        cb->prev_seq = 0;
                        cb->seq = 0;
                }
                ret = dumpit(skb, cb);
                if (ret)
                        break;
        }
        cb->family = idx;

        return skb->len ? : ret;
}

struct sk_buff *rtmsg_ifinfo_build_skb(int type, struct net_device *dev,
                                       unsigned int change,
                                       u32 event, gfp_t flags, int *new_nsid,
                                       int new_ifindex, u32 portid,
                                       const struct nlmsghdr *nlh)
{
        struct net *net = dev_net(dev);
        struct sk_buff *skb;
        int err = -ENOBUFS;
        u32 seq = 0;

        skb = nlmsg_new(if_nlmsg_size(dev, 0), flags);
        if (skb == NULL)
                goto errout;

        if (nlmsg_report(nlh))
                seq = nlmsg_seq(nlh);
        else
                portid = 0;

        err = rtnl_fill_ifinfo(skb, dev, dev_net(dev),
                               type, portid, seq, change, 0, 0, event,
                               new_nsid, new_ifindex, -1, flags);
        if (err < 0) {
                /* -EMSGSIZE implies BUG in if_nlmsg_size() */
                WARN_ON(err == -EMSGSIZE);
                kfree_skb(skb);
                goto errout;
        }
        return skb;
errout:
        rtnl_set_sk_err(net, RTNLGRP_LINK, err);
        return NULL;
}

void rtmsg_ifinfo_send(struct sk_buff *skb, struct net_device *dev, gfp_t flags,
                       u32 portid, const struct nlmsghdr *nlh)
{
        struct net *net = dev_net(dev);

        rtnl_notify(skb, net, portid, RTNLGRP_LINK, nlh, flags);
}

static void rtmsg_ifinfo_event(int type, struct net_device *dev,
                               unsigned int change, u32 event,
                               gfp_t flags, int *new_nsid, int new_ifindex,
                               u32 portid, const struct nlmsghdr *nlh)
{
        struct sk_buff *skb;

        if (dev->reg_state != NETREG_REGISTERED)
                return;

        skb = rtmsg_ifinfo_build_skb(type, dev, change, event, flags, new_nsid,
                                     new_ifindex, portid, nlh);
        if (skb)
                rtmsg_ifinfo_send(skb, dev, flags, portid, nlh);
}

void rtmsg_ifinfo(int type, struct net_device *dev, unsigned int change,
                  gfp_t flags, u32 portid, const struct nlmsghdr *nlh)
{
        rtmsg_ifinfo_event(type, dev, change, rtnl_get_event(0), flags,
                           NULL, 0, portid, nlh);
}

void rtmsg_ifinfo_newnet(int type, struct net_device *dev, unsigned int change,
                         gfp_t flags, int *new_nsid, int new_ifindex)
{
        rtmsg_ifinfo_event(type, dev, change, rtnl_get_event(0), flags,
                           new_nsid, new_ifindex, 0, NULL);
}

static int nlmsg_populate_fdb_fill(struct sk_buff *skb,
                                   struct net_device *dev,
                                   u8 *addr, u16 vid, u32 pid, u32 seq,
                                   int type, unsigned int flags,
                                   int nlflags, u16 ndm_state)
{
        struct nlmsghdr *nlh;
        struct ndmsg *ndm;

        nlh = nlmsg_put(skb, pid, seq, type, sizeof(*ndm), nlflags);
        if (!nlh)
                return -EMSGSIZE;

        ndm = nlmsg_data(nlh);
        ndm->ndm_family  = AF_BRIDGE;
        ndm->ndm_pad1         = 0;
        ndm->ndm_pad2    = 0;
        ndm->ndm_flags         = flags;
        ndm->ndm_type         = 0;
        ndm->ndm_ifindex = dev->ifindex;
        ndm->ndm_state   = ndm_state;

        if (nla_put(skb, NDA_LLADDR, dev->addr_len, addr))
                goto nla_put_failure;
        if (vid)
                if (nla_put(skb, NDA_VLAN, sizeof(u16), &vid))
                        goto nla_put_failure;

        nlmsg_end(skb, nlh);
        return 0;

nla_put_failure:
        nlmsg_cancel(skb, nlh);
        return -EMSGSIZE;
}

static inline size_t rtnl_fdb_nlmsg_size(const struct net_device *dev)
{
        return NLMSG_ALIGN(sizeof(struct ndmsg)) +
               nla_total_size(dev->addr_len) +        /* NDA_LLADDR */
               nla_total_size(sizeof(u16)) +        /* NDA_VLAN */
               0;
}

static void rtnl_fdb_notify(struct net_device *dev, u8 *addr, u16 vid, int type,
                            u16 ndm_state)
{
        struct net *net = dev_net(dev);
        struct sk_buff *skb;
        int err = -ENOBUFS;

        skb = nlmsg_new(rtnl_fdb_nlmsg_size(dev), GFP_ATOMIC);
        if (!skb)
                goto errout;

        err = nlmsg_populate_fdb_fill(skb, dev, addr, vid,
                                      0, 0, type, NTF_SELF, 0, ndm_state);
        if (err < 0) {
                kfree_skb(skb);
                goto errout;
        }

        rtnl_notify(skb, net, 0, RTNLGRP_NEIGH, NULL, GFP_ATOMIC);
        return;
errout:
        rtnl_set_sk_err(net, RTNLGRP_NEIGH, err);
}

/*
 * ndo_dflt_fdb_add - default netdevice operation to add an FDB entry
 */
int ndo_dflt_fdb_add(struct ndmsg *ndm,
                     struct nlattr *tb[],
                     struct net_device *dev,
                     const unsigned char *addr, u16 vid,
                     u16 flags)
{
        int err = -EINVAL;

        /* If aging addresses are supported device will need to
         * implement its own handler for this.
         */
        if (ndm->ndm_state && !(ndm->ndm_state & NUD_PERMANENT)) {
                netdev_info(dev, "default FDB implementation only supports local addresses\n");
                return err;
        }

        if (tb[NDA_FLAGS_EXT]) {
                netdev_info(dev, "invalid flags given to default FDB implementation\n");
                return err;
        }

        if (vid) {
                netdev_info(dev, "vlans aren't supported yet for dev_uc|mc_add()\n");
                return err;
        }

        if (is_unicast_ether_addr(addr) || is_link_local_ether_addr(addr))
                err = dev_uc_add_excl(dev, addr);
        else if (is_multicast_ether_addr(addr))
                err = dev_mc_add_excl(dev, addr);

        /* Only return duplicate errors if NLM_F_EXCL is set */
        if (err == -EEXIST && !(flags & NLM_F_EXCL))
                err = 0;

        return err;
}
EXPORT_SYMBOL(ndo_dflt_fdb_add);

static int fdb_vid_parse(struct nlattr *vlan_attr, u16 *p_vid,
                         struct netlink_ext_ack *extack)
{
        u16 vid = 0;

        if (vlan_attr) {
                if (nla_len(vlan_attr) != sizeof(u16)) {
                        NL_SET_ERR_MSG(extack, "invalid vlan attribute size");
                        return -EINVAL;
                }

                vid = nla_get_u16(vlan_attr);

                if (!vid || vid >= VLAN_VID_MASK) {
                        NL_SET_ERR_MSG(extack, "invalid vlan id");
                        return -EINVAL;
                }
        }
        *p_vid = vid;
        return 0;
}

static int rtnl_fdb_add(struct sk_buff *skb, struct nlmsghdr *nlh,
                        struct netlink_ext_ack *extack)
{
        struct net *net = sock_net(skb->sk);
        struct ndmsg *ndm;
        struct nlattr *tb[NDA_MAX+1];
        struct net_device *dev;
        u8 *addr;
        u16 vid;
        int err;

        err = nlmsg_parse_deprecated(nlh, sizeof(*ndm), tb, NDA_MAX, NULL,
                                     extack);
        if (err < 0)
                return err;

        ndm = nlmsg_data(nlh);
        if (ndm->ndm_ifindex == 0) {
                NL_SET_ERR_MSG(extack, "invalid ifindex");
                return -EINVAL;
        }

        dev = __dev_get_by_index(net, ndm->ndm_ifindex);
        if (dev == NULL) {
                NL_SET_ERR_MSG(extack, "unknown ifindex");
                return -ENODEV;
        }

        if (!tb[NDA_LLADDR] || nla_len(tb[NDA_LLADDR]) != ETH_ALEN) {
                NL_SET_ERR_MSG(extack, "invalid address");
                return -EINVAL;
        }

        if (dev->type != ARPHRD_ETHER) {
                NL_SET_ERR_MSG(extack, "FDB add only supported for Ethernet devices");
                return -EINVAL;
        }

        addr = nla_data(tb[NDA_LLADDR]);

        err = fdb_vid_parse(tb[NDA_VLAN], &vid, extack);
        if (err)
                return err;

        err = -EOPNOTSUPP;

        /* Support fdb on master device the net/bridge default case */
        if ((!ndm->ndm_flags || ndm->ndm_flags & NTF_MASTER) &&
            netif_is_bridge_port(dev)) {
                struct net_device *br_dev = netdev_master_upper_dev_get(dev);
                const struct net_device_ops *ops = br_dev->netdev_ops;
                bool notified = false;

                err = ops->ndo_fdb_add(ndm, tb, dev, addr, vid,
                                       nlh->nlmsg_flags, &notified, extack);
                if (err)
                        goto out;
                else
                        ndm->ndm_flags &= ~NTF_MASTER;
        }

        /* Embedded bridge, macvlan, and any other device support */
        if ((ndm->ndm_flags & NTF_SELF)) {
                bool notified = false;

                if (dev->netdev_ops->ndo_fdb_add)
                        err = dev->netdev_ops->ndo_fdb_add(ndm, tb, dev, addr,
                                                           vid,
                                                           nlh->nlmsg_flags,
                                                           &notified, extack);
                else
                        err = ndo_dflt_fdb_add(ndm, tb, dev, addr, vid,
                                               nlh->nlmsg_flags);

                if (!err && !notified) {
                        rtnl_fdb_notify(dev, addr, vid, RTM_NEWNEIGH,
                                        ndm->ndm_state);
                        ndm->ndm_flags &= ~NTF_SELF;
                }
        }
out:
        return err;
}

/*
 * ndo_dflt_fdb_del - default netdevice operation to delete an FDB entry
 */
int ndo_dflt_fdb_del(struct ndmsg *ndm,
                     struct nlattr *tb[],
                     struct net_device *dev,
                     const unsigned char *addr, u16 vid)
{
        int err = -EINVAL;

        /* If aging addresses are supported device will need to
         * implement its own handler for this.
         */
        if (!(ndm->ndm_state & NUD_PERMANENT)) {
                netdev_info(dev, "default FDB implementation only supports local addresses\n");
                return err;
        }

        if (is_unicast_ether_addr(addr) || is_link_local_ether_addr(addr))
                err = dev_uc_del(dev, addr);
        else if (is_multicast_ether_addr(addr))
                err = dev_mc_del(dev, addr);

        return err;
}
EXPORT_SYMBOL(ndo_dflt_fdb_del);

static int rtnl_fdb_del(struct sk_buff *skb, struct nlmsghdr *nlh,
                        struct netlink_ext_ack *extack)
{
        bool del_bulk = !!(nlh->nlmsg_flags & NLM_F_BULK);
        struct net *net = sock_net(skb->sk);
        const struct net_device_ops *ops;
        struct ndmsg *ndm;
        struct nlattr *tb[NDA_MAX+1];
        struct net_device *dev;
        __u8 *addr = NULL;
        int err;
        u16 vid;

        if (!netlink_capable(skb, CAP_NET_ADMIN))
                return -EPERM;

        if (!del_bulk) {
                err = nlmsg_parse_deprecated(nlh, sizeof(*ndm), tb, NDA_MAX,
                                             NULL, extack);
        } else {
                /* For bulk delete, the drivers will parse the message with
                 * policy.
                 */
                err = nlmsg_parse(nlh, sizeof(*ndm), tb, NDA_MAX, NULL, extack);
        }
        if (err < 0)
                return err;

        ndm = nlmsg_data(nlh);
        if (ndm->ndm_ifindex == 0) {
                NL_SET_ERR_MSG(extack, "invalid ifindex");
                return -EINVAL;
        }

        dev = __dev_get_by_index(net, ndm->ndm_ifindex);
        if (dev == NULL) {
                NL_SET_ERR_MSG(extack, "unknown ifindex");
                return -ENODEV;
        }

        if (!del_bulk) {
                if (!tb[NDA_LLADDR] || nla_len(tb[NDA_LLADDR]) != ETH_ALEN) {
                        NL_SET_ERR_MSG(extack, "invalid address");
                        return -EINVAL;
                }
                addr = nla_data(tb[NDA_LLADDR]);

                err = fdb_vid_parse(tb[NDA_VLAN], &vid, extack);
                if (err)
                        return err;
        }

        if (dev->type != ARPHRD_ETHER) {
                NL_SET_ERR_MSG(extack, "FDB delete only supported for Ethernet devices");
                return -EINVAL;
        }

        err = -EOPNOTSUPP;

        /* Support fdb on master device the net/bridge default case */
        if ((!ndm->ndm_flags || ndm->ndm_flags & NTF_MASTER) &&
            netif_is_bridge_port(dev)) {
                struct net_device *br_dev = netdev_master_upper_dev_get(dev);
                bool notified = false;

                ops = br_dev->netdev_ops;
                if (!del_bulk) {
                        if (ops->ndo_fdb_del)
                                err = ops->ndo_fdb_del(ndm, tb, dev, addr, vid,
                                                       &notified, extack);
                } else {
                        if (ops->ndo_fdb_del_bulk)
                                err = ops->ndo_fdb_del_bulk(nlh, dev, extack);
                }

                if (err)
                        goto out;
                else
                        ndm->ndm_flags &= ~NTF_MASTER;
        }

        /* Embedded bridge, macvlan, and any other device support */
        if (ndm->ndm_flags & NTF_SELF) {
                bool notified = false;

                ops = dev->netdev_ops;
                if (!del_bulk) {
                        if (ops->ndo_fdb_del)
                                err = ops->ndo_fdb_del(ndm, tb, dev, addr, vid,
                                                       &notified, extack);
                        else
                                err = ndo_dflt_fdb_del(ndm, tb, dev, addr, vid);
                } else {
                        /* in case err was cleared by NTF_MASTER call */
                        err = -EOPNOTSUPP;
                        if (ops->ndo_fdb_del_bulk)
                                err = ops->ndo_fdb_del_bulk(nlh, dev, extack);
                }

                if (!err) {
                        if (!del_bulk && !notified)
                                rtnl_fdb_notify(dev, addr, vid, RTM_DELNEIGH,
                                                ndm->ndm_state);
                        ndm->ndm_flags &= ~NTF_SELF;
                }
        }
out:
        return err;
}

static int nlmsg_populate_fdb(struct sk_buff *skb,
                              struct netlink_callback *cb,
                              struct net_device *dev,
                              int *idx,
                              struct netdev_hw_addr_list *list)
{
        struct ndo_fdb_dump_context *ctx = (void *)cb->ctx;
        struct netdev_hw_addr *ha;
        u32 portid, seq;
        int err;

        portid = NETLINK_CB(cb->skb).portid;
        seq = cb->nlh->nlmsg_seq;

        list_for_each_entry(ha, &list->list, list) {
                if (*idx < ctx->fdb_idx)
                        goto skip;

                err = nlmsg_populate_fdb_fill(skb, dev, ha->addr, 0,
                                              portid, seq,
                                              RTM_NEWNEIGH, NTF_SELF,
                                              NLM_F_MULTI, NUD_PERMANENT);
                if (err < 0)
                        return err;
skip:
                *idx += 1;
        }
        return 0;
}

/**
 * ndo_dflt_fdb_dump - default netdevice operation to dump an FDB table.
 * @skb: socket buffer to store message in
 * @cb: netlink callback
 * @dev: netdevice
 * @filter_dev: ignored
 * @idx: the number of FDB table entries dumped is added to *@idx
 *
 * Default netdevice operation to dump the existing unicast address list.
 * Returns number of addresses from list put in skb.
 */
int ndo_dflt_fdb_dump(struct sk_buff *skb,
                      struct netlink_callback *cb,
                      struct net_device *dev,
                      struct net_device *filter_dev,
                      int *idx)
{
        int err;

        if (dev->type != ARPHRD_ETHER)
                return -EINVAL;

        netif_addr_lock_bh(dev);
        err = nlmsg_populate_fdb(skb, cb, dev, idx, &dev->uc);
        if (err)
                goto out;
        err = nlmsg_populate_fdb(skb, cb, dev, idx, &dev->mc);
out:
        netif_addr_unlock_bh(dev);
        return err;
}
EXPORT_SYMBOL(ndo_dflt_fdb_dump);

static int valid_fdb_dump_strict(const struct nlmsghdr *nlh,
                                 int *br_idx, int *brport_idx,
                                 struct netlink_ext_ack *extack)
{
        struct nlattr *tb[NDA_MAX + 1];
        struct ndmsg *ndm;
        int err, i;

        ndm = nlmsg_payload(nlh, sizeof(*ndm));
        if (!ndm) {
                NL_SET_ERR_MSG(extack, "Invalid header for fdb dump request");
                return -EINVAL;
        }

        if (ndm->ndm_pad1  || ndm->ndm_pad2  || ndm->ndm_state ||
            ndm->ndm_flags || ndm->ndm_type) {
                NL_SET_ERR_MSG(extack, "Invalid values in header for fdb dump request");
                return -EINVAL;
        }

        err = nlmsg_parse_deprecated_strict(nlh, sizeof(struct ndmsg), tb,
                                            NDA_MAX, NULL, extack);
        if (err < 0)
                return err;

        *brport_idx = ndm->ndm_ifindex;
        for (i = 0; i <= NDA_MAX; ++i) {
                if (!tb[i])
                        continue;

                switch (i) {
                case NDA_IFINDEX:
                        if (nla_len(tb[i]) != sizeof(u32)) {
                                NL_SET_ERR_MSG(extack, "Invalid IFINDEX attribute in fdb dump request");
                                return -EINVAL;
                        }
                        *brport_idx = nla_get_u32(tb[NDA_IFINDEX]);
                        break;
                case NDA_MASTER:
                        if (nla_len(tb[i]) != sizeof(u32)) {
                                NL_SET_ERR_MSG(extack, "Invalid MASTER attribute in fdb dump request");
                                return -EINVAL;
                        }
                        *br_idx = nla_get_u32(tb[NDA_MASTER]);
                        break;
                default:
                        NL_SET_ERR_MSG(extack, "Unsupported attribute in fdb dump request");
                        return -EINVAL;
                }
        }

        return 0;
}

static int valid_fdb_dump_legacy(const struct nlmsghdr *nlh,
                                 int *br_idx, int *brport_idx,
                                 struct netlink_ext_ack *extack)
{
        struct nlattr *tb[IFLA_MAX+1];
        int err;

        /* A hack to preserve kernel<->userspace interface.
         * Before Linux v4.12 this code accepted ndmsg since iproute2 v3.3.0.
         * However, ndmsg is shorter than ifinfomsg thus nlmsg_parse() bails.
         * So, check for ndmsg with an optional u32 attribute (not used here).
         * Fortunately these sizes don't conflict with the size of ifinfomsg
         * with an optional attribute.
         */
        if (nlmsg_len(nlh) != sizeof(struct ndmsg) &&
            (nlmsg_len(nlh) != sizeof(struct ndmsg) +
             nla_attr_size(sizeof(u32)))) {
                struct ifinfomsg *ifm;

                err = nlmsg_parse_deprecated(nlh, sizeof(struct ifinfomsg),
                                             tb, IFLA_MAX, ifla_policy,
                                             extack);
                if (err < 0) {
                        return -EINVAL;
                } else if (err == 0) {
                        if (tb[IFLA_MASTER])
                                *br_idx = nla_get_u32(tb[IFLA_MASTER]);
                }

                ifm = nlmsg_data(nlh);
                *brport_idx = ifm->ifi_index;
        }
        return 0;
}

static int rtnl_fdb_dump(struct sk_buff *skb, struct netlink_callback *cb)
{
        const struct net_device_ops *ops = NULL, *cops = NULL;
        struct ndo_fdb_dump_context *ctx = (void *)cb->ctx;
        struct net_device *dev, *br_dev = NULL;
        struct net *net = sock_net(skb->sk);
        int brport_idx = 0;
        int br_idx = 0;
        int fidx = 0;
        int err;

        NL_ASSERT_CTX_FITS(struct ndo_fdb_dump_context);

        if (cb->strict_check)
                err = valid_fdb_dump_strict(cb->nlh, &br_idx, &brport_idx,
                                            cb->extack);
        else
                err = valid_fdb_dump_legacy(cb->nlh, &br_idx, &brport_idx,
                                            cb->extack);
        if (err < 0)
                return err;

        if (br_idx) {
                br_dev = __dev_get_by_index(net, br_idx);
                if (!br_dev)
                        return -ENODEV;

                ops = br_dev->netdev_ops;
        }

        for_each_netdev_dump(net, dev, ctx->ifindex) {
                if (brport_idx && (dev->ifindex != brport_idx))
                        continue;

                if (!br_idx) { /* user did not specify a specific bridge */
                        if (netif_is_bridge_port(dev)) {
                                br_dev = netdev_master_upper_dev_get(dev);
                                cops = br_dev->netdev_ops;
                        }
                } else {
                        if (dev != br_dev &&
                            !netif_is_bridge_port(dev))
                                continue;

                        if (br_dev != netdev_master_upper_dev_get(dev) &&
                            !netif_is_bridge_master(dev))
                                continue;
                        cops = ops;
                }

                if (netif_is_bridge_port(dev)) {
                        if (cops && cops->ndo_fdb_dump) {
                                err = cops->ndo_fdb_dump(skb, cb, br_dev, dev,
                                                        &fidx);
                                if (err == -EMSGSIZE)
                                        break;
                        }
                }

                if (dev->netdev_ops->ndo_fdb_dump)
                        err = dev->netdev_ops->ndo_fdb_dump(skb, cb, dev, NULL,
                                                            &fidx);
                else
                        err = ndo_dflt_fdb_dump(skb, cb, dev, NULL, &fidx);
                if (err == -EMSGSIZE)
                        break;

                cops = NULL;

                /* reset fdb offset to 0 for rest of the interfaces */
                ctx->fdb_idx = 0;
                fidx = 0;
        }

        ctx->fdb_idx = fidx;

        return skb->len;
}

static int valid_fdb_get_strict(const struct nlmsghdr *nlh,
                                struct nlattr **tb, u8 *ndm_flags,
                                int *br_idx, int *brport_idx, u8 **addr,
                                u16 *vid, struct netlink_ext_ack *extack)
{
        struct ndmsg *ndm;
        int err, i;

        ndm = nlmsg_payload(nlh, sizeof(*ndm));
        if (!ndm) {
                NL_SET_ERR_MSG(extack, "Invalid header for fdb get request");
                return -EINVAL;
        }

        if (ndm->ndm_pad1  || ndm->ndm_pad2  || ndm->ndm_state ||
            ndm->ndm_type) {
                NL_SET_ERR_MSG(extack, "Invalid values in header for fdb get request");
                return -EINVAL;
        }

        if (ndm->ndm_flags & ~(NTF_MASTER | NTF_SELF)) {
                NL_SET_ERR_MSG(extack, "Invalid flags in header for fdb get request");
                return -EINVAL;
        }

        err = nlmsg_parse_deprecated_strict(nlh, sizeof(struct ndmsg), tb,
                                            NDA_MAX, nda_policy, extack);
        if (err < 0)
                return err;

        *ndm_flags = ndm->ndm_flags;
        *brport_idx = ndm->ndm_ifindex;
        for (i = 0; i <= NDA_MAX; ++i) {
                if (!tb[i])
                        continue;

                switch (i) {
                case NDA_MASTER:
                        *br_idx = nla_get_u32(tb[i]);
                        break;
                case NDA_LLADDR:
                        if (nla_len(tb[i]) != ETH_ALEN) {
                                NL_SET_ERR_MSG(extack, "Invalid address in fdb get request");
                                return -EINVAL;
                        }
                        *addr = nla_data(tb[i]);
                        break;
                case NDA_VLAN:
                        err = fdb_vid_parse(tb[i], vid, extack);
                        if (err)
                                return err;
                        break;
                case NDA_VNI:
                        break;
                default:
                        NL_SET_ERR_MSG(extack, "Unsupported attribute in fdb get request");
                        return -EINVAL;
                }
        }

        return 0;
}

static int rtnl_fdb_get(struct sk_buff *in_skb, struct nlmsghdr *nlh,
                        struct netlink_ext_ack *extack)
{
        struct net_device *dev = NULL, *br_dev = NULL;
        const struct net_device_ops *ops = NULL;
        struct net *net = sock_net(in_skb->sk);
        struct nlattr *tb[NDA_MAX + 1];
        struct sk_buff *skb;
        int brport_idx = 0;
        u8 ndm_flags = 0;
        int br_idx = 0;
        u8 *addr = NULL;
        u16 vid = 0;
        int err;

        err = valid_fdb_get_strict(nlh, tb, &ndm_flags, &br_idx,
                                   &brport_idx, &addr, &vid, extack);
        if (err < 0)
                return err;

        if (!addr) {
                NL_SET_ERR_MSG(extack, "Missing lookup address for fdb get request");
                return -EINVAL;
        }

        if (brport_idx) {
                dev = __dev_get_by_index(net, brport_idx);
                if (!dev) {
                        NL_SET_ERR_MSG(extack, "Unknown device ifindex");
                        return -ENODEV;
                }
        }

        if (br_idx) {
                if (dev) {
                        NL_SET_ERR_MSG(extack, "Master and device are mutually exclusive");
                        return -EINVAL;
                }

                br_dev = __dev_get_by_index(net, br_idx);
                if (!br_dev) {
                        NL_SET_ERR_MSG(extack, "Invalid master ifindex");
                        return -EINVAL;
                }
                ops = br_dev->netdev_ops;
        }

        if (dev) {
                if (!ndm_flags || (ndm_flags & NTF_MASTER)) {
                        if (!netif_is_bridge_port(dev)) {
                                NL_SET_ERR_MSG(extack, "Device is not a bridge port");
                                return -EINVAL;
                        }
                        br_dev = netdev_master_upper_dev_get(dev);
                        if (!br_dev) {
                                NL_SET_ERR_MSG(extack, "Master of device not found");
                                return -EINVAL;
                        }
                        ops = br_dev->netdev_ops;
                } else {
                        if (!(ndm_flags & NTF_SELF)) {
                                NL_SET_ERR_MSG(extack, "Missing NTF_SELF");
                                return -EINVAL;
                        }
                        ops = dev->netdev_ops;
                }
        }

        if (!br_dev && !dev) {
                NL_SET_ERR_MSG(extack, "No device specified");
                return -ENODEV;
        }

        if (!ops || !ops->ndo_fdb_get) {
                NL_SET_ERR_MSG(extack, "Fdb get operation not supported by device");
                return -EOPNOTSUPP;
        }

        skb = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
        if (!skb)
                return -ENOBUFS;

        if (br_dev)
                dev = br_dev;
        err = ops->ndo_fdb_get(skb, tb, dev, addr, vid,
                               NETLINK_CB(in_skb).portid,
                               nlh->nlmsg_seq, extack);
        if (err)
                goto out;

        return rtnl_unicast(skb, net, NETLINK_CB(in_skb).portid);
out:
        kfree_skb(skb);
        return err;
}

static int brport_nla_put_flag(struct sk_buff *skb, u32 flags, u32 mask,
                               unsigned int attrnum, unsigned int flag)
{
        if (mask & flag)
                return nla_put_u8(skb, attrnum, !!(flags & flag));
        return 0;
}

int ndo_dflt_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq,
                            struct net_device *dev, u16 mode,
                            u32 flags, u32 mask, int nlflags,
                            u32 filter_mask,
                            int (*vlan_fill)(struct sk_buff *skb,
                                             struct net_device *dev,
                                             u32 filter_mask))
{
        struct nlmsghdr *nlh;
        struct ifinfomsg *ifm;
        struct nlattr *br_afspec;
        struct nlattr *protinfo;
        u8 operstate = netif_running(dev) ? dev->operstate : IF_OPER_DOWN;
        struct net_device *br_dev = netdev_master_upper_dev_get(dev);
        int err = 0;

        nlh = nlmsg_put(skb, pid, seq, RTM_NEWLINK, sizeof(*ifm), nlflags);
        if (nlh == NULL)
                return -EMSGSIZE;

        ifm = nlmsg_data(nlh);
        ifm->ifi_family = AF_BRIDGE;
        ifm->__ifi_pad = 0;
        ifm->ifi_type = dev->type;
        ifm->ifi_index = dev->ifindex;
        ifm->ifi_flags = dev_get_flags(dev);
        ifm->ifi_change = 0;


        if (nla_put_string(skb, IFLA_IFNAME, dev->name) ||
            nla_put_u32(skb, IFLA_MTU, dev->mtu) ||
            nla_put_u8(skb, IFLA_OPERSTATE, operstate) ||
            (br_dev &&
             nla_put_u32(skb, IFLA_MASTER, br_dev->ifindex)) ||
            (dev->addr_len &&
             nla_put(skb, IFLA_ADDRESS, dev->addr_len, dev->dev_addr)) ||
            (dev->ifindex != dev_get_iflink(dev) &&
             nla_put_u32(skb, IFLA_LINK, dev_get_iflink(dev))))
                goto nla_put_failure;

        br_afspec = nla_nest_start_noflag(skb, IFLA_AF_SPEC);
        if (!br_afspec)
                goto nla_put_failure;

        if (nla_put_u16(skb, IFLA_BRIDGE_FLAGS, BRIDGE_FLAGS_SELF)) {
                nla_nest_cancel(skb, br_afspec);
                goto nla_put_failure;
        }

        if (mode != BRIDGE_MODE_UNDEF) {
                if (nla_put_u16(skb, IFLA_BRIDGE_MODE, mode)) {
                        nla_nest_cancel(skb, br_afspec);
                        goto nla_put_failure;
                }
        }
        if (vlan_fill) {
                err = vlan_fill(skb, dev, filter_mask);
                if (err) {
                        nla_nest_cancel(skb, br_afspec);
                        goto nla_put_failure;
                }
        }
        nla_nest_end(skb, br_afspec);

        protinfo = nla_nest_start(skb, IFLA_PROTINFO);
        if (!protinfo)
                goto nla_put_failure;

        if (brport_nla_put_flag(skb, flags, mask,
                                IFLA_BRPORT_MODE, BR_HAIRPIN_MODE) ||
            brport_nla_put_flag(skb, flags, mask,
                                IFLA_BRPORT_GUARD, BR_BPDU_GUARD) ||
            brport_nla_put_flag(skb, flags, mask,
                                IFLA_BRPORT_FAST_LEAVE,
                                BR_MULTICAST_FAST_LEAVE) ||
            brport_nla_put_flag(skb, flags, mask,
                                IFLA_BRPORT_PROTECT, BR_ROOT_BLOCK) ||
            brport_nla_put_flag(skb, flags, mask,
                                IFLA_BRPORT_LEARNING, BR_LEARNING) ||
            brport_nla_put_flag(skb, flags, mask,
                                IFLA_BRPORT_LEARNING_SYNC, BR_LEARNING_SYNC) ||
            brport_nla_put_flag(skb, flags, mask,
                                IFLA_BRPORT_UNICAST_FLOOD, BR_FLOOD) ||
            brport_nla_put_flag(skb, flags, mask,
                                IFLA_BRPORT_PROXYARP, BR_PROXYARP) ||
            brport_nla_put_flag(skb, flags, mask,
                                IFLA_BRPORT_MCAST_FLOOD, BR_MCAST_FLOOD) ||
            brport_nla_put_flag(skb, flags, mask,
                                IFLA_BRPORT_BCAST_FLOOD, BR_BCAST_FLOOD)) {
                nla_nest_cancel(skb, protinfo);
                goto nla_put_failure;
        }

        nla_nest_end(skb, protinfo);

        nlmsg_end(skb, nlh);
        return 0;
nla_put_failure:
        nlmsg_cancel(skb, nlh);
        return err ? err : -EMSGSIZE;
}
EXPORT_SYMBOL_GPL(ndo_dflt_bridge_getlink);

static int valid_bridge_getlink_req(const struct nlmsghdr *nlh,
                                    bool strict_check, u32 *filter_mask,
                                    struct netlink_ext_ack *extack)
{
        struct nlattr *tb[IFLA_MAX+1];
        int err, i;

        if (strict_check) {
                struct ifinfomsg *ifm;

                ifm = nlmsg_payload(nlh, sizeof(*ifm));
                if (!ifm) {
                        NL_SET_ERR_MSG(extack, "Invalid header for bridge link dump");
                        return -EINVAL;
                }

                if (ifm->__ifi_pad || ifm->ifi_type || ifm->ifi_flags ||
                    ifm->ifi_change || ifm->ifi_index) {
                        NL_SET_ERR_MSG(extack, "Invalid values in header for bridge link dump request");
                        return -EINVAL;
                }

                err = nlmsg_parse_deprecated_strict(nlh,
                                                    sizeof(struct ifinfomsg),
                                                    tb, IFLA_MAX, ifla_policy,
                                                    extack);
        } else {
                err = nlmsg_parse_deprecated(nlh, sizeof(struct ifinfomsg),
                                             tb, IFLA_MAX, ifla_policy,
                                             extack);
        }
        if (err < 0)
                return err;

        /* new attributes should only be added with strict checking */
        for (i = 0; i <= IFLA_MAX; ++i) {
                if (!tb[i])
                        continue;

                switch (i) {
                case IFLA_EXT_MASK:
                        *filter_mask = nla_get_u32(tb[i]);
                        break;
                default:
                        if (strict_check) {
                                NL_SET_ERR_MSG(extack, "Unsupported attribute in bridge link dump request");
                                return -EINVAL;
                        }
                }
        }

        return 0;
}

static int rtnl_bridge_getlink(struct sk_buff *skb, struct netlink_callback *cb)
{
        const struct nlmsghdr *nlh = cb->nlh;
        struct net *net = sock_net(skb->sk);
        struct net_device *dev;
        int idx = 0;
        u32 portid = NETLINK_CB(cb->skb).portid;
        u32 seq = nlh->nlmsg_seq;
        u32 filter_mask = 0;
        int err;

        err = valid_bridge_getlink_req(nlh, cb->strict_check, &filter_mask,
                                       cb->extack);
        if (err < 0 && cb->strict_check)
                return err;

        rcu_read_lock();
        for_each_netdev_rcu(net, dev) {
                const struct net_device_ops *ops = dev->netdev_ops;
                struct net_device *br_dev = netdev_master_upper_dev_get(dev);

                if (br_dev && br_dev->netdev_ops->ndo_bridge_getlink) {
                        if (idx >= cb->args[0]) {
                                err = br_dev->netdev_ops->ndo_bridge_getlink(
                                                skb, portid, seq, dev,
                                                filter_mask, NLM_F_MULTI);
                                if (err < 0 && err != -EOPNOTSUPP) {
                                        if (likely(skb->len))
                                                break;

                                        goto out_err;
                                }
                        }
                        idx++;
                }

                if (ops->ndo_bridge_getlink) {
                        if (idx >= cb->args[0]) {
                                err = ops->ndo_bridge_getlink(skb, portid,
                                                              seq, dev,
                                                              filter_mask,
                                                              NLM_F_MULTI);
                                if (err < 0 && err != -EOPNOTSUPP) {
                                        if (likely(skb->len))
                                                break;

                                        goto out_err;
                                }
                        }
                        idx++;
                }
        }
        err = skb->len;
out_err:
        rcu_read_unlock();
        cb->args[0] = idx;

        return err;
}

static inline size_t bridge_nlmsg_size(void)
{
        return NLMSG_ALIGN(sizeof(struct ifinfomsg))
                + nla_total_size(IFNAMSIZ)        /* IFLA_IFNAME */
                + nla_total_size(MAX_ADDR_LEN)        /* IFLA_ADDRESS */
                + nla_total_size(sizeof(u32))        /* IFLA_MASTER */
                + nla_total_size(sizeof(u32))        /* IFLA_MTU */
                + nla_total_size(sizeof(u32))        /* IFLA_LINK */
                + nla_total_size(sizeof(u32))        /* IFLA_OPERSTATE */
                + nla_total_size(sizeof(u8))        /* IFLA_PROTINFO */
                + nla_total_size(sizeof(struct nlattr))        /* IFLA_AF_SPEC */
                + nla_total_size(sizeof(u16))        /* IFLA_BRIDGE_FLAGS */
                + nla_total_size(sizeof(u16));        /* IFLA_BRIDGE_MODE */
}

static int rtnl_bridge_notify(struct net_device *dev)
{
        struct net *net = dev_net(dev);
        struct sk_buff *skb;
        int err = -EOPNOTSUPP;

        if (!dev->netdev_ops->ndo_bridge_getlink)
                return 0;

        skb = nlmsg_new(bridge_nlmsg_size(), GFP_ATOMIC);
        if (!skb) {
                err = -ENOMEM;
                goto errout;
        }

        err = dev->netdev_ops->ndo_bridge_getlink(skb, 0, 0, dev, 0, 0);
        if (err < 0)
                goto errout;

        /* Notification info is only filled for bridge ports, not the bridge
         * device itself. Therefore, a zero notification length is valid and
         * should not result in an error.
         */
        if (!skb->len)
                goto errout;

        rtnl_notify(skb, net, 0, RTNLGRP_LINK, NULL, GFP_ATOMIC);
        return 0;
errout:
        WARN_ON(err == -EMSGSIZE);
        kfree_skb(skb);
        if (err)
                rtnl_set_sk_err(net, RTNLGRP_LINK, err);
        return err;
}

static int rtnl_bridge_setlink(struct sk_buff *skb, struct nlmsghdr *nlh,
                               struct netlink_ext_ack *extack)
{
        struct net *net = sock_net(skb->sk);
        struct ifinfomsg *ifm;
        struct net_device *dev;
        struct nlattr *br_spec, *attr, *br_flags_attr = NULL;
        int rem, err = -EOPNOTSUPP;
        u16 flags = 0;

        if (nlmsg_len(nlh) < sizeof(*ifm))
                return -EINVAL;

        ifm = nlmsg_data(nlh);
        if (ifm->ifi_family != AF_BRIDGE)
                return -EPFNOSUPPORT;

        dev = __dev_get_by_index(net, ifm->ifi_index);
        if (!dev) {
                NL_SET_ERR_MSG(extack, "unknown ifindex");
                return -ENODEV;
        }

        br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC);
        if (br_spec) {
                nla_for_each_nested(attr, br_spec, rem) {
                        if (nla_type(attr) == IFLA_BRIDGE_FLAGS && !br_flags_attr) {
                                if (nla_len(attr) < sizeof(flags))
                                        return -EINVAL;

                                br_flags_attr = attr;
                                flags = nla_get_u16(attr);
                        }

                        if (nla_type(attr) == IFLA_BRIDGE_MODE) {
                                if (nla_len(attr) < sizeof(u16))
                                        return -EINVAL;
                        }
                }
        }

        if (!flags || (flags & BRIDGE_FLAGS_MASTER)) {
                struct net_device *br_dev = netdev_master_upper_dev_get(dev);

                if (!br_dev || !br_dev->netdev_ops->ndo_bridge_setlink) {
                        err = -EOPNOTSUPP;
                        goto out;
                }

                err = br_dev->netdev_ops->ndo_bridge_setlink(dev, nlh, flags,
                                                             extack);
                if (err)
                        goto out;

                flags &= ~BRIDGE_FLAGS_MASTER;
        }

        if ((flags & BRIDGE_FLAGS_SELF)) {
                if (!dev->netdev_ops->ndo_bridge_setlink)
                        err = -EOPNOTSUPP;
                else
                        err = dev->netdev_ops->ndo_bridge_setlink(dev, nlh,
                                                                  flags,
                                                                  extack);
                if (!err) {
                        flags &= ~BRIDGE_FLAGS_SELF;

                        /* Generate event to notify upper layer of bridge
                         * change
                         */
                        err = rtnl_bridge_notify(dev);
                }
        }

        if (br_flags_attr)
                memcpy(nla_data(br_flags_attr), &flags, sizeof(flags));
out:
        return err;
}

static int rtnl_bridge_dellink(struct sk_buff *skb, struct nlmsghdr *nlh,
                               struct netlink_ext_ack *extack)
{
        struct net *net = sock_net(skb->sk);
        struct ifinfomsg *ifm;
        struct net_device *dev;
        struct nlattr *br_spec, *attr = NULL;
        int rem, err = -EOPNOTSUPP;
        u16 flags = 0;
        bool have_flags = false;

        if (nlmsg_len(nlh) < sizeof(*ifm))
                return -EINVAL;

        ifm = nlmsg_data(nlh);
        if (ifm->ifi_family != AF_BRIDGE)
                return -EPFNOSUPPORT;

        dev = __dev_get_by_index(net, ifm->ifi_index);
        if (!dev) {
                NL_SET_ERR_MSG(extack, "unknown ifindex");
                return -ENODEV;
        }

        br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC);
        if (br_spec) {
                nla_for_each_nested_type(attr, IFLA_BRIDGE_FLAGS, br_spec,
                                         rem) {
                        if (nla_len(attr) < sizeof(flags))
                                return -EINVAL;

                        have_flags = true;
                        flags = nla_get_u16(attr);
                        break;
                }
        }

        if (!flags || (flags & BRIDGE_FLAGS_MASTER)) {
                struct net_device *br_dev = netdev_master_upper_dev_get(dev);

                if (!br_dev || !br_dev->netdev_ops->ndo_bridge_dellink) {
                        err = -EOPNOTSUPP;
                        goto out;
                }

                err = br_dev->netdev_ops->ndo_bridge_dellink(dev, nlh, flags);
                if (err)
                        goto out;

                flags &= ~BRIDGE_FLAGS_MASTER;
        }

        if ((flags & BRIDGE_FLAGS_SELF)) {
                if (!dev->netdev_ops->ndo_bridge_dellink)
                        err = -EOPNOTSUPP;
                else
                        err = dev->netdev_ops->ndo_bridge_dellink(dev, nlh,
                                                                  flags);

                if (!err) {
                        flags &= ~BRIDGE_FLAGS_SELF;

                        /* Generate event to notify upper layer of bridge
                         * change
                         */
                        err = rtnl_bridge_notify(dev);
                }
        }

        if (have_flags)
                memcpy(nla_data(attr), &flags, sizeof(flags));
out:
        return err;
}

static bool stats_attr_valid(unsigned int mask, int attrid, int idxattr)
{
        return (mask & IFLA_STATS_FILTER_BIT(attrid)) &&
               (!idxattr || idxattr == attrid);
}

static bool
rtnl_offload_xstats_have_ndo(const struct net_device *dev, int attr_id)
{
        return dev->netdev_ops &&
               dev->netdev_ops->ndo_has_offload_stats &&
               dev->netdev_ops->ndo_get_offload_stats &&
               dev->netdev_ops->ndo_has_offload_stats(dev, attr_id);
}

static unsigned int
rtnl_offload_xstats_get_size_ndo(const struct net_device *dev, int attr_id)
{
        return rtnl_offload_xstats_have_ndo(dev, attr_id) ?
               sizeof(struct rtnl_link_stats64) : 0;
}

static int
rtnl_offload_xstats_fill_ndo(struct net_device *dev, int attr_id,
                             struct sk_buff *skb)
{
        unsigned int size = rtnl_offload_xstats_get_size_ndo(dev, attr_id);
        struct nlattr *attr = NULL;
        void *attr_data;
        int err;

        if (!size)
                return -ENODATA;

        attr = nla_reserve_64bit(skb, attr_id, size,
                                 IFLA_OFFLOAD_XSTATS_UNSPEC);
        if (!attr)
                return -EMSGSIZE;

        attr_data = nla_data(attr);
        memset(attr_data, 0, size);

        err = dev->netdev_ops->ndo_get_offload_stats(attr_id, dev, attr_data);
        if (err)
                return err;

        return 0;
}

static unsigned int
rtnl_offload_xstats_get_size_stats(const struct net_device *dev,
                                   enum netdev_offload_xstats_type type)
{
        bool enabled = netdev_offload_xstats_enabled(dev, type);

        return enabled ? sizeof(struct rtnl_hw_stats64) : 0;
}

struct rtnl_offload_xstats_request_used {
        bool request;
        bool used;
};

static int
rtnl_offload_xstats_get_stats(struct net_device *dev,
                              enum netdev_offload_xstats_type type,
                              struct rtnl_offload_xstats_request_used *ru,
                              struct rtnl_hw_stats64 *stats,
                              struct netlink_ext_ack *extack)
{
        bool request;
        bool used;
        int err;

        request = netdev_offload_xstats_enabled(dev, type);
        if (!request) {
                used = false;
                goto out;
        }

        err = netdev_offload_xstats_get(dev, type, stats, &used, extack);
        if (err)
                return err;

out:
        if (ru) {
                ru->request = request;
                ru->used = used;
        }
        return 0;
}

static int
rtnl_offload_xstats_fill_hw_s_info_one(struct sk_buff *skb, int attr_id,
                                       struct rtnl_offload_xstats_request_used *ru)
{
        struct nlattr *nest;

        nest = nla_nest_start(skb, attr_id);
        if (!nest)
                return -EMSGSIZE;

        if (nla_put_u8(skb, IFLA_OFFLOAD_XSTATS_HW_S_INFO_REQUEST, ru->request))
                goto nla_put_failure;

        if (nla_put_u8(skb, IFLA_OFFLOAD_XSTATS_HW_S_INFO_USED, ru->used))
                goto nla_put_failure;

        nla_nest_end(skb, nest);
        return 0;

nla_put_failure:
        nla_nest_cancel(skb, nest);
        return -EMSGSIZE;
}

static int
rtnl_offload_xstats_fill_hw_s_info(struct sk_buff *skb, struct net_device *dev,
                                   struct netlink_ext_ack *extack)
{
        enum netdev_offload_xstats_type t_l3 = NETDEV_OFFLOAD_XSTATS_TYPE_L3;
        struct rtnl_offload_xstats_request_used ru_l3;
        struct nlattr *nest;
        int err;

        err = rtnl_offload_xstats_get_stats(dev, t_l3, &ru_l3, NULL, extack);
        if (err)
                return err;

        nest = nla_nest_start(skb, IFLA_OFFLOAD_XSTATS_HW_S_INFO);
        if (!nest)
                return -EMSGSIZE;

        if (rtnl_offload_xstats_fill_hw_s_info_one(skb,
                                                   IFLA_OFFLOAD_XSTATS_L3_STATS,
                                                   &ru_l3))
                goto nla_put_failure;

        nla_nest_end(skb, nest);
        return 0;

nla_put_failure:
        nla_nest_cancel(skb, nest);
        return -EMSGSIZE;
}

static int rtnl_offload_xstats_fill(struct sk_buff *skb, struct net_device *dev,
                                    int *prividx, u32 off_filter_mask,
                                    struct netlink_ext_ack *extack)
{
        enum netdev_offload_xstats_type t_l3 = NETDEV_OFFLOAD_XSTATS_TYPE_L3;
        int attr_id_hw_s_info = IFLA_OFFLOAD_XSTATS_HW_S_INFO;
        int attr_id_l3_stats = IFLA_OFFLOAD_XSTATS_L3_STATS;
        int attr_id_cpu_hit = IFLA_OFFLOAD_XSTATS_CPU_HIT;
        bool have_data = false;
        int err;

        if (*prividx <= attr_id_cpu_hit &&
            (off_filter_mask &
             IFLA_STATS_FILTER_BIT(attr_id_cpu_hit))) {
                err = rtnl_offload_xstats_fill_ndo(dev, attr_id_cpu_hit, skb);
                if (!err) {
                        have_data = true;
                } else if (err != -ENODATA) {
                        *prividx = attr_id_cpu_hit;
                        return err;
                }
        }

        if (*prividx <= attr_id_hw_s_info &&
            (off_filter_mask & IFLA_STATS_FILTER_BIT(attr_id_hw_s_info))) {
                *prividx = attr_id_hw_s_info;

                err = rtnl_offload_xstats_fill_hw_s_info(skb, dev, extack);
                if (err)
                        return err;

                have_data = true;
                *prividx = 0;
        }

        if (*prividx <= attr_id_l3_stats &&
            (off_filter_mask & IFLA_STATS_FILTER_BIT(attr_id_l3_stats))) {
                unsigned int size_l3;
                struct nlattr *attr;

                *prividx = attr_id_l3_stats;

                size_l3 = rtnl_offload_xstats_get_size_stats(dev, t_l3);
                if (!size_l3)
                        goto skip_l3_stats;
                attr = nla_reserve_64bit(skb, attr_id_l3_stats, size_l3,
                                         IFLA_OFFLOAD_XSTATS_UNSPEC);
                if (!attr)
                        return -EMSGSIZE;

                err = rtnl_offload_xstats_get_stats(dev, t_l3, NULL,
                                                    nla_data(attr), extack);
                if (err)
                        return err;

                have_data = true;
skip_l3_stats:
                *prividx = 0;
        }

        if (!have_data)
                return -ENODATA;

        *prividx = 0;
        return 0;
}

static unsigned int
rtnl_offload_xstats_get_size_hw_s_info_one(const struct net_device *dev,
                                           enum netdev_offload_xstats_type type)
{
        return nla_total_size(0) +
                /* IFLA_OFFLOAD_XSTATS_HW_S_INFO_REQUEST */
                nla_total_size(sizeof(u8)) +
                /* IFLA_OFFLOAD_XSTATS_HW_S_INFO_USED */
                nla_total_size(sizeof(u8)) +
                0;
}

static unsigned int
rtnl_offload_xstats_get_size_hw_s_info(const struct net_device *dev)
{
        enum netdev_offload_xstats_type t_l3 = NETDEV_OFFLOAD_XSTATS_TYPE_L3;

        return nla_total_size(0) +
                /* IFLA_OFFLOAD_XSTATS_L3_STATS */
                rtnl_offload_xstats_get_size_hw_s_info_one(dev, t_l3) +
                0;
}

static int rtnl_offload_xstats_get_size(const struct net_device *dev,
                                        u32 off_filter_mask)
{
        enum netdev_offload_xstats_type t_l3 = NETDEV_OFFLOAD_XSTATS_TYPE_L3;
        int attr_id_cpu_hit = IFLA_OFFLOAD_XSTATS_CPU_HIT;
        int nla_size = 0;
        int size;

        if (off_filter_mask &
            IFLA_STATS_FILTER_BIT(attr_id_cpu_hit)) {
                size = rtnl_offload_xstats_get_size_ndo(dev, attr_id_cpu_hit);
                nla_size += nla_total_size_64bit(size);
        }

        if (off_filter_mask &
            IFLA_STATS_FILTER_BIT(IFLA_OFFLOAD_XSTATS_HW_S_INFO))
                nla_size += rtnl_offload_xstats_get_size_hw_s_info(dev);

        if (off_filter_mask &
            IFLA_STATS_FILTER_BIT(IFLA_OFFLOAD_XSTATS_L3_STATS)) {
                size = rtnl_offload_xstats_get_size_stats(dev, t_l3);
                nla_size += nla_total_size_64bit(size);
        }

        if (nla_size != 0)
                nla_size += nla_total_size(0);

        return nla_size;
}

struct rtnl_stats_dump_filters {
        /* mask[0] filters outer attributes. Then individual nests have their
         * filtering mask at the index of the nested attribute.
         */
        u32 mask[IFLA_STATS_MAX + 1];
};

static int rtnl_fill_statsinfo(struct sk_buff *skb, struct net_device *dev,
                               int type, u32 pid, u32 seq, u32 change,
                               unsigned int flags,
                               const struct rtnl_stats_dump_filters *filters,
                               int *idxattr, int *prividx,
                               struct netlink_ext_ack *extack)
{
        unsigned int filter_mask = filters->mask[0];
        struct if_stats_msg *ifsm;
        struct nlmsghdr *nlh;
        struct nlattr *attr;
        int s_prividx = *prividx;
        int err;

        ASSERT_RTNL();

        nlh = nlmsg_put(skb, pid, seq, type, sizeof(*ifsm), flags);
        if (!nlh)
                return -EMSGSIZE;

        ifsm = nlmsg_data(nlh);
        ifsm->family = PF_UNSPEC;
        ifsm->pad1 = 0;
        ifsm->pad2 = 0;
        ifsm->ifindex = dev->ifindex;
        ifsm->filter_mask = filter_mask;

        if (stats_attr_valid(filter_mask, IFLA_STATS_LINK_64, *idxattr)) {
                struct rtnl_link_stats64 *sp;

                attr = nla_reserve_64bit(skb, IFLA_STATS_LINK_64,
                                         sizeof(struct rtnl_link_stats64),
                                         IFLA_STATS_UNSPEC);
                if (!attr) {
                        err = -EMSGSIZE;
                        goto nla_put_failure;
                }

                sp = nla_data(attr);
                dev_get_stats(dev, sp);
        }

        if (stats_attr_valid(filter_mask, IFLA_STATS_LINK_XSTATS, *idxattr)) {
                const struct rtnl_link_ops *ops = dev->rtnl_link_ops;

                if (ops && ops->fill_linkxstats) {
                        *idxattr = IFLA_STATS_LINK_XSTATS;
                        attr = nla_nest_start_noflag(skb,
                                                     IFLA_STATS_LINK_XSTATS);
                        if (!attr) {
                                err = -EMSGSIZE;
                                goto nla_put_failure;
                        }

                        err = ops->fill_linkxstats(skb, dev, prividx, *idxattr);
                        nla_nest_end(skb, attr);
                        if (err)
                                goto nla_put_failure;
                        *idxattr = 0;
                }
        }

        if (stats_attr_valid(filter_mask, IFLA_STATS_LINK_XSTATS_SLAVE,
                             *idxattr)) {
                const struct rtnl_link_ops *ops = NULL;
                const struct net_device *master;

                master = netdev_master_upper_dev_get(dev);
                if (master)
                        ops = master->rtnl_link_ops;
                if (ops && ops->fill_linkxstats) {
                        *idxattr = IFLA_STATS_LINK_XSTATS_SLAVE;
                        attr = nla_nest_start_noflag(skb,
                                                     IFLA_STATS_LINK_XSTATS_SLAVE);
                        if (!attr) {
                                err = -EMSGSIZE;
                                goto nla_put_failure;
                        }

                        err = ops->fill_linkxstats(skb, dev, prividx, *idxattr);
                        nla_nest_end(skb, attr);
                        if (err)
                                goto nla_put_failure;
                        *idxattr = 0;
                }
        }

        if (stats_attr_valid(filter_mask, IFLA_STATS_LINK_OFFLOAD_XSTATS,
                             *idxattr)) {
                u32 off_filter_mask;

                off_filter_mask = filters->mask[IFLA_STATS_LINK_OFFLOAD_XSTATS];
                *idxattr = IFLA_STATS_LINK_OFFLOAD_XSTATS;
                attr = nla_nest_start_noflag(skb,
                                             IFLA_STATS_LINK_OFFLOAD_XSTATS);
                if (!attr) {
                        err = -EMSGSIZE;
                        goto nla_put_failure;
                }

                err = rtnl_offload_xstats_fill(skb, dev, prividx,
                                               off_filter_mask, extack);
                if (err == -ENODATA)
                        nla_nest_cancel(skb, attr);
                else
                        nla_nest_end(skb, attr);

                if (err && err != -ENODATA)
                        goto nla_put_failure;
                *idxattr = 0;
        }

        if (stats_attr_valid(filter_mask, IFLA_STATS_AF_SPEC, *idxattr)) {
                struct rtnl_af_ops *af_ops;

                *idxattr = IFLA_STATS_AF_SPEC;
                attr = nla_nest_start_noflag(skb, IFLA_STATS_AF_SPEC);
                if (!attr) {
                        err = -EMSGSIZE;
                        goto nla_put_failure;
                }

                rcu_read_lock();
                list_for_each_entry_rcu(af_ops, &rtnl_af_ops, list) {
                        if (af_ops->fill_stats_af) {
                                struct nlattr *af;

                                af = nla_nest_start_noflag(skb,
                                                           af_ops->family);
                                if (!af) {
                                        rcu_read_unlock();
                                        err = -EMSGSIZE;
                                        goto nla_put_failure;
                                }
                                err = af_ops->fill_stats_af(skb, dev);

                                if (err == -ENODATA) {
                                        nla_nest_cancel(skb, af);
                                } else if (err < 0) {
                                        rcu_read_unlock();
                                        goto nla_put_failure;
                                }

                                nla_nest_end(skb, af);
                        }
                }
                rcu_read_unlock();

                nla_nest_end(skb, attr);

                *idxattr = 0;
        }

        nlmsg_end(skb, nlh);

        return 0;

nla_put_failure:
        /* not a multi message or no progress mean a real error */
        if (!(flags & NLM_F_MULTI) || s_prividx == *prividx)
                nlmsg_cancel(skb, nlh);
        else
                nlmsg_end(skb, nlh);

        return err;
}

static size_t if_nlmsg_stats_size(const struct net_device *dev,
                                  const struct rtnl_stats_dump_filters *filters)
{
        size_t size = NLMSG_ALIGN(sizeof(struct if_stats_msg));
        unsigned int filter_mask = filters->mask[0];

        if (stats_attr_valid(filter_mask, IFLA_STATS_LINK_64, 0))
                size += nla_total_size_64bit(sizeof(struct rtnl_link_stats64));

        if (stats_attr_valid(filter_mask, IFLA_STATS_LINK_XSTATS, 0)) {
                const struct rtnl_link_ops *ops = dev->rtnl_link_ops;
                int attr = IFLA_STATS_LINK_XSTATS;

                if (ops && ops->get_linkxstats_size) {
                        size += nla_total_size(ops->get_linkxstats_size(dev,
                                                                        attr));
                        /* for IFLA_STATS_LINK_XSTATS */
                        size += nla_total_size(0);
                }
        }

        if (stats_attr_valid(filter_mask, IFLA_STATS_LINK_XSTATS_SLAVE, 0)) {
                struct net_device *_dev = (struct net_device *)dev;
                const struct rtnl_link_ops *ops = NULL;
                const struct net_device *master;

                /* netdev_master_upper_dev_get can't take const */
                master = netdev_master_upper_dev_get(_dev);
                if (master)
                        ops = master->rtnl_link_ops;
                if (ops && ops->get_linkxstats_size) {
                        int attr = IFLA_STATS_LINK_XSTATS_SLAVE;

                        size += nla_total_size(ops->get_linkxstats_size(dev,
                                                                        attr));
                        /* for IFLA_STATS_LINK_XSTATS_SLAVE */
                        size += nla_total_size(0);
                }
        }

        if (stats_attr_valid(filter_mask, IFLA_STATS_LINK_OFFLOAD_XSTATS, 0)) {
                u32 off_filter_mask;

                off_filter_mask = filters->mask[IFLA_STATS_LINK_OFFLOAD_XSTATS];
                size += rtnl_offload_xstats_get_size(dev, off_filter_mask);
        }

        if (stats_attr_valid(filter_mask, IFLA_STATS_AF_SPEC, 0)) {
                struct rtnl_af_ops *af_ops;

                /* for IFLA_STATS_AF_SPEC */
                size += nla_total_size(0);

                rcu_read_lock();
                list_for_each_entry_rcu(af_ops, &rtnl_af_ops, list) {
                        if (af_ops->get_stats_af_size) {
                                size += nla_total_size(
                                        af_ops->get_stats_af_size(dev));

                                /* for AF_* */
                                size += nla_total_size(0);
                        }
                }
                rcu_read_unlock();
        }

        return size;
}

#define RTNL_STATS_OFFLOAD_XSTATS_VALID ((1 << __IFLA_OFFLOAD_XSTATS_MAX) - 1)

static const struct nla_policy
rtnl_stats_get_policy_filters[IFLA_STATS_MAX + 1] = {
        [IFLA_STATS_LINK_OFFLOAD_XSTATS] =
                    NLA_POLICY_MASK(NLA_U32, RTNL_STATS_OFFLOAD_XSTATS_VALID),
};

static const struct nla_policy
rtnl_stats_get_policy[IFLA_STATS_GETSET_MAX + 1] = {
        [IFLA_STATS_GET_FILTERS] =
                    NLA_POLICY_NESTED(rtnl_stats_get_policy_filters),
};

static const struct nla_policy
ifla_stats_set_policy[IFLA_STATS_GETSET_MAX + 1] = {
        [IFLA_STATS_SET_OFFLOAD_XSTATS_L3_STATS] = NLA_POLICY_MAX(NLA_U8, 1),
};

static int rtnl_stats_get_parse_filters(struct nlattr *ifla_filters,
                                        struct rtnl_stats_dump_filters *filters,
                                        struct netlink_ext_ack *extack)
{
        struct nlattr *tb[IFLA_STATS_MAX + 1];
        int err;
        int at;

        err = nla_parse_nested(tb, IFLA_STATS_MAX, ifla_filters,
                               rtnl_stats_get_policy_filters, extack);
        if (err < 0)
                return err;

        for (at = 1; at <= IFLA_STATS_MAX; at++) {
                if (tb[at]) {
                        if (!(filters->mask[0] & IFLA_STATS_FILTER_BIT(at))) {
                                NL_SET_ERR_MSG(extack, "Filtered attribute not enabled in filter_mask");
                                return -EINVAL;
                        }
                        filters->mask[at] = nla_get_u32(tb[at]);
                }
        }

        return 0;
}

static int rtnl_stats_get_parse(const struct nlmsghdr *nlh,
                                u32 filter_mask,
                                struct rtnl_stats_dump_filters *filters,
                                struct netlink_ext_ack *extack)
{
        struct nlattr *tb[IFLA_STATS_GETSET_MAX + 1];
        int err;
        int i;

        filters->mask[0] = filter_mask;
        for (i = 1; i < ARRAY_SIZE(filters->mask); i++)
                filters->mask[i] = -1U;

        err = nlmsg_parse(nlh, sizeof(struct if_stats_msg), tb,
                          IFLA_STATS_GETSET_MAX, rtnl_stats_get_policy, extack);
        if (err < 0)
                return err;

        if (tb[IFLA_STATS_GET_FILTERS]) {
                err = rtnl_stats_get_parse_filters(tb[IFLA_STATS_GET_FILTERS],
                                                   filters, extack);
                if (err)
                        return err;
        }

        return 0;
}

static int rtnl_valid_stats_req(const struct nlmsghdr *nlh, bool strict_check,
                                bool is_dump, struct netlink_ext_ack *extack)
{
        struct if_stats_msg *ifsm;

        ifsm = nlmsg_payload(nlh, sizeof(*ifsm));
        if (!ifsm) {
                NL_SET_ERR_MSG(extack, "Invalid header for stats dump");
                return -EINVAL;
        }

        if (!strict_check)
                return 0;

        /* only requests using strict checks can pass data to influence
         * the dump. The legacy exception is filter_mask.
         */
        if (ifsm->pad1 || ifsm->pad2 || (is_dump && ifsm->ifindex)) {
                NL_SET_ERR_MSG(extack, "Invalid values in header for stats dump request");
                return -EINVAL;
        }
        if (ifsm->filter_mask >= IFLA_STATS_FILTER_BIT(IFLA_STATS_MAX + 1)) {
                NL_SET_ERR_MSG(extack, "Invalid stats requested through filter mask");
                return -EINVAL;
        }

        return 0;
}

static int rtnl_stats_get(struct sk_buff *skb, struct nlmsghdr *nlh,
                          struct netlink_ext_ack *extack)
{
        struct rtnl_stats_dump_filters filters;
        struct net *net = sock_net(skb->sk);
        struct net_device *dev = NULL;
        int idxattr = 0, prividx = 0;
        struct if_stats_msg *ifsm;
        struct sk_buff *nskb;
        int err;

        err = rtnl_valid_stats_req(nlh, netlink_strict_get_check(skb),
                                   false, extack);
        if (err)
                return err;

        ifsm = nlmsg_data(nlh);
        if (ifsm->ifindex > 0)
                dev = __dev_get_by_index(net, ifsm->ifindex);
        else
                return -EINVAL;

        if (!dev)
                return -ENODEV;

        if (!ifsm->filter_mask) {
                NL_SET_ERR_MSG(extack, "Filter mask must be set for stats get");
                return -EINVAL;
        }

        err = rtnl_stats_get_parse(nlh, ifsm->filter_mask, &filters, extack);
        if (err)
                return err;

        nskb = nlmsg_new(if_nlmsg_stats_size(dev, &filters), GFP_KERNEL);
        if (!nskb)
                return -ENOBUFS;

        err = rtnl_fill_statsinfo(nskb, dev, RTM_NEWSTATS,
                                  NETLINK_CB(skb).portid, nlh->nlmsg_seq, 0,
                                  0, &filters, &idxattr, &prividx, extack);
        if (err < 0) {
                /* -EMSGSIZE implies BUG in if_nlmsg_stats_size */
                WARN_ON(err == -EMSGSIZE);
                kfree_skb(nskb);
        } else {
                err = rtnl_unicast(nskb, net, NETLINK_CB(skb).portid);
        }

        return err;
}

static int rtnl_stats_dump(struct sk_buff *skb, struct netlink_callback *cb)
{
        struct netlink_ext_ack *extack = cb->extack;
        struct rtnl_stats_dump_filters filters;
        struct net *net = sock_net(skb->sk);
        unsigned int flags = NLM_F_MULTI;
        struct if_stats_msg *ifsm;
        struct {
                unsigned long ifindex;
                int idxattr;
                int prividx;
        } *ctx = (void *)cb->ctx;
        struct net_device *dev;
        int err;

        cb->seq = net->dev_base_seq;

        err = rtnl_valid_stats_req(cb->nlh, cb->strict_check, true, extack);
        if (err)
                return err;

        ifsm = nlmsg_data(cb->nlh);
        if (!ifsm->filter_mask) {
                NL_SET_ERR_MSG(extack, "Filter mask must be set for stats dump");
                return -EINVAL;
        }

        err = rtnl_stats_get_parse(cb->nlh, ifsm->filter_mask, &filters,
                                   extack);
        if (err)
                return err;

        for_each_netdev_dump(net, dev, ctx->ifindex) {
                err = rtnl_fill_statsinfo(skb, dev, RTM_NEWSTATS,
                                          NETLINK_CB(cb->skb).portid,
                                          cb->nlh->nlmsg_seq, 0,
                                          flags, &filters,
                                          &ctx->idxattr, &ctx->prividx,
                                          extack);
                /* If we ran out of room on the first message,
                 * we're in trouble.
                 */
                WARN_ON((err == -EMSGSIZE) && (skb->len == 0));

                if (err < 0)
                        break;
                ctx->prividx = 0;
                ctx->idxattr = 0;
                nl_dump_check_consistent(cb, nlmsg_hdr(skb));
        }

        return err;
}

void rtnl_offload_xstats_notify(struct net_device *dev)
{
        struct rtnl_stats_dump_filters response_filters = {};
        struct net *net = dev_net(dev);
        int idxattr = 0, prividx = 0;
        struct sk_buff *skb;
        int err = -ENOBUFS;

        ASSERT_RTNL();

        response_filters.mask[0] |=
                IFLA_STATS_FILTER_BIT(IFLA_STATS_LINK_OFFLOAD_XSTATS);
        response_filters.mask[IFLA_STATS_LINK_OFFLOAD_XSTATS] |=
                IFLA_STATS_FILTER_BIT(IFLA_OFFLOAD_XSTATS_HW_S_INFO);

        skb = nlmsg_new(if_nlmsg_stats_size(dev, &response_filters),
                        GFP_KERNEL);
        if (!skb)
                goto errout;

        err = rtnl_fill_statsinfo(skb, dev, RTM_NEWSTATS, 0, 0, 0, 0,
                                  &response_filters, &idxattr, &prividx, NULL);
        if (err < 0) {
                kfree_skb(skb);
                goto errout;
        }

        rtnl_notify(skb, net, 0, RTNLGRP_STATS, NULL, GFP_KERNEL);
        return;

errout:
        rtnl_set_sk_err(net, RTNLGRP_STATS, err);
}
EXPORT_SYMBOL(rtnl_offload_xstats_notify);

static int rtnl_stats_set(struct sk_buff *skb, struct nlmsghdr *nlh,
                          struct netlink_ext_ack *extack)
{
        enum netdev_offload_xstats_type t_l3 = NETDEV_OFFLOAD_XSTATS_TYPE_L3;
        struct rtnl_stats_dump_filters response_filters = {};
        struct nlattr *tb[IFLA_STATS_GETSET_MAX + 1];
        struct net *net = sock_net(skb->sk);
        struct net_device *dev = NULL;
        struct if_stats_msg *ifsm;
        bool notify = false;
        int err;

        err = rtnl_valid_stats_req(nlh, netlink_strict_get_check(skb),
                                   false, extack);
        if (err)
                return err;

        ifsm = nlmsg_data(nlh);
        if (ifsm->family != AF_UNSPEC) {
                NL_SET_ERR_MSG(extack, "Address family should be AF_UNSPEC");
                return -EINVAL;
        }

        if (ifsm->ifindex > 0)
                dev = __dev_get_by_index(net, ifsm->ifindex);
        else
                return -EINVAL;

        if (!dev)
                return -ENODEV;

        if (ifsm->filter_mask) {
                NL_SET_ERR_MSG(extack, "Filter mask must be 0 for stats set");
                return -EINVAL;
        }

        err = nlmsg_parse(nlh, sizeof(*ifsm), tb, IFLA_STATS_GETSET_MAX,
                          ifla_stats_set_policy, extack);
        if (err < 0)
                return err;

        if (tb[IFLA_STATS_SET_OFFLOAD_XSTATS_L3_STATS]) {
                u8 req = nla_get_u8(tb[IFLA_STATS_SET_OFFLOAD_XSTATS_L3_STATS]);

                if (req)
                        err = netdev_offload_xstats_enable(dev, t_l3, extack);
                else
                        err = netdev_offload_xstats_disable(dev, t_l3);

                if (!err)
                        notify = true;
                else if (err != -EALREADY)
                        return err;

                response_filters.mask[0] |=
                        IFLA_STATS_FILTER_BIT(IFLA_STATS_LINK_OFFLOAD_XSTATS);
                response_filters.mask[IFLA_STATS_LINK_OFFLOAD_XSTATS] |=
                        IFLA_STATS_FILTER_BIT(IFLA_OFFLOAD_XSTATS_HW_S_INFO);
        }

        if (notify)
                rtnl_offload_xstats_notify(dev);

        return 0;
}

static int rtnl_mdb_valid_dump_req(const struct nlmsghdr *nlh,
                                   struct netlink_ext_ack *extack)
{
        struct br_port_msg *bpm;

        bpm = nlmsg_payload(nlh, sizeof(*bpm));
        if (!bpm) {
                NL_SET_ERR_MSG(extack, "Invalid header for mdb dump request");
                return -EINVAL;
        }

        if (bpm->ifindex) {
                NL_SET_ERR_MSG(extack, "Filtering by device index is not supported for mdb dump request");
                return -EINVAL;
        }
        if (nlmsg_attrlen(nlh, sizeof(*bpm))) {
                NL_SET_ERR_MSG(extack, "Invalid data after header in mdb dump request");
                return -EINVAL;
        }

        return 0;
}

struct rtnl_mdb_dump_ctx {
        long idx;
};

static int rtnl_mdb_dump(struct sk_buff *skb, struct netlink_callback *cb)
{
        struct rtnl_mdb_dump_ctx *ctx = (void *)cb->ctx;
        struct net *net = sock_net(skb->sk);
        struct net_device *dev;
        int idx, s_idx;
        int err;

        NL_ASSERT_CTX_FITS(struct rtnl_mdb_dump_ctx);

        if (cb->strict_check) {
                err = rtnl_mdb_valid_dump_req(cb->nlh, cb->extack);
                if (err)
                        return err;
        }

        s_idx = ctx->idx;
        idx = 0;

        for_each_netdev(net, dev) {
                if (idx < s_idx)
                        goto skip;
                if (!dev->netdev_ops->ndo_mdb_dump)
                        goto skip;

                err = dev->netdev_ops->ndo_mdb_dump(dev, skb, cb);
                if (err == -EMSGSIZE)
                        goto out;
                /* Moving on to next device, reset markers and sequence
                 * counters since they are all maintained per-device.
                 */
                memset(cb->ctx, 0, sizeof(cb->ctx));
                cb->prev_seq = 0;
                cb->seq = 0;
skip:
                idx++;
        }

out:
        ctx->idx = idx;
        return skb->len;
}

static int rtnl_validate_mdb_entry_get(const struct nlattr *attr,
                                       struct netlink_ext_ack *extack)
{
        struct br_mdb_entry *entry = nla_data(attr);

        if (nla_len(attr) != sizeof(struct br_mdb_entry)) {
                NL_SET_ERR_MSG_ATTR(extack, attr, "Invalid attribute length");
                return -EINVAL;
        }

        if (entry->ifindex) {
                NL_SET_ERR_MSG(extack, "Entry ifindex cannot be specified");
                return -EINVAL;
        }

        if (entry->state) {
                NL_SET_ERR_MSG(extack, "Entry state cannot be specified");
                return -EINVAL;
        }

        if (entry->flags) {
                NL_SET_ERR_MSG(extack, "Entry flags cannot be specified");
                return -EINVAL;
        }

        if (entry->vid >= VLAN_VID_MASK) {
                NL_SET_ERR_MSG(extack, "Invalid entry VLAN id");
                return -EINVAL;
        }

        if (entry->addr.proto != htons(ETH_P_IP) &&
            entry->addr.proto != htons(ETH_P_IPV6) &&
            entry->addr.proto != 0) {
                NL_SET_ERR_MSG(extack, "Unknown entry protocol");
                return -EINVAL;
        }

        return 0;
}

static const struct nla_policy mdba_get_policy[MDBA_GET_ENTRY_MAX + 1] = {
        [MDBA_GET_ENTRY] = NLA_POLICY_VALIDATE_FN(NLA_BINARY,
                                                  rtnl_validate_mdb_entry_get,
                                                  sizeof(struct br_mdb_entry)),
        [MDBA_GET_ENTRY_ATTRS] = { .type = NLA_NESTED },
};

static int rtnl_mdb_get(struct sk_buff *in_skb, struct nlmsghdr *nlh,
                        struct netlink_ext_ack *extack)
{
        struct nlattr *tb[MDBA_GET_ENTRY_MAX + 1];
        struct net *net = sock_net(in_skb->sk);
        struct br_port_msg *bpm;
        struct net_device *dev;
        int err;

        err = nlmsg_parse(nlh, sizeof(struct br_port_msg), tb,
                          MDBA_GET_ENTRY_MAX, mdba_get_policy, extack);
        if (err)
                return err;

        bpm = nlmsg_data(nlh);
        if (!bpm->ifindex) {
                NL_SET_ERR_MSG(extack, "Invalid ifindex");
                return -EINVAL;
        }

        dev = __dev_get_by_index(net, bpm->ifindex);
        if (!dev) {
                NL_SET_ERR_MSG(extack, "Device doesn't exist");
                return -ENODEV;
        }

        if (NL_REQ_ATTR_CHECK(extack, NULL, tb, MDBA_GET_ENTRY)) {
                NL_SET_ERR_MSG(extack, "Missing MDBA_GET_ENTRY attribute");
                return -EINVAL;
        }

        if (!dev->netdev_ops->ndo_mdb_get) {
                NL_SET_ERR_MSG(extack, "Device does not support MDB operations");
                return -EOPNOTSUPP;
        }

        return dev->netdev_ops->ndo_mdb_get(dev, tb, NETLINK_CB(in_skb).portid,
                                            nlh->nlmsg_seq, extack);
}

static int rtnl_validate_mdb_entry(const struct nlattr *attr,
                                   struct netlink_ext_ack *extack)
{
        struct br_mdb_entry *entry = nla_data(attr);

        if (nla_len(attr) != sizeof(struct br_mdb_entry)) {
                NL_SET_ERR_MSG_ATTR(extack, attr, "Invalid attribute length");
                return -EINVAL;
        }

        if (entry->ifindex == 0) {
                NL_SET_ERR_MSG(extack, "Zero entry ifindex is not allowed");
                return -EINVAL;
        }

        if (entry->addr.proto == htons(ETH_P_IP)) {
                if (!ipv4_is_multicast(entry->addr.u.ip4) &&
                    !ipv4_is_zeronet(entry->addr.u.ip4)) {
                        NL_SET_ERR_MSG(extack, "IPv4 entry group address is not multicast or 0.0.0.0");
                        return -EINVAL;
                }
                if (ipv4_is_local_multicast(entry->addr.u.ip4)) {
                        NL_SET_ERR_MSG(extack, "IPv4 entry group address is local multicast");
                        return -EINVAL;
                }
#if IS_ENABLED(CONFIG_IPV6)
        } else if (entry->addr.proto == htons(ETH_P_IPV6)) {
                if (ipv6_addr_is_ll_all_nodes(&entry->addr.u.ip6)) {
                        NL_SET_ERR_MSG(extack, "IPv6 entry group address is link-local all nodes");
                        return -EINVAL;
                }
#endif
        } else if (entry->addr.proto == 0) {
                /* L2 mdb */
                if (!is_multicast_ether_addr(entry->addr.u.mac_addr)) {
                        NL_SET_ERR_MSG(extack, "L2 entry group is not multicast");
                        return -EINVAL;
                }
        } else {
                NL_SET_ERR_MSG(extack, "Unknown entry protocol");
                return -EINVAL;
        }

        if (entry->state != MDB_PERMANENT && entry->state != MDB_TEMPORARY) {
                NL_SET_ERR_MSG(extack, "Unknown entry state");
                return -EINVAL;
        }
        if (entry->vid >= VLAN_VID_MASK) {
                NL_SET_ERR_MSG(extack, "Invalid entry VLAN id");
                return -EINVAL;
        }

        return 0;
}

static const struct nla_policy mdba_policy[MDBA_SET_ENTRY_MAX + 1] = {
        [MDBA_SET_ENTRY_UNSPEC] = { .strict_start_type = MDBA_SET_ENTRY_ATTRS + 1 },
        [MDBA_SET_ENTRY] = NLA_POLICY_VALIDATE_FN(NLA_BINARY,
                                                  rtnl_validate_mdb_entry,
                                                  sizeof(struct br_mdb_entry)),
        [MDBA_SET_ENTRY_ATTRS] = { .type = NLA_NESTED },
};

static int rtnl_mdb_add(struct sk_buff *skb, struct nlmsghdr *nlh,
                        struct netlink_ext_ack *extack)
{
        struct nlattr *tb[MDBA_SET_ENTRY_MAX + 1];
        struct net *net = sock_net(skb->sk);
        struct br_port_msg *bpm;
        struct net_device *dev;
        int err;

        err = nlmsg_parse_deprecated(nlh, sizeof(*bpm), tb,
                                     MDBA_SET_ENTRY_MAX, mdba_policy, extack);
        if (err)
                return err;

        bpm = nlmsg_data(nlh);
        if (!bpm->ifindex) {
                NL_SET_ERR_MSG(extack, "Invalid ifindex");
                return -EINVAL;
        }

        dev = __dev_get_by_index(net, bpm->ifindex);
        if (!dev) {
                NL_SET_ERR_MSG(extack, "Device doesn't exist");
                return -ENODEV;
        }

        if (NL_REQ_ATTR_CHECK(extack, NULL, tb, MDBA_SET_ENTRY)) {
                NL_SET_ERR_MSG(extack, "Missing MDBA_SET_ENTRY attribute");
                return -EINVAL;
        }

        if (!dev->netdev_ops->ndo_mdb_add) {
                NL_SET_ERR_MSG(extack, "Device does not support MDB operations");
                return -EOPNOTSUPP;
        }

        return dev->netdev_ops->ndo_mdb_add(dev, tb, nlh->nlmsg_flags, extack);
}

static int rtnl_validate_mdb_entry_del_bulk(const struct nlattr *attr,
                                            struct netlink_ext_ack *extack)
{
        struct br_mdb_entry *entry = nla_data(attr);
        struct br_mdb_entry zero_entry = {};

        if (nla_len(attr) != sizeof(struct br_mdb_entry)) {
                NL_SET_ERR_MSG_ATTR(extack, attr, "Invalid attribute length");
                return -EINVAL;
        }

        if (entry->state != MDB_PERMANENT && entry->state != MDB_TEMPORARY) {
                NL_SET_ERR_MSG(extack, "Unknown entry state");
                return -EINVAL;
        }

        if (entry->flags) {
                NL_SET_ERR_MSG(extack, "Entry flags cannot be set");
                return -EINVAL;
        }

        if (entry->vid >= VLAN_N_VID - 1) {
                NL_SET_ERR_MSG(extack, "Invalid entry VLAN id");
                return -EINVAL;
        }

        if (memcmp(&entry->addr, &zero_entry.addr, sizeof(entry->addr))) {
                NL_SET_ERR_MSG(extack, "Entry address cannot be set");
                return -EINVAL;
        }

        return 0;
}

static const struct nla_policy mdba_del_bulk_policy[MDBA_SET_ENTRY_MAX + 1] = {
        [MDBA_SET_ENTRY] = NLA_POLICY_VALIDATE_FN(NLA_BINARY,
                                                  rtnl_validate_mdb_entry_del_bulk,
                                                  sizeof(struct br_mdb_entry)),
        [MDBA_SET_ENTRY_ATTRS] = { .type = NLA_NESTED },
};

static int rtnl_mdb_del(struct sk_buff *skb, struct nlmsghdr *nlh,
                        struct netlink_ext_ack *extack)
{
        bool del_bulk = !!(nlh->nlmsg_flags & NLM_F_BULK);
        struct nlattr *tb[MDBA_SET_ENTRY_MAX + 1];
        struct net *net = sock_net(skb->sk);
        struct br_port_msg *bpm;
        struct net_device *dev;
        int err;

        if (!del_bulk)
                err = nlmsg_parse_deprecated(nlh, sizeof(*bpm), tb,
                                             MDBA_SET_ENTRY_MAX, mdba_policy,
                                             extack);
        else
                err = nlmsg_parse(nlh, sizeof(*bpm), tb, MDBA_SET_ENTRY_MAX,
                                  mdba_del_bulk_policy, extack);
        if (err)
                return err;

        bpm = nlmsg_data(nlh);
        if (!bpm->ifindex) {
                NL_SET_ERR_MSG(extack, "Invalid ifindex");
                return -EINVAL;
        }

        dev = __dev_get_by_index(net, bpm->ifindex);
        if (!dev) {
                NL_SET_ERR_MSG(extack, "Device doesn't exist");
                return -ENODEV;
        }

        if (NL_REQ_ATTR_CHECK(extack, NULL, tb, MDBA_SET_ENTRY)) {
                NL_SET_ERR_MSG(extack, "Missing MDBA_SET_ENTRY attribute");
                return -EINVAL;
        }

        if (del_bulk) {
                if (!dev->netdev_ops->ndo_mdb_del_bulk) {
                        NL_SET_ERR_MSG(extack, "Device does not support MDB bulk deletion");
                        return -EOPNOTSUPP;
                }
                return dev->netdev_ops->ndo_mdb_del_bulk(dev, tb, extack);
        }

        if (!dev->netdev_ops->ndo_mdb_del) {
                NL_SET_ERR_MSG(extack, "Device does not support MDB operations");
                return -EOPNOTSUPP;
        }

        return dev->netdev_ops->ndo_mdb_del(dev, tb, extack);
}

/* Process one rtnetlink message. */

static int rtnl_dumpit(struct sk_buff *skb, struct netlink_callback *cb)
{
        const bool needs_lock = !(cb->flags & RTNL_FLAG_DUMP_UNLOCKED);
        rtnl_dumpit_func dumpit = cb->data;
        int err;

        /* Previous iteration have already finished, avoid calling->dumpit()
         * again, it may not expect to be called after it reached the end.
         */
        if (!dumpit)
                return 0;

        if (needs_lock)
                rtnl_lock();
        err = dumpit(skb, cb);
        if (needs_lock)
                rtnl_unlock();

        /* Old dump handlers used to send NLM_DONE as in a separate recvmsg().
         * Some applications which parse netlink manually depend on this.
         */
        if (cb->flags & RTNL_FLAG_DUMP_SPLIT_NLM_DONE) {
                if (err < 0 && err != -EMSGSIZE)
                        return err;
                if (!err)
                        cb->data = NULL;

                return skb->len;
        }
        return err;
}

static int rtnetlink_dump_start(struct sock *ssk, struct sk_buff *skb,
                                const struct nlmsghdr *nlh,
                                struct netlink_dump_control *control)
{
        if (control->flags & RTNL_FLAG_DUMP_SPLIT_NLM_DONE ||
            !(control->flags & RTNL_FLAG_DUMP_UNLOCKED)) {
                WARN_ON(control->data);
                control->data = control->dump;
                control->dump = rtnl_dumpit;
        }

        return netlink_dump_start(ssk, skb, nlh, control);
}

static int rtnetlink_rcv_msg(struct sk_buff *skb, struct nlmsghdr *nlh,
                             struct netlink_ext_ack *extack)
{
        struct net *net = sock_net(skb->sk);
        struct rtnl_link *link;
        enum rtnl_kinds kind;
        struct module *owner;
        int err = -EOPNOTSUPP;
        rtnl_doit_func doit;
        unsigned int flags;
        int family;
        int type;

        type = nlh->nlmsg_type;
        if (type > RTM_MAX)
                return -EOPNOTSUPP;

        type -= RTM_BASE;

        /* All the messages must have at least 1 byte length */
        if (nlmsg_len(nlh) < sizeof(struct rtgenmsg))
                return 0;

        family = ((struct rtgenmsg *)nlmsg_data(nlh))->rtgen_family;
        kind = rtnl_msgtype_kind(type);

        if (kind != RTNL_KIND_GET && !netlink_net_capable(skb, CAP_NET_ADMIN))
                return -EPERM;

        rcu_read_lock();
        if (kind == RTNL_KIND_GET && (nlh->nlmsg_flags & NLM_F_DUMP)) {
                struct sock *rtnl;
                rtnl_dumpit_func dumpit;
                u32 min_dump_alloc = 0;

                link = rtnl_get_link(family, type);
                if (!link || !link->dumpit) {
                        family = PF_UNSPEC;
                        link = rtnl_get_link(family, type);
                        if (!link || !link->dumpit)
                                goto err_unlock;
                }
                owner = link->owner;
                dumpit = link->dumpit;
                flags = link->flags;

                if (type == RTM_GETLINK - RTM_BASE)
                        min_dump_alloc = rtnl_calcit(skb, nlh);

                err = 0;
                /* need to do this before rcu_read_unlock() */
                if (!try_module_get(owner))
                        err = -EPROTONOSUPPORT;

                rcu_read_unlock();

                rtnl = net->rtnl;
                if (err == 0) {
                        struct netlink_dump_control c = {
                                .dump                = dumpit,
                                .min_dump_alloc        = min_dump_alloc,
                                .module                = owner,
                                .flags                = flags,
                        };
                        err = rtnetlink_dump_start(rtnl, skb, nlh, &c);
                        /* netlink_dump_start() will keep a reference on
                         * module if dump is still in progress.
                         */
                        module_put(owner);
                }
                return err;
        }

        link = rtnl_get_link(family, type);
        if (!link || !link->doit) {
                family = PF_UNSPEC;
                link = rtnl_get_link(PF_UNSPEC, type);
                if (!link || !link->doit)
                        goto out_unlock;
        }

        owner = link->owner;
        if (!try_module_get(owner)) {
                err = -EPROTONOSUPPORT;
                goto out_unlock;
        }

        flags = link->flags;
        if (kind == RTNL_KIND_DEL && (nlh->nlmsg_flags & NLM_F_BULK) &&
            !(flags & RTNL_FLAG_BULK_DEL_SUPPORTED)) {
                NL_SET_ERR_MSG(extack, "Bulk delete is not supported");
                module_put(owner);
                goto err_unlock;
        }

        if (flags & RTNL_FLAG_DOIT_UNLOCKED) {
                doit = link->doit;
                rcu_read_unlock();
                if (doit)
                        err = doit(skb, nlh, extack);
                module_put(owner);
                return err;
        }
        rcu_read_unlock();

        rtnl_lock();
        link = rtnl_get_link(family, type);
        if (link && link->doit)
                err = link->doit(skb, nlh, extack);
        rtnl_unlock();

        module_put(owner);

        return err;

out_unlock:
        rcu_read_unlock();
        return err;

err_unlock:
        rcu_read_unlock();
        return -EOPNOTSUPP;
}

static void rtnetlink_rcv(struct sk_buff *skb)
{
        netlink_rcv_skb(skb, &rtnetlink_rcv_msg);
}

static int rtnetlink_bind(struct net *net, int group)
{
        switch (group) {
        case RTNLGRP_IPV4_MROUTE_R:
        case RTNLGRP_IPV6_MROUTE_R:
                if (!ns_capable(net->user_ns, CAP_NET_ADMIN))
                        return -EPERM;
                break;
        }
        return 0;
}

static int rtnetlink_event(struct notifier_block *this, unsigned long event, void *ptr)
{
        struct net_device *dev = netdev_notifier_info_to_dev(ptr);

        switch (event) {
        case NETDEV_REBOOT:
        case NETDEV_CHANGEMTU:
        case NETDEV_CHANGEADDR:
        case NETDEV_CHANGENAME:
        case NETDEV_FEAT_CHANGE:
        case NETDEV_BONDING_FAILOVER:
        case NETDEV_POST_TYPE_CHANGE:
        case NETDEV_NOTIFY_PEERS:
        case NETDEV_CHANGEUPPER:
        case NETDEV_RESEND_IGMP:
        case NETDEV_CHANGEINFODATA:
        case NETDEV_CHANGELOWERSTATE:
        case NETDEV_CHANGE_TX_QUEUE_LEN:
                rtmsg_ifinfo_event(RTM_NEWLINK, dev, 0, rtnl_get_event(event),
                                   GFP_KERNEL, NULL, 0, 0, NULL);
                break;
        default:
                break;
        }
        return NOTIFY_DONE;
}

static struct notifier_block rtnetlink_dev_notifier = {
        .notifier_call        = rtnetlink_event,
};


static int __net_init rtnetlink_net_init(struct net *net)
{
        struct sock *sk;
        struct netlink_kernel_cfg cfg = {
                .groups                = RTNLGRP_MAX,
                .input                = rtnetlink_rcv,
                .flags                = NL_CFG_F_NONROOT_RECV,
                .bind                = rtnetlink_bind,
        };

        sk = netlink_kernel_create(net, NETLINK_ROUTE, &cfg);
        if (!sk)
                return -ENOMEM;
        net->rtnl = sk;
        return 0;
}

static void __net_exit rtnetlink_net_exit(struct net *net)
{
        netlink_kernel_release(net->rtnl);
        net->rtnl = NULL;
}

static struct pernet_operations rtnetlink_net_ops = {
        .init = rtnetlink_net_init,
        .exit = rtnetlink_net_exit,
};

static const struct rtnl_msg_handler rtnetlink_rtnl_msg_handlers[] __initconst = {
        {.msgtype = RTM_NEWLINK, .doit = rtnl_newlink,
         .flags = RTNL_FLAG_DOIT_PERNET},
        {.msgtype = RTM_DELLINK, .doit = rtnl_dellink,
         .flags = RTNL_FLAG_DOIT_PERNET_WIP},
        {.msgtype = RTM_GETLINK, .doit = rtnl_getlink,
         .dumpit = rtnl_dump_ifinfo, .flags = RTNL_FLAG_DUMP_SPLIT_NLM_DONE},
        {.msgtype = RTM_SETLINK, .doit = rtnl_setlink,
         .flags = RTNL_FLAG_DOIT_PERNET_WIP},
        {.msgtype = RTM_GETADDR, .dumpit = rtnl_dump_all},
        {.msgtype = RTM_GETROUTE, .dumpit = rtnl_dump_all},
        {.msgtype = RTM_GETNETCONF, .dumpit = rtnl_dump_all},
        {.msgtype = RTM_GETSTATS, .doit = rtnl_stats_get,
         .dumpit = rtnl_stats_dump},
        {.msgtype = RTM_SETSTATS, .doit = rtnl_stats_set},
        {.msgtype = RTM_NEWLINKPROP, .doit = rtnl_newlinkprop},
        {.msgtype = RTM_DELLINKPROP, .doit = rtnl_dellinkprop},
        {.protocol = PF_BRIDGE, .msgtype = RTM_GETLINK,
         .dumpit = rtnl_bridge_getlink},
        {.protocol = PF_BRIDGE, .msgtype = RTM_DELLINK,
         .doit = rtnl_bridge_dellink},
        {.protocol = PF_BRIDGE, .msgtype = RTM_SETLINK,
         .doit = rtnl_bridge_setlink},
        {.protocol = PF_BRIDGE, .msgtype = RTM_NEWNEIGH, .doit = rtnl_fdb_add},
        {.protocol = PF_BRIDGE, .msgtype = RTM_DELNEIGH, .doit = rtnl_fdb_del,
         .flags = RTNL_FLAG_BULK_DEL_SUPPORTED},
        {.protocol = PF_BRIDGE, .msgtype = RTM_GETNEIGH, .doit = rtnl_fdb_get,
         .dumpit = rtnl_fdb_dump},
        {.protocol = PF_BRIDGE, .msgtype = RTM_NEWMDB, .doit = rtnl_mdb_add},
        {.protocol = PF_BRIDGE, .msgtype = RTM_DELMDB, .doit = rtnl_mdb_del,
         .flags = RTNL_FLAG_BULK_DEL_SUPPORTED},
        {.protocol = PF_BRIDGE, .msgtype = RTM_GETMDB, .doit = rtnl_mdb_get,
         .dumpit = rtnl_mdb_dump},
};

void __init rtnetlink_init(void)
{
        if (register_pernet_subsys(&rtnetlink_net_ops))
                panic("rtnetlink_init: cannot initialize rtnetlink\n");

        register_netdevice_notifier(&rtnetlink_dev_notifier);

        rtnl_register_many(rtnetlink_rtnl_msg_handlers);
}
































































































































































































































































































































































































































































































































































































































































































   61 








































   31 






















   20 






















   17 






















   16 





























   34 


























    1 

































































    2 



















































































   16 














































































































































































































































































































































































































































































































































    8 























    4 















































































































































































































































































































































































































































































































































































































































































































































































































   21 






























   21 


























   17 





































































































































































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
/* SPDX-License-Identifier: GPL-2.0-or-later */
/* AF_RXRPC tracepoints
 *
 * Copyright (C) 2016 Red Hat, Inc. All Rights Reserved.
 * Written by David Howells (dhowells@redhat.com)
 */
#undef TRACE_SYSTEM
#define TRACE_SYSTEM rxrpc

#if !defined(_TRACE_RXRPC_H) || defined(TRACE_HEADER_MULTI_READ)
#define _TRACE_RXRPC_H

#include <linux/tracepoint.h>
#include <linux/errqueue.h>

/*
 * Declare tracing information enums and their string mappings for display.
 */
#define rxrpc_abort_reasons \
        /* AFS errors */                                                \
        EM(afs_abort_general_error,                "afs-error")                \
        EM(afs_abort_interrupted,                "afs-intr")                \
        EM(afs_abort_oom,                        "afs-oom")                \
        EM(afs_abort_op_not_supported,                "afs-op-notsupp")        \
        EM(afs_abort_probeuuid_negative,        "afs-probeuuid-neg")        \
        EM(afs_abort_send_data_error,                "afs-send-data")        \
        EM(afs_abort_unmarshal_error,                "afs-unmarshal")        \
        EM(afs_abort_unsupported_sec_class,        "afs-unsup-sec-class")        \
        /* rxperf errors */                                                \
        EM(rxperf_abort_general_error,                "rxperf-error")                \
        EM(rxperf_abort_oom,                        "rxperf-oom")                \
        EM(rxperf_abort_op_not_supported,        "rxperf-op-notsupp")        \
        EM(rxperf_abort_unmarshal_error,        "rxperf-unmarshal")        \
        /* RxKAD security errors */                                        \
        EM(rxkad_abort_1_short_check,                "rxkad1-short-check")        \
        EM(rxkad_abort_1_short_data,                "rxkad1-short-data")        \
        EM(rxkad_abort_1_short_encdata,                "rxkad1-short-encdata")        \
        EM(rxkad_abort_1_short_header,                "rxkad1-short-hdr")        \
        EM(rxkad_abort_2_short_check,                "rxkad2-short-check")        \
        EM(rxkad_abort_2_short_data,                "rxkad2-short-data")        \
        EM(rxkad_abort_2_short_header,                "rxkad2-short-hdr")        \
        EM(rxkad_abort_2_short_len,                "rxkad2-short-len")        \
        EM(rxkad_abort_bad_checksum,                "rxkad2-bad-cksum")        \
        EM(rxkad_abort_chall_key_expired,        "rxkad-chall-key-exp")        \
        EM(rxkad_abort_chall_level,                "rxkad-chall-level")        \
        EM(rxkad_abort_chall_no_key,                "rxkad-chall-nokey")        \
        EM(rxkad_abort_chall_short,                "rxkad-chall-short")        \
        EM(rxkad_abort_chall_version,                "rxkad-chall-version")        \
        EM(rxkad_abort_resp_bad_callid,                "rxkad-resp-bad-callid") \
        EM(rxkad_abort_resp_bad_checksum,        "rxkad-resp-bad-cksum")        \
        EM(rxkad_abort_resp_bad_param,                "rxkad-resp-bad-param")        \
        EM(rxkad_abort_resp_call_ctr,                "rxkad-resp-call-ctr") \
        EM(rxkad_abort_resp_call_state,                "rxkad-resp-call-state") \
        EM(rxkad_abort_resp_key_expired,        "rxkad-resp-key-exp")        \
        EM(rxkad_abort_resp_key_rejected,        "rxkad-resp-key-rej")        \
        EM(rxkad_abort_resp_level,                "rxkad-resp-level")        \
        EM(rxkad_abort_resp_nokey,                "rxkad-resp-nokey")        \
        EM(rxkad_abort_resp_ooseq,                "rxkad-resp-ooseq")        \
        EM(rxkad_abort_resp_short,                "rxkad-resp-short")        \
        EM(rxkad_abort_resp_short_tkt,                "rxkad-resp-short-tkt")        \
        EM(rxkad_abort_resp_tkt_aname,                "rxkad-resp-tk-aname")        \
        EM(rxkad_abort_resp_tkt_expired,        "rxkad-resp-tk-exp")        \
        EM(rxkad_abort_resp_tkt_future,                "rxkad-resp-tk-future")        \
        EM(rxkad_abort_resp_tkt_inst,                "rxkad-resp-tk-inst")        \
        EM(rxkad_abort_resp_tkt_len,                "rxkad-resp-tk-len")        \
        EM(rxkad_abort_resp_tkt_realm,                "rxkad-resp-tk-realm")        \
        EM(rxkad_abort_resp_tkt_short,                "rxkad-resp-tk-short")        \
        EM(rxkad_abort_resp_tkt_sinst,                "rxkad-resp-tk-sinst")        \
        EM(rxkad_abort_resp_tkt_sname,                "rxkad-resp-tk-sname")        \
        EM(rxkad_abort_resp_unknown_tkt,        "rxkad-resp-unknown-tkt") \
        EM(rxkad_abort_resp_version,                "rxkad-resp-version")        \
        /* RxGK security errors */                                        \
        EM(rxgk_abort_1_verify_mic_eproto,        "rxgk1-vfy-mic-eproto")        \
        EM(rxgk_abort_2_decrypt_eproto,                "rxgk2-dec-eproto")        \
        EM(rxgk_abort_2_short_data,                "rxgk2-short-data")        \
        EM(rxgk_abort_2_short_encdata,                "rxgk2-short-encdata")        \
        EM(rxgk_abort_2_short_header,                "rxgk2-short-hdr")        \
        EM(rxgk_abort_bad_key_number,                "rxgk-bad-key-num")        \
        EM(rxgk_abort_chall_key_expired,        "rxgk-chall-key-exp")        \
        EM(rxgk_abort_chall_no_key,                "rxgk-chall-nokey")        \
        EM(rxgk_abort_chall_short,                "rxgk-chall-short")        \
        EM(rxgk_abort_resp_auth_dec,                "rxgk-resp-auth-dec")        \
        EM(rxgk_abort_resp_bad_callid,                "rxgk-resp-bad-callid")        \
        EM(rxgk_abort_resp_bad_nonce,                "rxgk-resp-bad-nonce")        \
        EM(rxgk_abort_resp_bad_param,                "rxgk-resp-bad-param")        \
        EM(rxgk_abort_resp_call_ctr,                "rxgk-resp-call-ctr")        \
        EM(rxgk_abort_resp_call_state,                "rxgk-resp-call-state")        \
        EM(rxgk_abort_resp_internal_error,        "rxgk-resp-int-error")        \
        EM(rxgk_abort_resp_nopkg,                "rxgk-resp-nopkg")        \
        EM(rxgk_abort_resp_short_applen,        "rxgk-resp-short-applen") \
        EM(rxgk_abort_resp_short_auth,                "rxgk-resp-short-auth") \
        EM(rxgk_abort_resp_short_call_list,        "rxgk-resp-short-callls") \
        EM(rxgk_abort_resp_short_packet,        "rxgk-resp-short-packet") \
        EM(rxgk_abort_resp_short_yfs_klen,        "rxgk-resp-short-yfs-klen") \
        EM(rxgk_abort_resp_short_yfs_key,        "rxgk-resp-short-yfs-key") \
        EM(rxgk_abort_resp_short_yfs_tkt,        "rxgk-resp-short-yfs-tkt") \
        EM(rxgk_abort_resp_tok_dec,                "rxgk-resp-tok-dec")        \
        EM(rxgk_abort_resp_tok_internal_error,        "rxgk-resp-tok-int-err") \
        EM(rxgk_abort_resp_tok_keyerr,                "rxgk-resp-tok-keyerr")        \
        EM(rxgk_abort_resp_tok_nokey,                "rxgk-resp-tok-nokey")        \
        EM(rxgk_abort_resp_tok_nopkg,                "rxgk-resp-tok-nopkg")        \
        EM(rxgk_abort_resp_tok_short,                "rxgk-resp-tok-short")        \
        EM(rxgk_abort_resp_xdr_align,                "rxgk-resp-xdr-align")        \
        /* rxrpc errors */                                                \
        EM(rxrpc_abort_call_improper_term,        "call-improper-term")        \
        EM(rxrpc_abort_call_reset,                "call-reset")                \
        EM(rxrpc_abort_call_sendmsg,                "call-sendmsg")                \
        EM(rxrpc_abort_call_sock_release,        "call-sock-rel")        \
        EM(rxrpc_abort_call_sock_release_tba,        "call-sock-rel-tba")        \
        EM(rxrpc_abort_call_timeout,                "call-timeout")                \
        EM(rxrpc_abort_no_service_key,                "no-serv-key")                \
        EM(rxrpc_abort_nomem,                        "nomem")                \
        EM(rxrpc_abort_response_sendmsg,        "resp-sendmsg")                \
        EM(rxrpc_abort_service_not_offered,        "serv-not-offered")        \
        EM(rxrpc_abort_shut_down,                "shut-down")                \
        EM(rxrpc_abort_unsupported_security,        "unsup-sec")                \
        EM(rxrpc_badmsg_bad_abort,                "bad-abort")                \
        EM(rxrpc_badmsg_bad_jumbo,                "bad-jumbo")                \
        EM(rxrpc_badmsg_short_ack,                "short-ack")                \
        EM(rxrpc_badmsg_short_ack_trailer,        "short-ack-trailer")        \
        EM(rxrpc_badmsg_short_hdr,                "short-hdr")                \
        EM(rxrpc_badmsg_unsupported_packet,        "unsup-pkt")                \
        EM(rxrpc_badmsg_zero_call,                "zero-call")                \
        EM(rxrpc_badmsg_zero_seq,                "zero-seq")                \
        EM(rxrpc_badmsg_zero_service,                "zero-service")                \
        EM(rxrpc_eproto_ackr_outside_window,        "ackr-out-win")                \
        EM(rxrpc_eproto_ackr_sack_overflow,        "ackr-sack-over")        \
        EM(rxrpc_eproto_ackr_short_sack,        "ackr-short-sack")        \
        EM(rxrpc_eproto_ackr_zero,                "ackr-zero")                \
        EM(rxrpc_eproto_bad_upgrade,                "bad-upgrade")                \
        EM(rxrpc_eproto_data_after_last,        "data-after-last")        \
        EM(rxrpc_eproto_different_last,                "diff-last")                \
        EM(rxrpc_eproto_early_reply,                "early-reply")                \
        EM(rxrpc_eproto_improper_term,                "improper-term")        \
        EM(rxrpc_eproto_no_client_call,                "no-cl-call")                \
        EM(rxrpc_eproto_no_client_conn,                "no-cl-conn")                \
        EM(rxrpc_eproto_no_service_call,        "no-sv-call")                \
        EM(rxrpc_eproto_reupgrade,                "re-upgrade")                \
        EM(rxrpc_eproto_rxnull_challenge,        "rxnull-chall")                \
        EM(rxrpc_eproto_rxnull_response,        "rxnull-resp")                \
        EM(rxrpc_eproto_tx_rot_last,                "tx-rot-last")                \
        EM(rxrpc_eproto_unexpected_ack,                "unex-ack")                \
        EM(rxrpc_eproto_unexpected_ackall,        "unex-ackall")                \
        EM(rxrpc_eproto_unexpected_implicit_end, "unex-impl-end")        \
        EM(rxrpc_eproto_unexpected_reply,        "unex-reply")                \
        EM(rxrpc_eproto_wrong_security,                "wrong-sec")                \
        EM(rxrpc_recvmsg_excess_data,                "recvmsg-excess")        \
        EM(rxrpc_recvmsg_short_data,                "recvmsg-short")        \
        E_(rxrpc_sendmsg_late_send,                "sendmsg-late")

#define rxrpc_call_poke_traces \
        EM(rxrpc_call_poke_abort,                "Abort")        \
        EM(rxrpc_call_poke_complete,                "Compl")        \
        EM(rxrpc_call_poke_conn_abort,                "Conn-abort")        \
        EM(rxrpc_call_poke_error,                "Error")        \
        EM(rxrpc_call_poke_idle,                "Idle")                \
        EM(rxrpc_call_poke_rx_packet,                "Rx-packet")        \
        EM(rxrpc_call_poke_set_timeout,                "Set-timo")        \
        EM(rxrpc_call_poke_start,                "Start")        \
        EM(rxrpc_call_poke_timer,                "Timer")        \
        E_(rxrpc_call_poke_timer_now,                "Timer-now")

#define rxrpc_skb_traces \
        EM(rxrpc_skb_eaten_by_unshare,                "ETN unshare  ") \
        EM(rxrpc_skb_eaten_by_unshare_nomem,        "ETN unshar-nm") \
        EM(rxrpc_skb_get_call_rx,                "GET call-rx  ") \
        EM(rxrpc_skb_get_conn_secured,                "GET conn-secd") \
        EM(rxrpc_skb_get_conn_work,                "GET conn-work") \
        EM(rxrpc_skb_get_local_work,                "GET locl-work") \
        EM(rxrpc_skb_get_post_oob,                "GET post-oob ") \
        EM(rxrpc_skb_get_reject_work,                "GET rej-work ") \
        EM(rxrpc_skb_get_to_recvmsg,                "GET to-recv  ") \
        EM(rxrpc_skb_get_to_recvmsg_oos,        "GET to-recv-o") \
        EM(rxrpc_skb_new_encap_rcv,                "NEW encap-rcv") \
        EM(rxrpc_skb_new_error_report,                "NEW error-rpt") \
        EM(rxrpc_skb_new_jumbo_subpacket,        "NEW jumbo-sub") \
        EM(rxrpc_skb_new_response_rxgk,                "NEW resp-rxgk") \
        EM(rxrpc_skb_new_response_rxkad,        "NEW resp-rxkd") \
        EM(rxrpc_skb_new_unshared,                "NEW unshared ") \
        EM(rxrpc_skb_put_call_rx,                "PUT call-rx  ") \
        EM(rxrpc_skb_put_challenge,                "PUT challenge") \
        EM(rxrpc_skb_put_conn_secured,                "PUT conn-secd") \
        EM(rxrpc_skb_put_conn_work,                "PUT conn-work") \
        EM(rxrpc_skb_put_error_report,                "PUT error-rep") \
        EM(rxrpc_skb_put_input,                        "PUT input    ") \
        EM(rxrpc_skb_put_jumbo_subpacket,        "PUT jumbo-sub") \
        EM(rxrpc_skb_put_oob,                        "PUT oob      ") \
        EM(rxrpc_skb_put_purge,                        "PUT purge    ") \
        EM(rxrpc_skb_put_purge_oob,                "PUT purge-oob") \
        EM(rxrpc_skb_put_response,                "PUT response ") \
        EM(rxrpc_skb_put_rotate,                "PUT rotate   ") \
        EM(rxrpc_skb_put_unknown,                "PUT unknown  ") \
        EM(rxrpc_skb_see_conn_work,                "SEE conn-work") \
        EM(rxrpc_skb_see_oob_challenge,                "SEE oob-chall") \
        EM(rxrpc_skb_see_recvmsg,                "SEE recvmsg  ") \
        EM(rxrpc_skb_see_recvmsg_oob,                "SEE recvm-oob") \
        EM(rxrpc_skb_see_reject,                "SEE reject   ") \
        EM(rxrpc_skb_see_rotate,                "SEE rotate   ") \
        E_(rxrpc_skb_see_version,                "SEE version  ")

#define rxrpc_local_traces \
        EM(rxrpc_local_free,                        "FREE        ") \
        EM(rxrpc_local_get_call,                "GET call    ") \
        EM(rxrpc_local_get_client_conn,                "GET conn-cln") \
        EM(rxrpc_local_get_for_use,                "GET for-use ") \
        EM(rxrpc_local_get_peer,                "GET peer    ") \
        EM(rxrpc_local_get_prealloc_conn,        "GET conn-pre") \
        EM(rxrpc_local_new,                        "NEW         ") \
        EM(rxrpc_local_put_bind,                "PUT bind    ") \
        EM(rxrpc_local_put_call,                "PUT call    ") \
        EM(rxrpc_local_put_for_use,                "PUT for-use ") \
        EM(rxrpc_local_put_kill_conn,                "PUT conn-kil") \
        EM(rxrpc_local_put_peer,                "PUT peer    ") \
        EM(rxrpc_local_put_prealloc_peer,        "PUT peer-pre") \
        EM(rxrpc_local_put_release_sock,        "PUT rel-sock") \
        EM(rxrpc_local_stop,                        "STOP        ") \
        EM(rxrpc_local_stopped,                        "STOPPED     ") \
        EM(rxrpc_local_unuse_bind,                "UNU bind    ") \
        EM(rxrpc_local_unuse_conn_work,                "UNU conn-wrk") \
        EM(rxrpc_local_unuse_peer_keepalive,        "UNU peer-kpa") \
        EM(rxrpc_local_unuse_release_sock,        "UNU rel-sock") \
        EM(rxrpc_local_use_conn_work,                "USE conn-wrk") \
        EM(rxrpc_local_use_lookup,                "USE lookup  ") \
        E_(rxrpc_local_use_peer_keepalive,        "USE peer-kpa")

#define rxrpc_peer_traces \
        EM(rxrpc_peer_free,                        "FREE        ") \
        EM(rxrpc_peer_get_accept,                "GET accept  ") \
        EM(rxrpc_peer_get_application,                "GET app     ") \
        EM(rxrpc_peer_get_bundle,                "GET bundle  ") \
        EM(rxrpc_peer_get_call,                        "GET call    ") \
        EM(rxrpc_peer_get_client_conn,                "GET cln-conn") \
        EM(rxrpc_peer_get_input,                "GET input   ") \
        EM(rxrpc_peer_get_input_error,                "GET inpt-err") \
        EM(rxrpc_peer_get_keepalive,                "GET keepaliv") \
        EM(rxrpc_peer_get_lookup_client,        "GET look-cln") \
        EM(rxrpc_peer_get_service_conn,                "GET srv-conn") \
        EM(rxrpc_peer_new_client,                "NEW client  ") \
        EM(rxrpc_peer_new_prealloc,                "NEW prealloc") \
        EM(rxrpc_peer_put_application,                "PUT app     ") \
        EM(rxrpc_peer_put_bundle,                "PUT bundle  ") \
        EM(rxrpc_peer_put_call,                        "PUT call    ") \
        EM(rxrpc_peer_put_conn,                        "PUT conn    ") \
        EM(rxrpc_peer_put_input,                "PUT input   ") \
        EM(rxrpc_peer_put_input_error,                "PUT inpt-err") \
        E_(rxrpc_peer_put_keepalive,                "PUT keepaliv")

#define rxrpc_bundle_traces \
        EM(rxrpc_bundle_free,                        "FREE        ") \
        EM(rxrpc_bundle_get_client_call,        "GET clt-call") \
        EM(rxrpc_bundle_get_client_conn,        "GET clt-conn") \
        EM(rxrpc_bundle_get_service_conn,        "GET svc-conn") \
        EM(rxrpc_bundle_put_call,                "PUT call    ") \
        EM(rxrpc_bundle_put_conn,                "PUT conn    ") \
        EM(rxrpc_bundle_put_discard,                "PUT discard ") \
        E_(rxrpc_bundle_new,                        "NEW         ")

#define rxrpc_conn_traces \
        EM(rxrpc_conn_free,                        "FREE        ") \
        EM(rxrpc_conn_get_activate_call,        "GET act-call") \
        EM(rxrpc_conn_get_call_input,                "GET inp-call") \
        EM(rxrpc_conn_get_challenge_input,        "GET inp-chal") \
        EM(rxrpc_conn_get_conn_input,                "GET inp-conn") \
        EM(rxrpc_conn_get_idle,                        "GET idle    ") \
        EM(rxrpc_conn_get_poke_abort,                "GET pk-abort") \
        EM(rxrpc_conn_get_poke_response,        "GET response") \
        EM(rxrpc_conn_get_poke_secured,                "GET secured ") \
        EM(rxrpc_conn_get_poke_timer,                "GET poke    ") \
        EM(rxrpc_conn_get_service_conn,                "GET svc-conn") \
        EM(rxrpc_conn_new_client,                "NEW client  ") \
        EM(rxrpc_conn_new_service,                "NEW service ") \
        EM(rxrpc_conn_put_call,                        "PUT call    ") \
        EM(rxrpc_conn_put_call_input,                "PUT inp-call") \
        EM(rxrpc_conn_put_challenge_input,        "PUT inp-chal") \
        EM(rxrpc_conn_put_conn_input,                "PUT inp-conn") \
        EM(rxrpc_conn_put_discard_idle,                "PUT disc-idl") \
        EM(rxrpc_conn_put_local_dead,                "PUT loc-dead") \
        EM(rxrpc_conn_put_noreuse,                "PUT noreuse ") \
        EM(rxrpc_conn_put_oob,                        "PUT oob     ") \
        EM(rxrpc_conn_put_poke,                        "PUT poke    ") \
        EM(rxrpc_conn_put_service_reaped,        "PUT svc-reap") \
        EM(rxrpc_conn_put_unbundle,                "PUT unbundle") \
        EM(rxrpc_conn_put_unidle,                "PUT unidle  ") \
        EM(rxrpc_conn_put_work,                        "PUT work    ") \
        EM(rxrpc_conn_queue_challenge,                "QUE chall   ") \
        EM(rxrpc_conn_queue_retry_work,                "QUE retry-wk") \
        EM(rxrpc_conn_queue_rx_work,                "QUE rx-work ") \
        EM(rxrpc_conn_see_new_service_conn,        "SEE new-svc ") \
        EM(rxrpc_conn_see_reap_service,                "SEE reap-svc") \
        E_(rxrpc_conn_see_work,                        "SEE work    ")

#define rxrpc_client_traces \
        EM(rxrpc_client_activate_chans,                "Activa") \
        EM(rxrpc_client_alloc,                        "Alloc ") \
        EM(rxrpc_client_chan_activate,                "ChActv") \
        EM(rxrpc_client_chan_disconnect,        "ChDisc") \
        EM(rxrpc_client_chan_pass,                "ChPass") \
        EM(rxrpc_client_cleanup,                "Clean ") \
        EM(rxrpc_client_discard,                "Discar") \
        EM(rxrpc_client_exposed,                "Expose") \
        EM(rxrpc_client_replace,                "Replac") \
        EM(rxrpc_client_queue_new_call,                "Q-Call") \
        EM(rxrpc_client_to_active,                "->Actv") \
        E_(rxrpc_client_to_idle,                "->Idle")

#define rxrpc_call_traces \
        EM(rxrpc_call_get_io_thread,                "GET iothread") \
        EM(rxrpc_call_get_input,                "GET input   ") \
        EM(rxrpc_call_get_kernel_service,        "GET krnl-srv") \
        EM(rxrpc_call_get_notify_socket,        "GET notify  ") \
        EM(rxrpc_call_get_poke,                        "GET poke    ") \
        EM(rxrpc_call_get_recvmsg,                "GET recvmsg ") \
        EM(rxrpc_call_get_release_sock,                "GET rel-sock") \
        EM(rxrpc_call_get_sendmsg,                "GET sendmsg ") \
        EM(rxrpc_call_get_userid,                "GET user-id ") \
        EM(rxrpc_call_new_client,                "NEW client  ") \
        EM(rxrpc_call_new_prealloc_service,        "NEW prealloc") \
        EM(rxrpc_call_put_discard_prealloc,        "PUT disc-pre") \
        EM(rxrpc_call_put_discard_error,        "PUT disc-err") \
        EM(rxrpc_call_put_io_thread,                "PUT iothread") \
        EM(rxrpc_call_put_input,                "PUT input   ") \
        EM(rxrpc_call_put_kernel,                "PUT kernel  ") \
        EM(rxrpc_call_put_poke,                        "PUT poke    ") \
        EM(rxrpc_call_put_recvmsg,                "PUT recvmsg ") \
        EM(rxrpc_call_put_release_sock,                "PUT rls-sock") \
        EM(rxrpc_call_put_release_sock_tba,        "PUT rls-sk-a") \
        EM(rxrpc_call_put_sendmsg,                "PUT sendmsg ") \
        EM(rxrpc_call_put_unnotify,                "PUT unnotify") \
        EM(rxrpc_call_put_userid_exists,        "PUT u-exists") \
        EM(rxrpc_call_put_userid,                "PUT user-id ") \
        EM(rxrpc_call_see_accept,                "SEE accept  ") \
        EM(rxrpc_call_see_activate_client,        "SEE act-clnt") \
        EM(rxrpc_call_see_connect_failed,        "SEE con-fail") \
        EM(rxrpc_call_see_connected,                "SEE connect ") \
        EM(rxrpc_call_see_conn_abort,                "SEE conn-abt") \
        EM(rxrpc_call_see_disconnected,                "SEE disconn ") \
        EM(rxrpc_call_see_distribute_error,        "SEE dist-err") \
        EM(rxrpc_call_see_input,                "SEE input   ") \
        EM(rxrpc_call_see_release,                "SEE release ") \
        EM(rxrpc_call_see_userid_exists,        "SEE u-exists") \
        EM(rxrpc_call_see_waiting_call,                "SEE q-conn  ") \
        E_(rxrpc_call_see_zap,                        "SEE zap     ")

#define rxrpc_txqueue_traces \
        EM(rxrpc_txqueue_await_reply,                "AWR") \
        EM(rxrpc_txqueue_end,                        "END") \
        EM(rxrpc_txqueue_queue,                        "QUE") \
        EM(rxrpc_txqueue_queue_last,                "QLS") \
        EM(rxrpc_txqueue_rotate,                "ROT") \
        EM(rxrpc_txqueue_rotate_last,                "RLS") \
        E_(rxrpc_txqueue_wait,                        "WAI")

#define rxrpc_txdata_traces \
        EM(rxrpc_txdata_inject_loss,                " *INJ-LOSS*") \
        EM(rxrpc_txdata_new_data,                " ") \
        EM(rxrpc_txdata_retransmit,                " *RETRANS*") \
        EM(rxrpc_txdata_tlp_new_data,                " *TLP-NEW*") \
        E_(rxrpc_txdata_tlp_retransmit,                " *TLP-RETRANS*")

#define rxrpc_receive_traces \
        EM(rxrpc_receive_end,                        "END") \
        EM(rxrpc_receive_front,                        "FRN") \
        EM(rxrpc_receive_incoming,                "INC") \
        EM(rxrpc_receive_queue,                        "QUE") \
        EM(rxrpc_receive_queue_last,                "QLS") \
        EM(rxrpc_receive_queue_oos,                "QUO") \
        EM(rxrpc_receive_queue_oos_last,        "QOL") \
        EM(rxrpc_receive_oos,                        "OOS") \
        EM(rxrpc_receive_oos_last,                "OSL") \
        EM(rxrpc_receive_rotate,                "ROT") \
        E_(rxrpc_receive_rotate_last,                "RLS")

#define rxrpc_recvmsg_traces \
        EM(rxrpc_recvmsg_cont,                        "CONT") \
        EM(rxrpc_recvmsg_data_return,                "DATA") \
        EM(rxrpc_recvmsg_dequeue,                "DEQU") \
        EM(rxrpc_recvmsg_enter,                        "ENTR") \
        EM(rxrpc_recvmsg_full,                        "FULL") \
        EM(rxrpc_recvmsg_hole,                        "HOLE") \
        EM(rxrpc_recvmsg_next,                        "NEXT") \
        EM(rxrpc_recvmsg_oobq,                        "OOBQ") \
        EM(rxrpc_recvmsg_requeue,                "REQU") \
        EM(rxrpc_recvmsg_return,                "RETN") \
        EM(rxrpc_recvmsg_terminal,                "TERM") \
        EM(rxrpc_recvmsg_to_be_accepted,        "TBAC") \
        EM(rxrpc_recvmsg_unqueue,                "UNQU") \
        E_(rxrpc_recvmsg_wait,                        "WAIT")

#define rxrpc_rtt_tx_traces \
        EM(rxrpc_rtt_tx_cancel,                        "CNCE") \
        EM(rxrpc_rtt_tx_data,                        "DATA") \
        EM(rxrpc_rtt_tx_no_slot,                "FULL") \
        E_(rxrpc_rtt_tx_ping,                        "PING")

#define rxrpc_rtt_rx_traces \
        EM(rxrpc_rtt_rx_data_ack,                "DACK") \
        EM(rxrpc_rtt_rx_obsolete,                "OBSL") \
        EM(rxrpc_rtt_rx_lost,                        "LOST") \
        E_(rxrpc_rtt_rx_ping_response,                "PONG")

#define rxrpc_timer_traces \
        EM(rxrpc_timer_trace_delayed_ack,        "DelayAck ") \
        EM(rxrpc_timer_trace_expect_rx,                "ExpectRx ") \
        EM(rxrpc_timer_trace_hard,                "HardLimit") \
        EM(rxrpc_timer_trace_idle,                "IdleLimit") \
        EM(rxrpc_timer_trace_keepalive,                "KeepAlive") \
        EM(rxrpc_timer_trace_ping,                "DelayPing") \
        EM(rxrpc_timer_trace_rack_off,                "RACK-OFF ") \
        EM(rxrpc_timer_trace_rack_zwp,                "RACK-ZWP ") \
        EM(rxrpc_timer_trace_rack_reo,                "RACK-Reo ") \
        EM(rxrpc_timer_trace_rack_tlp_pto,        "TLP-PTO  ") \
        E_(rxrpc_timer_trace_rack_rto,                "RTO      ")

#define rxrpc_propose_ack_traces \
        EM(rxrpc_propose_ack_client_tx_end,        "ClTxEnd") \
        EM(rxrpc_propose_ack_delayed_ack,        "DlydAck") \
        EM(rxrpc_propose_ack_input_data,        "DataIn ") \
        EM(rxrpc_propose_ack_input_data_hole,        "DataInH") \
        EM(rxrpc_propose_ack_ping_for_keepalive, "KeepAlv") \
        EM(rxrpc_propose_ack_ping_for_lost_ack,        "LostAck") \
        EM(rxrpc_propose_ack_ping_for_lost_reply, "LostRpl") \
        EM(rxrpc_propose_ack_ping_for_0_retrans, "0-Retrn") \
        EM(rxrpc_propose_ack_ping_for_mtu_probe, "MTUProb") \
        EM(rxrpc_propose_ack_ping_for_old_rtt,        "OldRtt ") \
        EM(rxrpc_propose_ack_ping_for_params,        "Params ") \
        EM(rxrpc_propose_ack_ping_for_rtt,        "Rtt    ") \
        EM(rxrpc_propose_ack_processing_op,        "ProcOp ") \
        EM(rxrpc_propose_ack_respond_to_ack,        "Rsp2Ack") \
        EM(rxrpc_propose_ack_respond_to_ping,        "Rsp2Png") \
        EM(rxrpc_propose_ack_retransmit,        "Retrans") \
        EM(rxrpc_propose_ack_retry_tx,                "RetryTx") \
        EM(rxrpc_propose_ack_rotate_rx,                "RxAck  ") \
        EM(rxrpc_propose_ack_rx_idle,                "RxIdle ") \
        E_(rxrpc_propose_ack_terminal_ack,        "ClTerm ")

#define rxrpc_ca_states \
        EM(RXRPC_CA_CONGEST_AVOIDANCE,                "CongAvoid") \
        EM(RXRPC_CA_FAST_RETRANSMIT,                "FastReTx ") \
        EM(RXRPC_CA_PACKET_LOSS,                "PktLoss  ") \
        E_(RXRPC_CA_SLOW_START,                        "SlowStart")

#define rxrpc_congest_changes \
        EM(rxrpc_cong_begin_retransmission,        " Retrans") \
        EM(rxrpc_cong_cleared_nacks,                " Cleared") \
        EM(rxrpc_cong_new_low_nack,                " NewLowN") \
        EM(rxrpc_cong_no_change,                " -") \
        EM(rxrpc_cong_progress,                        " Progres") \
        EM(rxrpc_cong_idle_reset,                " IdleRes") \
        EM(rxrpc_cong_retransmit_again,                " ReTxAgn") \
        EM(rxrpc_cong_rtt_window_end,                " RttWinE") \
        E_(rxrpc_cong_saw_nack,                        " SawNack")

#define rxrpc_pkts \
        EM(0,                                        "?00") \
        EM(RXRPC_PACKET_TYPE_DATA,                "DATA") \
        EM(RXRPC_PACKET_TYPE_ACK,                "ACK") \
        EM(RXRPC_PACKET_TYPE_BUSY,                "BUSY") \
        EM(RXRPC_PACKET_TYPE_ABORT,                "ABORT") \
        EM(RXRPC_PACKET_TYPE_ACKALL,                "ACKALL") \
        EM(RXRPC_PACKET_TYPE_CHALLENGE,                "CHALL") \
        EM(RXRPC_PACKET_TYPE_RESPONSE,                "RESP") \
        EM(RXRPC_PACKET_TYPE_DEBUG,                "DEBUG") \
        EM(9,                                        "?09") \
        EM(10,                                        "?10") \
        EM(11,                                        "?11") \
        EM(12,                                        "?12") \
        EM(RXRPC_PACKET_TYPE_VERSION,                "VERSION") \
        EM(14,                                        "?14") \
        E_(15,                                        "?15")

#define rxrpc_ack_names \
        EM(0,                                        "-0-") \
        EM(RXRPC_ACK_REQUESTED,                        "REQ") \
        EM(RXRPC_ACK_DUPLICATE,                        "DUP") \
        EM(RXRPC_ACK_OUT_OF_SEQUENCE,                "OOS") \
        EM(RXRPC_ACK_EXCEEDS_WINDOW,                "WIN") \
        EM(RXRPC_ACK_NOSPACE,                        "MEM") \
        EM(RXRPC_ACK_PING,                        "PNG") \
        EM(RXRPC_ACK_PING_RESPONSE,                "PNR") \
        EM(RXRPC_ACK_DELAY,                        "DLY") \
        EM(RXRPC_ACK_IDLE,                        "IDL") \
        E_(RXRPC_ACK__INVALID,                        "-?-")

#define rxrpc_sack_traces \
        EM(rxrpc_sack_advance,                        "ADV")        \
        EM(rxrpc_sack_fill,                        "FIL")        \
        EM(rxrpc_sack_nack,                        "NAK")        \
        EM(rxrpc_sack_none,                        "---")        \
        E_(rxrpc_sack_oos,                        "OOS")

#define rxrpc_completions \
        EM(RXRPC_CALL_SUCCEEDED,                "Succeeded") \
        EM(RXRPC_CALL_REMOTELY_ABORTED,                "RemoteAbort") \
        EM(RXRPC_CALL_LOCALLY_ABORTED,                "LocalAbort") \
        EM(RXRPC_CALL_LOCAL_ERROR,                "LocalError") \
        E_(RXRPC_CALL_NETWORK_ERROR,                "NetError")

#define rxrpc_tx_points \
        EM(rxrpc_tx_point_call_abort,                "CallAbort") \
        EM(rxrpc_tx_point_call_ack,                "CallAck") \
        EM(rxrpc_tx_point_call_data_frag,        "CallDataFrag") \
        EM(rxrpc_tx_point_call_data_nofrag,        "CallDataNofrag") \
        EM(rxrpc_tx_point_call_final_resend,        "CallFinalResend") \
        EM(rxrpc_tx_point_conn_abort,                "ConnAbort") \
        EM(rxrpc_tx_point_reject,                "Reject") \
        EM(rxrpc_tx_point_rxgk_challenge,        "RxGKChall") \
        EM(rxrpc_tx_point_rxkad_challenge,        "RxkadChall") \
        EM(rxrpc_tx_point_response,                "Response") \
        EM(rxrpc_tx_point_version_keepalive,        "VerKeepalive") \
        E_(rxrpc_tx_point_version_reply,        "VerReply")

#define rxrpc_req_ack_traces \
        EM(rxrpc_reqack_ack_lost,                "ACK-LOST  ")        \
        EM(rxrpc_reqack_app_stall,                "APP-STALL ")        \
        EM(rxrpc_reqack_more_rtt,                "MORE-RTT  ")        \
        EM(rxrpc_reqack_no_srv_last,                "NO-SRVLAST")        \
        EM(rxrpc_reqack_old_rtt,                "OLD-RTT   ")        \
        EM(rxrpc_reqack_retrans,                "RETRANS   ")        \
        EM(rxrpc_reqack_slow_start,                "SLOW-START")        \
        E_(rxrpc_reqack_small_txwin,                "SMALL-TXWN")
/* ---- Must update size of stat_why_req_ack[] if more are added! */

#define rxrpc_txbuf_traces \
        EM(rxrpc_txbuf_alloc_data,                "ALLOC DATA ")        \
        EM(rxrpc_txbuf_alloc_response,                "ALLOC RESP ")        \
        EM(rxrpc_txbuf_free,                        "FREE       ")        \
        EM(rxrpc_txbuf_get_buffer,                "GET BUFFER ")        \
        EM(rxrpc_txbuf_get_trans,                "GET TRANS  ")        \
        EM(rxrpc_txbuf_get_retrans,                "GET RETRANS")        \
        EM(rxrpc_txbuf_put_cleaned,                "PUT CLEANED")        \
        EM(rxrpc_txbuf_put_nomem,                "PUT NOMEM  ")        \
        EM(rxrpc_txbuf_put_rotated,                "PUT ROTATED")        \
        EM(rxrpc_txbuf_put_response_tx,                "PUT RESP TX")        \
        EM(rxrpc_txbuf_put_send_aborted,        "PUT SEND-X ")        \
        EM(rxrpc_txbuf_put_trans,                "PUT TRANS  ")        \
        EM(rxrpc_txbuf_see_lost,                "SEE LOST   ")        \
        EM(rxrpc_txbuf_see_out_of_step,                "OUT-OF-STEP")        \
        E_(rxrpc_txbuf_see_send_more,                "SEE SEND+  ")

#define rxrpc_tq_traces \
        EM(rxrpc_tq_alloc,                        "ALLOC") \
        EM(rxrpc_tq_cleaned,                        "CLEAN") \
        EM(rxrpc_tq_decant,                        "DCNT ") \
        EM(rxrpc_tq_decant_advance,                "DCNT>") \
        EM(rxrpc_tq_queue,                        "QUEUE") \
        EM(rxrpc_tq_queue_dup,                        "QUE!!") \
        EM(rxrpc_tq_rotate,                        "ROT  ") \
        EM(rxrpc_tq_rotate_and_free,                "ROT-F") \
        EM(rxrpc_tq_rotate_and_keep,                "ROT-K") \
        EM(rxrpc_tq_transmit,                        "XMIT ") \
        E_(rxrpc_tq_transmit_advance,                "XMIT>")

#define rxrpc_pmtud_reduce_traces \
        EM(rxrpc_pmtud_reduce_ack,                "Ack  ")        \
        EM(rxrpc_pmtud_reduce_icmp,                "Icmp ")        \
        E_(rxrpc_pmtud_reduce_route,                "Route")

#define rxrpc_rotate_traces \
        EM(rxrpc_rotate_trace_hack,                "hard-ack")        \
        EM(rxrpc_rotate_trace_sack,                "soft-ack")        \
        E_(rxrpc_rotate_trace_snak,                "soft-nack")

#define rxrpc_rack_timer_modes \
        EM(RXRPC_CALL_RACKTIMER_OFF,                "---") \
        EM(RXRPC_CALL_RACKTIMER_RACK_REORDER,        "REO") \
        EM(RXRPC_CALL_RACKTIMER_TLP_PTO,        "TLP") \
        E_(RXRPC_CALL_RACKTIMER_RTO,                "RTO")

#define rxrpc_tlp_probe_traces \
        EM(rxrpc_tlp_probe_trace_busy,                "busy")                \
        EM(rxrpc_tlp_probe_trace_transmit_new,        "transmit-new")        \
        E_(rxrpc_tlp_probe_trace_retransmit,        "retransmit")

#define rxrpc_tlp_ack_traces \
        EM(rxrpc_tlp_ack_trace_acked,                "acked")        \
        EM(rxrpc_tlp_ack_trace_dup_acked,        "dup-acked")        \
        EM(rxrpc_tlp_ack_trace_hard_beyond,        "hard-beyond")        \
        EM(rxrpc_tlp_ack_trace_incomplete,        "incomplete")        \
        E_(rxrpc_tlp_ack_trace_new_data,        "new-data")

/*
 * Generate enums for tracing information.
 */
#ifndef __NETFS_DECLARE_TRACE_ENUMS_ONCE_ONLY
#define __NETFS_DECLARE_TRACE_ENUMS_ONCE_ONLY

#undef EM
#undef E_
#define EM(a, b) a,
#define E_(a, b) a

enum rxrpc_abort_reason                { rxrpc_abort_reasons } __mode(byte);
enum rxrpc_bundle_trace                { rxrpc_bundle_traces } __mode(byte);
enum rxrpc_call_poke_trace        { rxrpc_call_poke_traces } __mode(byte);
enum rxrpc_call_trace                { rxrpc_call_traces } __mode(byte);
enum rxrpc_client_trace                { rxrpc_client_traces } __mode(byte);
enum rxrpc_congest_change        { rxrpc_congest_changes } __mode(byte);
enum rxrpc_conn_trace                { rxrpc_conn_traces } __mode(byte);
enum rxrpc_local_trace                { rxrpc_local_traces } __mode(byte);
enum rxrpc_peer_trace                { rxrpc_peer_traces } __mode(byte);
enum rxrpc_pmtud_reduce_trace        { rxrpc_pmtud_reduce_traces } __mode(byte);
enum rxrpc_propose_ack_outcome        { rxrpc_propose_ack_outcomes } __mode(byte);
enum rxrpc_propose_ack_trace        { rxrpc_propose_ack_traces } __mode(byte);
enum rxrpc_receive_trace        { rxrpc_receive_traces } __mode(byte);
enum rxrpc_recvmsg_trace        { rxrpc_recvmsg_traces } __mode(byte);
enum rxrpc_req_ack_trace        { rxrpc_req_ack_traces } __mode(byte);
enum rxrpc_rotate_trace                { rxrpc_rotate_traces } __mode(byte);
enum rxrpc_rtt_rx_trace                { rxrpc_rtt_rx_traces } __mode(byte);
enum rxrpc_rtt_tx_trace                { rxrpc_rtt_tx_traces } __mode(byte);
enum rxrpc_sack_trace                { rxrpc_sack_traces } __mode(byte);
enum rxrpc_skb_trace                { rxrpc_skb_traces } __mode(byte);
enum rxrpc_timer_trace                { rxrpc_timer_traces } __mode(byte);
enum rxrpc_tlp_ack_trace        { rxrpc_tlp_ack_traces } __mode(byte);
enum rxrpc_tlp_probe_trace        { rxrpc_tlp_probe_traces } __mode(byte);
enum rxrpc_tq_trace                { rxrpc_tq_traces } __mode(byte);
enum rxrpc_tx_point                { rxrpc_tx_points } __mode(byte);
enum rxrpc_txbuf_trace                { rxrpc_txbuf_traces } __mode(byte);
enum rxrpc_txdata_trace                { rxrpc_txdata_traces } __mode(byte);
enum rxrpc_txqueue_trace        { rxrpc_txqueue_traces } __mode(byte);

#endif /* end __RXRPC_DECLARE_TRACE_ENUMS_ONCE_ONLY */

/*
 * Export enum symbols via userspace.
 */
#undef EM
#undef E_

#ifndef RXRPC_TRACE_ONLY_DEFINE_ENUMS

#define EM(a, b) TRACE_DEFINE_ENUM(a);
#define E_(a, b) TRACE_DEFINE_ENUM(a);

rxrpc_abort_reasons;
rxrpc_bundle_traces;
rxrpc_ca_states;
rxrpc_call_poke_traces;
rxrpc_call_traces;
rxrpc_client_traces;
rxrpc_congest_changes;
rxrpc_conn_traces;
rxrpc_local_traces;
rxrpc_pmtud_reduce_traces;
rxrpc_propose_ack_traces;
rxrpc_rack_timer_modes;
rxrpc_receive_traces;
rxrpc_recvmsg_traces;
rxrpc_req_ack_traces;
rxrpc_rotate_traces;
rxrpc_rtt_rx_traces;
rxrpc_rtt_tx_traces;
rxrpc_sack_traces;
rxrpc_skb_traces;
rxrpc_timer_traces;
rxrpc_tlp_ack_traces;
rxrpc_tlp_probe_traces;
rxrpc_tq_traces;
rxrpc_tx_points;
rxrpc_txbuf_traces;
rxrpc_txdata_traces;
rxrpc_txqueue_traces;

/*
 * Now redefine the EM() and E_() macros to map the enums to the strings that
 * will be printed in the output.
 */
#undef EM
#undef E_
#define EM(a, b)        { a, b },
#define E_(a, b)        { a, b }

TRACE_EVENT(rxrpc_local,
            TP_PROTO(unsigned int local_debug_id, enum rxrpc_local_trace op,
                     int ref, int usage),

            TP_ARGS(local_debug_id, op, ref, usage),

            TP_STRUCT__entry(
                    __field(unsigned int,        local)
                    __field(int,                op)
                    __field(int,                ref)
                    __field(int,                usage)
                             ),

            TP_fast_assign(
                    __entry->local = local_debug_id;
                    __entry->op = op;
                    __entry->ref = ref;
                    __entry->usage = usage;
                           ),

            TP_printk("L=%08x %s r=%d u=%d",
                      __entry->local,
                      __print_symbolic(__entry->op, rxrpc_local_traces),
                      __entry->ref,
                      __entry->usage)
            );

TRACE_EVENT(rxrpc_iothread_rx,
            TP_PROTO(struct rxrpc_local *local, unsigned int nr_rx),
            TP_ARGS(local, nr_rx),
            TP_STRUCT__entry(
                    __field(unsigned int,        local)
                    __field(unsigned int,        nr_rx)
                             ),
            TP_fast_assign(
                    __entry->local = local->debug_id;
                    __entry->nr_rx = nr_rx;
                           ),
            TP_printk("L=%08x nrx=%u", __entry->local, __entry->nr_rx)
            );

TRACE_EVENT(rxrpc_peer,
            TP_PROTO(unsigned int peer_debug_id, int ref, enum rxrpc_peer_trace why),

            TP_ARGS(peer_debug_id, ref, why),

            TP_STRUCT__entry(
                    __field(unsigned int,        peer)
                    __field(int,                ref)
                    __field(enum rxrpc_peer_trace, why)
                             ),

            TP_fast_assign(
                    __entry->peer = peer_debug_id;
                    __entry->ref = ref;
                    __entry->why = why;
                           ),

            TP_printk("P=%08x %s r=%d",
                      __entry->peer,
                      __print_symbolic(__entry->why, rxrpc_peer_traces),
                      __entry->ref)
            );

TRACE_EVENT(rxrpc_bundle,
            TP_PROTO(unsigned int bundle_debug_id, int ref, enum rxrpc_bundle_trace why),

            TP_ARGS(bundle_debug_id, ref, why),

            TP_STRUCT__entry(
                    __field(unsigned int,        bundle)
                    __field(int,                ref)
                    __field(int,                why)
                             ),

            TP_fast_assign(
                    __entry->bundle = bundle_debug_id;
                    __entry->ref = ref;
                    __entry->why = why;
                           ),

            TP_printk("CB=%08x %s r=%d",
                      __entry->bundle,
                      __print_symbolic(__entry->why, rxrpc_bundle_traces),
                      __entry->ref)
            );

TRACE_EVENT(rxrpc_conn,
            TP_PROTO(unsigned int conn_debug_id, int ref, enum rxrpc_conn_trace why),

            TP_ARGS(conn_debug_id, ref, why),

            TP_STRUCT__entry(
                    __field(unsigned int,        conn)
                    __field(int,                ref)
                    __field(int,                why)
                             ),

            TP_fast_assign(
                    __entry->conn = conn_debug_id;
                    __entry->ref = ref;
                    __entry->why = why;
                           ),

            TP_printk("C=%08x %s r=%d",
                      __entry->conn,
                      __print_symbolic(__entry->why, rxrpc_conn_traces),
                      __entry->ref)
            );

TRACE_EVENT(rxrpc_client,
            TP_PROTO(struct rxrpc_connection *conn, int channel,
                     enum rxrpc_client_trace op),

            TP_ARGS(conn, channel, op),

            TP_STRUCT__entry(
                    __field(unsigned int,                conn)
                    __field(u32,                        cid)
                    __field(int,                        channel)
                    __field(int,                        usage)
                    __field(enum rxrpc_client_trace,        op)
                             ),

            TP_fast_assign(
                    __entry->conn = conn ? conn->debug_id : 0;
                    __entry->channel = channel;
                    __entry->usage = conn ? refcount_read(&conn->ref) : -2;
                    __entry->op = op;
                    __entry->cid = conn ? conn->proto.cid : 0;
                           ),

            TP_printk("C=%08x h=%2d %s i=%08x u=%d",
                      __entry->conn,
                      __entry->channel,
                      __print_symbolic(__entry->op, rxrpc_client_traces),
                      __entry->cid,
                      __entry->usage)
            );

TRACE_EVENT(rxrpc_call,
            TP_PROTO(unsigned int call_debug_id, int ref, unsigned long aux,
                     enum rxrpc_call_trace why),

            TP_ARGS(call_debug_id, ref, aux, why),

            TP_STRUCT__entry(
                    __field(unsigned int,        call)
                    __field(int,                ref)
                    __field(int,                why)
                    __field(unsigned long,        aux)
                             ),

            TP_fast_assign(
                    __entry->call = call_debug_id;
                    __entry->ref = ref;
                    __entry->why = why;
                    __entry->aux = aux;
                           ),

            TP_printk("c=%08x %s r=%d a=%lx",
                      __entry->call,
                      __print_symbolic(__entry->why, rxrpc_call_traces),
                      __entry->ref,
                      __entry->aux)
            );

TRACE_EVENT(rxrpc_skb,
            TP_PROTO(struct sk_buff *skb, int usage, int mod_count,
                     enum rxrpc_skb_trace why),

            TP_ARGS(skb, usage, mod_count, why),

            TP_STRUCT__entry(
                    __field(struct sk_buff *,                skb)
                    __field(int,                        usage)
                    __field(int,                        mod_count)
                    __field(enum rxrpc_skb_trace,        why)
                             ),

            TP_fast_assign(
                    __entry->skb = skb;
                    __entry->usage = usage;
                    __entry->mod_count = mod_count;
                    __entry->why = why;
                           ),

            TP_printk("s=%p Rx %s u=%d m=%d",
                      __entry->skb,
                      __print_symbolic(__entry->why, rxrpc_skb_traces),
                      __entry->usage,
                      __entry->mod_count)
            );

TRACE_EVENT(rxrpc_rx_packet,
            TP_PROTO(struct rxrpc_skb_priv *sp),

            TP_ARGS(sp),

            TP_STRUCT__entry(
                    __field_struct(struct rxrpc_host_header,        hdr)
                             ),

            TP_fast_assign(
                    memcpy(&__entry->hdr, &sp->hdr, sizeof(__entry->hdr));
                           ),

            TP_printk("%08x:%08x:%08x:%04x %08x %08x %02x %02x %s",
                      __entry->hdr.epoch, __entry->hdr.cid,
                      __entry->hdr.callNumber, __entry->hdr.serviceId,
                      __entry->hdr.serial, __entry->hdr.seq,
                      __entry->hdr.securityIndex, __entry->hdr.flags,
                      __print_symbolic(__entry->hdr.type, rxrpc_pkts))
            );

TRACE_EVENT(rxrpc_rx_done,
            TP_PROTO(int result, int abort_code),

            TP_ARGS(result, abort_code),

            TP_STRUCT__entry(
                    __field(int,        result)
                    __field(int,        abort_code)
                             ),

            TP_fast_assign(
                    __entry->result = result;
                    __entry->abort_code = abort_code;
                           ),

            TP_printk("r=%d a=%d", __entry->result, __entry->abort_code)
            );

TRACE_EVENT(rxrpc_abort_call,
            TP_PROTO(const struct rxrpc_call *call, int abort_code),

            TP_ARGS(call, abort_code),

            TP_STRUCT__entry(
                    __field(unsigned int,                call_nr)
                    __field(enum rxrpc_abort_reason,        why)
                    __field(int,                        abort_code)
                    __field(int,                        error)
                             ),

            TP_fast_assign(
                    __entry->call_nr        = call->debug_id;
                    __entry->why        = call->send_abort_why;
                    __entry->abort_code        = abort_code;
                    __entry->error        = call->send_abort_err;
                           ),

            TP_printk("c=%08x a=%d e=%d %s",
                      __entry->call_nr,
                      __entry->abort_code, __entry->error,
                      __print_symbolic(__entry->why, rxrpc_abort_reasons))
            );

TRACE_EVENT(rxrpc_abort,
            TP_PROTO(unsigned int call_nr, enum rxrpc_abort_reason why,
                     u32 cid, u32 call_id, rxrpc_seq_t seq, int abort_code, int error),

            TP_ARGS(call_nr, why, cid, call_id, seq, abort_code, error),

            TP_STRUCT__entry(
                    __field(unsigned int,                call_nr)
                    __field(enum rxrpc_abort_reason,        why)
                    __field(u32,                        cid)
                    __field(u32,                        call_id)
                    __field(rxrpc_seq_t,                seq)
                    __field(int,                        abort_code)
                    __field(int,                        error)
                             ),

            TP_fast_assign(
                    __entry->call_nr = call_nr;
                    __entry->why = why;
                    __entry->cid = cid;
                    __entry->call_id = call_id;
                    __entry->abort_code = abort_code;
                    __entry->error = error;
                    __entry->seq = seq;
                           ),

            TP_printk("c=%08x %08x:%08x s=%u a=%d e=%d %s",
                      __entry->call_nr,
                      __entry->cid, __entry->call_id, __entry->seq,
                      __entry->abort_code, __entry->error,
                      __print_symbolic(__entry->why, rxrpc_abort_reasons))
            );

TRACE_EVENT(rxrpc_call_complete,
            TP_PROTO(struct rxrpc_call *call),

            TP_ARGS(call),

            TP_STRUCT__entry(
                    __field(unsigned int,                call)
                    __field(enum rxrpc_call_completion,        compl)
                    __field(int,                        error)
                    __field(u32,                        abort_code)
                             ),

            TP_fast_assign(
                    __entry->call = call->debug_id;
                    __entry->compl = call->completion;
                    __entry->error = call->error;
                    __entry->abort_code = call->abort_code;
                           ),

            TP_printk("c=%08x %s r=%d ac=%d",
                      __entry->call,
                      __print_symbolic(__entry->compl, rxrpc_completions),
                      __entry->error,
                      __entry->abort_code)
            );

TRACE_EVENT(rxrpc_txqueue,
            TP_PROTO(struct rxrpc_call *call, enum rxrpc_txqueue_trace why),

            TP_ARGS(call, why),

            TP_STRUCT__entry(
                    __field(unsigned int,                call)
                    __field(enum rxrpc_txqueue_trace,        why)
                    __field(rxrpc_seq_t,                tx_bottom)
                    __field(rxrpc_seq_t,                acks_hard_ack)
                    __field(rxrpc_seq_t,                tx_top)
                    __field(rxrpc_seq_t,                send_top)
                    __field(int,                        tx_winsize)
                             ),

            TP_fast_assign(
                    __entry->call = call->debug_id;
                    __entry->why = why;
                    __entry->tx_bottom = call->tx_bottom;
                    __entry->acks_hard_ack = call->acks_hard_ack;
                    __entry->tx_top = call->tx_top;
                    __entry->send_top = call->send_top;
                    __entry->tx_winsize = call->tx_winsize;
                           ),

            TP_printk("c=%08x %s b=%08x h=%08x n=%u/%u/%u/%u",
                      __entry->call,
                      __print_symbolic(__entry->why, rxrpc_txqueue_traces),
                      __entry->tx_bottom,
                      __entry->acks_hard_ack,
                      __entry->acks_hard_ack - __entry->tx_bottom,
                      __entry->tx_top - __entry->acks_hard_ack,
                      __entry->send_top - __entry->tx_top,
                      __entry->tx_winsize)
            );

TRACE_EVENT(rxrpc_transmit,
            TP_PROTO(struct rxrpc_call *call, rxrpc_seq_t send_top, int space),

            TP_ARGS(call, send_top, space),

            TP_STRUCT__entry(
                    __field(unsigned int,        call)
                    __field(rxrpc_seq_t,        seq)
                    __field(u16,                space)
                    __field(u16,                tx_winsize)
                    __field(u16,                cong_cwnd)
                    __field(u16,                cong_extra)
                    __field(u16,                in_flight)
                    __field(u16,                prepared)
                    __field(u16,                pmtud_jumbo)
                             ),

            TP_fast_assign(
                    __entry->call        = call->debug_id;
                    __entry->seq        = call->tx_top + 1;
                    __entry->space        = space;
                    __entry->tx_winsize        = call->tx_winsize;
                    __entry->cong_cwnd        = call->cong_cwnd;
                    __entry->cong_extra        = call->cong_extra;
                    __entry->prepared        = send_top - call->tx_bottom;
                    __entry->in_flight        = call->tx_top - call->tx_bottom;
                    __entry->pmtud_jumbo = call->peer->pmtud_jumbo;
                           ),

            TP_printk("c=%08x q=%08x sp=%u tw=%u cw=%u+%u pr=%u if=%u pj=%u",
                      __entry->call,
                      __entry->seq,
                      __entry->space,
                      __entry->tx_winsize,
                      __entry->cong_cwnd,
                      __entry->cong_extra,
                      __entry->prepared,
                      __entry->in_flight,
                      __entry->pmtud_jumbo)
            );

TRACE_EVENT(rxrpc_tx_rotate,
            TP_PROTO(struct rxrpc_call *call, rxrpc_seq_t seq, rxrpc_seq_t to),

            TP_ARGS(call, seq, to),

            TP_STRUCT__entry(
                    __field(unsigned int,        call)
                    __field(rxrpc_seq_t,        seq)
                    __field(rxrpc_seq_t,        to)
                    __field(rxrpc_seq_t,        top)
                             ),

            TP_fast_assign(
                    __entry->call        = call->debug_id;
                    __entry->seq        = seq;
                    __entry->to                = to;
                    __entry->top        = call->tx_top;
                           ),

            TP_printk("c=%08x q=%08x-%08x-%08x",
                      __entry->call,
                      __entry->seq,
                      __entry->to,
                      __entry->top)
            );

TRACE_EVENT(rxrpc_rx_data,
            TP_PROTO(unsigned int call, rxrpc_seq_t seq,
                     rxrpc_serial_t serial, u8 flags),

            TP_ARGS(call, seq, serial, flags),

            TP_STRUCT__entry(
                    __field(unsigned int,        call)
                    __field(rxrpc_seq_t,        seq)
                    __field(rxrpc_serial_t,        serial)
                    __field(u8,                        flags)
                             ),

            TP_fast_assign(
                    __entry->call = call;
                    __entry->seq = seq;
                    __entry->serial = serial;
                    __entry->flags = flags;
                           ),

            TP_printk("c=%08x DATA %08x q=%08x fl=%02x",
                      __entry->call,
                      __entry->serial,
                      __entry->seq,
                      __entry->flags)
            );

TRACE_EVENT(rxrpc_rx_ack,
            TP_PROTO(struct rxrpc_call *call, struct rxrpc_skb_priv *sp),

            TP_ARGS(call, sp),

            TP_STRUCT__entry(
                    __field(unsigned int,        call)
                    __field(rxrpc_serial_t,        serial)
                    __field(rxrpc_serial_t,        ack_serial)
                    __field(rxrpc_seq_t,        first)
                    __field(rxrpc_seq_t,        prev)
                    __field(u8,                        reason)
                    __field(u8,                        n_acks)
                    __field(u8,                        user_status)
                             ),

            TP_fast_assign(
                    __entry->call        = call->debug_id;
                    __entry->serial        = sp->hdr.serial;
                    __entry->user_status = sp->hdr.userStatus;
                    __entry->ack_serial = sp->ack.acked_serial;
                    __entry->first        = sp->ack.first_ack;
                    __entry->prev        = sp->ack.prev_ack;
                    __entry->reason        = sp->ack.reason;
                    __entry->n_acks        = sp->ack.nr_acks;
                           ),

            TP_printk("c=%08x %08x %s r=%08x us=%02x f=%08x p=%08x n=%u",
                      __entry->call,
                      __entry->serial,
                      __print_symbolic(__entry->reason, rxrpc_ack_names),
                      __entry->ack_serial,
                      __entry->user_status,
                      __entry->first,
                      __entry->prev,
                      __entry->n_acks)
            );

TRACE_EVENT(rxrpc_rx_abort,
            TP_PROTO(struct rxrpc_call *call, rxrpc_serial_t serial,
                     u32 abort_code),

            TP_ARGS(call, serial, abort_code),

            TP_STRUCT__entry(
                    __field(unsigned int,        call)
                    __field(rxrpc_serial_t,        serial)
                    __field(u32,                abort_code)
                             ),

            TP_fast_assign(
                    __entry->call = call->debug_id;
                    __entry->serial = serial;
                    __entry->abort_code = abort_code;
                           ),

            TP_printk("c=%08x ABORT %08x ac=%d",
                      __entry->call,
                      __entry->serial,
                      __entry->abort_code)
            );

TRACE_EVENT(rxrpc_rx_conn_abort,
            TP_PROTO(const struct rxrpc_connection *conn, const struct sk_buff *skb),

            TP_ARGS(conn, skb),

            TP_STRUCT__entry(
                    __field(unsigned int,        conn)
                    __field(rxrpc_serial_t,        serial)
                    __field(u32,                abort_code)
                             ),

            TP_fast_assign(
                    __entry->conn = conn->debug_id;
                    __entry->serial = rxrpc_skb(skb)->hdr.serial;
                    __entry->abort_code = skb->priority;
                           ),

            TP_printk("C=%08x ABORT %08x ac=%d",
                      __entry->conn,
                      __entry->serial,
                      __entry->abort_code)
            );

TRACE_EVENT(rxrpc_tx_challenge,
            TP_PROTO(struct rxrpc_connection *conn, rxrpc_serial_t serial,
                     u32 version, u32 nonce),

            TP_ARGS(conn, serial, version, nonce),

            TP_STRUCT__entry(
                    __field(unsigned int,        conn)
                    __field(rxrpc_serial_t,        serial)
                    __field(u32,                version)
                    __field(u32,                nonce)
                    __field(u16,                service_id)
                    __field(u8,                        security_ix)
                             ),

            TP_fast_assign(
                    __entry->conn = conn->debug_id;
                    __entry->serial = serial;
                    __entry->version = version;
                    __entry->nonce = nonce;
                    __entry->service_id = conn->service_id;
                    __entry->security_ix = conn->security_ix;
                           ),

            TP_printk("C=%08x CHALLENGE r=%08x sv=%u+%u v=%x n=%x",
                      __entry->conn,
                      __entry->serial,
                      __entry->service_id,
                      __entry->security_ix,
                      __entry->version,
                      __entry->nonce)
            );

TRACE_EVENT(rxrpc_rx_challenge,
            TP_PROTO(struct rxrpc_connection *conn, rxrpc_serial_t serial,
                     u32 version, u32 nonce, u32 min_level),

            TP_ARGS(conn, serial, version, nonce, min_level),

            TP_STRUCT__entry(
                    __field(unsigned int,        conn)
                    __field(rxrpc_serial_t,        serial)
                    __field(u32,                version)
                    __field(u32,                nonce)
                    __field(u32,                min_level)
                    __field(u16,                service_id)
                    __field(u8,                        security_ix)
                             ),

            TP_fast_assign(
                    __entry->conn = conn->debug_id;
                    __entry->serial = serial;
                    __entry->version = version;
                    __entry->nonce = nonce;
                    __entry->min_level = min_level;
                    __entry->service_id = conn->service_id;
                    __entry->security_ix = conn->security_ix;
                           ),

            TP_printk("C=%08x CHALLENGE r=%08x sv=%u+%u v=%x n=%x ml=%x",
                      __entry->conn,
                      __entry->serial,
                      __entry->service_id,
                      __entry->security_ix,
                      __entry->version,
                      __entry->nonce,
                      __entry->min_level)
            );

TRACE_EVENT(rxrpc_tx_response,
            TP_PROTO(struct rxrpc_connection *conn, rxrpc_serial_t serial,
                     struct rxrpc_skb_priv *rsp),

            TP_ARGS(conn, serial, rsp),

            TP_STRUCT__entry(
                    __field(unsigned int,        conn)
                    __field(rxrpc_serial_t,        serial)
                    __field(rxrpc_serial_t,        challenge)
                    __field(u32,                version)
                    __field(u32,                kvno)
                    __field(u16,                ticket_len)
                    __field(u16,                appdata_len)
                    __field(u16,                service_id)
                    __field(u8,                        security_ix)
                             ),

            TP_fast_assign(
                    __entry->conn        = conn->debug_id;
                    __entry->serial        = serial;
                    __entry->challenge        = rsp->resp.challenge_serial;
                    __entry->version        = rsp->resp.version;
                    __entry->kvno        = rsp->resp.kvno;
                    __entry->ticket_len = rsp->resp.ticket_len;
                    __entry->service_id = conn->service_id;
                    __entry->security_ix = conn->security_ix;
                           ),

            TP_printk("C=%08x RESPONSE r=%08x cr=%08x sv=%u+%u v=%x kv=%x tl=%u",
                      __entry->conn,
                      __entry->serial,
                      __entry->challenge,
                      __entry->service_id,
                      __entry->security_ix,
                      __entry->version,
                      __entry->kvno,
                      __entry->ticket_len)
            );

TRACE_EVENT(rxrpc_rx_response,
            TP_PROTO(struct rxrpc_connection *conn, rxrpc_serial_t serial,
                     u32 version, u32 kvno, u32 ticket_len),

            TP_ARGS(conn, serial, version, kvno, ticket_len),

            TP_STRUCT__entry(
                    __field(unsigned int,        conn)
                    __field(rxrpc_serial_t,        serial)
                    __field(u32,                version)
                    __field(u32,                kvno)
                    __field(u32,                ticket_len)
                    __field(u8,                        security_ix)
                             ),

            TP_fast_assign(
                    __entry->conn = conn->debug_id;
                    __entry->serial = serial;
                    __entry->version = version;
                    __entry->kvno = kvno;
                    __entry->ticket_len = ticket_len;
                    __entry->security_ix = conn->security_ix;
                           ),

            TP_printk("C=%08x RESPONSE r=%08x sx=%u v=%x kvno=%x tl=%x",
                      __entry->conn,
                      __entry->serial,
                      __entry->security_ix,
                      __entry->version,
                      __entry->kvno,
                      __entry->ticket_len)
            );

TRACE_EVENT(rxrpc_rx_rwind_change,
            TP_PROTO(struct rxrpc_call *call, rxrpc_serial_t serial,
                     u32 rwind, bool wake),

            TP_ARGS(call, serial, rwind, wake),

            TP_STRUCT__entry(
                    __field(unsigned int,        call)
                    __field(rxrpc_serial_t,        serial)
                    __field(u32,                rwind)
                    __field(bool,                wake)
                             ),

            TP_fast_assign(
                    __entry->call = call->debug_id;
                    __entry->serial = serial;
                    __entry->rwind = rwind;
                    __entry->wake = wake;
                           ),

            TP_printk("c=%08x %08x rw=%u%s",
                      __entry->call,
                      __entry->serial,
                      __entry->rwind,
                      __entry->wake ? " wake" : "")
            );

TRACE_EVENT(rxrpc_tx_packet,
            TP_PROTO(unsigned int call_id, struct rxrpc_wire_header *whdr,
                     enum rxrpc_tx_point where),

            TP_ARGS(call_id, whdr, where),

            TP_STRUCT__entry(
                    __field(unsigned int,                        call)
                    __field(enum rxrpc_tx_point,                where)
                    __field_struct(struct rxrpc_wire_header,        whdr)
                             ),

            TP_fast_assign(
                    __entry->call = call_id;
                    memcpy(&__entry->whdr, whdr, sizeof(__entry->whdr));
                    __entry->where = where;
                           ),

            TP_printk("c=%08x %08x:%08x:%08x:%04x %08x %08x %02x %02x %s %s",
                      __entry->call,
                      ntohl(__entry->whdr.epoch),
                      ntohl(__entry->whdr.cid),
                      ntohl(__entry->whdr.callNumber),
                      ntohs(__entry->whdr.serviceId),
                      ntohl(__entry->whdr.serial),
                      ntohl(__entry->whdr.seq),
                      __entry->whdr.type, __entry->whdr.flags,
                      __entry->whdr.type <= 15 ?
                      __print_symbolic(__entry->whdr.type, rxrpc_pkts) : "?UNK",
                      __print_symbolic(__entry->where, rxrpc_tx_points))
            );

TRACE_EVENT(rxrpc_tx_data,
            TP_PROTO(struct rxrpc_call *call, rxrpc_seq_t seq,
                     rxrpc_serial_t serial, unsigned int flags,
                     enum rxrpc_txdata_trace trace),

            TP_ARGS(call, seq, serial, flags, trace),

            TP_STRUCT__entry(
                    __field(unsigned int,        call)
                    __field(rxrpc_seq_t,        seq)
                    __field(rxrpc_serial_t,        serial)
                    __field(u32,                cid)
                    __field(u32,                call_id)
                    __field(u16,                flags)
                    __field(enum rxrpc_txdata_trace, trace)
                             ),

            TP_fast_assign(
                    __entry->call = call->debug_id;
                    __entry->cid = call->cid;
                    __entry->call_id = call->call_id;
                    __entry->seq = seq;
                    __entry->serial = serial;
                    __entry->flags = flags;
                    __entry->trace = trace;
                           ),

            TP_printk("c=%08x DATA %08x:%08x %08x q=%08x fl=%02x%s",
                      __entry->call,
                      __entry->cid,
                      __entry->call_id,
                      __entry->serial,
                      __entry->seq,
                      __entry->flags & RXRPC_TXBUF_WIRE_FLAGS,
                      __print_symbolic(__entry->trace, rxrpc_txdata_traces))
            );

TRACE_EVENT(rxrpc_tx_ack,
            TP_PROTO(unsigned int call, rxrpc_serial_t serial,
                     rxrpc_seq_t ack_first, rxrpc_serial_t ack_serial,
                     u8 reason, u8 n_acks, u16 rwind,
                     enum rxrpc_propose_ack_trace trace),

            TP_ARGS(call, serial, ack_first, ack_serial, reason, n_acks, rwind, trace),

            TP_STRUCT__entry(
                    __field(unsigned int,        call)
                    __field(rxrpc_serial_t,        serial)
                    __field(rxrpc_seq_t,        ack_first)
                    __field(rxrpc_serial_t,        ack_serial)
                    __field(u8,                        reason)
                    __field(u8,                        n_acks)
                    __field(u16,                rwind)
                    __field(enum rxrpc_propose_ack_trace, trace)
                             ),

            TP_fast_assign(
                    __entry->call = call;
                    __entry->serial = serial;
                    __entry->ack_first = ack_first;
                    __entry->ack_serial = ack_serial;
                    __entry->reason = reason;
                    __entry->n_acks = n_acks;
                    __entry->rwind = rwind;
                    __entry->trace = trace;
                           ),

            TP_printk(" c=%08x ACK  %08x %s f=%08x r=%08x n=%u rw=%u %s",
                      __entry->call,
                      __entry->serial,
                      __print_symbolic(__entry->reason, rxrpc_ack_names),
                      __entry->ack_first,
                      __entry->ack_serial,
                      __entry->n_acks,
                      __entry->rwind,
                      __print_symbolic(__entry->trace, rxrpc_propose_ack_traces))
            );

TRACE_EVENT(rxrpc_receive,
            TP_PROTO(struct rxrpc_call *call, enum rxrpc_receive_trace why,
                     rxrpc_serial_t serial, rxrpc_seq_t seq),

            TP_ARGS(call, why, serial, seq),

            TP_STRUCT__entry(
                    __field(unsigned int,                call)
                    __field(enum rxrpc_receive_trace,        why)
                    __field(rxrpc_serial_t,                serial)
                    __field(rxrpc_seq_t,                seq)
                    __field(rxrpc_seq_t,                window)
                    __field(rxrpc_seq_t,                wtop)
                             ),

            TP_fast_assign(
                    __entry->call = call->debug_id;
                    __entry->why = why;
                    __entry->serial = serial;
                    __entry->seq = seq;
                    __entry->window = call->ackr_window;
                    __entry->wtop = call->ackr_wtop;
                           ),

            TP_printk("c=%08x %s r=%08x q=%08x w=%08x-%08x",
                      __entry->call,
                      __print_symbolic(__entry->why, rxrpc_receive_traces),
                      __entry->serial,
                      __entry->seq,
                      __entry->window,
                      __entry->wtop)
            );

TRACE_EVENT(rxrpc_recvmsg,
            TP_PROTO(unsigned int call_debug_id, enum rxrpc_recvmsg_trace why,
                     int ret),

            TP_ARGS(call_debug_id, why, ret),

            TP_STRUCT__entry(
                    __field(unsigned int,                call)
                    __field(enum rxrpc_recvmsg_trace,        why)
                    __field(int,                        ret)
                             ),

            TP_fast_assign(
                    __entry->call = call_debug_id;
                    __entry->why = why;
                    __entry->ret = ret;
                           ),

            TP_printk("c=%08x %s ret=%d",
                      __entry->call,
                      __print_symbolic(__entry->why, rxrpc_recvmsg_traces),
                      __entry->ret)
            );

TRACE_EVENT(rxrpc_recvdata,
            TP_PROTO(struct rxrpc_call *call, enum rxrpc_recvmsg_trace why,
                     rxrpc_seq_t seq, unsigned int offset, unsigned int len,
                     int ret),

            TP_ARGS(call, why, seq, offset, len, ret),

            TP_STRUCT__entry(
                    __field(unsigned int,                call)
                    __field(enum rxrpc_recvmsg_trace,        why)
                    __field(rxrpc_seq_t,                seq)
                    __field(unsigned int,                offset)
                    __field(unsigned int,                len)
                    __field(int,                        ret)
                             ),

            TP_fast_assign(
                    __entry->call = call ? call->debug_id : 0;
                    __entry->why = why;
                    __entry->seq = seq;
                    __entry->offset = offset;
                    __entry->len = len;
                    __entry->ret = ret;
                           ),

            TP_printk("c=%08x %s q=%08x o=%u l=%u ret=%d",
                      __entry->call,
                      __print_symbolic(__entry->why, rxrpc_recvmsg_traces),
                      __entry->seq,
                      __entry->offset,
                      __entry->len,
                      __entry->ret)
            );

TRACE_EVENT(rxrpc_rtt_tx,
            TP_PROTO(struct rxrpc_call *call, enum rxrpc_rtt_tx_trace why,
                     int slot, rxrpc_serial_t send_serial),

            TP_ARGS(call, why, slot, send_serial),

            TP_STRUCT__entry(
                    __field(unsigned int,                call)
                    __field(enum rxrpc_rtt_tx_trace,        why)
                    __field(int,                        slot)
                    __field(rxrpc_serial_t,                send_serial)
                             ),

            TP_fast_assign(
                    __entry->call = call->debug_id;
                    __entry->why = why;
                    __entry->slot = slot;
                    __entry->send_serial = send_serial;
                           ),

            TP_printk("c=%08x [%d] %s sr=%08x",
                      __entry->call,
                      __entry->slot,
                      __print_symbolic(__entry->why, rxrpc_rtt_tx_traces),
                      __entry->send_serial)
            );

TRACE_EVENT(rxrpc_rtt_rx,
            TP_PROTO(struct rxrpc_call *call, enum rxrpc_rtt_rx_trace why,
                     int slot,
                     rxrpc_serial_t send_serial, rxrpc_serial_t resp_serial,
                     u32 rtt, u32 srtt, u32 rto),

            TP_ARGS(call, why, slot, send_serial, resp_serial, rtt, srtt, rto),

            TP_STRUCT__entry(
                    __field(unsigned int,                call)
                    __field(enum rxrpc_rtt_rx_trace,        why)
                    __field(int,                        slot)
                    __field(rxrpc_serial_t,                send_serial)
                    __field(rxrpc_serial_t,                resp_serial)
                    __field(u32,                        rtt)
                    __field(u32,                        srtt)
                    __field(u32,                        rto)
                    __field(u32,                        min_rtt)
                             ),

            TP_fast_assign(
                    __entry->call = call->debug_id;
                    __entry->why = why;
                    __entry->slot = slot;
                    __entry->send_serial = send_serial;
                    __entry->resp_serial = resp_serial;
                    __entry->rtt = rtt;
                    __entry->srtt = srtt;
                    __entry->rto = rto;
                    __entry->min_rtt = minmax_get(&call->min_rtt)
                           ),

            TP_printk("c=%08x [%d] %s sr=%08x rr=%08x rtt=%u srtt=%u rto=%u min=%u",
                      __entry->call,
                      __entry->slot,
                      __print_symbolic(__entry->why, rxrpc_rtt_rx_traces),
                      __entry->send_serial,
                      __entry->resp_serial,
                      __entry->rtt,
                      __entry->srtt / 8,
                      __entry->rto,
                      __entry->min_rtt)
            );

TRACE_EVENT(rxrpc_timer_set,
            TP_PROTO(struct rxrpc_call *call, ktime_t delay,
                     enum rxrpc_timer_trace why),

            TP_ARGS(call, delay, why),

            TP_STRUCT__entry(
                    __field(unsigned int,                call)
                    __field(enum rxrpc_timer_trace,        why)
                    __field(ktime_t,                        delay)
                             ),

            TP_fast_assign(
                    __entry->call                = call->debug_id;
                    __entry->why                = why;
                    __entry->delay                = delay;
                           ),

            TP_printk("c=%08x %s to=%lld",
                      __entry->call,
                      __print_symbolic(__entry->why, rxrpc_timer_traces),
                      ktime_to_us(__entry->delay))
            );

TRACE_EVENT(rxrpc_timer_exp,
            TP_PROTO(struct rxrpc_call *call, ktime_t delay,
                     enum rxrpc_timer_trace why),

            TP_ARGS(call, delay, why),

            TP_STRUCT__entry(
                    __field(unsigned int,                call)
                    __field(enum rxrpc_timer_trace,        why)
                    __field(ktime_t,                        delay)
                             ),

            TP_fast_assign(
                    __entry->call                = call->debug_id;
                    __entry->why                = why;
                    __entry->delay                = delay;
                           ),

            TP_printk("c=%08x %s to=%lld",
                      __entry->call,
                      __print_symbolic(__entry->why, rxrpc_timer_traces),
                      ktime_to_us(__entry->delay))
            );

TRACE_EVENT(rxrpc_timer_can,
            TP_PROTO(struct rxrpc_call *call, enum rxrpc_timer_trace why),

            TP_ARGS(call, why),

            TP_STRUCT__entry(
                    __field(unsigned int,                call)
                    __field(enum rxrpc_timer_trace,        why)
                             ),

            TP_fast_assign(
                    __entry->call                = call->debug_id;
                    __entry->why                = why;
                           ),

            TP_printk("c=%08x %s",
                      __entry->call,
                      __print_symbolic(__entry->why, rxrpc_timer_traces))
            );

TRACE_EVENT(rxrpc_timer_restart,
            TP_PROTO(struct rxrpc_call *call, ktime_t delay, unsigned long delayj),

            TP_ARGS(call, delay, delayj),

            TP_STRUCT__entry(
                    __field(unsigned int,                call)
                    __field(unsigned long,                delayj)
                    __field(ktime_t,                        delay)
                             ),

            TP_fast_assign(
                    __entry->call                = call->debug_id;
                    __entry->delayj                = delayj;
                    __entry->delay                = delay;
                           ),

            TP_printk("c=%08x to=%lld j=%ld",
                      __entry->call,
                      ktime_to_us(__entry->delay),
                      __entry->delayj)
            );

TRACE_EVENT(rxrpc_timer_expired,
            TP_PROTO(struct rxrpc_call *call),

            TP_ARGS(call),

            TP_STRUCT__entry(
                    __field(unsigned int,        call)
                             ),

            TP_fast_assign(
                    __entry->call                = call->debug_id;
                           ),

            TP_printk("c=%08x EXPIRED",
                      __entry->call)
            );

TRACE_EVENT(rxrpc_rx_lose,
            TP_PROTO(struct rxrpc_skb_priv *sp),

            TP_ARGS(sp),

            TP_STRUCT__entry(
                    __field_struct(struct rxrpc_host_header,        hdr)
                             ),

            TP_fast_assign(
                    memcpy(&__entry->hdr, &sp->hdr, sizeof(__entry->hdr));
                           ),

            TP_printk("%08x:%08x:%08x:%04x %08x %08x %02x %02x %s *LOSE*",
                      __entry->hdr.epoch, __entry->hdr.cid,
                      __entry->hdr.callNumber, __entry->hdr.serviceId,
                      __entry->hdr.serial, __entry->hdr.seq,
                      __entry->hdr.type, __entry->hdr.flags,
                      __entry->hdr.type <= 15 ?
                      __print_symbolic(__entry->hdr.type, rxrpc_pkts) : "?UNK")
            );

TRACE_EVENT(rxrpc_propose_ack,
            TP_PROTO(struct rxrpc_call *call, enum rxrpc_propose_ack_trace why,
                     u8 ack_reason, rxrpc_serial_t serial),

            TP_ARGS(call, why, ack_reason, serial),

            TP_STRUCT__entry(
                    __field(unsigned int,                        call)
                    __field(enum rxrpc_propose_ack_trace,        why)
                    __field(rxrpc_serial_t,                        serial)
                    __field(u8,                                        ack_reason)
                             ),

            TP_fast_assign(
                    __entry->call        = call->debug_id;
                    __entry->why        = why;
                    __entry->serial        = serial;
                    __entry->ack_reason        = ack_reason;
                           ),

            TP_printk("c=%08x %s %s r=%08x",
                      __entry->call,
                      __print_symbolic(__entry->why, rxrpc_propose_ack_traces),
                      __print_symbolic(__entry->ack_reason, rxrpc_ack_names),
                      __entry->serial)
            );

TRACE_EVENT(rxrpc_send_ack,
            TP_PROTO(struct rxrpc_call *call, enum rxrpc_propose_ack_trace why,
                     u8 ack_reason, rxrpc_serial_t serial),

            TP_ARGS(call, why, ack_reason, serial),

            TP_STRUCT__entry(
                    __field(unsigned int,                        call)
                    __field(enum rxrpc_propose_ack_trace,        why)
                    __field(rxrpc_serial_t,                        serial)
                    __field(u8,                                        ack_reason)
                             ),

            TP_fast_assign(
                    __entry->call        = call->debug_id;
                    __entry->why        = why;
                    __entry->serial        = serial;
                    __entry->ack_reason        = ack_reason;
                           ),

            TP_printk("c=%08x %s %s r=%08x",
                      __entry->call,
                      __print_symbolic(__entry->why, rxrpc_propose_ack_traces),
                      __print_symbolic(__entry->ack_reason, rxrpc_ack_names),
                      __entry->serial)
            );

TRACE_EVENT(rxrpc_drop_ack,
            TP_PROTO(struct rxrpc_call *call, enum rxrpc_propose_ack_trace why,
                     u8 ack_reason, rxrpc_serial_t serial, bool nobuf),

            TP_ARGS(call, why, ack_reason, serial, nobuf),

            TP_STRUCT__entry(
                    __field(unsigned int,                        call)
                    __field(enum rxrpc_propose_ack_trace,        why)
                    __field(rxrpc_serial_t,                        serial)
                    __field(u8,                                        ack_reason)
                    __field(bool,                                nobuf)
                             ),

            TP_fast_assign(
                    __entry->call        = call->debug_id;
                    __entry->why        = why;
                    __entry->serial        = serial;
                    __entry->ack_reason        = ack_reason;
                    __entry->nobuf        = nobuf;
                           ),

            TP_printk("c=%08x %s %s r=%08x nbf=%u",
                      __entry->call,
                      __print_symbolic(__entry->why, rxrpc_propose_ack_traces),
                      __print_symbolic(__entry->ack_reason, rxrpc_ack_names),
                      __entry->serial, __entry->nobuf)
            );

TRACE_EVENT(rxrpc_retransmit,
            TP_PROTO(struct rxrpc_call *call,
                     struct rxrpc_send_data_req *req,
                     struct rxrpc_txbuf *txb),

            TP_ARGS(call, req, txb),

            TP_STRUCT__entry(
                    __field(unsigned int,        call)
                    __field(unsigned int,        qbase)
                    __field(rxrpc_seq_t,        seq)
                    __field(rxrpc_serial_t,        serial)
                             ),

            TP_fast_assign(
                    __entry->call = call->debug_id;
                    __entry->qbase = req->tq->qbase;
                    __entry->seq = req->seq;
                    __entry->serial = txb->serial;
                           ),

            TP_printk("c=%08x tq=%x q=%x r=%x",
                      __entry->call,
                      __entry->qbase,
                      __entry->seq,
                      __entry->serial)
            );

TRACE_EVENT(rxrpc_congest,
            TP_PROTO(struct rxrpc_call *call, struct rxrpc_ack_summary *summary),

            TP_ARGS(call, summary),

            TP_STRUCT__entry(
                    __field(unsigned int,                        call)
                    __field(enum rxrpc_ca_state,                ca_state)
                    __field(rxrpc_seq_t,                        hard_ack)
                    __field(rxrpc_seq_t,                        top)
                    __field(rxrpc_seq_t,                        lowest_nak)
                    __field(u16,                                nr_sacks)
                    __field(u16,                                nr_snacks)
                    __field(u16,                                cwnd)
                    __field(u16,                                ssthresh)
                    __field(u16,                                cumul_acks)
                    __field(u16,                                dup_acks)
                    __field_struct(struct rxrpc_ack_summary,        sum)
                             ),

            TP_fast_assign(
                    __entry->call        = call->debug_id;
                    __entry->ca_state        = call->cong_ca_state;
                    __entry->hard_ack        = call->acks_hard_ack;
                    __entry->top        = call->tx_top;
                    __entry->lowest_nak        = call->acks_lowest_nak;
                    __entry->nr_sacks        = call->acks_nr_sacks;
                    __entry->nr_snacks        = call->acks_nr_snacks;
                    __entry->cwnd        = call->cong_cwnd;
                    __entry->ssthresh        = call->cong_ssthresh;
                    __entry->cumul_acks        = call->cong_cumul_acks;
                    __entry->dup_acks        = call->cong_dup_acks;
                    memcpy(&__entry->sum, summary, sizeof(__entry->sum));
                           ),

            TP_printk("c=%08x r=%08x %s q=%08x %s cw=%u ss=%u A=%u+%u/%u+%u r=%u b=%u u=%u d=%u l=%x%s%s%s",
                      __entry->call,
                      __entry->sum.acked_serial,
                      __print_symbolic(__entry->sum.ack_reason, rxrpc_ack_names),
                      __entry->hard_ack,
                      __print_symbolic(__entry->ca_state, rxrpc_ca_states),
                      __entry->cwnd,
                      __entry->ssthresh,
                      __entry->nr_sacks, __entry->sum.nr_new_sacks,
                      __entry->nr_snacks, __entry->sum.nr_new_snacks,
                      __entry->sum.nr_new_hacks,
                      __entry->top - __entry->hard_ack,
                      __entry->cumul_acks,
                      __entry->dup_acks,
                      __entry->lowest_nak, __entry->sum.new_low_snack ? "!" : "",
                      __print_symbolic(__entry->sum.change, rxrpc_congest_changes),
                      __entry->sum.retrans_timeo ? " rTxTo" : "")
            );

TRACE_EVENT(rxrpc_reset_cwnd,
            TP_PROTO(struct rxrpc_call *call, ktime_t since_last_tx, ktime_t rtt),

            TP_ARGS(call, since_last_tx, rtt),

            TP_STRUCT__entry(
                    __field(unsigned int,                call)
                    __field(enum rxrpc_ca_state,        ca_state)
                    __field(unsigned short,                cwnd)
                    __field(unsigned short,                extra)
                    __field(rxrpc_seq_t,                hard_ack)
                    __field(rxrpc_seq_t,                prepared)
                    __field(ktime_t,                        since_last_tx)
                    __field(ktime_t,                        rtt)
                    __field(bool,                        has_data)
                             ),

            TP_fast_assign(
                    __entry->call        = call->debug_id;
                    __entry->ca_state        = call->cong_ca_state;
                    __entry->cwnd        = call->cong_cwnd;
                    __entry->extra        = call->cong_extra;
                    __entry->hard_ack        = call->acks_hard_ack;
                    __entry->prepared        = call->send_top - call->tx_bottom;
                    __entry->since_last_tx = since_last_tx;
                    __entry->rtt        = rtt;
                    __entry->has_data        = call->tx_bottom != call->tx_top;
                           ),

            TP_printk("c=%08x q=%08x %s cw=%u+%u pr=%u tm=%llu/%llu d=%u",
                      __entry->call,
                      __entry->hard_ack,
                      __print_symbolic(__entry->ca_state, rxrpc_ca_states),
                      __entry->cwnd,
                      __entry->extra,
                      __entry->prepared,
                      ktime_to_us(__entry->since_last_tx),
                      ktime_to_us(__entry->rtt),
                      __entry->has_data)
            );

TRACE_EVENT(rxrpc_disconnect_call,
            TP_PROTO(struct rxrpc_call *call),

            TP_ARGS(call),

            TP_STRUCT__entry(
                    __field(unsigned int,        call)
                    __field(u32,                abort_code)
                             ),

            TP_fast_assign(
                    __entry->call = call->debug_id;
                    __entry->abort_code = call->abort_code;
                           ),

            TP_printk("c=%08x ab=%08x",
                      __entry->call,
                      __entry->abort_code)
            );

TRACE_EVENT(rxrpc_improper_term,
            TP_PROTO(struct rxrpc_call *call),

            TP_ARGS(call),

            TP_STRUCT__entry(
                    __field(unsigned int,        call)
                    __field(u32,                abort_code)
                             ),

            TP_fast_assign(
                    __entry->call = call->debug_id;
                    __entry->abort_code = call->abort_code;
                           ),

            TP_printk("c=%08x ab=%08x",
                      __entry->call,
                      __entry->abort_code)
            );

TRACE_EVENT(rxrpc_connect_call,
            TP_PROTO(struct rxrpc_call *call),

            TP_ARGS(call),

            TP_STRUCT__entry(
                    __field(unsigned int,                call)
                    __field(unsigned long,                user_call_ID)
                    __field(u32,                        cid)
                    __field(u32,                        call_id)
                    __field_struct(struct sockaddr_rxrpc, srx)
                             ),

            TP_fast_assign(
                    __entry->call = call->debug_id;
                    __entry->user_call_ID = call->user_call_ID;
                    __entry->cid = call->cid;
                    __entry->call_id = call->call_id;
                    __entry->srx = call->dest_srx;
                           ),

            TP_printk("c=%08x u=%p %08x:%08x dst=%pISp",
                      __entry->call,
                      (void *)__entry->user_call_ID,
                      __entry->cid,
                      __entry->call_id,
                      &__entry->srx.transport)
            );

TRACE_EVENT(rxrpc_apply_acks,
            TP_PROTO(struct rxrpc_call *call, struct rxrpc_txqueue *tq),

            TP_ARGS(call, tq),

            TP_STRUCT__entry(
                    __field(unsigned int,        call)
                    __field(unsigned int,        nr_rep)
                    __field(rxrpc_seq_t,        qbase)
                    __field(unsigned long,        acks)
                             ),

            TP_fast_assign(
                    __entry->call = call->debug_id;
                    __entry->qbase = tq->qbase;
                    __entry->acks = tq->segment_acked;
                    __entry->nr_rep = tq->nr_reported_acks;
                           ),

            TP_printk("c=%08x tq=%x acks=%016lx rep=%u",
                      __entry->call,
                      __entry->qbase,
                      __entry->acks,
                      __entry->nr_rep)
            );

TRACE_EVENT(rxrpc_resend,
            TP_PROTO(struct rxrpc_call *call, rxrpc_serial_t ack_serial),

            TP_ARGS(call, ack_serial),

            TP_STRUCT__entry(
                    __field(unsigned int,        call)
                    __field(rxrpc_seq_t,        seq)
                    __field(rxrpc_seq_t,        transmitted)
                    __field(rxrpc_serial_t,        ack_serial)
                             ),

            TP_fast_assign(
                    __entry->call = call->debug_id;
                    __entry->seq = call->acks_hard_ack;
                    __entry->transmitted = call->tx_transmitted;
                    __entry->ack_serial = ack_serial;
                           ),

            TP_printk("c=%08x r=%x q=%x tq=%x",
                      __entry->call,
                      __entry->ack_serial,
                      __entry->seq,
                      __entry->transmitted)
            );

TRACE_EVENT(rxrpc_resend_lost,
            TP_PROTO(struct rxrpc_call *call, struct rxrpc_txqueue *tq, unsigned long lost),

            TP_ARGS(call, tq, lost),

            TP_STRUCT__entry(
                    __field(unsigned int,        call)
                    __field(rxrpc_seq_t,        qbase)
                    __field(u8,                        nr_rep)
                    __field(unsigned long,        lost)
                             ),

            TP_fast_assign(
                    __entry->call = call->debug_id;
                    __entry->qbase = tq->qbase;
                    __entry->nr_rep = tq->nr_reported_acks;
                    __entry->lost = lost;
                           ),

            TP_printk("c=%08x tq=%x lost=%016lx nr=%u",
                      __entry->call,
                      __entry->qbase,
                      __entry->lost,
                      __entry->nr_rep)
            );

TRACE_EVENT(rxrpc_rotate,
            TP_PROTO(struct rxrpc_call *call, struct rxrpc_txqueue *tq,
                     struct rxrpc_ack_summary *summary, rxrpc_seq_t seq,
                     enum rxrpc_rotate_trace trace),

            TP_ARGS(call, tq, summary, seq, trace),

            TP_STRUCT__entry(
                    __field(unsigned int,        call)
                    __field(rxrpc_seq_t,        qbase)
                    __field(rxrpc_seq_t,        seq)
                    __field(unsigned int,        nr_rep)
                    __field(enum rxrpc_rotate_trace, trace)
                             ),

            TP_fast_assign(
                    __entry->call = call->debug_id;
                    __entry->qbase = tq->qbase;
                    __entry->seq = seq;
                    __entry->nr_rep = tq->nr_reported_acks;
                    __entry->trace = trace;
                           ),

            TP_printk("c=%08x tq=%x q=%x nr=%x %s",
                      __entry->call,
                      __entry->qbase,
                      __entry->seq,
                      __entry->nr_rep,
                      __print_symbolic(__entry->trace, rxrpc_rotate_traces))
            );

TRACE_EVENT(rxrpc_rx_icmp,
            TP_PROTO(struct rxrpc_peer *peer, struct sock_extended_err *ee,
                     struct sockaddr_rxrpc *srx),

            TP_ARGS(peer, ee, srx),

            TP_STRUCT__entry(
                    __field(unsigned int,                        peer)
                    __field_struct(struct sock_extended_err,        ee)
                    __field_struct(struct sockaddr_rxrpc,        srx)
                             ),

            TP_fast_assign(
                    __entry->peer = peer->debug_id;
                    memcpy(&__entry->ee, ee, sizeof(__entry->ee));
                    memcpy(&__entry->srx, srx, sizeof(__entry->srx));
                           ),

            TP_printk("P=%08x o=%u t=%u c=%u i=%u d=%u e=%d %pISp",
                      __entry->peer,
                      __entry->ee.ee_origin,
                      __entry->ee.ee_type,
                      __entry->ee.ee_code,
                      __entry->ee.ee_info,
                      __entry->ee.ee_data,
                      __entry->ee.ee_errno,
                      &__entry->srx.transport)
            );

TRACE_EVENT(rxrpc_tx_fail,
            TP_PROTO(unsigned int debug_id, rxrpc_serial_t serial, int ret,
                     enum rxrpc_tx_point where),

            TP_ARGS(debug_id, serial, ret, where),

            TP_STRUCT__entry(
                    __field(unsigned int,                debug_id)
                    __field(rxrpc_serial_t,                serial)
                    __field(int,                        ret)
                    __field(enum rxrpc_tx_point,        where)
                             ),

            TP_fast_assign(
                    __entry->debug_id = debug_id;
                    __entry->serial = serial;
                    __entry->ret = ret;
                    __entry->where = where;
                           ),

            TP_printk("c=%08x r=%x ret=%d %s",
                      __entry->debug_id,
                      __entry->serial,
                      __entry->ret,
                      __print_symbolic(__entry->where, rxrpc_tx_points))
            );

TRACE_EVENT(rxrpc_call_reset,
            TP_PROTO(struct rxrpc_call *call),

            TP_ARGS(call),

            TP_STRUCT__entry(
                    __field(unsigned int,        debug_id)
                    __field(u32,                cid)
                    __field(u32,                call_id)
                    __field(rxrpc_serial_t,        call_serial)
                    __field(rxrpc_serial_t,        conn_serial)
                    __field(rxrpc_seq_t,        tx_seq)
                    __field(rxrpc_seq_t,        rx_seq)
                             ),

            TP_fast_assign(
                    __entry->debug_id = call->debug_id;
                    __entry->cid = call->cid;
                    __entry->call_id = call->call_id;
                    __entry->call_serial = call->rx_serial;
                    __entry->conn_serial = call->conn->hi_serial;
                    __entry->tx_seq = call->acks_hard_ack;
                    __entry->rx_seq = call->rx_highest_seq;
                           ),

            TP_printk("c=%08x %08x:%08x r=%08x/%08x tx=%08x rx=%08x",
                      __entry->debug_id,
                      __entry->cid, __entry->call_id,
                      __entry->call_serial, __entry->conn_serial,
                      __entry->tx_seq, __entry->rx_seq)
            );

TRACE_EVENT(rxrpc_notify_socket,
            TP_PROTO(unsigned int debug_id, rxrpc_serial_t serial),

            TP_ARGS(debug_id, serial),

            TP_STRUCT__entry(
                    __field(unsigned int,        debug_id)
                    __field(rxrpc_serial_t,        serial)
                             ),

            TP_fast_assign(
                    __entry->debug_id = debug_id;
                    __entry->serial = serial;
                           ),

            TP_printk("c=%08x r=%08x",
                      __entry->debug_id,
                      __entry->serial)
            );

TRACE_EVENT(rxrpc_rx_discard_ack,
            TP_PROTO(struct rxrpc_call *call, rxrpc_serial_t serial,
                     rxrpc_seq_t hard_ack, rxrpc_seq_t prev_pkt),

            TP_ARGS(call, serial, hard_ack, prev_pkt),

            TP_STRUCT__entry(
                    __field(unsigned int,        debug_id)
                    __field(rxrpc_serial_t,        serial)
                    __field(rxrpc_seq_t,        hard_ack)
                    __field(rxrpc_seq_t,        prev_pkt)
                    __field(rxrpc_seq_t,        acks_hard_ack)
                    __field(rxrpc_seq_t,        acks_prev_seq)
                             ),

            TP_fast_assign(
                    __entry->debug_id                = call->debug_id;
                    __entry->serial                = serial;
                    __entry->hard_ack                = hard_ack;
                    __entry->prev_pkt                = prev_pkt;
                    __entry->acks_hard_ack        = call->acks_hard_ack;
                    __entry->acks_prev_seq        = call->acks_prev_seq;
                           ),

            TP_printk("c=%08x r=%08x %08x<%08x %08x<%08x",
                      __entry->debug_id,
                      __entry->serial,
                      __entry->hard_ack,
                      __entry->acks_hard_ack,
                      __entry->prev_pkt,
                      __entry->acks_prev_seq)
            );

TRACE_EVENT(rxrpc_req_ack,
            TP_PROTO(unsigned int call_debug_id, rxrpc_seq_t seq,
                     enum rxrpc_req_ack_trace why),

            TP_ARGS(call_debug_id, seq, why),

            TP_STRUCT__entry(
                    __field(unsigned int,                call_debug_id)
                    __field(rxrpc_seq_t,                seq)
                    __field(enum rxrpc_req_ack_trace,        why)
                             ),

            TP_fast_assign(
                    __entry->call_debug_id = call_debug_id;
                    __entry->seq = seq;
                    __entry->why = why;
                           ),

            TP_printk("c=%08x q=%08x REQ-%s",
                      __entry->call_debug_id,
                      __entry->seq,
                      __print_symbolic(__entry->why, rxrpc_req_ack_traces))
            );

TRACE_EVENT(rxrpc_txbuf,
            TP_PROTO(unsigned int debug_id,
                     unsigned int call_debug_id, rxrpc_seq_t seq,
                     int ref, enum rxrpc_txbuf_trace what),

            TP_ARGS(debug_id, call_debug_id, seq, ref, what),

            TP_STRUCT__entry(
                    __field(unsigned int,                debug_id)
                    __field(unsigned int,                call_debug_id)
                    __field(rxrpc_seq_t,                seq)
                    __field(int,                        ref)
                    __field(enum rxrpc_txbuf_trace,        what)
                             ),

            TP_fast_assign(
                    __entry->debug_id = debug_id;
                    __entry->call_debug_id = call_debug_id;
                    __entry->seq = seq;
                    __entry->ref = ref;
                    __entry->what = what;
                           ),

            TP_printk("B=%08x c=%08x q=%08x %s r=%d",
                      __entry->debug_id,
                      __entry->call_debug_id,
                      __entry->seq,
                      __print_symbolic(__entry->what, rxrpc_txbuf_traces),
                      __entry->ref)
            );

TRACE_EVENT(rxrpc_tq,
            TP_PROTO(struct rxrpc_call *call, struct rxrpc_txqueue *tq,
                     rxrpc_seq_t seq, enum rxrpc_tq_trace trace),

            TP_ARGS(call, tq, seq, trace),

            TP_STRUCT__entry(
                    __field(unsigned int,                call_debug_id)
                    __field(rxrpc_seq_t,                qbase)
                    __field(rxrpc_seq_t,                seq)
                    __field(enum rxrpc_tq_trace,        trace)
                             ),

            TP_fast_assign(
                    __entry->call_debug_id = call->debug_id;
                    __entry->qbase = tq ? tq->qbase : call->tx_qbase;
                    __entry->seq = seq;
                    __entry->trace = trace;
                           ),

            TP_printk("c=%08x bq=%08x q=%08x %s",
                      __entry->call_debug_id,
                      __entry->qbase,
                      __entry->seq,
                      __print_symbolic(__entry->trace, rxrpc_tq_traces))
            );

TRACE_EVENT(rxrpc_poke_call,
            TP_PROTO(struct rxrpc_call *call, bool busy,
                     enum rxrpc_call_poke_trace what),

            TP_ARGS(call, busy, what),

            TP_STRUCT__entry(
                    __field(unsigned int,                call_debug_id)
                    __field(bool,                        busy)
                    __field(enum rxrpc_call_poke_trace,        what)
                             ),

            TP_fast_assign(
                    __entry->call_debug_id = call->debug_id;
                    __entry->busy = busy;
                    __entry->what = what;
                           ),

            TP_printk("c=%08x %s%s",
                      __entry->call_debug_id,
                      __print_symbolic(__entry->what, rxrpc_call_poke_traces),
                      __entry->busy ? "!" : "")
            );

TRACE_EVENT(rxrpc_call_poked,
            TP_PROTO(struct rxrpc_call *call),

            TP_ARGS(call),

            TP_STRUCT__entry(
                    __field(unsigned int,        call_debug_id)
                             ),

            TP_fast_assign(
                    __entry->call_debug_id = call->debug_id;
                           ),

            TP_printk("c=%08x",
                      __entry->call_debug_id)
            );

TRACE_EVENT(rxrpc_sack,
            TP_PROTO(struct rxrpc_call *call, rxrpc_seq_t seq,
                     unsigned int sack, enum rxrpc_sack_trace what),

            TP_ARGS(call, seq, sack, what),

            TP_STRUCT__entry(
                    __field(unsigned int,                call_debug_id)
                    __field(rxrpc_seq_t,                seq)
                    __field(unsigned int,                sack)
                    __field(enum rxrpc_sack_trace,        what)
                             ),

            TP_fast_assign(
                    __entry->call_debug_id = call->debug_id;
                    __entry->seq = seq;
                    __entry->sack = sack;
                    __entry->what = what;
                           ),

            TP_printk("c=%08x q=%08x %s k=%x",
                      __entry->call_debug_id,
                      __entry->seq,
                      __print_symbolic(__entry->what, rxrpc_sack_traces),
                      __entry->sack)
            );

TRACE_EVENT(rxrpc_pmtud_tx,
            TP_PROTO(struct rxrpc_call *call),

            TP_ARGS(call),

            TP_STRUCT__entry(
                    __field(unsigned int,        peer_debug_id)
                    __field(unsigned int,        call_debug_id)
                    __field(rxrpc_serial_t,        ping_serial)
                    __field(unsigned short,        pmtud_trial)
                    __field(unsigned short,        pmtud_good)
                    __field(unsigned short,        pmtud_bad)
                             ),

            TP_fast_assign(
                    __entry->peer_debug_id = call->peer->debug_id;
                    __entry->call_debug_id = call->debug_id;
                    __entry->ping_serial = call->conn->pmtud_probe;
                    __entry->pmtud_trial = call->peer->pmtud_trial;
                    __entry->pmtud_good = call->peer->pmtud_good;
                    __entry->pmtud_bad = call->peer->pmtud_bad;
                           ),

            TP_printk("P=%08x c=%08x pr=%08x %u-%u-%u",
                      __entry->peer_debug_id,
                      __entry->call_debug_id,
                      __entry->ping_serial,
                      __entry->pmtud_good,
                      __entry->pmtud_trial,
                      __entry->pmtud_bad)
            );

TRACE_EVENT(rxrpc_pmtud_rx,
            TP_PROTO(struct rxrpc_connection *conn, rxrpc_serial_t resp_serial),

            TP_ARGS(conn, resp_serial),

            TP_STRUCT__entry(
                    __field(unsigned int,        peer_debug_id)
                    __field(unsigned int,        call_debug_id)
                    __field(rxrpc_serial_t,        ping_serial)
                    __field(rxrpc_serial_t,        resp_serial)
                    __field(unsigned short,        max_data)
                    __field(u8,                        jumbo_max)
                             ),

            TP_fast_assign(
                    __entry->peer_debug_id = conn->peer->debug_id;
                    __entry->call_debug_id = conn->pmtud_call;
                    __entry->ping_serial = conn->pmtud_probe;
                    __entry->resp_serial = resp_serial;
                    __entry->max_data = conn->peer->max_data;
                    __entry->jumbo_max = conn->peer->pmtud_jumbo;
                           ),

            TP_printk("P=%08x c=%08x pr=%08x rr=%08x max=%u jm=%u",
                      __entry->peer_debug_id,
                      __entry->call_debug_id,
                      __entry->ping_serial,
                      __entry->resp_serial,
                      __entry->max_data,
                      __entry->jumbo_max)
            );

TRACE_EVENT(rxrpc_pmtud_lost,
            TP_PROTO(struct rxrpc_connection *conn, rxrpc_serial_t resp_serial),

            TP_ARGS(conn, resp_serial),

            TP_STRUCT__entry(
                    __field(unsigned int,        peer_debug_id)
                    __field(unsigned int,        call_debug_id)
                    __field(rxrpc_serial_t,        ping_serial)
                    __field(rxrpc_serial_t,        resp_serial)
                             ),

            TP_fast_assign(
                    __entry->peer_debug_id = conn->peer->debug_id;
                    __entry->call_debug_id = conn->pmtud_call;
                    __entry->ping_serial = conn->pmtud_probe;
                    __entry->resp_serial = resp_serial;
                           ),

            TP_printk("P=%08x c=%08x pr=%08x rr=%08x",
                      __entry->peer_debug_id,
                      __entry->call_debug_id,
                      __entry->ping_serial,
                      __entry->resp_serial)
            );

TRACE_EVENT(rxrpc_pmtud_reduce,
            TP_PROTO(struct rxrpc_peer *peer, rxrpc_serial_t serial,
                     unsigned int max_data, enum rxrpc_pmtud_reduce_trace reason),

            TP_ARGS(peer, serial, max_data, reason),

            TP_STRUCT__entry(
                    __field(unsigned int,        peer_debug_id)
                    __field(rxrpc_serial_t,        serial)
                    __field(unsigned int,        max_data)
                    __field(enum rxrpc_pmtud_reduce_trace, reason)
                             ),

            TP_fast_assign(
                    __entry->peer_debug_id = peer->debug_id;
                    __entry->serial = serial;
                    __entry->max_data = max_data;
                    __entry->reason = reason;
                           ),

            TP_printk("P=%08x %s r=%08x m=%u",
                      __entry->peer_debug_id,
                      __print_symbolic(__entry->reason, rxrpc_pmtud_reduce_traces),
                      __entry->serial, __entry->max_data)
            );

TRACE_EVENT(rxrpc_rack,
            TP_PROTO(struct rxrpc_call *call, ktime_t timo),

            TP_ARGS(call, timo),

            TP_STRUCT__entry(
                    __field(unsigned int,        call)
                    __field(rxrpc_serial_t,        ack_serial)
                    __field(rxrpc_seq_t,        seq)
                    __field(enum rxrpc_rack_timer_mode, mode)
                    __field(unsigned short,        nr_sent)
                    __field(unsigned short,        nr_lost)
                    __field(unsigned short,        nr_resent)
                    __field(unsigned short,        nr_sacked)
                    __field(ktime_t,                timo)
                             ),

            TP_fast_assign(
                    __entry->call        = call->debug_id;
                    __entry->ack_serial        = call->rx_serial;
                    __entry->seq        = call->rack_end_seq;
                    __entry->mode        = call->rack_timer_mode;
                    __entry->nr_sent        = call->tx_nr_sent;
                    __entry->nr_lost        = call->tx_nr_lost;
                    __entry->nr_resent        = call->tx_nr_resent;
                    __entry->nr_sacked        = call->acks_nr_sacks;
                    __entry->timo        = timo;
                           ),

            TP_printk("c=%08x r=%08x q=%08x %s slrs=%u,%u,%u,%u t=%lld",
                      __entry->call, __entry->ack_serial, __entry->seq,
                      __print_symbolic(__entry->mode, rxrpc_rack_timer_modes),
                      __entry->nr_sent, __entry->nr_lost,
                      __entry->nr_resent, __entry->nr_sacked,
                      ktime_to_us(__entry->timo))
            );

TRACE_EVENT(rxrpc_rack_update,
            TP_PROTO(struct rxrpc_call *call, struct rxrpc_ack_summary *summary),

            TP_ARGS(call, summary),

            TP_STRUCT__entry(
                    __field(unsigned int,        call)
                    __field(rxrpc_serial_t,        ack_serial)
                    __field(rxrpc_seq_t,        seq)
                    __field(int,                xmit_ts)
                             ),

            TP_fast_assign(
                    __entry->call        = call->debug_id;
                    __entry->ack_serial        = call->rx_serial;
                    __entry->seq        = call->rack_end_seq;
                    __entry->xmit_ts        = ktime_sub(call->acks_latest_ts, call->rack_xmit_ts);
                           ),

            TP_printk("c=%08x r=%08x q=%08x xt=%lld",
                      __entry->call, __entry->ack_serial, __entry->seq,
                      ktime_to_us(__entry->xmit_ts))
            );

TRACE_EVENT(rxrpc_rack_scan_loss,
            TP_PROTO(struct rxrpc_call *call),

            TP_ARGS(call),

            TP_STRUCT__entry(
                    __field(unsigned int,        call)
                    __field(ktime_t,                rack_rtt)
                    __field(ktime_t,                rack_reo_wnd)
                             ),

            TP_fast_assign(
                    __entry->call                = call->debug_id;
                    __entry->rack_rtt                = call->rack_rtt;
                    __entry->rack_reo_wnd        = call->rack_reo_wnd;
                           ),

            TP_printk("c=%08x rtt=%lld reow=%lld",
                      __entry->call, ktime_to_us(__entry->rack_rtt),
                      ktime_to_us(__entry->rack_reo_wnd))
            );

TRACE_EVENT(rxrpc_rack_scan_loss_tq,
            TP_PROTO(struct rxrpc_call *call, const struct rxrpc_txqueue *tq,
                     unsigned long nacks),

            TP_ARGS(call, tq, nacks),

            TP_STRUCT__entry(
                    __field(unsigned int,        call)
                    __field(rxrpc_seq_t,        qbase)
                    __field(unsigned long,        nacks)
                    __field(unsigned long,        lost)
                    __field(unsigned long,        retrans)
                             ),

            TP_fast_assign(
                    __entry->call        = call->debug_id;
                    __entry->qbase        = tq->qbase;
                    __entry->nacks        = nacks;
                    __entry->lost        = tq->segment_lost;
                    __entry->retrans        = tq->segment_retransmitted;
                           ),

            TP_printk("c=%08x q=%08x n=%lx l=%lx r=%lx",
                      __entry->call, __entry->qbase,
                      __entry->nacks, __entry->lost, __entry->retrans)
            );

TRACE_EVENT(rxrpc_rack_detect_loss,
            TP_PROTO(struct rxrpc_call *call, struct rxrpc_ack_summary *summary,
                     rxrpc_seq_t seq),

            TP_ARGS(call, summary, seq),

            TP_STRUCT__entry(
                    __field(unsigned int,        call)
                    __field(rxrpc_serial_t,        ack_serial)
                    __field(rxrpc_seq_t,        seq)
                             ),

            TP_fast_assign(
                    __entry->call        = call->debug_id;
                    __entry->ack_serial        = call->rx_serial;
                    __entry->seq        = seq;
                           ),

            TP_printk("c=%08x r=%08x q=%08x",
                      __entry->call, __entry->ack_serial, __entry->seq)
            );

TRACE_EVENT(rxrpc_rack_mark_loss_tq,
            TP_PROTO(struct rxrpc_call *call, const struct rxrpc_txqueue *tq),

            TP_ARGS(call, tq),

            TP_STRUCT__entry(
                    __field(unsigned int,        call)
                    __field(rxrpc_seq_t,        qbase)
                    __field(rxrpc_seq_t,        trans)
                    __field(unsigned long,        acked)
                    __field(unsigned long,        lost)
                    __field(unsigned long,        retrans)
                             ),

            TP_fast_assign(
                    __entry->call        = call->debug_id;
                    __entry->qbase        = tq->qbase;
                    __entry->trans        = call->tx_transmitted;
                    __entry->acked        = tq->segment_acked;
                    __entry->lost        = tq->segment_lost;
                    __entry->retrans        = tq->segment_retransmitted;
                           ),

            TP_printk("c=%08x tq=%08x txq=%08x a=%lx l=%lx r=%lx",
                      __entry->call, __entry->qbase, __entry->trans,
                      __entry->acked, __entry->lost, __entry->retrans)
            );

TRACE_EVENT(rxrpc_tlp_probe,
            TP_PROTO(struct rxrpc_call *call, enum rxrpc_tlp_probe_trace trace),

            TP_ARGS(call, trace),

            TP_STRUCT__entry(
                    __field(unsigned int,                call)
                    __field(rxrpc_serial_t,                serial)
                    __field(rxrpc_seq_t,                seq)
                    __field(enum rxrpc_tlp_probe_trace,        trace)
                             ),

            TP_fast_assign(
                    __entry->call        = call->debug_id;
                    __entry->serial        = call->tlp_serial;
                    __entry->seq        = call->tlp_seq;
                    __entry->trace        = trace;
                           ),

            TP_printk("c=%08x r=%08x pq=%08x %s",
                      __entry->call, __entry->serial, __entry->seq,
                      __print_symbolic(__entry->trace, rxrpc_tlp_probe_traces))
            );

TRACE_EVENT(rxrpc_tlp_ack,
            TP_PROTO(struct rxrpc_call *call, struct rxrpc_ack_summary *summary,
                     enum rxrpc_tlp_ack_trace trace),

            TP_ARGS(call, summary, trace),

            TP_STRUCT__entry(
                    __field(unsigned int,                call)
                    __field(rxrpc_serial_t,                serial)
                    __field(rxrpc_seq_t,                tlp_seq)
                    __field(rxrpc_seq_t,                hard_ack)
                    __field(enum rxrpc_tlp_ack_trace,        trace)
                             ),

            TP_fast_assign(
                    __entry->call        = call->debug_id;
                    __entry->serial        = call->tlp_serial;
                    __entry->tlp_seq        = call->tlp_seq;
                    __entry->hard_ack        = call->acks_hard_ack;
                    __entry->trace        = trace;
                           ),

            TP_printk("c=%08x r=%08x pq=%08x hq=%08x %s",
                      __entry->call, __entry->serial,
                      __entry->tlp_seq, __entry->hard_ack,
                      __print_symbolic(__entry->trace, rxrpc_tlp_ack_traces))
            );

TRACE_EVENT(rxrpc_rack_timer,
            TP_PROTO(struct rxrpc_call *call, ktime_t delay, bool exp),

            TP_ARGS(call, delay, exp),

            TP_STRUCT__entry(
                    __field(unsigned int,                call)
                    __field(bool,                        exp)
                    __field(enum rxrpc_rack_timer_mode,        mode)
                    __field(ktime_t,                        delay)
                             ),

            TP_fast_assign(
                    __entry->call                = call->debug_id;
                    __entry->exp                = exp;
                    __entry->mode                = call->rack_timer_mode;
                    __entry->delay                = delay;
                           ),

            TP_printk("c=%08x %s %s to=%lld",
                      __entry->call,
                      __entry->exp ? "Exp" : "Set",
                      __print_symbolic(__entry->mode, rxrpc_rack_timer_modes),
                      ktime_to_us(__entry->delay))
            );

TRACE_EVENT(rxrpc_rxgk_rekey,
            TP_PROTO(struct rxrpc_connection *conn,
                     unsigned int current_key, unsigned int requested_key),

            TP_ARGS(conn, current_key, requested_key),

            TP_STRUCT__entry(
                    __field(unsigned int,        conn)
                    __field(unsigned int,        current_key)
                    __field(unsigned int,        requested_key)
                             ),

            TP_fast_assign(
                    __entry->conn                = conn->debug_id;
                    __entry->current_key        = current_key;
                    __entry->requested_key        = requested_key;
                           ),

            TP_printk("C=%08x cur=%x req=%x",
                      __entry->conn,
                      __entry->current_key,
                      __entry->requested_key)
            );

#undef EM
#undef E_

#endif /* RXRPC_TRACE_ONLY_DEFINE_ENUMS */
#endif /* _TRACE_RXRPC_H */

/* This part must be outside protection */
#include <trace/define_trace.h>










































































    1 


    1 












































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
// SPDX-License-Identifier: GPL-2.0-only
/* Xtables module to match packets using a BPF filter.
 * Copyright 2013 Google Inc.
 * Written by Willem de Bruijn <willemb@google.com>
 */

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#include <linux/module.h>
#include <linux/syscalls.h>
#include <linux/skbuff.h>
#include <linux/filter.h>
#include <linux/bpf.h>

#include <linux/netfilter/xt_bpf.h>
#include <linux/netfilter/x_tables.h>

MODULE_AUTHOR("Willem de Bruijn <willemb@google.com>");
MODULE_DESCRIPTION("Xtables: BPF filter match");
MODULE_LICENSE("GPL");
MODULE_ALIAS("ipt_bpf");
MODULE_ALIAS("ip6t_bpf");

static int __bpf_mt_check_bytecode(struct sock_filter *insns, __u16 len,
                                   struct bpf_prog **ret)
{
        struct sock_fprog_kern program;

        if (len > XT_BPF_MAX_NUM_INSTR)
                return -EINVAL;

        program.len = len;
        program.filter = insns;

        if (bpf_prog_create(ret, &program)) {
                pr_info_ratelimited("check failed: parse error\n");
                return -EINVAL;
        }

        return 0;
}

static int __bpf_mt_check_fd(int fd, struct bpf_prog **ret)
{
        struct bpf_prog *prog;

        prog = bpf_prog_get_type(fd, BPF_PROG_TYPE_SOCKET_FILTER);
        if (IS_ERR(prog))
                return PTR_ERR(prog);

        *ret = prog;
        return 0;
}

static int __bpf_mt_check_path(const char *path, struct bpf_prog **ret)
{
        if (strnlen(path, XT_BPF_PATH_MAX) == XT_BPF_PATH_MAX)
                return -EINVAL;

        *ret = bpf_prog_get_type_path(path, BPF_PROG_TYPE_SOCKET_FILTER);
        return PTR_ERR_OR_ZERO(*ret);
}

static int bpf_mt_check(const struct xt_mtchk_param *par)
{
        struct xt_bpf_info *info = par->matchinfo;

        return __bpf_mt_check_bytecode(info->bpf_program,
                                       info->bpf_program_num_elem,
                                       &info->filter);
}

static int bpf_mt_check_v1(const struct xt_mtchk_param *par)
{
        struct xt_bpf_info_v1 *info = par->matchinfo;

        if (info->mode == XT_BPF_MODE_BYTECODE)
                return __bpf_mt_check_bytecode(info->bpf_program,
                                               info->bpf_program_num_elem,
                                               &info->filter);
        else if (info->mode == XT_BPF_MODE_FD_ELF)
                return __bpf_mt_check_fd(info->fd, &info->filter);
        else if (info->mode == XT_BPF_MODE_PATH_PINNED)
                return __bpf_mt_check_path(info->path, &info->filter);
        else
                return -EINVAL;
}

static bool bpf_mt(const struct sk_buff *skb, struct xt_action_param *par)
{
        const struct xt_bpf_info *info = par->matchinfo;

        return bpf_prog_run(info->filter, skb);
}

static bool bpf_mt_v1(const struct sk_buff *skb, struct xt_action_param *par)
{
        const struct xt_bpf_info_v1 *info = par->matchinfo;

        return !!bpf_prog_run_save_cb(info->filter, (struct sk_buff *) skb);
}

static void bpf_mt_destroy(const struct xt_mtdtor_param *par)
{
        const struct xt_bpf_info *info = par->matchinfo;

        bpf_prog_destroy(info->filter);
}

static void bpf_mt_destroy_v1(const struct xt_mtdtor_param *par)
{
        const struct xt_bpf_info_v1 *info = par->matchinfo;

        bpf_prog_destroy(info->filter);
}

static struct xt_match bpf_mt_reg[] __read_mostly = {
        {
                .name                = "bpf",
                .revision        = 0,
                .family                = NFPROTO_UNSPEC,
                .checkentry        = bpf_mt_check,
                .match                = bpf_mt,
                .destroy        = bpf_mt_destroy,
                .matchsize        = sizeof(struct xt_bpf_info),
                .usersize        = offsetof(struct xt_bpf_info, filter),
                .me                = THIS_MODULE,
        },
        {
                .name                = "bpf",
                .revision        = 1,
                .family                = NFPROTO_UNSPEC,
                .checkentry        = bpf_mt_check_v1,
                .match                = bpf_mt_v1,
                .destroy        = bpf_mt_destroy_v1,
                .matchsize        = sizeof(struct xt_bpf_info_v1),
                .usersize        = offsetof(struct xt_bpf_info_v1, filter),
                .me                = THIS_MODULE,
        },
};

static int __init bpf_mt_init(void)
{
        return xt_register_matches(bpf_mt_reg, ARRAY_SIZE(bpf_mt_reg));
}

static void __exit bpf_mt_exit(void)
{
        xt_unregister_matches(bpf_mt_reg, ARRAY_SIZE(bpf_mt_reg));
}

module_init(bpf_mt_init);
module_exit(bpf_mt_exit);


































































































































































































































    1 












































































































































































































































































































































































































































































































































































































































































































































































    1 








    1 
































































    1 















    1 





























































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
// SPDX-License-Identifier: GPL-2.0+
/*
 *        mxuport.c - MOXA UPort series driver
 *
 *        Copyright (c) 2006 Moxa Technologies Co., Ltd.
 *        Copyright (c) 2013 Andrew Lunn <andrew@lunn.ch>
 *
 *        Supports the following Moxa USB to serial converters:
 *         2 ports : UPort 1250, UPort 1250I
 *         4 ports : UPort 1410, UPort 1450, UPort 1450I
 *         8 ports : UPort 1610-8, UPort 1650-8
 *        16 ports : UPort 1610-16, UPort 1650-16
 */

#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/firmware.h>
#include <linux/jiffies.h>
#include <linux/serial.h>
#include <linux/serial_reg.h>
#include <linux/slab.h>
#include <linux/tty.h>
#include <linux/tty_driver.h>
#include <linux/tty_flip.h>
#include <linux/uaccess.h>
#include <linux/usb.h>
#include <linux/usb/serial.h>
#include <linux/unaligned.h>

/* Definitions for the vendor ID and device ID */
#define MX_USBSERIAL_VID        0x110A
#define MX_UPORT1250_PID        0x1250
#define MX_UPORT1251_PID        0x1251
#define MX_UPORT1410_PID        0x1410
#define MX_UPORT1450_PID        0x1450
#define MX_UPORT1451_PID        0x1451
#define MX_UPORT1618_PID        0x1618
#define MX_UPORT1658_PID        0x1658
#define MX_UPORT1613_PID        0x1613
#define MX_UPORT1653_PID        0x1653

/* Definitions for USB info */
#define HEADER_SIZE                4
#define EVENT_LENGTH                8
#define DOWN_BLOCK_SIZE                64

/* Definitions for firmware info */
#define VER_ADDR_1                0x20
#define VER_ADDR_2                0x24
#define VER_ADDR_3                0x28

/* Definitions for USB vendor request */
#define RQ_VENDOR_NONE                        0x00
#define RQ_VENDOR_SET_BAUD                0x01 /* Set baud rate */
#define RQ_VENDOR_SET_LINE                0x02 /* Set line status */
#define RQ_VENDOR_SET_CHARS                0x03 /* Set Xon/Xoff chars */
#define RQ_VENDOR_SET_RTS                0x04 /* Set RTS */
#define RQ_VENDOR_SET_DTR                0x05 /* Set DTR */
#define RQ_VENDOR_SET_XONXOFF                0x06 /* Set auto Xon/Xoff */
#define RQ_VENDOR_SET_RX_HOST_EN        0x07 /* Set RX host enable */
#define RQ_VENDOR_SET_OPEN                0x08 /* Set open/close port */
#define RQ_VENDOR_PURGE                        0x09 /* Purge Rx/Tx buffer */
#define RQ_VENDOR_SET_MCR                0x0A /* Set MCR register */
#define RQ_VENDOR_SET_BREAK                0x0B /* Set Break signal */

#define RQ_VENDOR_START_FW_DOWN                0x0C /* Start firmware download */
#define RQ_VENDOR_STOP_FW_DOWN                0x0D /* Stop firmware download */
#define RQ_VENDOR_QUERY_FW_READY        0x0E /* Query if new firmware ready */

#define RQ_VENDOR_SET_FIFO_DISABLE        0x0F /* Set fifo disable */
#define RQ_VENDOR_SET_INTERFACE                0x10 /* Set interface */
#define RQ_VENDOR_SET_HIGH_PERFOR        0x11 /* Set hi-performance */

#define RQ_VENDOR_ERASE_BLOCK                0x12 /* Erase flash block */
#define RQ_VENDOR_WRITE_PAGE                0x13 /* Write flash page */
#define RQ_VENDOR_PREPARE_WRITE                0x14 /* Prepare write flash */
#define RQ_VENDOR_CONFIRM_WRITE                0x15 /* Confirm write flash */
#define RQ_VENDOR_LOCATE                0x16 /* Locate the device */

#define RQ_VENDOR_START_ROM_DOWN        0x17 /* Start firmware download */
#define RQ_VENDOR_ROM_DATA                0x18 /* Rom file data */
#define RQ_VENDOR_STOP_ROM_DOWN                0x19 /* Stop firmware download */
#define RQ_VENDOR_FW_DATA                0x20 /* Firmware data */

#define RQ_VENDOR_RESET_DEVICE                0x23 /* Try to reset the device */
#define RQ_VENDOR_QUERY_FW_CONFIG        0x24

#define RQ_VENDOR_GET_VERSION                0x81 /* Get firmware version */
#define RQ_VENDOR_GET_PAGE                0x82 /* Read flash page */
#define RQ_VENDOR_GET_ROM_PROC                0x83 /* Get ROM process state */

#define RQ_VENDOR_GET_INQUEUE                0x84 /* Data in input buffer */
#define RQ_VENDOR_GET_OUTQUEUE                0x85 /* Data in output buffer */

#define RQ_VENDOR_GET_MSR                0x86 /* Get modem status register */

/* Definitions for UPort event type */
#define UPORT_EVENT_NONE                0 /* None */
#define UPORT_EVENT_TXBUF_THRESHOLD        1 /* Tx buffer threshold */
#define UPORT_EVENT_SEND_NEXT                2 /* Send next */
#define UPORT_EVENT_MSR                        3 /* Modem status */
#define UPORT_EVENT_LSR                        4 /* Line status */
#define UPORT_EVENT_MCR                        5 /* Modem control */

/* Definitions for serial event type */
#define SERIAL_EV_CTS                        0x0008        /* CTS changed state */
#define SERIAL_EV_DSR                        0x0010        /* DSR changed state */
#define SERIAL_EV_RLSD                        0x0020        /* RLSD changed state */

/* Definitions for modem control event type */
#define SERIAL_EV_XOFF                        0x40        /* XOFF received */

/* Definitions for line control of communication */
#define MX_WORDLENGTH_5                        5
#define MX_WORDLENGTH_6                        6
#define MX_WORDLENGTH_7                        7
#define MX_WORDLENGTH_8                        8

#define MX_PARITY_NONE                        0
#define MX_PARITY_ODD                        1
#define MX_PARITY_EVEN                        2
#define MX_PARITY_MARK                        3
#define MX_PARITY_SPACE                        4

#define MX_STOP_BITS_1                        0
#define MX_STOP_BITS_1_5                1
#define MX_STOP_BITS_2                        2

#define MX_RTS_DISABLE                        0x0
#define MX_RTS_ENABLE                        0x1
#define MX_RTS_HW                        0x2
#define MX_RTS_NO_CHANGE                0x3 /* Flag, not valid register value*/

#define MX_INT_RS232                        0
#define MX_INT_2W_RS485                        1
#define MX_INT_RS422                        2
#define MX_INT_4W_RS485                        3

/* Definitions for holding reason */
#define MX_WAIT_FOR_CTS                        0x0001
#define MX_WAIT_FOR_DSR                        0x0002
#define MX_WAIT_FOR_DCD                        0x0004
#define MX_WAIT_FOR_XON                        0x0008
#define MX_WAIT_FOR_START_TX                0x0010
#define MX_WAIT_FOR_UNTHROTTLE                0x0020
#define MX_WAIT_FOR_LOW_WATER                0x0040
#define MX_WAIT_FOR_SEND_NEXT                0x0080

#define MX_UPORT_2_PORT                        BIT(0)
#define MX_UPORT_4_PORT                        BIT(1)
#define MX_UPORT_8_PORT                        BIT(2)
#define MX_UPORT_16_PORT                BIT(3)

/* This structure holds all of the local port information */
struct mxuport_port {
        u8 mcr_state;                /* Last MCR state */
        u8 msr_state;                /* Last MSR state */
        struct mutex mutex;        /* Protects mcr_state */
        spinlock_t spinlock;        /* Protects msr_state */
};

/* Table of devices that work with this driver */
static const struct usb_device_id mxuport_idtable[] = {
        { USB_DEVICE(MX_USBSERIAL_VID, MX_UPORT1250_PID),
          .driver_info = MX_UPORT_2_PORT },
        { USB_DEVICE(MX_USBSERIAL_VID, MX_UPORT1251_PID),
          .driver_info = MX_UPORT_2_PORT },
        { USB_DEVICE(MX_USBSERIAL_VID, MX_UPORT1410_PID),
          .driver_info = MX_UPORT_4_PORT },
        { USB_DEVICE(MX_USBSERIAL_VID, MX_UPORT1450_PID),
          .driver_info = MX_UPORT_4_PORT },
        { USB_DEVICE(MX_USBSERIAL_VID, MX_UPORT1451_PID),
          .driver_info = MX_UPORT_4_PORT },
        { USB_DEVICE(MX_USBSERIAL_VID, MX_UPORT1618_PID),
          .driver_info = MX_UPORT_8_PORT },
        { USB_DEVICE(MX_USBSERIAL_VID, MX_UPORT1658_PID),
          .driver_info = MX_UPORT_8_PORT },
        { USB_DEVICE(MX_USBSERIAL_VID, MX_UPORT1613_PID),
          .driver_info = MX_UPORT_16_PORT },
        { USB_DEVICE(MX_USBSERIAL_VID, MX_UPORT1653_PID),
          .driver_info = MX_UPORT_16_PORT },
        {}                        /* Terminating entry */
};

MODULE_DEVICE_TABLE(usb, mxuport_idtable);

/*
 * Add a four byte header containing the port number and the number of
 * bytes of data in the message. Return the number of bytes in the
 * buffer.
 */
static int mxuport_prepare_write_buffer(struct usb_serial_port *port,
                                        void *dest, size_t size)
{
        u8 *buf = dest;
        int count;

        count = kfifo_out_locked(&port->write_fifo, buf + HEADER_SIZE,
                                 size - HEADER_SIZE,
                                 &port->lock);

        put_unaligned_be16(port->port_number, buf);
        put_unaligned_be16(count, buf + 2);

        dev_dbg(&port->dev, "%s - size %zd count %d\n", __func__,
                size, count);

        return count + HEADER_SIZE;
}

/* Read the given buffer in from the control pipe. */
static int mxuport_recv_ctrl_urb(struct usb_serial *serial,
                                 u8 request, u16 value, u16 index,
                                 u8 *data, size_t size)
{
        int status;

        status = usb_control_msg(serial->dev,
                                 usb_rcvctrlpipe(serial->dev, 0),
                                 request,
                                 (USB_DIR_IN | USB_TYPE_VENDOR |
                                  USB_RECIP_DEVICE), value, index,
                                 data, size,
                                 USB_CTRL_GET_TIMEOUT);
        if (status < 0) {
                dev_err(&serial->interface->dev,
                        "%s - usb_control_msg failed (%d)\n",
                        __func__, status);
                return status;
        }

        if (status != size) {
                dev_err(&serial->interface->dev,
                        "%s - short read (%d / %zd)\n",
                        __func__, status, size);
                return -EIO;
        }

        return status;
}

/* Write the given buffer out to the control pipe.  */
static int mxuport_send_ctrl_data_urb(struct usb_serial *serial,
                                      u8 request,
                                      u16 value, u16 index,
                                      u8 *data, size_t size)
{
        int status;

        status = usb_control_msg(serial->dev,
                                 usb_sndctrlpipe(serial->dev, 0),
                                 request,
                                 (USB_DIR_OUT | USB_TYPE_VENDOR |
                                  USB_RECIP_DEVICE), value, index,
                                 data, size,
                                 USB_CTRL_SET_TIMEOUT);
        if (status < 0) {
                dev_err(&serial->interface->dev,
                        "%s - usb_control_msg failed (%d)\n",
                        __func__, status);
                return status;
        }

        return 0;
}

/* Send a vendor request without any data */
static int mxuport_send_ctrl_urb(struct usb_serial *serial,
                                 u8 request, u16 value, u16 index)
{
        return mxuport_send_ctrl_data_urb(serial, request, value, index,
                                          NULL, 0);
}

/*
 * mxuport_throttle - throttle function of driver
 *
 * This function is called by the tty driver when it wants to stop the
 * data being read from the port. Since all the data comes over one
 * bulk in endpoint, we cannot stop submitting urbs by setting
 * port->throttle. Instead tell the device to stop sending us data for
 * the port.
 */
static void mxuport_throttle(struct tty_struct *tty)
{
        struct usb_serial_port *port = tty->driver_data;
        struct usb_serial *serial = port->serial;

        dev_dbg(&port->dev, "%s\n", __func__);

        mxuport_send_ctrl_urb(serial, RQ_VENDOR_SET_RX_HOST_EN,
                              0, port->port_number);
}

/*
 * mxuport_unthrottle - unthrottle function of driver
 *
 * This function is called by the tty driver when it wants to resume
 * the data being read from the port. Tell the device it can resume
 * sending us received data from the port.
 */
static void mxuport_unthrottle(struct tty_struct *tty)
{

        struct usb_serial_port *port = tty->driver_data;
        struct usb_serial *serial = port->serial;

        dev_dbg(&port->dev, "%s\n", __func__);

        mxuport_send_ctrl_urb(serial, RQ_VENDOR_SET_RX_HOST_EN,
                              1, port->port_number);
}

/*
 * Processes one chunk of data received for a port.  Mostly a copy of
 * usb_serial_generic_process_read_urb().
 */
static void mxuport_process_read_urb_data(struct usb_serial_port *port,
                                          char *data, int size)
{
        int i;

        if (port->sysrq) {
                for (i = 0; i < size; i++, data++) {
                        if (!usb_serial_handle_sysrq_char(port, *data))
                                tty_insert_flip_char(&port->port, *data,
                                                     TTY_NORMAL);
                }
        } else {
                tty_insert_flip_string(&port->port, data, size);
        }
        tty_flip_buffer_push(&port->port);
}

static void mxuport_msr_event(struct usb_serial_port *port, u8 buf[4])
{
        struct mxuport_port *mxport = usb_get_serial_port_data(port);
        u8 rcv_msr_hold = buf[2] & 0xF0;
        u16 rcv_msr_event = get_unaligned_be16(buf);
        unsigned long flags;

        if (rcv_msr_event == 0)
                return;

        /* Update MSR status */
        spin_lock_irqsave(&mxport->spinlock, flags);

        dev_dbg(&port->dev, "%s - current MSR status = 0x%x\n",
                __func__, mxport->msr_state);

        if (rcv_msr_hold & UART_MSR_CTS) {
                mxport->msr_state |= UART_MSR_CTS;
                dev_dbg(&port->dev, "%s - CTS high\n", __func__);
        } else {
                mxport->msr_state &= ~UART_MSR_CTS;
                dev_dbg(&port->dev, "%s - CTS low\n", __func__);
        }

        if (rcv_msr_hold & UART_MSR_DSR) {
                mxport->msr_state |= UART_MSR_DSR;
                dev_dbg(&port->dev, "%s - DSR high\n", __func__);
        } else {
                mxport->msr_state &= ~UART_MSR_DSR;
                dev_dbg(&port->dev, "%s - DSR low\n", __func__);
        }

        if (rcv_msr_hold & UART_MSR_DCD) {
                mxport->msr_state |= UART_MSR_DCD;
                dev_dbg(&port->dev, "%s - DCD high\n", __func__);
        } else {
                mxport->msr_state &= ~UART_MSR_DCD;
                dev_dbg(&port->dev, "%s - DCD low\n", __func__);
        }
        spin_unlock_irqrestore(&mxport->spinlock, flags);

        if (rcv_msr_event &
            (SERIAL_EV_CTS | SERIAL_EV_DSR | SERIAL_EV_RLSD)) {

                if (rcv_msr_event & SERIAL_EV_CTS) {
                        port->icount.cts++;
                        dev_dbg(&port->dev, "%s - CTS change\n", __func__);
                }

                if (rcv_msr_event & SERIAL_EV_DSR) {
                        port->icount.dsr++;
                        dev_dbg(&port->dev, "%s - DSR change\n", __func__);
                }

                if (rcv_msr_event & SERIAL_EV_RLSD) {
                        port->icount.dcd++;
                        dev_dbg(&port->dev, "%s - DCD change\n", __func__);
                }
                wake_up_interruptible(&port->port.delta_msr_wait);
        }
}

static void mxuport_lsr_event(struct usb_serial_port *port, u8 buf[4])
{
        u8 lsr_event = buf[2];

        if (lsr_event & UART_LSR_BI) {
                port->icount.brk++;
                dev_dbg(&port->dev, "%s - break error\n", __func__);
        }

        if (lsr_event & UART_LSR_FE) {
                port->icount.frame++;
                dev_dbg(&port->dev, "%s - frame error\n", __func__);
        }

        if (lsr_event & UART_LSR_PE) {
                port->icount.parity++;
                dev_dbg(&port->dev, "%s - parity error\n", __func__);
        }

        if (lsr_event & UART_LSR_OE) {
                port->icount.overrun++;
                dev_dbg(&port->dev, "%s - overrun error\n", __func__);
        }
}

/*
 * When something interesting happens, modem control lines XON/XOFF
 * etc, the device sends an event. Process these events.
 */
static void mxuport_process_read_urb_event(struct usb_serial_port *port,
                                           u8 buf[4], u32 event)
{
        dev_dbg(&port->dev, "%s - receive event : %04x\n", __func__, event);

        switch (event) {
        case UPORT_EVENT_SEND_NEXT:
                /*
                 * Sent as part of the flow control on device buffers.
                 * Not currently used.
                 */
                break;
        case UPORT_EVENT_MSR:
                mxuport_msr_event(port, buf);
                break;
        case UPORT_EVENT_LSR:
                mxuport_lsr_event(port, buf);
                break;
        case UPORT_EVENT_MCR:
                /*
                 * Event to indicate a change in XON/XOFF from the
                 * peer.  Currently not used. We just continue
                 * sending the device data and it will buffer it if
                 * needed. This event could be used for flow control
                 * between the host and the device.
                 */
                break;
        default:
                dev_dbg(&port->dev, "Unexpected event\n");
                break;
        }
}

/*
 * One URB can contain data for multiple ports. Demultiplex the data,
 * checking the port exists, is opened and the message is valid.
 */
static void mxuport_process_read_urb_demux_data(struct urb *urb)
{
        struct usb_serial_port *port = urb->context;
        struct usb_serial *serial = port->serial;
        u8 *data = urb->transfer_buffer;
        u8 *end = data + urb->actual_length;
        struct usb_serial_port *demux_port;
        u8 *ch;
        u16 rcv_port;
        u16 rcv_len;

        while (data < end) {
                if (data + HEADER_SIZE > end) {
                        dev_warn(&port->dev, "%s - message with short header\n",
                                 __func__);
                        return;
                }

                rcv_port = get_unaligned_be16(data);
                if (rcv_port >= serial->num_ports) {
                        dev_warn(&port->dev, "%s - message for invalid port\n",
                                 __func__);
                        return;
                }

                demux_port = serial->port[rcv_port];
                rcv_len = get_unaligned_be16(data + 2);
                if (!rcv_len || data + HEADER_SIZE + rcv_len > end) {
                        dev_warn(&port->dev, "%s - short data\n", __func__);
                        return;
                }

                if (tty_port_initialized(&demux_port->port)) {
                        ch = data + HEADER_SIZE;
                        mxuport_process_read_urb_data(demux_port, ch, rcv_len);
                } else {
                        dev_dbg(&demux_port->dev, "%s - data for closed port\n",
                                __func__);
                }
                data += HEADER_SIZE + rcv_len;
        }
}

/*
 * One URB can contain events for multiple ports. Demultiplex the event,
 * checking the port exists, and is opened.
 */
static void mxuport_process_read_urb_demux_event(struct urb *urb)
{
        struct usb_serial_port *port = urb->context;
        struct usb_serial *serial = port->serial;
        u8 *data = urb->transfer_buffer;
        u8 *end = data + urb->actual_length;
        struct usb_serial_port *demux_port;
        u8 *ch;
        u16 rcv_port;
        u16 rcv_event;

        while (data < end) {
                if (data + EVENT_LENGTH > end) {
                        dev_warn(&port->dev, "%s - message with short event\n",
                                 __func__);
                        return;
                }

                rcv_port = get_unaligned_be16(data);
                if (rcv_port >= serial->num_ports) {
                        dev_warn(&port->dev, "%s - message for invalid port\n",
                                 __func__);
                        return;
                }

                demux_port = serial->port[rcv_port];
                if (tty_port_initialized(&demux_port->port)) {
                        ch = data + HEADER_SIZE;
                        rcv_event = get_unaligned_be16(data + 2);
                        mxuport_process_read_urb_event(demux_port, ch,
                                                       rcv_event);
                } else {
                        dev_dbg(&demux_port->dev,
                                "%s - event for closed port\n", __func__);
                }
                data += EVENT_LENGTH;
        }
}

/*
 * This is called when we have received data on the bulk in
 * endpoint. Depending on which port it was received on, it can
 * contain serial data or events.
 */
static void mxuport_process_read_urb(struct urb *urb)
{
        struct usb_serial_port *port = urb->context;
        struct usb_serial *serial = port->serial;

        if (port == serial->port[0])
                mxuport_process_read_urb_demux_data(urb);

        if (port == serial->port[1])
                mxuport_process_read_urb_demux_event(urb);
}

/*
 * Ask the device how many bytes it has queued to be sent out. If
 * there are none, return true.
 */
static bool mxuport_tx_empty(struct usb_serial_port *port)
{
        struct usb_serial *serial = port->serial;
        bool is_empty = true;
        u32 txlen;
        u8 *len_buf;
        int err;

        len_buf = kzalloc(4, GFP_KERNEL);
        if (!len_buf)
                goto out;

        err = mxuport_recv_ctrl_urb(serial, RQ_VENDOR_GET_OUTQUEUE, 0,
                                    port->port_number, len_buf, 4);
        if (err < 0)
                goto out;

        txlen = get_unaligned_be32(len_buf);
        dev_dbg(&port->dev, "%s - tx len = %u\n", __func__, txlen);

        if (txlen != 0)
                is_empty = false;

out:
        kfree(len_buf);
        return is_empty;
}

static int mxuport_set_mcr(struct usb_serial_port *port, u8 mcr_state)
{
        struct usb_serial *serial = port->serial;
        int err;

        dev_dbg(&port->dev, "%s - %02x\n", __func__, mcr_state);

        err = mxuport_send_ctrl_urb(serial, RQ_VENDOR_SET_MCR,
                                    mcr_state, port->port_number);
        if (err)
                dev_err(&port->dev, "%s - failed to change MCR\n", __func__);

        return err;
}

static int mxuport_set_dtr(struct usb_serial_port *port, int on)
{
        struct mxuport_port *mxport = usb_get_serial_port_data(port);
        struct usb_serial *serial = port->serial;
        int err;

        mutex_lock(&mxport->mutex);

        err = mxuport_send_ctrl_urb(serial, RQ_VENDOR_SET_DTR,
                                    !!on, port->port_number);
        if (!err) {
                if (on)
                        mxport->mcr_state |= UART_MCR_DTR;
                else
                        mxport->mcr_state &= ~UART_MCR_DTR;
        }

        mutex_unlock(&mxport->mutex);

        return err;
}

static int mxuport_set_rts(struct usb_serial_port *port, u8 state)
{
        struct mxuport_port *mxport = usb_get_serial_port_data(port);
        struct usb_serial *serial = port->serial;
        int err;
        u8 mcr_state;

        mutex_lock(&mxport->mutex);
        mcr_state = mxport->mcr_state;

        switch (state) {
        case MX_RTS_DISABLE:
                mcr_state &= ~UART_MCR_RTS;
                break;
        case MX_RTS_ENABLE:
                mcr_state |= UART_MCR_RTS;
                break;
        case MX_RTS_HW:
                /*
                 * Do not update mxport->mcr_state when doing hardware
                 * flow control.
                 */
                break;
        default:
                /*
                 * Should not happen, but somebody might try passing
                 * MX_RTS_NO_CHANGE, which is not valid.
                 */
                err = -EINVAL;
                goto out;
        }
        err = mxuport_send_ctrl_urb(serial, RQ_VENDOR_SET_RTS,
                                    state, port->port_number);
        if (!err)
                mxport->mcr_state = mcr_state;

out:
        mutex_unlock(&mxport->mutex);

        return err;
}

static void mxuport_dtr_rts(struct usb_serial_port *port, int on)
{
        struct mxuport_port *mxport = usb_get_serial_port_data(port);
        u8 mcr_state;
        int err;

        mutex_lock(&mxport->mutex);
        mcr_state = mxport->mcr_state;

        if (on)
                mcr_state |= (UART_MCR_RTS | UART_MCR_DTR);
        else
                mcr_state &= ~(UART_MCR_RTS | UART_MCR_DTR);

        err = mxuport_set_mcr(port, mcr_state);
        if (!err)
                mxport->mcr_state = mcr_state;

        mutex_unlock(&mxport->mutex);
}

static int mxuport_tiocmset(struct tty_struct *tty, unsigned int set,
                            unsigned int clear)
{
        struct usb_serial_port *port = tty->driver_data;
        struct mxuport_port *mxport = usb_get_serial_port_data(port);
        int err;
        u8 mcr_state;

        mutex_lock(&mxport->mutex);
        mcr_state = mxport->mcr_state;

        if (set & TIOCM_RTS)
                mcr_state |= UART_MCR_RTS;

        if (set & TIOCM_DTR)
                mcr_state |= UART_MCR_DTR;

        if (clear & TIOCM_RTS)
                mcr_state &= ~UART_MCR_RTS;

        if (clear & TIOCM_DTR)
                mcr_state &= ~UART_MCR_DTR;

        err = mxuport_set_mcr(port, mcr_state);
        if (!err)
                mxport->mcr_state = mcr_state;

        mutex_unlock(&mxport->mutex);

        return err;
}

static int mxuport_tiocmget(struct tty_struct *tty)
{
        struct mxuport_port *mxport;
        struct usb_serial_port *port = tty->driver_data;
        unsigned int result;
        unsigned long flags;
        unsigned int msr;
        unsigned int mcr;

        mxport = usb_get_serial_port_data(port);

        mutex_lock(&mxport->mutex);
        spin_lock_irqsave(&mxport->spinlock, flags);

        msr = mxport->msr_state;
        mcr = mxport->mcr_state;

        spin_unlock_irqrestore(&mxport->spinlock, flags);
        mutex_unlock(&mxport->mutex);

        result = (((mcr & UART_MCR_DTR) ? TIOCM_DTR : 0) |        /* 0x002 */
                  ((mcr & UART_MCR_RTS) ? TIOCM_RTS : 0) |        /* 0x004 */
                  ((msr & UART_MSR_CTS) ? TIOCM_CTS : 0) |        /* 0x020 */
                  ((msr & UART_MSR_DCD) ? TIOCM_CAR : 0) |        /* 0x040 */
                  ((msr & UART_MSR_RI) ? TIOCM_RI : 0) |        /* 0x080 */
                  ((msr & UART_MSR_DSR) ? TIOCM_DSR : 0));        /* 0x100 */

        dev_dbg(&port->dev, "%s - 0x%04x\n", __func__, result);

        return result;
}

static int mxuport_set_termios_flow(struct tty_struct *tty,
                                    const struct ktermios *old_termios,
                                    struct usb_serial_port *port,
                                    struct usb_serial *serial)
{
        u8 xon = START_CHAR(tty);
        u8 xoff = STOP_CHAR(tty);
        int enable;
        int err;
        u8 *buf;
        u8 rts;

        buf = kmalloc(2, GFP_KERNEL);
        if (!buf)
                return -ENOMEM;

        /* S/W flow control settings */
        if (I_IXOFF(tty) || I_IXON(tty)) {
                enable = 1;
                buf[0] = xon;
                buf[1] = xoff;

                err = mxuport_send_ctrl_data_urb(serial, RQ_VENDOR_SET_CHARS,
                                                 0, port->port_number,
                                                 buf, 2);
                if (err)
                        goto out;

                dev_dbg(&port->dev, "%s - XON = 0x%02x, XOFF = 0x%02x\n",
                        __func__, xon, xoff);
        } else {
                enable = 0;
        }

        err = mxuport_send_ctrl_urb(serial, RQ_VENDOR_SET_XONXOFF,
                                    enable, port->port_number);
        if (err)
                goto out;

        rts = MX_RTS_NO_CHANGE;

        /* H/W flow control settings */
        if (!old_termios ||
            C_CRTSCTS(tty) != (old_termios->c_cflag & CRTSCTS)) {
                if (C_CRTSCTS(tty))
                        rts = MX_RTS_HW;
                else
                        rts = MX_RTS_ENABLE;
        }

        if (C_BAUD(tty)) {
                if (old_termios && (old_termios->c_cflag & CBAUD) == B0) {
                        /* Raise DTR and RTS */
                        if (C_CRTSCTS(tty))
                                rts = MX_RTS_HW;
                        else
                                rts = MX_RTS_ENABLE;
                        mxuport_set_dtr(port, 1);
                }
        } else {
                /* Drop DTR and RTS */
                rts = MX_RTS_DISABLE;
                mxuport_set_dtr(port, 0);
        }

        if (rts != MX_RTS_NO_CHANGE)
                err = mxuport_set_rts(port, rts);

out:
        kfree(buf);
        return err;
}

static void mxuport_set_termios(struct tty_struct *tty,
                                struct usb_serial_port *port,
                                const struct ktermios *old_termios)
{
        struct usb_serial *serial = port->serial;
        u8 *buf;
        u8 data_bits;
        u8 stop_bits;
        u8 parity;
        int baud;
        int err;

        if (old_termios &&
            !tty_termios_hw_change(&tty->termios, old_termios) &&
            tty->termios.c_iflag == old_termios->c_iflag) {
                dev_dbg(&port->dev, "%s - nothing to change\n", __func__);
                return;
        }

        buf = kmalloc(4, GFP_KERNEL);
        if (!buf)
                return;

        /* Set data bit of termios */
        switch (C_CSIZE(tty)) {
        case CS5:
                data_bits = MX_WORDLENGTH_5;
                break;
        case CS6:
                data_bits = MX_WORDLENGTH_6;
                break;
        case CS7:
                data_bits = MX_WORDLENGTH_7;
                break;
        case CS8:
        default:
                data_bits = MX_WORDLENGTH_8;
                break;
        }

        /* Set parity of termios */
        if (C_PARENB(tty)) {
                if (C_CMSPAR(tty)) {
                        if (C_PARODD(tty))
                                parity = MX_PARITY_MARK;
                        else
                                parity = MX_PARITY_SPACE;
                } else {
                        if (C_PARODD(tty))
                                parity = MX_PARITY_ODD;
                        else
                                parity = MX_PARITY_EVEN;
                }
        } else {
                parity = MX_PARITY_NONE;
        }

        /* Set stop bit of termios */
        if (C_CSTOPB(tty))
                stop_bits = MX_STOP_BITS_2;
        else
                stop_bits = MX_STOP_BITS_1;

        buf[0] = data_bits;
        buf[1] = parity;
        buf[2] = stop_bits;
        buf[3] = 0;

        err = mxuport_send_ctrl_data_urb(serial, RQ_VENDOR_SET_LINE,
                                         0, port->port_number, buf, 4);
        if (err)
                goto out;

        err = mxuport_set_termios_flow(tty, old_termios, port, serial);
        if (err)
                goto out;

        baud = tty_get_baud_rate(tty);
        if (!baud)
                baud = 9600;

        /* Note: Little Endian */
        put_unaligned_le32(baud, buf);

        err = mxuport_send_ctrl_data_urb(serial, RQ_VENDOR_SET_BAUD,
                                         0, port->port_number,
                                         buf, 4);
        if (err)
                goto out;

        dev_dbg(&port->dev, "baud_rate        : %d\n", baud);
        dev_dbg(&port->dev, "data_bits        : %d\n", data_bits);
        dev_dbg(&port->dev, "parity        : %d\n", parity);
        dev_dbg(&port->dev, "stop_bits        : %d\n", stop_bits);

out:
        kfree(buf);
}

/*
 * Determine how many ports this device has dynamically.  It will be
 * called after the probe() callback is called, but before attach().
 */
static int mxuport_calc_num_ports(struct usb_serial *serial,
                                        struct usb_serial_endpoints *epds)
{
        unsigned long features = (unsigned long)usb_get_serial_data(serial);
        int num_ports;
        int i;

        if (features & MX_UPORT_2_PORT) {
                num_ports = 2;
        } else if (features & MX_UPORT_4_PORT) {
                num_ports = 4;
        } else if (features & MX_UPORT_8_PORT) {
                num_ports = 8;
        } else if (features & MX_UPORT_16_PORT) {
                num_ports = 16;
        } else {
                dev_warn(&serial->interface->dev,
                                "unknown device, assuming two ports\n");
                num_ports = 2;
        }

        /*
         * Setup bulk-out endpoint multiplexing. All ports share the same
         * bulk-out endpoint.
         */
        BUILD_BUG_ON(ARRAY_SIZE(epds->bulk_out) < 16);

        for (i = 1; i < num_ports; ++i)
                epds->bulk_out[i] = epds->bulk_out[0];

        epds->num_bulk_out = num_ports;

        return num_ports;
}

/* Get the version of the firmware currently running. */
static int mxuport_get_fw_version(struct usb_serial *serial, u32 *version)
{
        u8 *ver_buf;
        int err;

        ver_buf = kzalloc(4, GFP_KERNEL);
        if (!ver_buf)
                return -ENOMEM;

        /* Get firmware version from SDRAM */
        err = mxuport_recv_ctrl_urb(serial, RQ_VENDOR_GET_VERSION, 0, 0,
                                    ver_buf, 4);
        if (err != 4) {
                err = -EIO;
                goto out;
        }

        *version = (ver_buf[0] << 16) | (ver_buf[1] << 8) | ver_buf[2];
        err = 0;
out:
        kfree(ver_buf);
        return err;
}

/* Given a firmware blob, download it to the device. */
static int mxuport_download_fw(struct usb_serial *serial,
                               const struct firmware *fw_p)
{
        u8 *fw_buf;
        size_t txlen;
        size_t fwidx;
        int err;

        fw_buf = kmalloc(DOWN_BLOCK_SIZE, GFP_KERNEL);
        if (!fw_buf)
                return -ENOMEM;

        dev_dbg(&serial->interface->dev, "Starting firmware download...\n");
        err = mxuport_send_ctrl_urb(serial, RQ_VENDOR_START_FW_DOWN, 0, 0);
        if (err)
                goto out;

        fwidx = 0;
        do {
                txlen = min_t(size_t, (fw_p->size - fwidx), DOWN_BLOCK_SIZE);

                memcpy(fw_buf, &fw_p->data[fwidx], txlen);
                err = mxuport_send_ctrl_data_urb(serial, RQ_VENDOR_FW_DATA,
                                                 0, 0, fw_buf, txlen);
                if (err) {
                        mxuport_send_ctrl_urb(serial, RQ_VENDOR_STOP_FW_DOWN,
                                              0, 0);
                        goto out;
                }

                fwidx += txlen;
                usleep_range(1000, 2000);

        } while (fwidx < fw_p->size);

        msleep(1000);
        err = mxuport_send_ctrl_urb(serial, RQ_VENDOR_STOP_FW_DOWN, 0, 0);
        if (err)
                goto out;

        msleep(1000);
        err = mxuport_send_ctrl_urb(serial, RQ_VENDOR_QUERY_FW_READY, 0, 0);

out:
        kfree(fw_buf);
        return err;
}

static int mxuport_probe(struct usb_serial *serial,
                         const struct usb_device_id *id)
{
        u16 productid = le16_to_cpu(serial->dev->descriptor.idProduct);
        const struct firmware *fw_p = NULL;
        u32 version;
        int local_ver;
        char buf[32];
        int err;

        /* Load our firmware */
        err = mxuport_send_ctrl_urb(serial, RQ_VENDOR_QUERY_FW_CONFIG, 0, 0);
        if (err) {
                mxuport_send_ctrl_urb(serial, RQ_VENDOR_RESET_DEVICE, 0, 0);
                return err;
        }

        err = mxuport_get_fw_version(serial, &version);
        if (err < 0)
                return err;

        dev_dbg(&serial->interface->dev, "Device firmware version v%x.%x.%x\n",
                (version & 0xff0000) >> 16,
                (version & 0xff00) >> 8,
                (version & 0xff));

        snprintf(buf, sizeof(buf) - 1, "moxa/moxa-%04x.fw", productid);

        err = request_firmware(&fw_p, buf, &serial->interface->dev);
        if (err) {
                dev_warn(&serial->interface->dev, "Firmware %s not found\n",
                         buf);

                /* Use the firmware already in the device */
                err = 0;
        } else {
                local_ver = ((fw_p->data[VER_ADDR_1] << 16) |
                             (fw_p->data[VER_ADDR_2] << 8) |
                             fw_p->data[VER_ADDR_3]);
                dev_dbg(&serial->interface->dev,
                        "Available firmware version v%x.%x.%x\n",
                        fw_p->data[VER_ADDR_1], fw_p->data[VER_ADDR_2],
                        fw_p->data[VER_ADDR_3]);
                if (local_ver > version) {
                        err = mxuport_download_fw(serial, fw_p);
                        if (err)
                                goto out;
                        err  = mxuport_get_fw_version(serial, &version);
                        if (err < 0)
                                goto out;
                }
        }

        dev_info(&serial->interface->dev,
                 "Using device firmware version v%x.%x.%x\n",
                 (version & 0xff0000) >> 16,
                 (version & 0xff00) >> 8,
                 (version & 0xff));

        /*
         * Contains the features of this hardware. Store away for
         * later use, eg, number of ports.
         */
        usb_set_serial_data(serial, (void *)id->driver_info);
out:
        if (fw_p)
                release_firmware(fw_p);
        return err;
}


static int mxuport_port_probe(struct usb_serial_port *port)
{
        struct usb_serial *serial = port->serial;
        struct mxuport_port *mxport;
        int err;

        mxport = devm_kzalloc(&port->dev, sizeof(struct mxuport_port),
                              GFP_KERNEL);
        if (!mxport)
                return -ENOMEM;

        mutex_init(&mxport->mutex);
        spin_lock_init(&mxport->spinlock);

        /* Set the port private data */
        usb_set_serial_port_data(port, mxport);

        /* Set FIFO (Enable) */
        err = mxuport_send_ctrl_urb(serial, RQ_VENDOR_SET_FIFO_DISABLE,
                                    0, port->port_number);
        if (err)
                return err;

        /* Set transmission mode (Hi-Performance) */
        err = mxuport_send_ctrl_urb(serial, RQ_VENDOR_SET_HIGH_PERFOR,
                                    0, port->port_number);
        if (err)
                return err;

        /* Set interface (RS-232) */
        return mxuport_send_ctrl_urb(serial, RQ_VENDOR_SET_INTERFACE,
                                     MX_INT_RS232,
                                     port->port_number);
}

static int mxuport_attach(struct usb_serial *serial)
{
        struct usb_serial_port *port0 = serial->port[0];
        struct usb_serial_port *port1 = serial->port[1];
        int err;

        /*
         * All data from the ports is received on the first bulk in
         * endpoint, with a multiplex header. The second bulk in is
         * used for events.
         *
         * Start to read from the device.
         */
        err = usb_serial_generic_submit_read_urbs(port0, GFP_KERNEL);
        if (err)
                return err;

        err = usb_serial_generic_submit_read_urbs(port1, GFP_KERNEL);
        if (err) {
                usb_serial_generic_close(port0);
                return err;
        }

        return 0;
}

static void mxuport_release(struct usb_serial *serial)
{
        struct usb_serial_port *port0 = serial->port[0];
        struct usb_serial_port *port1 = serial->port[1];

        usb_serial_generic_close(port1);
        usb_serial_generic_close(port0);
}

static int mxuport_open(struct tty_struct *tty, struct usb_serial_port *port)
{
        struct mxuport_port *mxport = usb_get_serial_port_data(port);
        struct usb_serial *serial = port->serial;
        int err;

        /* Set receive host (enable) */
        err = mxuport_send_ctrl_urb(serial, RQ_VENDOR_SET_RX_HOST_EN,
                                    1, port->port_number);
        if (err)
                return err;

        err = mxuport_send_ctrl_urb(serial, RQ_VENDOR_SET_OPEN,
                                    1, port->port_number);
        if (err) {
                mxuport_send_ctrl_urb(serial, RQ_VENDOR_SET_RX_HOST_EN,
                                      0, port->port_number);
                return err;
        }

        /* Initial port termios */
        if (tty)
                mxuport_set_termios(tty, port, NULL);

        /*
         * TODO: use RQ_VENDOR_GET_MSR, once we know what it
         * returns.
         */
        mxport->msr_state = 0;

        return err;
}

static void mxuport_close(struct usb_serial_port *port)
{
        struct usb_serial *serial = port->serial;

        mxuport_send_ctrl_urb(serial, RQ_VENDOR_SET_OPEN, 0,
                              port->port_number);

        mxuport_send_ctrl_urb(serial, RQ_VENDOR_SET_RX_HOST_EN, 0,
                              port->port_number);
}

/* Send a break to the port. */
static int mxuport_break_ctl(struct tty_struct *tty, int break_state)
{
        struct usb_serial_port *port = tty->driver_data;
        struct usb_serial *serial = port->serial;
        int enable;

        if (break_state == -1) {
                enable = 1;
                dev_dbg(&port->dev, "%s - sending break\n", __func__);
        } else {
                enable = 0;
                dev_dbg(&port->dev, "%s - clearing break\n", __func__);
        }

        return mxuport_send_ctrl_urb(serial, RQ_VENDOR_SET_BREAK,
                                     enable, port->port_number);
}

static int mxuport_resume(struct usb_serial *serial)
{
        struct usb_serial_port *port;
        int c = 0;
        int i;
        int r;

        for (i = 0; i < 2; i++) {
                port = serial->port[i];

                r = usb_serial_generic_submit_read_urbs(port, GFP_NOIO);
                if (r < 0)
                        c++;
        }

        for (i = 0; i < serial->num_ports; i++) {
                port = serial->port[i];
                if (!tty_port_initialized(&port->port))
                        continue;

                r = usb_serial_generic_write_start(port, GFP_NOIO);
                if (r < 0)
                        c++;
        }

        return c ? -EIO : 0;
}

static struct usb_serial_driver mxuport_device = {
        .driver = {
                .name =                "mxuport",
        },
        .description                = "MOXA UPort",
        .id_table                = mxuport_idtable,
        .num_bulk_in                = 2,
        .num_bulk_out                = 1,
        .probe                        = mxuport_probe,
        .port_probe                = mxuport_port_probe,
        .attach                        = mxuport_attach,
        .release                = mxuport_release,
        .calc_num_ports                = mxuport_calc_num_ports,
        .open                        = mxuport_open,
        .close                        = mxuport_close,
        .set_termios                = mxuport_set_termios,
        .break_ctl                = mxuport_break_ctl,
        .tx_empty                = mxuport_tx_empty,
        .tiocmiwait                = usb_serial_generic_tiocmiwait,
        .get_icount                = usb_serial_generic_get_icount,
        .throttle                = mxuport_throttle,
        .unthrottle                = mxuport_unthrottle,
        .tiocmget                = mxuport_tiocmget,
        .tiocmset                = mxuport_tiocmset,
        .dtr_rts                = mxuport_dtr_rts,
        .process_read_urb        = mxuport_process_read_urb,
        .prepare_write_buffer        = mxuport_prepare_write_buffer,
        .resume                        = mxuport_resume,
};

static struct usb_serial_driver *const serial_drivers[] = {
        &mxuport_device, NULL
};

module_usb_serial_driver(serial_drivers, mxuport_idtable);

MODULE_AUTHOR("Andrew Lunn <andrew@lunn.ch>");
MODULE_AUTHOR("<support@moxa.com>");
MODULE_DESCRIPTION("Moxa UPORT USB Serial driver");
MODULE_LICENSE("GPL");


















































































































































































































































































































































































































    2 













































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
// SPDX-License-Identifier: GPL-2.0
/* Copyright (C) 2017 Netronome Systems, Inc.
 * Copyright (C) 2019 Mellanox Technologies. All rights reserved
 */

#include <linux/completion.h>
#include <linux/device.h>
#include <linux/idr.h>
#include <linux/kernel.h>
#include <linux/list.h>
#include <linux/mutex.h>
#include <linux/refcount.h>
#include <linux/slab.h>
#include <linux/sysfs.h>

#include "netdevsim.h"

static DEFINE_IDA(nsim_bus_dev_ids);
static LIST_HEAD(nsim_bus_dev_list);
static DEFINE_MUTEX(nsim_bus_dev_list_lock);
static bool nsim_bus_enable;
static refcount_t nsim_bus_devs; /* Including the bus itself. */
static DECLARE_COMPLETION(nsim_bus_devs_released);

static struct nsim_bus_dev *to_nsim_bus_dev(struct device *dev)
{
        return container_of(dev, struct nsim_bus_dev, dev);
}

static ssize_t
nsim_bus_dev_numvfs_store(struct device *dev, struct device_attribute *attr,
                          const char *buf, size_t count)
{
        struct nsim_bus_dev *nsim_bus_dev = to_nsim_bus_dev(dev);
        unsigned int num_vfs;
        int ret;

        ret = kstrtouint(buf, 0, &num_vfs);
        if (ret)
                return ret;

        device_lock(dev);
        ret = -ENOENT;
        if (dev_get_drvdata(dev))
                ret = nsim_drv_configure_vfs(nsim_bus_dev, num_vfs);
        device_unlock(dev);

        return ret ? ret : count;
}

static ssize_t
nsim_bus_dev_numvfs_show(struct device *dev,
                         struct device_attribute *attr, char *buf)
{
        struct nsim_bus_dev *nsim_bus_dev = to_nsim_bus_dev(dev);

        return sprintf(buf, "%u\n", nsim_bus_dev->num_vfs);
}

static struct device_attribute nsim_bus_dev_numvfs_attr =
        __ATTR(sriov_numvfs, 0664, nsim_bus_dev_numvfs_show,
               nsim_bus_dev_numvfs_store);

static ssize_t
new_port_store(struct device *dev, struct device_attribute *attr,
               const char *buf, size_t count)
{
        struct nsim_bus_dev *nsim_bus_dev = to_nsim_bus_dev(dev);
        unsigned int port_index;
        int ret;

        /* Prevent to use nsim_bus_dev before initialization. */
        if (!smp_load_acquire(&nsim_bus_dev->init))
                return -EBUSY;
        ret = kstrtouint(buf, 0, &port_index);
        if (ret)
                return ret;

        ret = nsim_drv_port_add(nsim_bus_dev, NSIM_DEV_PORT_TYPE_PF, port_index);
        return ret ? ret : count;
}

static struct device_attribute nsim_bus_dev_new_port_attr = __ATTR_WO(new_port);

static ssize_t
del_port_store(struct device *dev, struct device_attribute *attr,
               const char *buf, size_t count)
{
        struct nsim_bus_dev *nsim_bus_dev = to_nsim_bus_dev(dev);
        unsigned int port_index;
        int ret;

        /* Prevent to use nsim_bus_dev before initialization. */
        if (!smp_load_acquire(&nsim_bus_dev->init))
                return -EBUSY;
        ret = kstrtouint(buf, 0, &port_index);
        if (ret)
                return ret;

        ret = nsim_drv_port_del(nsim_bus_dev, NSIM_DEV_PORT_TYPE_PF, port_index);
        return ret ? ret : count;
}

static struct device_attribute nsim_bus_dev_del_port_attr = __ATTR_WO(del_port);

static struct attribute *nsim_bus_dev_attrs[] = {
        &nsim_bus_dev_numvfs_attr.attr,
        &nsim_bus_dev_new_port_attr.attr,
        &nsim_bus_dev_del_port_attr.attr,
        NULL,
};

static const struct attribute_group nsim_bus_dev_attr_group = {
        .attrs = nsim_bus_dev_attrs,
};

static const struct attribute_group *nsim_bus_dev_attr_groups[] = {
        &nsim_bus_dev_attr_group,
        NULL,
};

static void nsim_bus_dev_release(struct device *dev)
{
        struct nsim_bus_dev *nsim_bus_dev;

        nsim_bus_dev = container_of(dev, struct nsim_bus_dev, dev);
        kfree(nsim_bus_dev);
        if (refcount_dec_and_test(&nsim_bus_devs))
                complete(&nsim_bus_devs_released);
}

static const struct device_type nsim_bus_dev_type = {
        .groups = nsim_bus_dev_attr_groups,
        .release = nsim_bus_dev_release,
};

static struct nsim_bus_dev *
nsim_bus_dev_new(unsigned int id, unsigned int port_count, unsigned int num_queues);

static ssize_t
new_device_store(const struct bus_type *bus, const char *buf, size_t count)
{
        unsigned int id, port_count, num_queues;
        struct nsim_bus_dev *nsim_bus_dev;
        int err;

        err = sscanf(buf, "%u %u %u", &id, &port_count, &num_queues);
        switch (err) {
        case 1:
                port_count = 1;
                fallthrough;
        case 2:
                num_queues = 1;
                fallthrough;
        case 3:
                if (id > INT_MAX) {
                        pr_err("Value of \"id\" is too big.\n");
                        return -EINVAL;
                }
                break;
        default:
                pr_err("Format for adding new device is \"id port_count num_queues\" (uint uint unit).\n");
                return -EINVAL;
        }

        mutex_lock(&nsim_bus_dev_list_lock);
        /* Prevent to use resource before initialization. */
        if (!smp_load_acquire(&nsim_bus_enable)) {
                err = -EBUSY;
                goto err;
        }

        nsim_bus_dev = nsim_bus_dev_new(id, port_count, num_queues);
        if (IS_ERR(nsim_bus_dev)) {
                err = PTR_ERR(nsim_bus_dev);
                goto err;
        }

        refcount_inc(&nsim_bus_devs);
        /* Allow using nsim_bus_dev */
        smp_store_release(&nsim_bus_dev->init, true);

        list_add_tail(&nsim_bus_dev->list, &nsim_bus_dev_list);
        mutex_unlock(&nsim_bus_dev_list_lock);

        return count;
err:
        mutex_unlock(&nsim_bus_dev_list_lock);
        return err;
}
static BUS_ATTR_WO(new_device);

static void nsim_bus_dev_del(struct nsim_bus_dev *nsim_bus_dev);

static ssize_t
del_device_store(const struct bus_type *bus, const char *buf, size_t count)
{
        struct nsim_bus_dev *nsim_bus_dev, *tmp;
        unsigned int id;
        int err;

        err = sscanf(buf, "%u", &id);
        switch (err) {
        case 1:
                if (id > INT_MAX) {
                        pr_err("Value of \"id\" is too big.\n");
                        return -EINVAL;
                }
                break;
        default:
                pr_err("Format for deleting device is \"id\" (uint).\n");
                return -EINVAL;
        }

        err = -ENOENT;
        mutex_lock(&nsim_bus_dev_list_lock);
        /* Prevent to use resource before initialization. */
        if (!smp_load_acquire(&nsim_bus_enable)) {
                mutex_unlock(&nsim_bus_dev_list_lock);
                return -EBUSY;
        }
        list_for_each_entry_safe(nsim_bus_dev, tmp, &nsim_bus_dev_list, list) {
                if (nsim_bus_dev->dev.id != id)
                        continue;
                list_del(&nsim_bus_dev->list);
                nsim_bus_dev_del(nsim_bus_dev);
                err = 0;
                break;
        }
        mutex_unlock(&nsim_bus_dev_list_lock);
        return !err ? count : err;
}
static BUS_ATTR_WO(del_device);

static ssize_t link_device_store(const struct bus_type *bus, const char *buf, size_t count)
{
        struct netdevsim *nsim_a, *nsim_b, *peer;
        struct net_device *dev_a, *dev_b;
        unsigned int ifidx_a, ifidx_b;
        int netnsfd_a, netnsfd_b, err;
        struct net *ns_a, *ns_b;

        err = sscanf(buf, "%d:%u %d:%u", &netnsfd_a, &ifidx_a, &netnsfd_b,
                     &ifidx_b);
        if (err != 4) {
                pr_err("Format for linking two devices is \"netnsfd_a:ifidx_a netnsfd_b:ifidx_b\" (int uint int uint).\n");
                return -EINVAL;
        }

        ns_a = get_net_ns_by_fd(netnsfd_a);
        if (IS_ERR(ns_a)) {
                pr_err("Could not find netns with fd: %d\n", netnsfd_a);
                return -EINVAL;
        }

        ns_b = get_net_ns_by_fd(netnsfd_b);
        if (IS_ERR(ns_b)) {
                pr_err("Could not find netns with fd: %d\n", netnsfd_b);
                put_net(ns_a);
                return -EINVAL;
        }

        err = -EINVAL;
        rtnl_lock();
        dev_a = __dev_get_by_index(ns_a, ifidx_a);
        if (!dev_a) {
                pr_err("Could not find device with ifindex %u in netnsfd %d\n",
                       ifidx_a, netnsfd_a);
                goto out_err;
        }

        if (!netdev_is_nsim(dev_a)) {
                pr_err("Device with ifindex %u in netnsfd %d is not a netdevsim\n",
                       ifidx_a, netnsfd_a);
                goto out_err;
        }

        dev_b = __dev_get_by_index(ns_b, ifidx_b);
        if (!dev_b) {
                pr_err("Could not find device with ifindex %u in netnsfd %d\n",
                       ifidx_b, netnsfd_b);
                goto out_err;
        }

        if (!netdev_is_nsim(dev_b)) {
                pr_err("Device with ifindex %u in netnsfd %d is not a netdevsim\n",
                       ifidx_b, netnsfd_b);
                goto out_err;
        }

        if (dev_a == dev_b) {
                pr_err("Cannot link a netdevsim to itself\n");
                goto out_err;
        }

        err = -EBUSY;
        nsim_a = netdev_priv(dev_a);
        peer = rtnl_dereference(nsim_a->peer);
        if (peer) {
                pr_err("Netdevsim %d:%u is already linked\n", netnsfd_a,
                       ifidx_a);
                goto out_err;
        }

        nsim_b = netdev_priv(dev_b);
        peer = rtnl_dereference(nsim_b->peer);
        if (peer) {
                pr_err("Netdevsim %d:%u is already linked\n", netnsfd_b,
                       ifidx_b);
                goto out_err;
        }

        err = 0;
        rcu_assign_pointer(nsim_a->peer, nsim_b);
        rcu_assign_pointer(nsim_b->peer, nsim_a);

out_err:
        put_net(ns_b);
        put_net(ns_a);
        rtnl_unlock();

        return !err ? count : err;
}
static BUS_ATTR_WO(link_device);

static ssize_t unlink_device_store(const struct bus_type *bus, const char *buf, size_t count)
{
        struct netdevsim *nsim, *peer;
        struct net_device *dev;
        unsigned int ifidx;
        int netnsfd, err;
        struct net *ns;

        err = sscanf(buf, "%u:%u", &netnsfd, &ifidx);
        if (err != 2) {
                pr_err("Format for unlinking a device is \"netnsfd:ifidx\" (int uint).\n");
                return -EINVAL;
        }

        ns = get_net_ns_by_fd(netnsfd);
        if (IS_ERR(ns)) {
                pr_err("Could not find netns with fd: %d\n", netnsfd);
                return -EINVAL;
        }

        err = -EINVAL;
        rtnl_lock();
        dev = __dev_get_by_index(ns, ifidx);
        if (!dev) {
                pr_err("Could not find device with ifindex %u in netnsfd %d\n",
                       ifidx, netnsfd);
                goto out_put_netns;
        }

        if (!netdev_is_nsim(dev)) {
                pr_err("Device with ifindex %u in netnsfd %d is not a netdevsim\n",
                       ifidx, netnsfd);
                goto out_put_netns;
        }

        nsim = netdev_priv(dev);
        peer = rtnl_dereference(nsim->peer);
        if (!peer)
                goto out_put_netns;

        err = 0;
        RCU_INIT_POINTER(nsim->peer, NULL);
        RCU_INIT_POINTER(peer->peer, NULL);

out_put_netns:
        put_net(ns);
        rtnl_unlock();

        return !err ? count : err;
}
static BUS_ATTR_WO(unlink_device);

static struct attribute *nsim_bus_attrs[] = {
        &bus_attr_new_device.attr,
        &bus_attr_del_device.attr,
        &bus_attr_link_device.attr,
        &bus_attr_unlink_device.attr,
        NULL
};
ATTRIBUTE_GROUPS(nsim_bus);

static int nsim_bus_probe(struct device *dev)
{
        struct nsim_bus_dev *nsim_bus_dev = to_nsim_bus_dev(dev);

        return nsim_drv_probe(nsim_bus_dev);
}

static void nsim_bus_remove(struct device *dev)
{
        struct nsim_bus_dev *nsim_bus_dev = to_nsim_bus_dev(dev);

        nsim_drv_remove(nsim_bus_dev);
}

static int nsim_num_vf(struct device *dev)
{
        struct nsim_bus_dev *nsim_bus_dev = to_nsim_bus_dev(dev);

        return nsim_bus_dev->num_vfs;
}

static const struct bus_type nsim_bus = {
        .name                = DRV_NAME,
        .dev_name        = DRV_NAME,
        .bus_groups        = nsim_bus_groups,
        .probe                = nsim_bus_probe,
        .remove                = nsim_bus_remove,
        .num_vf                = nsim_num_vf,
};

#define NSIM_BUS_DEV_MAX_VFS 4

static struct nsim_bus_dev *
nsim_bus_dev_new(unsigned int id, unsigned int port_count, unsigned int num_queues)
{
        struct nsim_bus_dev *nsim_bus_dev;
        int err;

        nsim_bus_dev = kzalloc(sizeof(*nsim_bus_dev), GFP_KERNEL);
        if (!nsim_bus_dev)
                return ERR_PTR(-ENOMEM);

        err = ida_alloc_range(&nsim_bus_dev_ids, id, id, GFP_KERNEL);
        if (err < 0)
                goto err_nsim_bus_dev_free;
        nsim_bus_dev->dev.id = err;
        nsim_bus_dev->dev.bus = &nsim_bus;
        nsim_bus_dev->dev.type = &nsim_bus_dev_type;
        nsim_bus_dev->port_count = port_count;
        nsim_bus_dev->num_queues = num_queues;
        nsim_bus_dev->initial_net = current->nsproxy->net_ns;
        nsim_bus_dev->max_vfs = NSIM_BUS_DEV_MAX_VFS;
        /* Disallow using nsim_bus_dev */
        smp_store_release(&nsim_bus_dev->init, false);

        err = device_register(&nsim_bus_dev->dev);
        if (err)
                goto err_nsim_bus_dev_id_free;

        return nsim_bus_dev;

err_nsim_bus_dev_id_free:
        ida_free(&nsim_bus_dev_ids, nsim_bus_dev->dev.id);
        put_device(&nsim_bus_dev->dev);
        nsim_bus_dev = NULL;
err_nsim_bus_dev_free:
        kfree(nsim_bus_dev);
        return ERR_PTR(err);
}

static void nsim_bus_dev_del(struct nsim_bus_dev *nsim_bus_dev)
{
        /* Disallow using nsim_bus_dev */
        smp_store_release(&nsim_bus_dev->init, false);
        ida_free(&nsim_bus_dev_ids, nsim_bus_dev->dev.id);
        device_unregister(&nsim_bus_dev->dev);
}

static struct device_driver nsim_driver = {
        .name                = DRV_NAME,
        .bus                = &nsim_bus,
        .owner                = THIS_MODULE,
};

int nsim_bus_init(void)
{
        int err;

        err = bus_register(&nsim_bus);
        if (err)
                return err;
        err = driver_register(&nsim_driver);
        if (err)
                goto err_bus_unregister;
        refcount_set(&nsim_bus_devs, 1);
        /* Allow using resources */
        smp_store_release(&nsim_bus_enable, true);
        return 0;

err_bus_unregister:
        bus_unregister(&nsim_bus);
        return err;
}

void nsim_bus_exit(void)
{
        struct nsim_bus_dev *nsim_bus_dev, *tmp;

        /* Disallow using resources */
        smp_store_release(&nsim_bus_enable, false);
        if (refcount_dec_and_test(&nsim_bus_devs))
                complete(&nsim_bus_devs_released);

        mutex_lock(&nsim_bus_dev_list_lock);
        list_for_each_entry_safe(nsim_bus_dev, tmp, &nsim_bus_dev_list, list) {
                list_del(&nsim_bus_dev->list);
                nsim_bus_dev_del(nsim_bus_dev);
        }
        mutex_unlock(&nsim_bus_dev_list_lock);

        wait_for_completion(&nsim_bus_devs_released);

        driver_unregister(&nsim_driver);
        bus_unregister(&nsim_bus);
}










   35 



































   79 























1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
/* SPDX-License-Identifier: GPL-2.0 */
#undef TRACE_SYSTEM
#define TRACE_SYSTEM mmap

#if !defined(_TRACE_MMAP_H) || defined(TRACE_HEADER_MULTI_READ)
#define _TRACE_MMAP_H

#include <linux/tracepoint.h>

TRACE_EVENT(vm_unmapped_area,

        TP_PROTO(unsigned long addr, struct vm_unmapped_area_info *info),

        TP_ARGS(addr, info),

        TP_STRUCT__entry(
                __field(unsigned long,        addr)
                __field(unsigned long,        total_vm)
                __field(unsigned long,        flags)
                __field(unsigned long,        length)
                __field(unsigned long,        low_limit)
                __field(unsigned long,        high_limit)
                __field(unsigned long,        align_mask)
                __field(unsigned long,        align_offset)
        ),

        TP_fast_assign(
                __entry->addr = addr;
                __entry->total_vm = current->mm->total_vm;
                __entry->flags = info->flags;
                __entry->length = info->length;
                __entry->low_limit = info->low_limit;
                __entry->high_limit = info->high_limit;
                __entry->align_mask = info->align_mask;
                __entry->align_offset = info->align_offset;
        ),

        TP_printk("addr=0x%lx err=%ld total_vm=0x%lx flags=0x%lx len=0x%lx lo=0x%lx hi=0x%lx mask=0x%lx ofs=0x%lx",
                IS_ERR_VALUE(__entry->addr) ? 0 : __entry->addr,
                IS_ERR_VALUE(__entry->addr) ? __entry->addr : 0,
                __entry->total_vm, __entry->flags, __entry->length,
                __entry->low_limit, __entry->high_limit, __entry->align_mask,
                __entry->align_offset)
);

TRACE_EVENT(exit_mmap,
        TP_PROTO(struct mm_struct *mm),

        TP_ARGS(mm),

        TP_STRUCT__entry(
                        __field(struct mm_struct *, mm)
                        __field(struct maple_tree *, mt)
        ),

        TP_fast_assign(
                       __entry->mm                = mm;
                       __entry->mt                = &mm->mm_mt;
        ),

        TP_printk("mt_mod %p, DESTROY",
                  __entry->mt
        )
);

#endif

/* This part must be outside protection */
#include <trace/define_trace.h>





































   20 




  940 









    1 

    2 






































  948 









  956 


























1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
// SPDX-License-Identifier: GPL-2.0
#include <linux/kernel.h>
#include <linux/export.h>
#include <linux/uaccess.h>
#include <linux/mm.h>
#include <linux/bitops.h>

#include <asm/word-at-a-time.h>

/*
 * Do a strnlen, return length of string *with* final '\0'.
 * 'count' is the user-supplied count, while 'max' is the
 * address space maximum.
 *
 * Return 0 for exceptions (which includes hitting the address
 * space maximum), or 'count+1' if hitting the user-supplied
 * maximum count.
 *
 * NOTE! We can sometimes overshoot the user-supplied maximum
 * if it fits in a aligned 'long'. The caller needs to check
 * the return value against "> max".
 */
static __always_inline long do_strnlen_user(const char __user *src, unsigned long count, unsigned long max)
{
        const struct word_at_a_time constants = WORD_AT_A_TIME_CONSTANTS;
        unsigned long align, res = 0;
        unsigned long c;

        /*
         * Do everything aligned. But that means that we
         * need to also expand the maximum..
         */
        align = (sizeof(unsigned long) - 1) & (unsigned long)src;
        src -= align;
        max += align;

        unsafe_get_user(c, (unsigned long __user *)src, efault);
        c |= aligned_byte_mask(align);

        for (;;) {
                unsigned long data;
                if (has_zero(c, &data, &constants)) {
                        data = prep_zero_mask(c, data, &constants);
                        data = create_zero_mask(data);
                        return res + find_zero(data) + 1 - align;
                }
                res += sizeof(unsigned long);
                /* We already handled 'unsigned long' bytes. Did we do it all ? */
                if (unlikely(max <= sizeof(unsigned long)))
                        break;
                max -= sizeof(unsigned long);
                unsafe_get_user(c, (unsigned long __user *)(src+res), efault);
        }
        res -= align;

        /*
         * Uhhuh. We hit 'max'. But was that the user-specified maximum
         * too? If so, return the marker for "too long".
         */
        if (res >= count)
                return count+1;

        /*
         * Nope: we hit the address space limit, and we still had more
         * characters the caller would have wanted. That's 0.
         */
efault:
        return 0;
}

/**
 * strnlen_user: - Get the size of a user string INCLUDING final NUL.
 * @str: The string to measure.
 * @count: Maximum count (including NUL character)
 *
 * Context: User context only. This function may sleep if pagefaults are
 *          enabled.
 *
 * Get the size of a NUL-terminated string in user space.
 *
 * Returns the size of the string INCLUDING the terminating NUL.
 * If the string is too long, returns a number larger than @count. User
 * has to check the return value against "> count".
 * On exception (or invalid count), returns 0.
 *
 * NOTE! You should basically never use this function. There is
 * almost never any valid case for using the length of a user space
 * string, since the string can be changed at any time by other
 * threads. Use "strncpy_from_user()" instead to get a stable copy
 * of the string.
 */
long strnlen_user(const char __user *str, long count)
{
        unsigned long max_addr, src_addr;

        if (unlikely(count <= 0))
                return 0;

        if (can_do_masked_user_access()) {
                long retval;

                str = masked_user_access_begin(str);
                retval = do_strnlen_user(str, count, count);
                user_read_access_end();
                return retval;
        }

        max_addr = TASK_SIZE_MAX;
        src_addr = (unsigned long)untagged_addr(str);
        if (likely(src_addr < max_addr)) {
                unsigned long max = max_addr - src_addr;
                long retval;

                /*
                 * Truncate 'max' to the user-specified limit, so that
                 * we only have one limit we need to check in the loop
                 */
                if (max > count)
                        max = count;

                if (user_read_access_begin(str, max)) {
                        retval = do_strnlen_user(str, count, max);
                        user_read_access_end();
                        return retval;
                }
        }
        return 0;
}
EXPORT_SYMBOL(strnlen_user);











    1 

















1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
// SPDX-License-Identifier: GPL-2.0
#include <linux/cpufreq.h>
#include <linux/fs.h>
#include <linux/init.h>
#include <linux/proc_fs.h>
#include <linux/seq_file.h>

extern const struct seq_operations cpuinfo_op;

static int cpuinfo_open(struct inode *inode, struct file *file)
{
        return seq_open(file, &cpuinfo_op);
}

static const struct proc_ops cpuinfo_proc_ops = {
        .proc_flags        = PROC_ENTRY_PERMANENT,
        .proc_open        = cpuinfo_open,
        .proc_read_iter        = seq_read_iter,
        .proc_lseek        = seq_lseek,
        .proc_release        = seq_release,
};

static int __init proc_cpuinfo_init(void)
{
        proc_create("cpuinfo", 0, NULL, &cpuinfo_proc_ops);
        return 0;
}
fs_initcall(proc_cpuinfo_init);
































  108 


































 6035 
    2 

















    4 

















 2713 


















   26 






















































   79 



























































































    1 




































































































































































 5993 
    3 
















    5 










































































































































  236 
    2 
















 1799 








































































































































































































































    1 



























































































    1 


















 5289 





































































































































































































































































































































  396 












































  334 











































































































































































    1 




















 5249 


















   96 


















  126 


















  189 


















    6 















    2 


































    8 























































   21 



































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































    1 
































































































































































  640 


































































    5 






















































































































































































































































  150 















   87 


































 5164 










































































































































































































  588 




































































































































































  318 










































































































































































































































































































































































































































    2 












































































































































































































































































































































 2097 




















































































 4835 












































 4811 





















 4816 






















































































































































































































































   73 















































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965
2966
2967
2968
2969
2970
2971
2972
2973
2974
2975
2976
2977
2978
2979
2980
2981
2982
2983
2984
2985
2986
2987
2988
2989
2990
2991
2992
2993
2994
2995
2996
2997
2998
2999
3000
3001
3002
3003
3004
3005
3006
3007
3008
3009
3010
3011
3012
3013
3014
3015
3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029
3030
3031
3032
3033
3034
3035
3036
3037
3038
3039
3040
3041
3042
3043
3044
3045
3046
3047
3048
3049
3050
3051
3052
3053
3054
3055
3056
3057
3058
3059
3060
3061
3062
3063
3064
3065
3066
3067
3068
3069
3070
3071
3072
3073
3074
3075
3076
3077
3078
3079
3080
3081
3082
3083
3084
3085
3086
3087
3088
3089
3090
3091
3092
3093
3094
3095
3096
3097
3098
3099
3100
3101
3102
3103
3104
3105
3106
3107
3108
3109
3110
3111
3112
3113
3114
3115
3116
3117
3118
3119
3120
3121
3122
3123
3124
3125
3126
3127
3128
3129
3130
3131
3132
3133
3134
3135
3136
3137
3138
3139
3140
3141
3142
3143
3144
3145
3146
3147
3148
3149
3150
3151
3152
3153
3154
3155
3156
3157
3158
3159
3160
3161
3162
3163
3164
3165
3166
3167
3168
3169
3170
3171
3172
3173
3174
3175
3176
3177
3178
3179
3180
3181
3182
3183
3184
3185
3186
3187
3188
3189
3190
3191
3192
3193
3194
3195
3196
3197
3198
3199
3200
3201
3202
3203
3204
3205
3206
3207
3208
3209
3210
3211
3212
3213
3214
3215
3216
3217
3218
3219
3220
3221
3222
3223
3224
3225
3226
3227
3228
3229
3230
3231
3232
3233
3234
3235
3236
3237
3238
3239
3240
3241
3242
3243
3244
3245
3246
3247
3248
3249
3250
3251
3252
3253
3254
3255
3256
3257
3258
3259
3260
3261
3262
3263
3264
3265
3266
3267
3268
3269
3270
3271
3272
3273
3274
3275
3276
3277
3278
3279
3280
3281
3282
3283
3284
3285
3286
3287
3288
3289
3290
3291
3292
3293
3294
3295
3296
3297
3298
3299
3300
3301
3302
3303
3304
3305
3306
3307
3308
3309
3310
3311
3312
3313
3314
3315
3316
3317
3318
3319
3320
3321
3322
3323
3324
3325
3326
3327
3328
3329
3330
3331
3332
3333
3334
3335
3336
3337
3338
3339
3340
3341
3342
3343
3344
3345
3346
3347
3348
3349
3350
3351
3352
3353
3354
3355
3356
3357
3358
3359
3360
3361
3362
3363
3364
3365
3366
3367
3368
3369
3370
3371
3372
3373
3374
3375
3376
3377
3378
3379
3380
3381
3382
3383
3384
3385
3386
3387
3388
3389
3390
3391
3392
3393
3394
3395
3396
3397
3398
3399
3400
3401
3402
3403
3404
3405
3406
3407
3408
3409
3410
3411
3412
3413
3414
3415
3416
3417
3418
3419
3420
3421
3422
3423
3424
3425
3426
3427
3428
3429
3430
3431
3432
3433
3434
3435
3436
3437
3438
3439
3440
3441
3442
3443
3444
3445
3446
3447
3448
3449
3450
3451
3452
3453
3454
3455
3456
3457
3458
3459
3460
3461
3462
3463
3464
3465
3466
3467
3468
3469
3470
3471
3472
3473
3474
3475
3476
3477
3478
3479
3480
3481
3482
3483
3484
3485
3486
3487
3488
3489
3490
3491
3492
3493
3494
3495
3496
3497
3498
3499
3500
3501
3502
3503
3504
3505
3506
3507
3508
3509
3510
3511
3512
3513
3514
3515
3516
3517
3518
3519
3520
3521
3522
3523
3524
3525
3526
3527
3528
3529
3530
3531
3532
3533
3534
3535
3536
3537
3538
3539
3540
3541
3542
3543
3544
3545
3546
3547
3548
3549
3550
3551
3552
3553
3554
3555
3556
3557
3558
3559
3560
3561
3562
3563
3564
3565
3566
3567
3568
3569
3570
3571
3572
3573
3574
3575
3576
3577
3578
3579
3580
3581
3582
3583
3584
3585
3586
3587
3588
3589
3590
3591
3592
3593
3594
3595
3596
3597
3598
3599
3600
3601
3602
3603
3604
3605
3606
3607
3608
3609
3610
3611
3612
3613
3614
3615
3616
3617
3618
3619
3620
3621
3622
3623
3624
3625
3626
3627
3628
3629
3630
3631
3632
3633
3634
3635
3636
3637
3638
3639
3640
3641
3642
3643
3644
3645
3646
3647
3648
3649
3650
3651
3652
3653
3654
3655
3656
3657
3658
3659
3660
3661
3662
3663
3664
3665
3666
3667
3668
3669
3670
3671
3672
3673
3674
3675
3676
3677
3678
3679
3680
3681
3682
3683
3684
3685
3686
3687
3688
3689
3690
3691
3692
3693
3694
3695
3696
3697
3698
3699
3700
3701
3702
3703
3704
3705
3706
3707
3708
3709
3710
3711
3712
3713
3714
3715
3716
3717
3718
3719
3720
3721
3722
3723
3724
3725
3726
3727
3728
3729
3730
3731
3732
3733
3734
3735
3736
3737
3738
3739
3740
3741
3742
3743
3744
3745
3746
3747
3748
3749
3750
3751
3752
3753
3754
3755
3756
3757
3758
3759
3760
3761
3762
3763
3764
3765
3766
3767
3768
3769
3770
3771
3772
3773
3774
3775
3776
3777
3778
3779
3780
3781
3782
3783
3784
3785
3786
3787
3788
3789
3790
3791
3792
3793
3794
3795
3796
3797
3798
3799
3800
3801
3802
3803
3804
3805
3806
3807
3808
3809
3810
3811
3812
3813
3814
3815
3816
3817
3818
3819
3820
3821
3822
3823
3824
3825
3826
3827
3828
3829
3830
3831
3832
3833
3834
3835
3836
3837
3838
3839
3840
3841
3842
3843
3844
3845
3846
3847
3848
3849
3850
3851
3852
3853
3854
3855
3856
3857
3858
3859
3860
3861
3862
3863
3864
3865
3866
3867
3868
3869
3870
3871
3872
3873
3874
3875
3876
3877
3878
3879
3880
3881
3882
3883
3884
3885
3886
3887
3888
3889
3890
3891
3892
3893
3894
3895
3896
3897
3898
3899
3900
3901
3902
3903
3904
3905
3906
3907
3908
3909
3910
3911
3912
3913
3914
3915
3916
3917
3918
3919
3920
3921
3922
3923
3924
3925
3926
3927
3928
3929
3930
3931
3932
3933
3934
3935
3936
3937
3938
3939
3940
3941
3942
3943
3944
3945
3946
3947
3948
3949
3950
3951
3952
3953
3954
3955
3956
3957
3958
3959
3960
3961
3962
3963
3964
3965
3966
3967
3968
3969
3970
3971
3972
3973
3974
3975
3976
3977
3978
3979
3980
3981
3982
3983
3984
3985
3986
3987
3988
3989
3990
3991
3992
3993
3994
3995
3996
3997
3998
3999
4000
4001
4002
4003
4004
4005
4006
4007
4008
4009
4010
4011
4012
4013
4014
4015
4016
4017
4018
4019
4020
4021
4022
4023
4024
4025
4026
4027
4028
4029
4030
4031
4032
4033
4034
4035
4036
4037
4038
4039
4040
4041
4042
4043
4044
4045
4046
4047
4048
4049
4050
4051
4052
4053
4054
4055
4056
4057
4058
4059
4060
4061
4062
4063
4064
4065
4066
4067
4068
4069
4070
4071
4072
4073
4074
4075
4076
4077
4078
4079
4080
4081
4082
4083
4084
4085
4086
4087
4088
4089
4090
4091
4092
4093
4094
4095
4096
4097
4098
4099
4100
4101
4102
4103
4104
4105
4106
4107
4108
4109
4110
4111
4112
4113
4114
4115
4116
4117
4118
4119
4120
4121
4122
4123
4124
4125
4126
4127
4128
4129
4130
4131
4132
4133
4134
4135
4136
4137
4138
4139
4140
4141
4142
4143
4144
4145
4146
4147
4148
4149
4150
4151
4152
4153
4154
4155
4156
4157
4158
4159
4160
4161
4162
4163
4164
4165
4166
4167
4168
4169
4170
4171
4172
4173
4174
4175
4176
4177
4178
4179
4180
4181
4182
4183
4184
4185
4186
4187
4188
4189
4190
4191
4192
4193
4194
4195
4196
4197
4198
4199
4200
4201
4202
4203
4204
4205
4206
4207
4208
4209
4210
4211
4212
4213
4214
4215
4216
4217
4218
4219
4220
4221
4222
4223
4224
4225
4226
4227
4228
4229
4230
4231
4232
4233
4234
4235
4236
4237
4238
4239
4240
4241
4242
4243
4244
4245
4246
4247
4248
4249
4250
4251
4252
4253
4254
4255
4256
4257
4258
4259
4260
4261
4262
4263
4264
4265
4266
4267
4268
4269
4270
4271
4272
4273
4274
4275
4276
4277
4278
4279
4280
4281
4282
4283
4284
4285
4286
4287
4288
4289
4290
4291
4292
4293
4294
4295
4296
4297
4298
4299
4300
4301
4302
4303
4304
4305
4306
4307
4308
4309
4310
4311
4312
4313
4314
4315
4316
4317
4318
4319
4320
4321
4322
4323
4324
4325
4326
4327
4328
4329
4330
4331
4332
4333
4334
4335
4336
4337
4338
4339
4340
4341
4342
4343
4344
4345
4346
4347
4348
4349
4350
4351
4352
4353
4354
4355
4356
4357
4358
4359
4360
4361
4362
4363
4364
4365
4366
4367
4368
4369
4370
4371
4372
4373
4374
4375
4376
4377
4378
4379
4380
4381
4382
4383
4384
4385
4386
4387
4388
4389
4390
4391
4392
4393
4394
4395
4396
4397
4398
4399
4400
4401
4402
4403
4404
4405
4406
4407
4408
4409
4410
4411
4412
4413
4414
4415
4416
4417
4418
4419
4420
4421
4422
4423
4424
4425
4426
4427
4428
4429
4430
4431
4432
4433
4434
4435
4436
4437
4438
4439
4440
4441
4442
4443
4444
4445
4446
4447
4448
4449
4450
4451
4452
4453
4454
4455
4456
4457
4458
4459
4460
4461
4462
4463
4464
4465
4466
4467
4468
4469
4470
4471
4472
4473
4474
4475
4476
4477
4478
4479
4480
4481
4482
4483
4484
4485
4486
4487
4488
4489
4490
4491
4492
4493
4494
4495
4496
4497
4498
4499
4500
4501
4502
4503
4504
4505
4506
4507
4508
4509
4510
4511
4512
4513
4514
4515
4516
4517
4518
4519
4520
4521
4522
4523
4524
4525
4526
4527
4528
4529
4530
4531
4532
4533
4534
4535
4536
4537
4538
4539
4540
4541
4542
4543
4544
4545
4546
4547
4548
4549
4550
4551
4552
4553
4554
4555
4556
4557
4558
4559
4560
4561
4562
4563
4564
4565
4566
4567
4568
4569
4570
4571
4572
4573
4574
4575
4576
4577
4578
4579
4580
4581
4582
4583
4584
4585
4586
4587
4588
4589
4590
4591
4592
4593
4594
4595
4596
4597
4598
4599
4600
4601
4602
4603
4604
4605
4606
4607
4608
4609
4610
4611
4612
4613
4614
4615
4616
4617
4618
4619
4620
4621
4622
4623
4624
4625
4626
4627
4628
4629
4630
4631
4632
4633
4634
4635
4636
4637
4638
4639
4640
4641
4642
4643
4644
4645
4646
4647
4648
4649
4650
4651
4652
4653
4654
4655
4656
4657
4658
4659
4660
4661
4662
4663
4664
4665
4666
4667
4668
4669
4670
4671
4672
4673
4674
4675
4676
4677
4678
4679
4680
4681
4682
4683
4684
4685
4686
4687
4688
4689
4690
4691
4692
4693
4694
4695
4696
4697
4698
4699
4700
4701
4702
4703
4704
4705
4706
4707
4708
4709
4710
4711
4712
4713
4714
4715
4716
4717
4718
4719
4720
4721
4722
4723
4724
4725
4726
4727
4728
4729
4730
4731
4732
4733
4734
4735
4736
4737
4738
4739
4740
4741
4742
4743
4744
4745
4746
4747
4748
4749
4750
4751
4752
4753
4754
4755
4756
4757
4758
4759
4760
4761
4762
4763
4764
4765
4766
4767
4768
4769
4770
4771
4772
4773
4774
4775
4776
4777
4778
4779
4780
4781
4782
4783
4784
4785
4786
4787
4788
4789
4790
4791
4792
4793
4794
4795
4796
4797
4798
4799
4800
4801
4802
4803
4804
4805
4806
4807
4808
4809
4810
4811
4812
4813
4814
4815
4816
4817
4818
4819
4820
4821
4822
4823
4824
4825
4826
4827
4828
4829
4830
4831
4832
4833
4834
4835
4836
4837
4838
4839
4840
4841
4842
4843
4844
4845
4846
4847
4848
4849
4850
4851
4852
4853
4854
4855
4856
4857
4858
4859
4860
4861
4862
4863
4864
4865
4866
4867
4868
4869
4870
4871
4872
4873
4874
4875
4876
4877
4878
4879
4880
4881
4882
4883
4884
4885
4886
4887
4888
4889
4890
4891
4892
4893
4894
4895
4896
4897
4898
4899
4900
4901
4902
4903
4904
4905
4906
4907
4908
4909
4910
4911
4912
4913
4914
4915
4916
4917
4918
4919
4920
4921
4922
4923
4924
4925
4926
4927
4928
4929
4930
4931
4932
4933
4934
4935
4936
4937
4938
4939
4940
4941
4942
4943
4944
4945
4946
4947
4948
4949
4950
4951
4952
4953
4954
4955
4956
4957
4958
4959
4960
4961
4962
4963
4964
4965
4966
4967
4968
4969
4970
4971
4972
4973
4974
4975
4976
4977
4978
4979
4980
4981
4982
4983
4984
4985
4986
4987
4988
4989
4990
4991
4992
4993
4994
4995
4996
4997
4998
4999
5000
5001
5002
5003
5004
5005
5006
5007
5008
5009
5010
5011
5012
5013
5014
5015
5016
5017
5018
5019
5020
5021
5022
5023
5024
5025
5026
5027
5028
5029
5030
5031
5032
5033
5034
5035
5036
5037
5038
5039
5040
5041
5042
5043
5044
5045
5046
5047
5048
5049
5050
5051
5052
5053
// SPDX-License-Identifier: GPL-2.0

// Generated by scripts/atomic/gen-atomic-instrumented.sh
// DO NOT MODIFY THIS FILE DIRECTLY

/*
 * This file provoides atomic operations with explicit instrumentation (e.g.
 * KASAN, KCSAN), which should be used unless it is necessary to avoid
 * instrumentation. Where it is necessary to aovid instrumenation, the
 * raw_atomic*() operations should be used.
 */
#ifndef _LINUX_ATOMIC_INSTRUMENTED_H
#define _LINUX_ATOMIC_INSTRUMENTED_H

#include <linux/build_bug.h>
#include <linux/compiler.h>
#include <linux/instrumented.h>

/**
 * atomic_read() - atomic load with relaxed ordering
 * @v: pointer to atomic_t
 *
 * Atomically loads the value of @v with relaxed ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_read() there.
 *
 * Return: The value loaded from @v.
 */
static __always_inline int
atomic_read(const atomic_t *v)
{
        instrument_atomic_read(v, sizeof(*v));
        return raw_atomic_read(v);
}

/**
 * atomic_read_acquire() - atomic load with acquire ordering
 * @v: pointer to atomic_t
 *
 * Atomically loads the value of @v with acquire ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_read_acquire() there.
 *
 * Return: The value loaded from @v.
 */
static __always_inline int
atomic_read_acquire(const atomic_t *v)
{
        instrument_atomic_read(v, sizeof(*v));
        return raw_atomic_read_acquire(v);
}

/**
 * atomic_set() - atomic set with relaxed ordering
 * @v: pointer to atomic_t
 * @i: int value to assign
 *
 * Atomically sets @v to @i with relaxed ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_set() there.
 *
 * Return: Nothing.
 */
static __always_inline void
atomic_set(atomic_t *v, int i)
{
        instrument_atomic_write(v, sizeof(*v));
        raw_atomic_set(v, i);
}

/**
 * atomic_set_release() - atomic set with release ordering
 * @v: pointer to atomic_t
 * @i: int value to assign
 *
 * Atomically sets @v to @i with release ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_set_release() there.
 *
 * Return: Nothing.
 */
static __always_inline void
atomic_set_release(atomic_t *v, int i)
{
        kcsan_release();
        instrument_atomic_write(v, sizeof(*v));
        raw_atomic_set_release(v, i);
}

/**
 * atomic_add() - atomic add with relaxed ordering
 * @i: int value to add
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v + @i) with relaxed ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_add() there.
 *
 * Return: Nothing.
 */
static __always_inline void
atomic_add(int i, atomic_t *v)
{
        instrument_atomic_read_write(v, sizeof(*v));
        raw_atomic_add(i, v);
}

/**
 * atomic_add_return() - atomic add with full ordering
 * @i: int value to add
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v + @i) with full ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_add_return() there.
 *
 * Return: The updated value of @v.
 */
static __always_inline int
atomic_add_return(int i, atomic_t *v)
{
        kcsan_mb();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_add_return(i, v);
}

/**
 * atomic_add_return_acquire() - atomic add with acquire ordering
 * @i: int value to add
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v + @i) with acquire ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_add_return_acquire() there.
 *
 * Return: The updated value of @v.
 */
static __always_inline int
atomic_add_return_acquire(int i, atomic_t *v)
{
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_add_return_acquire(i, v);
}

/**
 * atomic_add_return_release() - atomic add with release ordering
 * @i: int value to add
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v + @i) with release ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_add_return_release() there.
 *
 * Return: The updated value of @v.
 */
static __always_inline int
atomic_add_return_release(int i, atomic_t *v)
{
        kcsan_release();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_add_return_release(i, v);
}

/**
 * atomic_add_return_relaxed() - atomic add with relaxed ordering
 * @i: int value to add
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v + @i) with relaxed ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_add_return_relaxed() there.
 *
 * Return: The updated value of @v.
 */
static __always_inline int
atomic_add_return_relaxed(int i, atomic_t *v)
{
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_add_return_relaxed(i, v);
}

/**
 * atomic_fetch_add() - atomic add with full ordering
 * @i: int value to add
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v + @i) with full ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_fetch_add() there.
 *
 * Return: The original value of @v.
 */
static __always_inline int
atomic_fetch_add(int i, atomic_t *v)
{
        kcsan_mb();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_fetch_add(i, v);
}

/**
 * atomic_fetch_add_acquire() - atomic add with acquire ordering
 * @i: int value to add
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v + @i) with acquire ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_fetch_add_acquire() there.
 *
 * Return: The original value of @v.
 */
static __always_inline int
atomic_fetch_add_acquire(int i, atomic_t *v)
{
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_fetch_add_acquire(i, v);
}

/**
 * atomic_fetch_add_release() - atomic add with release ordering
 * @i: int value to add
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v + @i) with release ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_fetch_add_release() there.
 *
 * Return: The original value of @v.
 */
static __always_inline int
atomic_fetch_add_release(int i, atomic_t *v)
{
        kcsan_release();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_fetch_add_release(i, v);
}

/**
 * atomic_fetch_add_relaxed() - atomic add with relaxed ordering
 * @i: int value to add
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v + @i) with relaxed ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_fetch_add_relaxed() there.
 *
 * Return: The original value of @v.
 */
static __always_inline int
atomic_fetch_add_relaxed(int i, atomic_t *v)
{
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_fetch_add_relaxed(i, v);
}

/**
 * atomic_sub() - atomic subtract with relaxed ordering
 * @i: int value to subtract
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v - @i) with relaxed ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_sub() there.
 *
 * Return: Nothing.
 */
static __always_inline void
atomic_sub(int i, atomic_t *v)
{
        instrument_atomic_read_write(v, sizeof(*v));
        raw_atomic_sub(i, v);
}

/**
 * atomic_sub_return() - atomic subtract with full ordering
 * @i: int value to subtract
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v - @i) with full ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_sub_return() there.
 *
 * Return: The updated value of @v.
 */
static __always_inline int
atomic_sub_return(int i, atomic_t *v)
{
        kcsan_mb();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_sub_return(i, v);
}

/**
 * atomic_sub_return_acquire() - atomic subtract with acquire ordering
 * @i: int value to subtract
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v - @i) with acquire ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_sub_return_acquire() there.
 *
 * Return: The updated value of @v.
 */
static __always_inline int
atomic_sub_return_acquire(int i, atomic_t *v)
{
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_sub_return_acquire(i, v);
}

/**
 * atomic_sub_return_release() - atomic subtract with release ordering
 * @i: int value to subtract
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v - @i) with release ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_sub_return_release() there.
 *
 * Return: The updated value of @v.
 */
static __always_inline int
atomic_sub_return_release(int i, atomic_t *v)
{
        kcsan_release();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_sub_return_release(i, v);
}

/**
 * atomic_sub_return_relaxed() - atomic subtract with relaxed ordering
 * @i: int value to subtract
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v - @i) with relaxed ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_sub_return_relaxed() there.
 *
 * Return: The updated value of @v.
 */
static __always_inline int
atomic_sub_return_relaxed(int i, atomic_t *v)
{
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_sub_return_relaxed(i, v);
}

/**
 * atomic_fetch_sub() - atomic subtract with full ordering
 * @i: int value to subtract
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v - @i) with full ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_fetch_sub() there.
 *
 * Return: The original value of @v.
 */
static __always_inline int
atomic_fetch_sub(int i, atomic_t *v)
{
        kcsan_mb();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_fetch_sub(i, v);
}

/**
 * atomic_fetch_sub_acquire() - atomic subtract with acquire ordering
 * @i: int value to subtract
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v - @i) with acquire ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_fetch_sub_acquire() there.
 *
 * Return: The original value of @v.
 */
static __always_inline int
atomic_fetch_sub_acquire(int i, atomic_t *v)
{
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_fetch_sub_acquire(i, v);
}

/**
 * atomic_fetch_sub_release() - atomic subtract with release ordering
 * @i: int value to subtract
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v - @i) with release ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_fetch_sub_release() there.
 *
 * Return: The original value of @v.
 */
static __always_inline int
atomic_fetch_sub_release(int i, atomic_t *v)
{
        kcsan_release();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_fetch_sub_release(i, v);
}

/**
 * atomic_fetch_sub_relaxed() - atomic subtract with relaxed ordering
 * @i: int value to subtract
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v - @i) with relaxed ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_fetch_sub_relaxed() there.
 *
 * Return: The original value of @v.
 */
static __always_inline int
atomic_fetch_sub_relaxed(int i, atomic_t *v)
{
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_fetch_sub_relaxed(i, v);
}

/**
 * atomic_inc() - atomic increment with relaxed ordering
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v + 1) with relaxed ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_inc() there.
 *
 * Return: Nothing.
 */
static __always_inline void
atomic_inc(atomic_t *v)
{
        instrument_atomic_read_write(v, sizeof(*v));
        raw_atomic_inc(v);
}

/**
 * atomic_inc_return() - atomic increment with full ordering
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v + 1) with full ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_inc_return() there.
 *
 * Return: The updated value of @v.
 */
static __always_inline int
atomic_inc_return(atomic_t *v)
{
        kcsan_mb();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_inc_return(v);
}

/**
 * atomic_inc_return_acquire() - atomic increment with acquire ordering
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v + 1) with acquire ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_inc_return_acquire() there.
 *
 * Return: The updated value of @v.
 */
static __always_inline int
atomic_inc_return_acquire(atomic_t *v)
{
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_inc_return_acquire(v);
}

/**
 * atomic_inc_return_release() - atomic increment with release ordering
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v + 1) with release ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_inc_return_release() there.
 *
 * Return: The updated value of @v.
 */
static __always_inline int
atomic_inc_return_release(atomic_t *v)
{
        kcsan_release();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_inc_return_release(v);
}

/**
 * atomic_inc_return_relaxed() - atomic increment with relaxed ordering
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v + 1) with relaxed ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_inc_return_relaxed() there.
 *
 * Return: The updated value of @v.
 */
static __always_inline int
atomic_inc_return_relaxed(atomic_t *v)
{
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_inc_return_relaxed(v);
}

/**
 * atomic_fetch_inc() - atomic increment with full ordering
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v + 1) with full ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_fetch_inc() there.
 *
 * Return: The original value of @v.
 */
static __always_inline int
atomic_fetch_inc(atomic_t *v)
{
        kcsan_mb();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_fetch_inc(v);
}

/**
 * atomic_fetch_inc_acquire() - atomic increment with acquire ordering
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v + 1) with acquire ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_fetch_inc_acquire() there.
 *
 * Return: The original value of @v.
 */
static __always_inline int
atomic_fetch_inc_acquire(atomic_t *v)
{
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_fetch_inc_acquire(v);
}

/**
 * atomic_fetch_inc_release() - atomic increment with release ordering
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v + 1) with release ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_fetch_inc_release() there.
 *
 * Return: The original value of @v.
 */
static __always_inline int
atomic_fetch_inc_release(atomic_t *v)
{
        kcsan_release();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_fetch_inc_release(v);
}

/**
 * atomic_fetch_inc_relaxed() - atomic increment with relaxed ordering
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v + 1) with relaxed ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_fetch_inc_relaxed() there.
 *
 * Return: The original value of @v.
 */
static __always_inline int
atomic_fetch_inc_relaxed(atomic_t *v)
{
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_fetch_inc_relaxed(v);
}

/**
 * atomic_dec() - atomic decrement with relaxed ordering
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v - 1) with relaxed ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_dec() there.
 *
 * Return: Nothing.
 */
static __always_inline void
atomic_dec(atomic_t *v)
{
        instrument_atomic_read_write(v, sizeof(*v));
        raw_atomic_dec(v);
}

/**
 * atomic_dec_return() - atomic decrement with full ordering
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v - 1) with full ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_dec_return() there.
 *
 * Return: The updated value of @v.
 */
static __always_inline int
atomic_dec_return(atomic_t *v)
{
        kcsan_mb();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_dec_return(v);
}

/**
 * atomic_dec_return_acquire() - atomic decrement with acquire ordering
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v - 1) with acquire ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_dec_return_acquire() there.
 *
 * Return: The updated value of @v.
 */
static __always_inline int
atomic_dec_return_acquire(atomic_t *v)
{
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_dec_return_acquire(v);
}

/**
 * atomic_dec_return_release() - atomic decrement with release ordering
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v - 1) with release ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_dec_return_release() there.
 *
 * Return: The updated value of @v.
 */
static __always_inline int
atomic_dec_return_release(atomic_t *v)
{
        kcsan_release();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_dec_return_release(v);
}

/**
 * atomic_dec_return_relaxed() - atomic decrement with relaxed ordering
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v - 1) with relaxed ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_dec_return_relaxed() there.
 *
 * Return: The updated value of @v.
 */
static __always_inline int
atomic_dec_return_relaxed(atomic_t *v)
{
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_dec_return_relaxed(v);
}

/**
 * atomic_fetch_dec() - atomic decrement with full ordering
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v - 1) with full ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_fetch_dec() there.
 *
 * Return: The original value of @v.
 */
static __always_inline int
atomic_fetch_dec(atomic_t *v)
{
        kcsan_mb();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_fetch_dec(v);
}

/**
 * atomic_fetch_dec_acquire() - atomic decrement with acquire ordering
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v - 1) with acquire ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_fetch_dec_acquire() there.
 *
 * Return: The original value of @v.
 */
static __always_inline int
atomic_fetch_dec_acquire(atomic_t *v)
{
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_fetch_dec_acquire(v);
}

/**
 * atomic_fetch_dec_release() - atomic decrement with release ordering
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v - 1) with release ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_fetch_dec_release() there.
 *
 * Return: The original value of @v.
 */
static __always_inline int
atomic_fetch_dec_release(atomic_t *v)
{
        kcsan_release();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_fetch_dec_release(v);
}

/**
 * atomic_fetch_dec_relaxed() - atomic decrement with relaxed ordering
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v - 1) with relaxed ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_fetch_dec_relaxed() there.
 *
 * Return: The original value of @v.
 */
static __always_inline int
atomic_fetch_dec_relaxed(atomic_t *v)
{
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_fetch_dec_relaxed(v);
}

/**
 * atomic_and() - atomic bitwise AND with relaxed ordering
 * @i: int value
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v & @i) with relaxed ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_and() there.
 *
 * Return: Nothing.
 */
static __always_inline void
atomic_and(int i, atomic_t *v)
{
        instrument_atomic_read_write(v, sizeof(*v));
        raw_atomic_and(i, v);
}

/**
 * atomic_fetch_and() - atomic bitwise AND with full ordering
 * @i: int value
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v & @i) with full ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_fetch_and() there.
 *
 * Return: The original value of @v.
 */
static __always_inline int
atomic_fetch_and(int i, atomic_t *v)
{
        kcsan_mb();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_fetch_and(i, v);
}

/**
 * atomic_fetch_and_acquire() - atomic bitwise AND with acquire ordering
 * @i: int value
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v & @i) with acquire ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_fetch_and_acquire() there.
 *
 * Return: The original value of @v.
 */
static __always_inline int
atomic_fetch_and_acquire(int i, atomic_t *v)
{
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_fetch_and_acquire(i, v);
}

/**
 * atomic_fetch_and_release() - atomic bitwise AND with release ordering
 * @i: int value
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v & @i) with release ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_fetch_and_release() there.
 *
 * Return: The original value of @v.
 */
static __always_inline int
atomic_fetch_and_release(int i, atomic_t *v)
{
        kcsan_release();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_fetch_and_release(i, v);
}

/**
 * atomic_fetch_and_relaxed() - atomic bitwise AND with relaxed ordering
 * @i: int value
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v & @i) with relaxed ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_fetch_and_relaxed() there.
 *
 * Return: The original value of @v.
 */
static __always_inline int
atomic_fetch_and_relaxed(int i, atomic_t *v)
{
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_fetch_and_relaxed(i, v);
}

/**
 * atomic_andnot() - atomic bitwise AND NOT with relaxed ordering
 * @i: int value
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v & ~@i) with relaxed ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_andnot() there.
 *
 * Return: Nothing.
 */
static __always_inline void
atomic_andnot(int i, atomic_t *v)
{
        instrument_atomic_read_write(v, sizeof(*v));
        raw_atomic_andnot(i, v);
}

/**
 * atomic_fetch_andnot() - atomic bitwise AND NOT with full ordering
 * @i: int value
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v & ~@i) with full ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_fetch_andnot() there.
 *
 * Return: The original value of @v.
 */
static __always_inline int
atomic_fetch_andnot(int i, atomic_t *v)
{
        kcsan_mb();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_fetch_andnot(i, v);
}

/**
 * atomic_fetch_andnot_acquire() - atomic bitwise AND NOT with acquire ordering
 * @i: int value
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v & ~@i) with acquire ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_fetch_andnot_acquire() there.
 *
 * Return: The original value of @v.
 */
static __always_inline int
atomic_fetch_andnot_acquire(int i, atomic_t *v)
{
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_fetch_andnot_acquire(i, v);
}

/**
 * atomic_fetch_andnot_release() - atomic bitwise AND NOT with release ordering
 * @i: int value
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v & ~@i) with release ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_fetch_andnot_release() there.
 *
 * Return: The original value of @v.
 */
static __always_inline int
atomic_fetch_andnot_release(int i, atomic_t *v)
{
        kcsan_release();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_fetch_andnot_release(i, v);
}

/**
 * atomic_fetch_andnot_relaxed() - atomic bitwise AND NOT with relaxed ordering
 * @i: int value
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v & ~@i) with relaxed ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_fetch_andnot_relaxed() there.
 *
 * Return: The original value of @v.
 */
static __always_inline int
atomic_fetch_andnot_relaxed(int i, atomic_t *v)
{
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_fetch_andnot_relaxed(i, v);
}

/**
 * atomic_or() - atomic bitwise OR with relaxed ordering
 * @i: int value
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v | @i) with relaxed ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_or() there.
 *
 * Return: Nothing.
 */
static __always_inline void
atomic_or(int i, atomic_t *v)
{
        instrument_atomic_read_write(v, sizeof(*v));
        raw_atomic_or(i, v);
}

/**
 * atomic_fetch_or() - atomic bitwise OR with full ordering
 * @i: int value
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v | @i) with full ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_fetch_or() there.
 *
 * Return: The original value of @v.
 */
static __always_inline int
atomic_fetch_or(int i, atomic_t *v)
{
        kcsan_mb();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_fetch_or(i, v);
}

/**
 * atomic_fetch_or_acquire() - atomic bitwise OR with acquire ordering
 * @i: int value
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v | @i) with acquire ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_fetch_or_acquire() there.
 *
 * Return: The original value of @v.
 */
static __always_inline int
atomic_fetch_or_acquire(int i, atomic_t *v)
{
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_fetch_or_acquire(i, v);
}

/**
 * atomic_fetch_or_release() - atomic bitwise OR with release ordering
 * @i: int value
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v | @i) with release ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_fetch_or_release() there.
 *
 * Return: The original value of @v.
 */
static __always_inline int
atomic_fetch_or_release(int i, atomic_t *v)
{
        kcsan_release();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_fetch_or_release(i, v);
}

/**
 * atomic_fetch_or_relaxed() - atomic bitwise OR with relaxed ordering
 * @i: int value
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v | @i) with relaxed ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_fetch_or_relaxed() there.
 *
 * Return: The original value of @v.
 */
static __always_inline int
atomic_fetch_or_relaxed(int i, atomic_t *v)
{
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_fetch_or_relaxed(i, v);
}

/**
 * atomic_xor() - atomic bitwise XOR with relaxed ordering
 * @i: int value
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v ^ @i) with relaxed ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_xor() there.
 *
 * Return: Nothing.
 */
static __always_inline void
atomic_xor(int i, atomic_t *v)
{
        instrument_atomic_read_write(v, sizeof(*v));
        raw_atomic_xor(i, v);
}

/**
 * atomic_fetch_xor() - atomic bitwise XOR with full ordering
 * @i: int value
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v ^ @i) with full ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_fetch_xor() there.
 *
 * Return: The original value of @v.
 */
static __always_inline int
atomic_fetch_xor(int i, atomic_t *v)
{
        kcsan_mb();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_fetch_xor(i, v);
}

/**
 * atomic_fetch_xor_acquire() - atomic bitwise XOR with acquire ordering
 * @i: int value
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v ^ @i) with acquire ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_fetch_xor_acquire() there.
 *
 * Return: The original value of @v.
 */
static __always_inline int
atomic_fetch_xor_acquire(int i, atomic_t *v)
{
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_fetch_xor_acquire(i, v);
}

/**
 * atomic_fetch_xor_release() - atomic bitwise XOR with release ordering
 * @i: int value
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v ^ @i) with release ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_fetch_xor_release() there.
 *
 * Return: The original value of @v.
 */
static __always_inline int
atomic_fetch_xor_release(int i, atomic_t *v)
{
        kcsan_release();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_fetch_xor_release(i, v);
}

/**
 * atomic_fetch_xor_relaxed() - atomic bitwise XOR with relaxed ordering
 * @i: int value
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v ^ @i) with relaxed ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_fetch_xor_relaxed() there.
 *
 * Return: The original value of @v.
 */
static __always_inline int
atomic_fetch_xor_relaxed(int i, atomic_t *v)
{
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_fetch_xor_relaxed(i, v);
}

/**
 * atomic_xchg() - atomic exchange with full ordering
 * @v: pointer to atomic_t
 * @new: int value to assign
 *
 * Atomically updates @v to @new with full ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_xchg() there.
 *
 * Return: The original value of @v.
 */
static __always_inline int
atomic_xchg(atomic_t *v, int new)
{
        kcsan_mb();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_xchg(v, new);
}

/**
 * atomic_xchg_acquire() - atomic exchange with acquire ordering
 * @v: pointer to atomic_t
 * @new: int value to assign
 *
 * Atomically updates @v to @new with acquire ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_xchg_acquire() there.
 *
 * Return: The original value of @v.
 */
static __always_inline int
atomic_xchg_acquire(atomic_t *v, int new)
{
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_xchg_acquire(v, new);
}

/**
 * atomic_xchg_release() - atomic exchange with release ordering
 * @v: pointer to atomic_t
 * @new: int value to assign
 *
 * Atomically updates @v to @new with release ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_xchg_release() there.
 *
 * Return: The original value of @v.
 */
static __always_inline int
atomic_xchg_release(atomic_t *v, int new)
{
        kcsan_release();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_xchg_release(v, new);
}

/**
 * atomic_xchg_relaxed() - atomic exchange with relaxed ordering
 * @v: pointer to atomic_t
 * @new: int value to assign
 *
 * Atomically updates @v to @new with relaxed ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_xchg_relaxed() there.
 *
 * Return: The original value of @v.
 */
static __always_inline int
atomic_xchg_relaxed(atomic_t *v, int new)
{
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_xchg_relaxed(v, new);
}

/**
 * atomic_cmpxchg() - atomic compare and exchange with full ordering
 * @v: pointer to atomic_t
 * @old: int value to compare with
 * @new: int value to assign
 *
 * If (@v == @old), atomically updates @v to @new with full ordering.
 * Otherwise, @v is not modified and relaxed ordering is provided.
 *
 * Unsafe to use in noinstr code; use raw_atomic_cmpxchg() there.
 *
 * Return: The original value of @v.
 */
static __always_inline int
atomic_cmpxchg(atomic_t *v, int old, int new)
{
        kcsan_mb();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_cmpxchg(v, old, new);
}

/**
 * atomic_cmpxchg_acquire() - atomic compare and exchange with acquire ordering
 * @v: pointer to atomic_t
 * @old: int value to compare with
 * @new: int value to assign
 *
 * If (@v == @old), atomically updates @v to @new with acquire ordering.
 * Otherwise, @v is not modified and relaxed ordering is provided.
 *
 * Unsafe to use in noinstr code; use raw_atomic_cmpxchg_acquire() there.
 *
 * Return: The original value of @v.
 */
static __always_inline int
atomic_cmpxchg_acquire(atomic_t *v, int old, int new)
{
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_cmpxchg_acquire(v, old, new);
}

/**
 * atomic_cmpxchg_release() - atomic compare and exchange with release ordering
 * @v: pointer to atomic_t
 * @old: int value to compare with
 * @new: int value to assign
 *
 * If (@v == @old), atomically updates @v to @new with release ordering.
 * Otherwise, @v is not modified and relaxed ordering is provided.
 *
 * Unsafe to use in noinstr code; use raw_atomic_cmpxchg_release() there.
 *
 * Return: The original value of @v.
 */
static __always_inline int
atomic_cmpxchg_release(atomic_t *v, int old, int new)
{
        kcsan_release();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_cmpxchg_release(v, old, new);
}

/**
 * atomic_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering
 * @v: pointer to atomic_t
 * @old: int value to compare with
 * @new: int value to assign
 *
 * If (@v == @old), atomically updates @v to @new with relaxed ordering.
 * Otherwise, @v is not modified and relaxed ordering is provided.
 *
 * Unsafe to use in noinstr code; use raw_atomic_cmpxchg_relaxed() there.
 *
 * Return: The original value of @v.
 */
static __always_inline int
atomic_cmpxchg_relaxed(atomic_t *v, int old, int new)
{
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_cmpxchg_relaxed(v, old, new);
}

/**
 * atomic_try_cmpxchg() - atomic compare and exchange with full ordering
 * @v: pointer to atomic_t
 * @old: pointer to int value to compare with
 * @new: int value to assign
 *
 * If (@v == @old), atomically updates @v to @new with full ordering.
 * Otherwise, @v is not modified, @old is updated to the current value of @v,
 * and relaxed ordering is provided.
 *
 * Unsafe to use in noinstr code; use raw_atomic_try_cmpxchg() there.
 *
 * Return: @true if the exchange occured, @false otherwise.
 */
static __always_inline bool
atomic_try_cmpxchg(atomic_t *v, int *old, int new)
{
        kcsan_mb();
        instrument_atomic_read_write(v, sizeof(*v));
        instrument_atomic_read_write(old, sizeof(*old));
        return raw_atomic_try_cmpxchg(v, old, new);
}

/**
 * atomic_try_cmpxchg_acquire() - atomic compare and exchange with acquire ordering
 * @v: pointer to atomic_t
 * @old: pointer to int value to compare with
 * @new: int value to assign
 *
 * If (@v == @old), atomically updates @v to @new with acquire ordering.
 * Otherwise, @v is not modified, @old is updated to the current value of @v,
 * and relaxed ordering is provided.
 *
 * Unsafe to use in noinstr code; use raw_atomic_try_cmpxchg_acquire() there.
 *
 * Return: @true if the exchange occured, @false otherwise.
 */
static __always_inline bool
atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new)
{
        instrument_atomic_read_write(v, sizeof(*v));
        instrument_atomic_read_write(old, sizeof(*old));
        return raw_atomic_try_cmpxchg_acquire(v, old, new);
}

/**
 * atomic_try_cmpxchg_release() - atomic compare and exchange with release ordering
 * @v: pointer to atomic_t
 * @old: pointer to int value to compare with
 * @new: int value to assign
 *
 * If (@v == @old), atomically updates @v to @new with release ordering.
 * Otherwise, @v is not modified, @old is updated to the current value of @v,
 * and relaxed ordering is provided.
 *
 * Unsafe to use in noinstr code; use raw_atomic_try_cmpxchg_release() there.
 *
 * Return: @true if the exchange occured, @false otherwise.
 */
static __always_inline bool
atomic_try_cmpxchg_release(atomic_t *v, int *old, int new)
{
        kcsan_release();
        instrument_atomic_read_write(v, sizeof(*v));
        instrument_atomic_read_write(old, sizeof(*old));
        return raw_atomic_try_cmpxchg_release(v, old, new);
}

/**
 * atomic_try_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering
 * @v: pointer to atomic_t
 * @old: pointer to int value to compare with
 * @new: int value to assign
 *
 * If (@v == @old), atomically updates @v to @new with relaxed ordering.
 * Otherwise, @v is not modified, @old is updated to the current value of @v,
 * and relaxed ordering is provided.
 *
 * Unsafe to use in noinstr code; use raw_atomic_try_cmpxchg_relaxed() there.
 *
 * Return: @true if the exchange occured, @false otherwise.
 */
static __always_inline bool
atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new)
{
        instrument_atomic_read_write(v, sizeof(*v));
        instrument_atomic_read_write(old, sizeof(*old));
        return raw_atomic_try_cmpxchg_relaxed(v, old, new);
}

/**
 * atomic_sub_and_test() - atomic subtract and test if zero with full ordering
 * @i: int value to subtract
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v - @i) with full ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_sub_and_test() there.
 *
 * Return: @true if the resulting value of @v is zero, @false otherwise.
 */
static __always_inline bool
atomic_sub_and_test(int i, atomic_t *v)
{
        kcsan_mb();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_sub_and_test(i, v);
}

/**
 * atomic_dec_and_test() - atomic decrement and test if zero with full ordering
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v - 1) with full ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_dec_and_test() there.
 *
 * Return: @true if the resulting value of @v is zero, @false otherwise.
 */
static __always_inline bool
atomic_dec_and_test(atomic_t *v)
{
        kcsan_mb();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_dec_and_test(v);
}

/**
 * atomic_inc_and_test() - atomic increment and test if zero with full ordering
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v + 1) with full ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_inc_and_test() there.
 *
 * Return: @true if the resulting value of @v is zero, @false otherwise.
 */
static __always_inline bool
atomic_inc_and_test(atomic_t *v)
{
        kcsan_mb();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_inc_and_test(v);
}

/**
 * atomic_add_negative() - atomic add and test if negative with full ordering
 * @i: int value to add
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v + @i) with full ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_add_negative() there.
 *
 * Return: @true if the resulting value of @v is negative, @false otherwise.
 */
static __always_inline bool
atomic_add_negative(int i, atomic_t *v)
{
        kcsan_mb();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_add_negative(i, v);
}

/**
 * atomic_add_negative_acquire() - atomic add and test if negative with acquire ordering
 * @i: int value to add
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v + @i) with acquire ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_add_negative_acquire() there.
 *
 * Return: @true if the resulting value of @v is negative, @false otherwise.
 */
static __always_inline bool
atomic_add_negative_acquire(int i, atomic_t *v)
{
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_add_negative_acquire(i, v);
}

/**
 * atomic_add_negative_release() - atomic add and test if negative with release ordering
 * @i: int value to add
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v + @i) with release ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_add_negative_release() there.
 *
 * Return: @true if the resulting value of @v is negative, @false otherwise.
 */
static __always_inline bool
atomic_add_negative_release(int i, atomic_t *v)
{
        kcsan_release();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_add_negative_release(i, v);
}

/**
 * atomic_add_negative_relaxed() - atomic add and test if negative with relaxed ordering
 * @i: int value to add
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v + @i) with relaxed ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_add_negative_relaxed() there.
 *
 * Return: @true if the resulting value of @v is negative, @false otherwise.
 */
static __always_inline bool
atomic_add_negative_relaxed(int i, atomic_t *v)
{
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_add_negative_relaxed(i, v);
}

/**
 * atomic_fetch_add_unless() - atomic add unless value with full ordering
 * @v: pointer to atomic_t
 * @a: int value to add
 * @u: int value to compare with
 *
 * If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
 * Otherwise, @v is not modified and relaxed ordering is provided.
 *
 * Unsafe to use in noinstr code; use raw_atomic_fetch_add_unless() there.
 *
 * Return: The original value of @v.
 */
static __always_inline int
atomic_fetch_add_unless(atomic_t *v, int a, int u)
{
        kcsan_mb();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_fetch_add_unless(v, a, u);
}

/**
 * atomic_add_unless() - atomic add unless value with full ordering
 * @v: pointer to atomic_t
 * @a: int value to add
 * @u: int value to compare with
 *
 * If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
 * Otherwise, @v is not modified and relaxed ordering is provided.
 *
 * Unsafe to use in noinstr code; use raw_atomic_add_unless() there.
 *
 * Return: @true if @v was updated, @false otherwise.
 */
static __always_inline bool
atomic_add_unless(atomic_t *v, int a, int u)
{
        kcsan_mb();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_add_unless(v, a, u);
}

/**
 * atomic_inc_not_zero() - atomic increment unless zero with full ordering
 * @v: pointer to atomic_t
 *
 * If (@v != 0), atomically updates @v to (@v + 1) with full ordering.
 * Otherwise, @v is not modified and relaxed ordering is provided.
 *
 * Unsafe to use in noinstr code; use raw_atomic_inc_not_zero() there.
 *
 * Return: @true if @v was updated, @false otherwise.
 */
static __always_inline bool
atomic_inc_not_zero(atomic_t *v)
{
        kcsan_mb();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_inc_not_zero(v);
}

/**
 * atomic_inc_unless_negative() - atomic increment unless negative with full ordering
 * @v: pointer to atomic_t
 *
 * If (@v >= 0), atomically updates @v to (@v + 1) with full ordering.
 * Otherwise, @v is not modified and relaxed ordering is provided.
 *
 * Unsafe to use in noinstr code; use raw_atomic_inc_unless_negative() there.
 *
 * Return: @true if @v was updated, @false otherwise.
 */
static __always_inline bool
atomic_inc_unless_negative(atomic_t *v)
{
        kcsan_mb();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_inc_unless_negative(v);
}

/**
 * atomic_dec_unless_positive() - atomic decrement unless positive with full ordering
 * @v: pointer to atomic_t
 *
 * If (@v <= 0), atomically updates @v to (@v - 1) with full ordering.
 * Otherwise, @v is not modified and relaxed ordering is provided.
 *
 * Unsafe to use in noinstr code; use raw_atomic_dec_unless_positive() there.
 *
 * Return: @true if @v was updated, @false otherwise.
 */
static __always_inline bool
atomic_dec_unless_positive(atomic_t *v)
{
        kcsan_mb();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_dec_unless_positive(v);
}

/**
 * atomic_dec_if_positive() - atomic decrement if positive with full ordering
 * @v: pointer to atomic_t
 *
 * If (@v > 0), atomically updates @v to (@v - 1) with full ordering.
 * Otherwise, @v is not modified and relaxed ordering is provided.
 *
 * Unsafe to use in noinstr code; use raw_atomic_dec_if_positive() there.
 *
 * Return: The old value of (@v - 1), regardless of whether @v was updated.
 */
static __always_inline int
atomic_dec_if_positive(atomic_t *v)
{
        kcsan_mb();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_dec_if_positive(v);
}

/**
 * atomic64_read() - atomic load with relaxed ordering
 * @v: pointer to atomic64_t
 *
 * Atomically loads the value of @v with relaxed ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic64_read() there.
 *
 * Return: The value loaded from @v.
 */
static __always_inline s64
atomic64_read(const atomic64_t *v)
{
        instrument_atomic_read(v, sizeof(*v));
        return raw_atomic64_read(v);
}

/**
 * atomic64_read_acquire() - atomic load with acquire ordering
 * @v: pointer to atomic64_t
 *
 * Atomically loads the value of @v with acquire ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic64_read_acquire() there.
 *
 * Return: The value loaded from @v.
 */
static __always_inline s64
atomic64_read_acquire(const atomic64_t *v)
{
        instrument_atomic_read(v, sizeof(*v));
        return raw_atomic64_read_acquire(v);
}

/**
 * atomic64_set() - atomic set with relaxed ordering
 * @v: pointer to atomic64_t
 * @i: s64 value to assign
 *
 * Atomically sets @v to @i with relaxed ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic64_set() there.
 *
 * Return: Nothing.
 */
static __always_inline void
atomic64_set(atomic64_t *v, s64 i)
{
        instrument_atomic_write(v, sizeof(*v));
        raw_atomic64_set(v, i);
}

/**
 * atomic64_set_release() - atomic set with release ordering
 * @v: pointer to atomic64_t
 * @i: s64 value to assign
 *
 * Atomically sets @v to @i with release ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic64_set_release() there.
 *
 * Return: Nothing.
 */
static __always_inline void
atomic64_set_release(atomic64_t *v, s64 i)
{
        kcsan_release();
        instrument_atomic_write(v, sizeof(*v));
        raw_atomic64_set_release(v, i);
}

/**
 * atomic64_add() - atomic add with relaxed ordering
 * @i: s64 value to add
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v + @i) with relaxed ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic64_add() there.
 *
 * Return: Nothing.
 */
static __always_inline void
atomic64_add(s64 i, atomic64_t *v)
{
        instrument_atomic_read_write(v, sizeof(*v));
        raw_atomic64_add(i, v);
}

/**
 * atomic64_add_return() - atomic add with full ordering
 * @i: s64 value to add
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v + @i) with full ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic64_add_return() there.
 *
 * Return: The updated value of @v.
 */
static __always_inline s64
atomic64_add_return(s64 i, atomic64_t *v)
{
        kcsan_mb();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic64_add_return(i, v);
}

/**
 * atomic64_add_return_acquire() - atomic add with acquire ordering
 * @i: s64 value to add
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v + @i) with acquire ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic64_add_return_acquire() there.
 *
 * Return: The updated value of @v.
 */
static __always_inline s64
atomic64_add_return_acquire(s64 i, atomic64_t *v)
{
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic64_add_return_acquire(i, v);
}

/**
 * atomic64_add_return_release() - atomic add with release ordering
 * @i: s64 value to add
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v + @i) with release ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic64_add_return_release() there.
 *
 * Return: The updated value of @v.
 */
static __always_inline s64
atomic64_add_return_release(s64 i, atomic64_t *v)
{
        kcsan_release();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic64_add_return_release(i, v);
}

/**
 * atomic64_add_return_relaxed() - atomic add with relaxed ordering
 * @i: s64 value to add
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v + @i) with relaxed ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic64_add_return_relaxed() there.
 *
 * Return: The updated value of @v.
 */
static __always_inline s64
atomic64_add_return_relaxed(s64 i, atomic64_t *v)
{
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic64_add_return_relaxed(i, v);
}

/**
 * atomic64_fetch_add() - atomic add with full ordering
 * @i: s64 value to add
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v + @i) with full ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic64_fetch_add() there.
 *
 * Return: The original value of @v.
 */
static __always_inline s64
atomic64_fetch_add(s64 i, atomic64_t *v)
{
        kcsan_mb();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic64_fetch_add(i, v);
}

/**
 * atomic64_fetch_add_acquire() - atomic add with acquire ordering
 * @i: s64 value to add
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v + @i) with acquire ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic64_fetch_add_acquire() there.
 *
 * Return: The original value of @v.
 */
static __always_inline s64
atomic64_fetch_add_acquire(s64 i, atomic64_t *v)
{
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic64_fetch_add_acquire(i, v);
}

/**
 * atomic64_fetch_add_release() - atomic add with release ordering
 * @i: s64 value to add
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v + @i) with release ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic64_fetch_add_release() there.
 *
 * Return: The original value of @v.
 */
static __always_inline s64
atomic64_fetch_add_release(s64 i, atomic64_t *v)
{
        kcsan_release();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic64_fetch_add_release(i, v);
}

/**
 * atomic64_fetch_add_relaxed() - atomic add with relaxed ordering
 * @i: s64 value to add
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v + @i) with relaxed ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic64_fetch_add_relaxed() there.
 *
 * Return: The original value of @v.
 */
static __always_inline s64
atomic64_fetch_add_relaxed(s64 i, atomic64_t *v)
{
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic64_fetch_add_relaxed(i, v);
}

/**
 * atomic64_sub() - atomic subtract with relaxed ordering
 * @i: s64 value to subtract
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v - @i) with relaxed ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic64_sub() there.
 *
 * Return: Nothing.
 */
static __always_inline void
atomic64_sub(s64 i, atomic64_t *v)
{
        instrument_atomic_read_write(v, sizeof(*v));
        raw_atomic64_sub(i, v);
}

/**
 * atomic64_sub_return() - atomic subtract with full ordering
 * @i: s64 value to subtract
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v - @i) with full ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic64_sub_return() there.
 *
 * Return: The updated value of @v.
 */
static __always_inline s64
atomic64_sub_return(s64 i, atomic64_t *v)
{
        kcsan_mb();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic64_sub_return(i, v);
}

/**
 * atomic64_sub_return_acquire() - atomic subtract with acquire ordering
 * @i: s64 value to subtract
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v - @i) with acquire ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic64_sub_return_acquire() there.
 *
 * Return: The updated value of @v.
 */
static __always_inline s64
atomic64_sub_return_acquire(s64 i, atomic64_t *v)
{
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic64_sub_return_acquire(i, v);
}

/**
 * atomic64_sub_return_release() - atomic subtract with release ordering
 * @i: s64 value to subtract
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v - @i) with release ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic64_sub_return_release() there.
 *
 * Return: The updated value of @v.
 */
static __always_inline s64
atomic64_sub_return_release(s64 i, atomic64_t *v)
{
        kcsan_release();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic64_sub_return_release(i, v);
}

/**
 * atomic64_sub_return_relaxed() - atomic subtract with relaxed ordering
 * @i: s64 value to subtract
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v - @i) with relaxed ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic64_sub_return_relaxed() there.
 *
 * Return: The updated value of @v.
 */
static __always_inline s64
atomic64_sub_return_relaxed(s64 i, atomic64_t *v)
{
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic64_sub_return_relaxed(i, v);
}

/**
 * atomic64_fetch_sub() - atomic subtract with full ordering
 * @i: s64 value to subtract
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v - @i) with full ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic64_fetch_sub() there.
 *
 * Return: The original value of @v.
 */
static __always_inline s64
atomic64_fetch_sub(s64 i, atomic64_t *v)
{
        kcsan_mb();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic64_fetch_sub(i, v);
}

/**
 * atomic64_fetch_sub_acquire() - atomic subtract with acquire ordering
 * @i: s64 value to subtract
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v - @i) with acquire ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic64_fetch_sub_acquire() there.
 *
 * Return: The original value of @v.
 */
static __always_inline s64
atomic64_fetch_sub_acquire(s64 i, atomic64_t *v)
{
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic64_fetch_sub_acquire(i, v);
}

/**
 * atomic64_fetch_sub_release() - atomic subtract with release ordering
 * @i: s64 value to subtract
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v - @i) with release ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic64_fetch_sub_release() there.
 *
 * Return: The original value of @v.
 */
static __always_inline s64
atomic64_fetch_sub_release(s64 i, atomic64_t *v)
{
        kcsan_release();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic64_fetch_sub_release(i, v);
}

/**
 * atomic64_fetch_sub_relaxed() - atomic subtract with relaxed ordering
 * @i: s64 value to subtract
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v - @i) with relaxed ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic64_fetch_sub_relaxed() there.
 *
 * Return: The original value of @v.
 */
static __always_inline s64
atomic64_fetch_sub_relaxed(s64 i, atomic64_t *v)
{
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic64_fetch_sub_relaxed(i, v);
}

/**
 * atomic64_inc() - atomic increment with relaxed ordering
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v + 1) with relaxed ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic64_inc() there.
 *
 * Return: Nothing.
 */
static __always_inline void
atomic64_inc(atomic64_t *v)
{
        instrument_atomic_read_write(v, sizeof(*v));
        raw_atomic64_inc(v);
}

/**
 * atomic64_inc_return() - atomic increment with full ordering
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v + 1) with full ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic64_inc_return() there.
 *
 * Return: The updated value of @v.
 */
static __always_inline s64
atomic64_inc_return(atomic64_t *v)
{
        kcsan_mb();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic64_inc_return(v);
}

/**
 * atomic64_inc_return_acquire() - atomic increment with acquire ordering
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v + 1) with acquire ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic64_inc_return_acquire() there.
 *
 * Return: The updated value of @v.
 */
static __always_inline s64
atomic64_inc_return_acquire(atomic64_t *v)
{
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic64_inc_return_acquire(v);
}

/**
 * atomic64_inc_return_release() - atomic increment with release ordering
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v + 1) with release ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic64_inc_return_release() there.
 *
 * Return: The updated value of @v.
 */
static __always_inline s64
atomic64_inc_return_release(atomic64_t *v)
{
        kcsan_release();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic64_inc_return_release(v);
}

/**
 * atomic64_inc_return_relaxed() - atomic increment with relaxed ordering
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v + 1) with relaxed ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic64_inc_return_relaxed() there.
 *
 * Return: The updated value of @v.
 */
static __always_inline s64
atomic64_inc_return_relaxed(atomic64_t *v)
{
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic64_inc_return_relaxed(v);
}

/**
 * atomic64_fetch_inc() - atomic increment with full ordering
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v + 1) with full ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic64_fetch_inc() there.
 *
 * Return: The original value of @v.
 */
static __always_inline s64
atomic64_fetch_inc(atomic64_t *v)
{
        kcsan_mb();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic64_fetch_inc(v);
}

/**
 * atomic64_fetch_inc_acquire() - atomic increment with acquire ordering
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v + 1) with acquire ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic64_fetch_inc_acquire() there.
 *
 * Return: The original value of @v.
 */
static __always_inline s64
atomic64_fetch_inc_acquire(atomic64_t *v)
{
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic64_fetch_inc_acquire(v);
}

/**
 * atomic64_fetch_inc_release() - atomic increment with release ordering
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v + 1) with release ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic64_fetch_inc_release() there.
 *
 * Return: The original value of @v.
 */
static __always_inline s64
atomic64_fetch_inc_release(atomic64_t *v)
{
        kcsan_release();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic64_fetch_inc_release(v);
}

/**
 * atomic64_fetch_inc_relaxed() - atomic increment with relaxed ordering
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v + 1) with relaxed ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic64_fetch_inc_relaxed() there.
 *
 * Return: The original value of @v.
 */
static __always_inline s64
atomic64_fetch_inc_relaxed(atomic64_t *v)
{
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic64_fetch_inc_relaxed(v);
}

/**
 * atomic64_dec() - atomic decrement with relaxed ordering
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v - 1) with relaxed ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic64_dec() there.
 *
 * Return: Nothing.
 */
static __always_inline void
atomic64_dec(atomic64_t *v)
{
        instrument_atomic_read_write(v, sizeof(*v));
        raw_atomic64_dec(v);
}

/**
 * atomic64_dec_return() - atomic decrement with full ordering
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v - 1) with full ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic64_dec_return() there.
 *
 * Return: The updated value of @v.
 */
static __always_inline s64
atomic64_dec_return(atomic64_t *v)
{
        kcsan_mb();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic64_dec_return(v);
}

/**
 * atomic64_dec_return_acquire() - atomic decrement with acquire ordering
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v - 1) with acquire ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic64_dec_return_acquire() there.
 *
 * Return: The updated value of @v.
 */
static __always_inline s64
atomic64_dec_return_acquire(atomic64_t *v)
{
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic64_dec_return_acquire(v);
}

/**
 * atomic64_dec_return_release() - atomic decrement with release ordering
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v - 1) with release ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic64_dec_return_release() there.
 *
 * Return: The updated value of @v.
 */
static __always_inline s64
atomic64_dec_return_release(atomic64_t *v)
{
        kcsan_release();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic64_dec_return_release(v);
}

/**
 * atomic64_dec_return_relaxed() - atomic decrement with relaxed ordering
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v - 1) with relaxed ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic64_dec_return_relaxed() there.
 *
 * Return: The updated value of @v.
 */
static __always_inline s64
atomic64_dec_return_relaxed(atomic64_t *v)
{
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic64_dec_return_relaxed(v);
}

/**
 * atomic64_fetch_dec() - atomic decrement with full ordering
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v - 1) with full ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic64_fetch_dec() there.
 *
 * Return: The original value of @v.
 */
static __always_inline s64
atomic64_fetch_dec(atomic64_t *v)
{
        kcsan_mb();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic64_fetch_dec(v);
}

/**
 * atomic64_fetch_dec_acquire() - atomic decrement with acquire ordering
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v - 1) with acquire ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic64_fetch_dec_acquire() there.
 *
 * Return: The original value of @v.
 */
static __always_inline s64
atomic64_fetch_dec_acquire(atomic64_t *v)
{
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic64_fetch_dec_acquire(v);
}

/**
 * atomic64_fetch_dec_release() - atomic decrement with release ordering
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v - 1) with release ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic64_fetch_dec_release() there.
 *
 * Return: The original value of @v.
 */
static __always_inline s64
atomic64_fetch_dec_release(atomic64_t *v)
{
        kcsan_release();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic64_fetch_dec_release(v);
}

/**
 * atomic64_fetch_dec_relaxed() - atomic decrement with relaxed ordering
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v - 1) with relaxed ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic64_fetch_dec_relaxed() there.
 *
 * Return: The original value of @v.
 */
static __always_inline s64
atomic64_fetch_dec_relaxed(atomic64_t *v)
{
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic64_fetch_dec_relaxed(v);
}

/**
 * atomic64_and() - atomic bitwise AND with relaxed ordering
 * @i: s64 value
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v & @i) with relaxed ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic64_and() there.
 *
 * Return: Nothing.
 */
static __always_inline void
atomic64_and(s64 i, atomic64_t *v)
{
        instrument_atomic_read_write(v, sizeof(*v));
        raw_atomic64_and(i, v);
}

/**
 * atomic64_fetch_and() - atomic bitwise AND with full ordering
 * @i: s64 value
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v & @i) with full ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic64_fetch_and() there.
 *
 * Return: The original value of @v.
 */
static __always_inline s64
atomic64_fetch_and(s64 i, atomic64_t *v)
{
        kcsan_mb();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic64_fetch_and(i, v);
}

/**
 * atomic64_fetch_and_acquire() - atomic bitwise AND with acquire ordering
 * @i: s64 value
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v & @i) with acquire ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic64_fetch_and_acquire() there.
 *
 * Return: The original value of @v.
 */
static __always_inline s64
atomic64_fetch_and_acquire(s64 i, atomic64_t *v)
{
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic64_fetch_and_acquire(i, v);
}

/**
 * atomic64_fetch_and_release() - atomic bitwise AND with release ordering
 * @i: s64 value
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v & @i) with release ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic64_fetch_and_release() there.
 *
 * Return: The original value of @v.
 */
static __always_inline s64
atomic64_fetch_and_release(s64 i, atomic64_t *v)
{
        kcsan_release();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic64_fetch_and_release(i, v);
}

/**
 * atomic64_fetch_and_relaxed() - atomic bitwise AND with relaxed ordering
 * @i: s64 value
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v & @i) with relaxed ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic64_fetch_and_relaxed() there.
 *
 * Return: The original value of @v.
 */
static __always_inline s64
atomic64_fetch_and_relaxed(s64 i, atomic64_t *v)
{
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic64_fetch_and_relaxed(i, v);
}

/**
 * atomic64_andnot() - atomic bitwise AND NOT with relaxed ordering
 * @i: s64 value
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v & ~@i) with relaxed ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic64_andnot() there.
 *
 * Return: Nothing.
 */
static __always_inline void
atomic64_andnot(s64 i, atomic64_t *v)
{
        instrument_atomic_read_write(v, sizeof(*v));
        raw_atomic64_andnot(i, v);
}

/**
 * atomic64_fetch_andnot() - atomic bitwise AND NOT with full ordering
 * @i: s64 value
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v & ~@i) with full ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic64_fetch_andnot() there.
 *
 * Return: The original value of @v.
 */
static __always_inline s64
atomic64_fetch_andnot(s64 i, atomic64_t *v)
{
        kcsan_mb();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic64_fetch_andnot(i, v);
}

/**
 * atomic64_fetch_andnot_acquire() - atomic bitwise AND NOT with acquire ordering
 * @i: s64 value
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v & ~@i) with acquire ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic64_fetch_andnot_acquire() there.
 *
 * Return: The original value of @v.
 */
static __always_inline s64
atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v)
{
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic64_fetch_andnot_acquire(i, v);
}

/**
 * atomic64_fetch_andnot_release() - atomic bitwise AND NOT with release ordering
 * @i: s64 value
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v & ~@i) with release ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic64_fetch_andnot_release() there.
 *
 * Return: The original value of @v.
 */
static __always_inline s64
atomic64_fetch_andnot_release(s64 i, atomic64_t *v)
{
        kcsan_release();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic64_fetch_andnot_release(i, v);
}

/**
 * atomic64_fetch_andnot_relaxed() - atomic bitwise AND NOT with relaxed ordering
 * @i: s64 value
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v & ~@i) with relaxed ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic64_fetch_andnot_relaxed() there.
 *
 * Return: The original value of @v.
 */
static __always_inline s64
atomic64_fetch_andnot_relaxed(s64 i, atomic64_t *v)
{
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic64_fetch_andnot_relaxed(i, v);
}

/**
 * atomic64_or() - atomic bitwise OR with relaxed ordering
 * @i: s64 value
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v | @i) with relaxed ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic64_or() there.
 *
 * Return: Nothing.
 */
static __always_inline void
atomic64_or(s64 i, atomic64_t *v)
{
        instrument_atomic_read_write(v, sizeof(*v));
        raw_atomic64_or(i, v);
}

/**
 * atomic64_fetch_or() - atomic bitwise OR with full ordering
 * @i: s64 value
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v | @i) with full ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic64_fetch_or() there.
 *
 * Return: The original value of @v.
 */
static __always_inline s64
atomic64_fetch_or(s64 i, atomic64_t *v)
{
        kcsan_mb();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic64_fetch_or(i, v);
}

/**
 * atomic64_fetch_or_acquire() - atomic bitwise OR with acquire ordering
 * @i: s64 value
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v | @i) with acquire ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic64_fetch_or_acquire() there.
 *
 * Return: The original value of @v.
 */
static __always_inline s64
atomic64_fetch_or_acquire(s64 i, atomic64_t *v)
{
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic64_fetch_or_acquire(i, v);
}

/**
 * atomic64_fetch_or_release() - atomic bitwise OR with release ordering
 * @i: s64 value
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v | @i) with release ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic64_fetch_or_release() there.
 *
 * Return: The original value of @v.
 */
static __always_inline s64
atomic64_fetch_or_release(s64 i, atomic64_t *v)
{
        kcsan_release();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic64_fetch_or_release(i, v);
}

/**
 * atomic64_fetch_or_relaxed() - atomic bitwise OR with relaxed ordering
 * @i: s64 value
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v | @i) with relaxed ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic64_fetch_or_relaxed() there.
 *
 * Return: The original value of @v.
 */
static __always_inline s64
atomic64_fetch_or_relaxed(s64 i, atomic64_t *v)
{
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic64_fetch_or_relaxed(i, v);
}

/**
 * atomic64_xor() - atomic bitwise XOR with relaxed ordering
 * @i: s64 value
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v ^ @i) with relaxed ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic64_xor() there.
 *
 * Return: Nothing.
 */
static __always_inline void
atomic64_xor(s64 i, atomic64_t *v)
{
        instrument_atomic_read_write(v, sizeof(*v));
        raw_atomic64_xor(i, v);
}

/**
 * atomic64_fetch_xor() - atomic bitwise XOR with full ordering
 * @i: s64 value
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v ^ @i) with full ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic64_fetch_xor() there.
 *
 * Return: The original value of @v.
 */
static __always_inline s64
atomic64_fetch_xor(s64 i, atomic64_t *v)
{
        kcsan_mb();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic64_fetch_xor(i, v);
}

/**
 * atomic64_fetch_xor_acquire() - atomic bitwise XOR with acquire ordering
 * @i: s64 value
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v ^ @i) with acquire ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic64_fetch_xor_acquire() there.
 *
 * Return: The original value of @v.
 */
static __always_inline s64
atomic64_fetch_xor_acquire(s64 i, atomic64_t *v)
{
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic64_fetch_xor_acquire(i, v);
}

/**
 * atomic64_fetch_xor_release() - atomic bitwise XOR with release ordering
 * @i: s64 value
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v ^ @i) with release ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic64_fetch_xor_release() there.
 *
 * Return: The original value of @v.
 */
static __always_inline s64
atomic64_fetch_xor_release(s64 i, atomic64_t *v)
{
        kcsan_release();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic64_fetch_xor_release(i, v);
}

/**
 * atomic64_fetch_xor_relaxed() - atomic bitwise XOR with relaxed ordering
 * @i: s64 value
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v ^ @i) with relaxed ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic64_fetch_xor_relaxed() there.
 *
 * Return: The original value of @v.
 */
static __always_inline s64
atomic64_fetch_xor_relaxed(s64 i, atomic64_t *v)
{
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic64_fetch_xor_relaxed(i, v);
}

/**
 * atomic64_xchg() - atomic exchange with full ordering
 * @v: pointer to atomic64_t
 * @new: s64 value to assign
 *
 * Atomically updates @v to @new with full ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic64_xchg() there.
 *
 * Return: The original value of @v.
 */
static __always_inline s64
atomic64_xchg(atomic64_t *v, s64 new)
{
        kcsan_mb();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic64_xchg(v, new);
}

/**
 * atomic64_xchg_acquire() - atomic exchange with acquire ordering
 * @v: pointer to atomic64_t
 * @new: s64 value to assign
 *
 * Atomically updates @v to @new with acquire ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic64_xchg_acquire() there.
 *
 * Return: The original value of @v.
 */
static __always_inline s64
atomic64_xchg_acquire(atomic64_t *v, s64 new)
{
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic64_xchg_acquire(v, new);
}

/**
 * atomic64_xchg_release() - atomic exchange with release ordering
 * @v: pointer to atomic64_t
 * @new: s64 value to assign
 *
 * Atomically updates @v to @new with release ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic64_xchg_release() there.
 *
 * Return: The original value of @v.
 */
static __always_inline s64
atomic64_xchg_release(atomic64_t *v, s64 new)
{
        kcsan_release();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic64_xchg_release(v, new);
}

/**
 * atomic64_xchg_relaxed() - atomic exchange with relaxed ordering
 * @v: pointer to atomic64_t
 * @new: s64 value to assign
 *
 * Atomically updates @v to @new with relaxed ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic64_xchg_relaxed() there.
 *
 * Return: The original value of @v.
 */
static __always_inline s64
atomic64_xchg_relaxed(atomic64_t *v, s64 new)
{
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic64_xchg_relaxed(v, new);
}

/**
 * atomic64_cmpxchg() - atomic compare and exchange with full ordering
 * @v: pointer to atomic64_t
 * @old: s64 value to compare with
 * @new: s64 value to assign
 *
 * If (@v == @old), atomically updates @v to @new with full ordering.
 * Otherwise, @v is not modified and relaxed ordering is provided.
 *
 * Unsafe to use in noinstr code; use raw_atomic64_cmpxchg() there.
 *
 * Return: The original value of @v.
 */
static __always_inline s64
atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new)
{
        kcsan_mb();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic64_cmpxchg(v, old, new);
}

/**
 * atomic64_cmpxchg_acquire() - atomic compare and exchange with acquire ordering
 * @v: pointer to atomic64_t
 * @old: s64 value to compare with
 * @new: s64 value to assign
 *
 * If (@v == @old), atomically updates @v to @new with acquire ordering.
 * Otherwise, @v is not modified and relaxed ordering is provided.
 *
 * Unsafe to use in noinstr code; use raw_atomic64_cmpxchg_acquire() there.
 *
 * Return: The original value of @v.
 */
static __always_inline s64
atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new)
{
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic64_cmpxchg_acquire(v, old, new);
}

/**
 * atomic64_cmpxchg_release() - atomic compare and exchange with release ordering
 * @v: pointer to atomic64_t
 * @old: s64 value to compare with
 * @new: s64 value to assign
 *
 * If (@v == @old), atomically updates @v to @new with release ordering.
 * Otherwise, @v is not modified and relaxed ordering is provided.
 *
 * Unsafe to use in noinstr code; use raw_atomic64_cmpxchg_release() there.
 *
 * Return: The original value of @v.
 */
static __always_inline s64
atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new)
{
        kcsan_release();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic64_cmpxchg_release(v, old, new);
}

/**
 * atomic64_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering
 * @v: pointer to atomic64_t
 * @old: s64 value to compare with
 * @new: s64 value to assign
 *
 * If (@v == @old), atomically updates @v to @new with relaxed ordering.
 * Otherwise, @v is not modified and relaxed ordering is provided.
 *
 * Unsafe to use in noinstr code; use raw_atomic64_cmpxchg_relaxed() there.
 *
 * Return: The original value of @v.
 */
static __always_inline s64
atomic64_cmpxchg_relaxed(atomic64_t *v, s64 old, s64 new)
{
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic64_cmpxchg_relaxed(v, old, new);
}

/**
 * atomic64_try_cmpxchg() - atomic compare and exchange with full ordering
 * @v: pointer to atomic64_t
 * @old: pointer to s64 value to compare with
 * @new: s64 value to assign
 *
 * If (@v == @old), atomically updates @v to @new with full ordering.
 * Otherwise, @v is not modified, @old is updated to the current value of @v,
 * and relaxed ordering is provided.
 *
 * Unsafe to use in noinstr code; use raw_atomic64_try_cmpxchg() there.
 *
 * Return: @true if the exchange occured, @false otherwise.
 */
static __always_inline bool
atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new)
{
        kcsan_mb();
        instrument_atomic_read_write(v, sizeof(*v));
        instrument_atomic_read_write(old, sizeof(*old));
        return raw_atomic64_try_cmpxchg(v, old, new);
}

/**
 * atomic64_try_cmpxchg_acquire() - atomic compare and exchange with acquire ordering
 * @v: pointer to atomic64_t
 * @old: pointer to s64 value to compare with
 * @new: s64 value to assign
 *
 * If (@v == @old), atomically updates @v to @new with acquire ordering.
 * Otherwise, @v is not modified, @old is updated to the current value of @v,
 * and relaxed ordering is provided.
 *
 * Unsafe to use in noinstr code; use raw_atomic64_try_cmpxchg_acquire() there.
 *
 * Return: @true if the exchange occured, @false otherwise.
 */
static __always_inline bool
atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new)
{
        instrument_atomic_read_write(v, sizeof(*v));
        instrument_atomic_read_write(old, sizeof(*old));
        return raw_atomic64_try_cmpxchg_acquire(v, old, new);
}

/**
 * atomic64_try_cmpxchg_release() - atomic compare and exchange with release ordering
 * @v: pointer to atomic64_t
 * @old: pointer to s64 value to compare with
 * @new: s64 value to assign
 *
 * If (@v == @old), atomically updates @v to @new with release ordering.
 * Otherwise, @v is not modified, @old is updated to the current value of @v,
 * and relaxed ordering is provided.
 *
 * Unsafe to use in noinstr code; use raw_atomic64_try_cmpxchg_release() there.
 *
 * Return: @true if the exchange occured, @false otherwise.
 */
static __always_inline bool
atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new)
{
        kcsan_release();
        instrument_atomic_read_write(v, sizeof(*v));
        instrument_atomic_read_write(old, sizeof(*old));
        return raw_atomic64_try_cmpxchg_release(v, old, new);
}

/**
 * atomic64_try_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering
 * @v: pointer to atomic64_t
 * @old: pointer to s64 value to compare with
 * @new: s64 value to assign
 *
 * If (@v == @old), atomically updates @v to @new with relaxed ordering.
 * Otherwise, @v is not modified, @old is updated to the current value of @v,
 * and relaxed ordering is provided.
 *
 * Unsafe to use in noinstr code; use raw_atomic64_try_cmpxchg_relaxed() there.
 *
 * Return: @true if the exchange occured, @false otherwise.
 */
static __always_inline bool
atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new)
{
        instrument_atomic_read_write(v, sizeof(*v));
        instrument_atomic_read_write(old, sizeof(*old));
        return raw_atomic64_try_cmpxchg_relaxed(v, old, new);
}

/**
 * atomic64_sub_and_test() - atomic subtract and test if zero with full ordering
 * @i: s64 value to subtract
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v - @i) with full ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic64_sub_and_test() there.
 *
 * Return: @true if the resulting value of @v is zero, @false otherwise.
 */
static __always_inline bool
atomic64_sub_and_test(s64 i, atomic64_t *v)
{
        kcsan_mb();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic64_sub_and_test(i, v);
}

/**
 * atomic64_dec_and_test() - atomic decrement and test if zero with full ordering
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v - 1) with full ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic64_dec_and_test() there.
 *
 * Return: @true if the resulting value of @v is zero, @false otherwise.
 */
static __always_inline bool
atomic64_dec_and_test(atomic64_t *v)
{
        kcsan_mb();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic64_dec_and_test(v);
}

/**
 * atomic64_inc_and_test() - atomic increment and test if zero with full ordering
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v + 1) with full ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic64_inc_and_test() there.
 *
 * Return: @true if the resulting value of @v is zero, @false otherwise.
 */
static __always_inline bool
atomic64_inc_and_test(atomic64_t *v)
{
        kcsan_mb();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic64_inc_and_test(v);
}

/**
 * atomic64_add_negative() - atomic add and test if negative with full ordering
 * @i: s64 value to add
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v + @i) with full ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic64_add_negative() there.
 *
 * Return: @true if the resulting value of @v is negative, @false otherwise.
 */
static __always_inline bool
atomic64_add_negative(s64 i, atomic64_t *v)
{
        kcsan_mb();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic64_add_negative(i, v);
}

/**
 * atomic64_add_negative_acquire() - atomic add and test if negative with acquire ordering
 * @i: s64 value to add
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v + @i) with acquire ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic64_add_negative_acquire() there.
 *
 * Return: @true if the resulting value of @v is negative, @false otherwise.
 */
static __always_inline bool
atomic64_add_negative_acquire(s64 i, atomic64_t *v)
{
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic64_add_negative_acquire(i, v);
}

/**
 * atomic64_add_negative_release() - atomic add and test if negative with release ordering
 * @i: s64 value to add
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v + @i) with release ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic64_add_negative_release() there.
 *
 * Return: @true if the resulting value of @v is negative, @false otherwise.
 */
static __always_inline bool
atomic64_add_negative_release(s64 i, atomic64_t *v)
{
        kcsan_release();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic64_add_negative_release(i, v);
}

/**
 * atomic64_add_negative_relaxed() - atomic add and test if negative with relaxed ordering
 * @i: s64 value to add
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v + @i) with relaxed ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic64_add_negative_relaxed() there.
 *
 * Return: @true if the resulting value of @v is negative, @false otherwise.
 */
static __always_inline bool
atomic64_add_negative_relaxed(s64 i, atomic64_t *v)
{
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic64_add_negative_relaxed(i, v);
}

/**
 * atomic64_fetch_add_unless() - atomic add unless value with full ordering
 * @v: pointer to atomic64_t
 * @a: s64 value to add
 * @u: s64 value to compare with
 *
 * If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
 * Otherwise, @v is not modified and relaxed ordering is provided.
 *
 * Unsafe to use in noinstr code; use raw_atomic64_fetch_add_unless() there.
 *
 * Return: The original value of @v.
 */
static __always_inline s64
atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)
{
        kcsan_mb();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic64_fetch_add_unless(v, a, u);
}

/**
 * atomic64_add_unless() - atomic add unless value with full ordering
 * @v: pointer to atomic64_t
 * @a: s64 value to add
 * @u: s64 value to compare with
 *
 * If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
 * Otherwise, @v is not modified and relaxed ordering is provided.
 *
 * Unsafe to use in noinstr code; use raw_atomic64_add_unless() there.
 *
 * Return: @true if @v was updated, @false otherwise.
 */
static __always_inline bool
atomic64_add_unless(atomic64_t *v, s64 a, s64 u)
{
        kcsan_mb();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic64_add_unless(v, a, u);
}

/**
 * atomic64_inc_not_zero() - atomic increment unless zero with full ordering
 * @v: pointer to atomic64_t
 *
 * If (@v != 0), atomically updates @v to (@v + 1) with full ordering.
 * Otherwise, @v is not modified and relaxed ordering is provided.
 *
 * Unsafe to use in noinstr code; use raw_atomic64_inc_not_zero() there.
 *
 * Return: @true if @v was updated, @false otherwise.
 */
static __always_inline bool
atomic64_inc_not_zero(atomic64_t *v)
{
        kcsan_mb();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic64_inc_not_zero(v);
}

/**
 * atomic64_inc_unless_negative() - atomic increment unless negative with full ordering
 * @v: pointer to atomic64_t
 *
 * If (@v >= 0), atomically updates @v to (@v + 1) with full ordering.
 * Otherwise, @v is not modified and relaxed ordering is provided.
 *
 * Unsafe to use in noinstr code; use raw_atomic64_inc_unless_negative() there.
 *
 * Return: @true if @v was updated, @false otherwise.
 */
static __always_inline bool
atomic64_inc_unless_negative(atomic64_t *v)
{
        kcsan_mb();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic64_inc_unless_negative(v);
}

/**
 * atomic64_dec_unless_positive() - atomic decrement unless positive with full ordering
 * @v: pointer to atomic64_t
 *
 * If (@v <= 0), atomically updates @v to (@v - 1) with full ordering.
 * Otherwise, @v is not modified and relaxed ordering is provided.
 *
 * Unsafe to use in noinstr code; use raw_atomic64_dec_unless_positive() there.
 *
 * Return: @true if @v was updated, @false otherwise.
 */
static __always_inline bool
atomic64_dec_unless_positive(atomic64_t *v)
{
        kcsan_mb();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic64_dec_unless_positive(v);
}

/**
 * atomic64_dec_if_positive() - atomic decrement if positive with full ordering
 * @v: pointer to atomic64_t
 *
 * If (@v > 0), atomically updates @v to (@v - 1) with full ordering.
 * Otherwise, @v is not modified and relaxed ordering is provided.
 *
 * Unsafe to use in noinstr code; use raw_atomic64_dec_if_positive() there.
 *
 * Return: The old value of (@v - 1), regardless of whether @v was updated.
 */
static __always_inline s64
atomic64_dec_if_positive(atomic64_t *v)
{
        kcsan_mb();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic64_dec_if_positive(v);
}

/**
 * atomic_long_read() - atomic load with relaxed ordering
 * @v: pointer to atomic_long_t
 *
 * Atomically loads the value of @v with relaxed ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_long_read() there.
 *
 * Return: The value loaded from @v.
 */
static __always_inline long
atomic_long_read(const atomic_long_t *v)
{
        instrument_atomic_read(v, sizeof(*v));
        return raw_atomic_long_read(v);
}

/**
 * atomic_long_read_acquire() - atomic load with acquire ordering
 * @v: pointer to atomic_long_t
 *
 * Atomically loads the value of @v with acquire ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_long_read_acquire() there.
 *
 * Return: The value loaded from @v.
 */
static __always_inline long
atomic_long_read_acquire(const atomic_long_t *v)
{
        instrument_atomic_read(v, sizeof(*v));
        return raw_atomic_long_read_acquire(v);
}

/**
 * atomic_long_set() - atomic set with relaxed ordering
 * @v: pointer to atomic_long_t
 * @i: long value to assign
 *
 * Atomically sets @v to @i with relaxed ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_long_set() there.
 *
 * Return: Nothing.
 */
static __always_inline void
atomic_long_set(atomic_long_t *v, long i)
{
        instrument_atomic_write(v, sizeof(*v));
        raw_atomic_long_set(v, i);
}

/**
 * atomic_long_set_release() - atomic set with release ordering
 * @v: pointer to atomic_long_t
 * @i: long value to assign
 *
 * Atomically sets @v to @i with release ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_long_set_release() there.
 *
 * Return: Nothing.
 */
static __always_inline void
atomic_long_set_release(atomic_long_t *v, long i)
{
        kcsan_release();
        instrument_atomic_write(v, sizeof(*v));
        raw_atomic_long_set_release(v, i);
}

/**
 * atomic_long_add() - atomic add with relaxed ordering
 * @i: long value to add
 * @v: pointer to atomic_long_t
 *
 * Atomically updates @v to (@v + @i) with relaxed ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_long_add() there.
 *
 * Return: Nothing.
 */
static __always_inline void
atomic_long_add(long i, atomic_long_t *v)
{
        instrument_atomic_read_write(v, sizeof(*v));
        raw_atomic_long_add(i, v);
}

/**
 * atomic_long_add_return() - atomic add with full ordering
 * @i: long value to add
 * @v: pointer to atomic_long_t
 *
 * Atomically updates @v to (@v + @i) with full ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_long_add_return() there.
 *
 * Return: The updated value of @v.
 */
static __always_inline long
atomic_long_add_return(long i, atomic_long_t *v)
{
        kcsan_mb();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_long_add_return(i, v);
}

/**
 * atomic_long_add_return_acquire() - atomic add with acquire ordering
 * @i: long value to add
 * @v: pointer to atomic_long_t
 *
 * Atomically updates @v to (@v + @i) with acquire ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_long_add_return_acquire() there.
 *
 * Return: The updated value of @v.
 */
static __always_inline long
atomic_long_add_return_acquire(long i, atomic_long_t *v)
{
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_long_add_return_acquire(i, v);
}

/**
 * atomic_long_add_return_release() - atomic add with release ordering
 * @i: long value to add
 * @v: pointer to atomic_long_t
 *
 * Atomically updates @v to (@v + @i) with release ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_long_add_return_release() there.
 *
 * Return: The updated value of @v.
 */
static __always_inline long
atomic_long_add_return_release(long i, atomic_long_t *v)
{
        kcsan_release();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_long_add_return_release(i, v);
}

/**
 * atomic_long_add_return_relaxed() - atomic add with relaxed ordering
 * @i: long value to add
 * @v: pointer to atomic_long_t
 *
 * Atomically updates @v to (@v + @i) with relaxed ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_long_add_return_relaxed() there.
 *
 * Return: The updated value of @v.
 */
static __always_inline long
atomic_long_add_return_relaxed(long i, atomic_long_t *v)
{
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_long_add_return_relaxed(i, v);
}

/**
 * atomic_long_fetch_add() - atomic add with full ordering
 * @i: long value to add
 * @v: pointer to atomic_long_t
 *
 * Atomically updates @v to (@v + @i) with full ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_add() there.
 *
 * Return: The original value of @v.
 */
static __always_inline long
atomic_long_fetch_add(long i, atomic_long_t *v)
{
        kcsan_mb();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_long_fetch_add(i, v);
}

/**
 * atomic_long_fetch_add_acquire() - atomic add with acquire ordering
 * @i: long value to add
 * @v: pointer to atomic_long_t
 *
 * Atomically updates @v to (@v + @i) with acquire ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_add_acquire() there.
 *
 * Return: The original value of @v.
 */
static __always_inline long
atomic_long_fetch_add_acquire(long i, atomic_long_t *v)
{
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_long_fetch_add_acquire(i, v);
}

/**
 * atomic_long_fetch_add_release() - atomic add with release ordering
 * @i: long value to add
 * @v: pointer to atomic_long_t
 *
 * Atomically updates @v to (@v + @i) with release ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_add_release() there.
 *
 * Return: The original value of @v.
 */
static __always_inline long
atomic_long_fetch_add_release(long i, atomic_long_t *v)
{
        kcsan_release();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_long_fetch_add_release(i, v);
}

/**
 * atomic_long_fetch_add_relaxed() - atomic add with relaxed ordering
 * @i: long value to add
 * @v: pointer to atomic_long_t
 *
 * Atomically updates @v to (@v + @i) with relaxed ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_add_relaxed() there.
 *
 * Return: The original value of @v.
 */
static __always_inline long
atomic_long_fetch_add_relaxed(long i, atomic_long_t *v)
{
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_long_fetch_add_relaxed(i, v);
}

/**
 * atomic_long_sub() - atomic subtract with relaxed ordering
 * @i: long value to subtract
 * @v: pointer to atomic_long_t
 *
 * Atomically updates @v to (@v - @i) with relaxed ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_long_sub() there.
 *
 * Return: Nothing.
 */
static __always_inline void
atomic_long_sub(long i, atomic_long_t *v)
{
        instrument_atomic_read_write(v, sizeof(*v));
        raw_atomic_long_sub(i, v);
}

/**
 * atomic_long_sub_return() - atomic subtract with full ordering
 * @i: long value to subtract
 * @v: pointer to atomic_long_t
 *
 * Atomically updates @v to (@v - @i) with full ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_long_sub_return() there.
 *
 * Return: The updated value of @v.
 */
static __always_inline long
atomic_long_sub_return(long i, atomic_long_t *v)
{
        kcsan_mb();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_long_sub_return(i, v);
}

/**
 * atomic_long_sub_return_acquire() - atomic subtract with acquire ordering
 * @i: long value to subtract
 * @v: pointer to atomic_long_t
 *
 * Atomically updates @v to (@v - @i) with acquire ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_long_sub_return_acquire() there.
 *
 * Return: The updated value of @v.
 */
static __always_inline long
atomic_long_sub_return_acquire(long i, atomic_long_t *v)
{
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_long_sub_return_acquire(i, v);
}

/**
 * atomic_long_sub_return_release() - atomic subtract with release ordering
 * @i: long value to subtract
 * @v: pointer to atomic_long_t
 *
 * Atomically updates @v to (@v - @i) with release ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_long_sub_return_release() there.
 *
 * Return: The updated value of @v.
 */
static __always_inline long
atomic_long_sub_return_release(long i, atomic_long_t *v)
{
        kcsan_release();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_long_sub_return_release(i, v);
}

/**
 * atomic_long_sub_return_relaxed() - atomic subtract with relaxed ordering
 * @i: long value to subtract
 * @v: pointer to atomic_long_t
 *
 * Atomically updates @v to (@v - @i) with relaxed ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_long_sub_return_relaxed() there.
 *
 * Return: The updated value of @v.
 */
static __always_inline long
atomic_long_sub_return_relaxed(long i, atomic_long_t *v)
{
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_long_sub_return_relaxed(i, v);
}

/**
 * atomic_long_fetch_sub() - atomic subtract with full ordering
 * @i: long value to subtract
 * @v: pointer to atomic_long_t
 *
 * Atomically updates @v to (@v - @i) with full ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_sub() there.
 *
 * Return: The original value of @v.
 */
static __always_inline long
atomic_long_fetch_sub(long i, atomic_long_t *v)
{
        kcsan_mb();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_long_fetch_sub(i, v);
}

/**
 * atomic_long_fetch_sub_acquire() - atomic subtract with acquire ordering
 * @i: long value to subtract
 * @v: pointer to atomic_long_t
 *
 * Atomically updates @v to (@v - @i) with acquire ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_sub_acquire() there.
 *
 * Return: The original value of @v.
 */
static __always_inline long
atomic_long_fetch_sub_acquire(long i, atomic_long_t *v)
{
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_long_fetch_sub_acquire(i, v);
}

/**
 * atomic_long_fetch_sub_release() - atomic subtract with release ordering
 * @i: long value to subtract
 * @v: pointer to atomic_long_t
 *
 * Atomically updates @v to (@v - @i) with release ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_sub_release() there.
 *
 * Return: The original value of @v.
 */
static __always_inline long
atomic_long_fetch_sub_release(long i, atomic_long_t *v)
{
        kcsan_release();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_long_fetch_sub_release(i, v);
}

/**
 * atomic_long_fetch_sub_relaxed() - atomic subtract with relaxed ordering
 * @i: long value to subtract
 * @v: pointer to atomic_long_t
 *
 * Atomically updates @v to (@v - @i) with relaxed ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_sub_relaxed() there.
 *
 * Return: The original value of @v.
 */
static __always_inline long
atomic_long_fetch_sub_relaxed(long i, atomic_long_t *v)
{
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_long_fetch_sub_relaxed(i, v);
}

/**
 * atomic_long_inc() - atomic increment with relaxed ordering
 * @v: pointer to atomic_long_t
 *
 * Atomically updates @v to (@v + 1) with relaxed ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_long_inc() there.
 *
 * Return: Nothing.
 */
static __always_inline void
atomic_long_inc(atomic_long_t *v)
{
        instrument_atomic_read_write(v, sizeof(*v));
        raw_atomic_long_inc(v);
}

/**
 * atomic_long_inc_return() - atomic increment with full ordering
 * @v: pointer to atomic_long_t
 *
 * Atomically updates @v to (@v + 1) with full ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_long_inc_return() there.
 *
 * Return: The updated value of @v.
 */
static __always_inline long
atomic_long_inc_return(atomic_long_t *v)
{
        kcsan_mb();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_long_inc_return(v);
}

/**
 * atomic_long_inc_return_acquire() - atomic increment with acquire ordering
 * @v: pointer to atomic_long_t
 *
 * Atomically updates @v to (@v + 1) with acquire ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_long_inc_return_acquire() there.
 *
 * Return: The updated value of @v.
 */
static __always_inline long
atomic_long_inc_return_acquire(atomic_long_t *v)
{
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_long_inc_return_acquire(v);
}

/**
 * atomic_long_inc_return_release() - atomic increment with release ordering
 * @v: pointer to atomic_long_t
 *
 * Atomically updates @v to (@v + 1) with release ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_long_inc_return_release() there.
 *
 * Return: The updated value of @v.
 */
static __always_inline long
atomic_long_inc_return_release(atomic_long_t *v)
{
        kcsan_release();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_long_inc_return_release(v);
}

/**
 * atomic_long_inc_return_relaxed() - atomic increment with relaxed ordering
 * @v: pointer to atomic_long_t
 *
 * Atomically updates @v to (@v + 1) with relaxed ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_long_inc_return_relaxed() there.
 *
 * Return: The updated value of @v.
 */
static __always_inline long
atomic_long_inc_return_relaxed(atomic_long_t *v)
{
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_long_inc_return_relaxed(v);
}

/**
 * atomic_long_fetch_inc() - atomic increment with full ordering
 * @v: pointer to atomic_long_t
 *
 * Atomically updates @v to (@v + 1) with full ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_inc() there.
 *
 * Return: The original value of @v.
 */
static __always_inline long
atomic_long_fetch_inc(atomic_long_t *v)
{
        kcsan_mb();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_long_fetch_inc(v);
}

/**
 * atomic_long_fetch_inc_acquire() - atomic increment with acquire ordering
 * @v: pointer to atomic_long_t
 *
 * Atomically updates @v to (@v + 1) with acquire ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_inc_acquire() there.
 *
 * Return: The original value of @v.
 */
static __always_inline long
atomic_long_fetch_inc_acquire(atomic_long_t *v)
{
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_long_fetch_inc_acquire(v);
}

/**
 * atomic_long_fetch_inc_release() - atomic increment with release ordering
 * @v: pointer to atomic_long_t
 *
 * Atomically updates @v to (@v + 1) with release ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_inc_release() there.
 *
 * Return: The original value of @v.
 */
static __always_inline long
atomic_long_fetch_inc_release(atomic_long_t *v)
{
        kcsan_release();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_long_fetch_inc_release(v);
}

/**
 * atomic_long_fetch_inc_relaxed() - atomic increment with relaxed ordering
 * @v: pointer to atomic_long_t
 *
 * Atomically updates @v to (@v + 1) with relaxed ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_inc_relaxed() there.
 *
 * Return: The original value of @v.
 */
static __always_inline long
atomic_long_fetch_inc_relaxed(atomic_long_t *v)
{
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_long_fetch_inc_relaxed(v);
}

/**
 * atomic_long_dec() - atomic decrement with relaxed ordering
 * @v: pointer to atomic_long_t
 *
 * Atomically updates @v to (@v - 1) with relaxed ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_long_dec() there.
 *
 * Return: Nothing.
 */
static __always_inline void
atomic_long_dec(atomic_long_t *v)
{
        instrument_atomic_read_write(v, sizeof(*v));
        raw_atomic_long_dec(v);
}

/**
 * atomic_long_dec_return() - atomic decrement with full ordering
 * @v: pointer to atomic_long_t
 *
 * Atomically updates @v to (@v - 1) with full ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_long_dec_return() there.
 *
 * Return: The updated value of @v.
 */
static __always_inline long
atomic_long_dec_return(atomic_long_t *v)
{
        kcsan_mb();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_long_dec_return(v);
}

/**
 * atomic_long_dec_return_acquire() - atomic decrement with acquire ordering
 * @v: pointer to atomic_long_t
 *
 * Atomically updates @v to (@v - 1) with acquire ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_long_dec_return_acquire() there.
 *
 * Return: The updated value of @v.
 */
static __always_inline long
atomic_long_dec_return_acquire(atomic_long_t *v)
{
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_long_dec_return_acquire(v);
}

/**
 * atomic_long_dec_return_release() - atomic decrement with release ordering
 * @v: pointer to atomic_long_t
 *
 * Atomically updates @v to (@v - 1) with release ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_long_dec_return_release() there.
 *
 * Return: The updated value of @v.
 */
static __always_inline long
atomic_long_dec_return_release(atomic_long_t *v)
{
        kcsan_release();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_long_dec_return_release(v);
}

/**
 * atomic_long_dec_return_relaxed() - atomic decrement with relaxed ordering
 * @v: pointer to atomic_long_t
 *
 * Atomically updates @v to (@v - 1) with relaxed ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_long_dec_return_relaxed() there.
 *
 * Return: The updated value of @v.
 */
static __always_inline long
atomic_long_dec_return_relaxed(atomic_long_t *v)
{
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_long_dec_return_relaxed(v);
}

/**
 * atomic_long_fetch_dec() - atomic decrement with full ordering
 * @v: pointer to atomic_long_t
 *
 * Atomically updates @v to (@v - 1) with full ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_dec() there.
 *
 * Return: The original value of @v.
 */
static __always_inline long
atomic_long_fetch_dec(atomic_long_t *v)
{
        kcsan_mb();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_long_fetch_dec(v);
}

/**
 * atomic_long_fetch_dec_acquire() - atomic decrement with acquire ordering
 * @v: pointer to atomic_long_t
 *
 * Atomically updates @v to (@v - 1) with acquire ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_dec_acquire() there.
 *
 * Return: The original value of @v.
 */
static __always_inline long
atomic_long_fetch_dec_acquire(atomic_long_t *v)
{
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_long_fetch_dec_acquire(v);
}

/**
 * atomic_long_fetch_dec_release() - atomic decrement with release ordering
 * @v: pointer to atomic_long_t
 *
 * Atomically updates @v to (@v - 1) with release ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_dec_release() there.
 *
 * Return: The original value of @v.
 */
static __always_inline long
atomic_long_fetch_dec_release(atomic_long_t *v)
{
        kcsan_release();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_long_fetch_dec_release(v);
}

/**
 * atomic_long_fetch_dec_relaxed() - atomic decrement with relaxed ordering
 * @v: pointer to atomic_long_t
 *
 * Atomically updates @v to (@v - 1) with relaxed ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_dec_relaxed() there.
 *
 * Return: The original value of @v.
 */
static __always_inline long
atomic_long_fetch_dec_relaxed(atomic_long_t *v)
{
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_long_fetch_dec_relaxed(v);
}

/**
 * atomic_long_and() - atomic bitwise AND with relaxed ordering
 * @i: long value
 * @v: pointer to atomic_long_t
 *
 * Atomically updates @v to (@v & @i) with relaxed ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_long_and() there.
 *
 * Return: Nothing.
 */
static __always_inline void
atomic_long_and(long i, atomic_long_t *v)
{
        instrument_atomic_read_write(v, sizeof(*v));
        raw_atomic_long_and(i, v);
}

/**
 * atomic_long_fetch_and() - atomic bitwise AND with full ordering
 * @i: long value
 * @v: pointer to atomic_long_t
 *
 * Atomically updates @v to (@v & @i) with full ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_and() there.
 *
 * Return: The original value of @v.
 */
static __always_inline long
atomic_long_fetch_and(long i, atomic_long_t *v)
{
        kcsan_mb();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_long_fetch_and(i, v);
}

/**
 * atomic_long_fetch_and_acquire() - atomic bitwise AND with acquire ordering
 * @i: long value
 * @v: pointer to atomic_long_t
 *
 * Atomically updates @v to (@v & @i) with acquire ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_and_acquire() there.
 *
 * Return: The original value of @v.
 */
static __always_inline long
atomic_long_fetch_and_acquire(long i, atomic_long_t *v)
{
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_long_fetch_and_acquire(i, v);
}

/**
 * atomic_long_fetch_and_release() - atomic bitwise AND with release ordering
 * @i: long value
 * @v: pointer to atomic_long_t
 *
 * Atomically updates @v to (@v & @i) with release ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_and_release() there.
 *
 * Return: The original value of @v.
 */
static __always_inline long
atomic_long_fetch_and_release(long i, atomic_long_t *v)
{
        kcsan_release();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_long_fetch_and_release(i, v);
}

/**
 * atomic_long_fetch_and_relaxed() - atomic bitwise AND with relaxed ordering
 * @i: long value
 * @v: pointer to atomic_long_t
 *
 * Atomically updates @v to (@v & @i) with relaxed ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_and_relaxed() there.
 *
 * Return: The original value of @v.
 */
static __always_inline long
atomic_long_fetch_and_relaxed(long i, atomic_long_t *v)
{
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_long_fetch_and_relaxed(i, v);
}

/**
 * atomic_long_andnot() - atomic bitwise AND NOT with relaxed ordering
 * @i: long value
 * @v: pointer to atomic_long_t
 *
 * Atomically updates @v to (@v & ~@i) with relaxed ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_long_andnot() there.
 *
 * Return: Nothing.
 */
static __always_inline void
atomic_long_andnot(long i, atomic_long_t *v)
{
        instrument_atomic_read_write(v, sizeof(*v));
        raw_atomic_long_andnot(i, v);
}

/**
 * atomic_long_fetch_andnot() - atomic bitwise AND NOT with full ordering
 * @i: long value
 * @v: pointer to atomic_long_t
 *
 * Atomically updates @v to (@v & ~@i) with full ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_andnot() there.
 *
 * Return: The original value of @v.
 */
static __always_inline long
atomic_long_fetch_andnot(long i, atomic_long_t *v)
{
        kcsan_mb();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_long_fetch_andnot(i, v);
}

/**
 * atomic_long_fetch_andnot_acquire() - atomic bitwise AND NOT with acquire ordering
 * @i: long value
 * @v: pointer to atomic_long_t
 *
 * Atomically updates @v to (@v & ~@i) with acquire ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_andnot_acquire() there.
 *
 * Return: The original value of @v.
 */
static __always_inline long
atomic_long_fetch_andnot_acquire(long i, atomic_long_t *v)
{
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_long_fetch_andnot_acquire(i, v);
}

/**
 * atomic_long_fetch_andnot_release() - atomic bitwise AND NOT with release ordering
 * @i: long value
 * @v: pointer to atomic_long_t
 *
 * Atomically updates @v to (@v & ~@i) with release ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_andnot_release() there.
 *
 * Return: The original value of @v.
 */
static __always_inline long
atomic_long_fetch_andnot_release(long i, atomic_long_t *v)
{
        kcsan_release();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_long_fetch_andnot_release(i, v);
}

/**
 * atomic_long_fetch_andnot_relaxed() - atomic bitwise AND NOT with relaxed ordering
 * @i: long value
 * @v: pointer to atomic_long_t
 *
 * Atomically updates @v to (@v & ~@i) with relaxed ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_andnot_relaxed() there.
 *
 * Return: The original value of @v.
 */
static __always_inline long
atomic_long_fetch_andnot_relaxed(long i, atomic_long_t *v)
{
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_long_fetch_andnot_relaxed(i, v);
}

/**
 * atomic_long_or() - atomic bitwise OR with relaxed ordering
 * @i: long value
 * @v: pointer to atomic_long_t
 *
 * Atomically updates @v to (@v | @i) with relaxed ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_long_or() there.
 *
 * Return: Nothing.
 */
static __always_inline void
atomic_long_or(long i, atomic_long_t *v)
{
        instrument_atomic_read_write(v, sizeof(*v));
        raw_atomic_long_or(i, v);
}

/**
 * atomic_long_fetch_or() - atomic bitwise OR with full ordering
 * @i: long value
 * @v: pointer to atomic_long_t
 *
 * Atomically updates @v to (@v | @i) with full ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_or() there.
 *
 * Return: The original value of @v.
 */
static __always_inline long
atomic_long_fetch_or(long i, atomic_long_t *v)
{
        kcsan_mb();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_long_fetch_or(i, v);
}

/**
 * atomic_long_fetch_or_acquire() - atomic bitwise OR with acquire ordering
 * @i: long value
 * @v: pointer to atomic_long_t
 *
 * Atomically updates @v to (@v | @i) with acquire ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_or_acquire() there.
 *
 * Return: The original value of @v.
 */
static __always_inline long
atomic_long_fetch_or_acquire(long i, atomic_long_t *v)
{
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_long_fetch_or_acquire(i, v);
}

/**
 * atomic_long_fetch_or_release() - atomic bitwise OR with release ordering
 * @i: long value
 * @v: pointer to atomic_long_t
 *
 * Atomically updates @v to (@v | @i) with release ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_or_release() there.
 *
 * Return: The original value of @v.
 */
static __always_inline long
atomic_long_fetch_or_release(long i, atomic_long_t *v)
{
        kcsan_release();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_long_fetch_or_release(i, v);
}

/**
 * atomic_long_fetch_or_relaxed() - atomic bitwise OR with relaxed ordering
 * @i: long value
 * @v: pointer to atomic_long_t
 *
 * Atomically updates @v to (@v | @i) with relaxed ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_or_relaxed() there.
 *
 * Return: The original value of @v.
 */
static __always_inline long
atomic_long_fetch_or_relaxed(long i, atomic_long_t *v)
{
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_long_fetch_or_relaxed(i, v);
}

/**
 * atomic_long_xor() - atomic bitwise XOR with relaxed ordering
 * @i: long value
 * @v: pointer to atomic_long_t
 *
 * Atomically updates @v to (@v ^ @i) with relaxed ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_long_xor() there.
 *
 * Return: Nothing.
 */
static __always_inline void
atomic_long_xor(long i, atomic_long_t *v)
{
        instrument_atomic_read_write(v, sizeof(*v));
        raw_atomic_long_xor(i, v);
}

/**
 * atomic_long_fetch_xor() - atomic bitwise XOR with full ordering
 * @i: long value
 * @v: pointer to atomic_long_t
 *
 * Atomically updates @v to (@v ^ @i) with full ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_xor() there.
 *
 * Return: The original value of @v.
 */
static __always_inline long
atomic_long_fetch_xor(long i, atomic_long_t *v)
{
        kcsan_mb();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_long_fetch_xor(i, v);
}

/**
 * atomic_long_fetch_xor_acquire() - atomic bitwise XOR with acquire ordering
 * @i: long value
 * @v: pointer to atomic_long_t
 *
 * Atomically updates @v to (@v ^ @i) with acquire ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_xor_acquire() there.
 *
 * Return: The original value of @v.
 */
static __always_inline long
atomic_long_fetch_xor_acquire(long i, atomic_long_t *v)
{
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_long_fetch_xor_acquire(i, v);
}

/**
 * atomic_long_fetch_xor_release() - atomic bitwise XOR with release ordering
 * @i: long value
 * @v: pointer to atomic_long_t
 *
 * Atomically updates @v to (@v ^ @i) with release ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_xor_release() there.
 *
 * Return: The original value of @v.
 */
static __always_inline long
atomic_long_fetch_xor_release(long i, atomic_long_t *v)
{
        kcsan_release();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_long_fetch_xor_release(i, v);
}

/**
 * atomic_long_fetch_xor_relaxed() - atomic bitwise XOR with relaxed ordering
 * @i: long value
 * @v: pointer to atomic_long_t
 *
 * Atomically updates @v to (@v ^ @i) with relaxed ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_xor_relaxed() there.
 *
 * Return: The original value of @v.
 */
static __always_inline long
atomic_long_fetch_xor_relaxed(long i, atomic_long_t *v)
{
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_long_fetch_xor_relaxed(i, v);
}

/**
 * atomic_long_xchg() - atomic exchange with full ordering
 * @v: pointer to atomic_long_t
 * @new: long value to assign
 *
 * Atomically updates @v to @new with full ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_long_xchg() there.
 *
 * Return: The original value of @v.
 */
static __always_inline long
atomic_long_xchg(atomic_long_t *v, long new)
{
        kcsan_mb();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_long_xchg(v, new);
}

/**
 * atomic_long_xchg_acquire() - atomic exchange with acquire ordering
 * @v: pointer to atomic_long_t
 * @new: long value to assign
 *
 * Atomically updates @v to @new with acquire ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_long_xchg_acquire() there.
 *
 * Return: The original value of @v.
 */
static __always_inline long
atomic_long_xchg_acquire(atomic_long_t *v, long new)
{
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_long_xchg_acquire(v, new);
}

/**
 * atomic_long_xchg_release() - atomic exchange with release ordering
 * @v: pointer to atomic_long_t
 * @new: long value to assign
 *
 * Atomically updates @v to @new with release ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_long_xchg_release() there.
 *
 * Return: The original value of @v.
 */
static __always_inline long
atomic_long_xchg_release(atomic_long_t *v, long new)
{
        kcsan_release();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_long_xchg_release(v, new);
}

/**
 * atomic_long_xchg_relaxed() - atomic exchange with relaxed ordering
 * @v: pointer to atomic_long_t
 * @new: long value to assign
 *
 * Atomically updates @v to @new with relaxed ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_long_xchg_relaxed() there.
 *
 * Return: The original value of @v.
 */
static __always_inline long
atomic_long_xchg_relaxed(atomic_long_t *v, long new)
{
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_long_xchg_relaxed(v, new);
}

/**
 * atomic_long_cmpxchg() - atomic compare and exchange with full ordering
 * @v: pointer to atomic_long_t
 * @old: long value to compare with
 * @new: long value to assign
 *
 * If (@v == @old), atomically updates @v to @new with full ordering.
 * Otherwise, @v is not modified and relaxed ordering is provided.
 *
 * Unsafe to use in noinstr code; use raw_atomic_long_cmpxchg() there.
 *
 * Return: The original value of @v.
 */
static __always_inline long
atomic_long_cmpxchg(atomic_long_t *v, long old, long new)
{
        kcsan_mb();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_long_cmpxchg(v, old, new);
}

/**
 * atomic_long_cmpxchg_acquire() - atomic compare and exchange with acquire ordering
 * @v: pointer to atomic_long_t
 * @old: long value to compare with
 * @new: long value to assign
 *
 * If (@v == @old), atomically updates @v to @new with acquire ordering.
 * Otherwise, @v is not modified and relaxed ordering is provided.
 *
 * Unsafe to use in noinstr code; use raw_atomic_long_cmpxchg_acquire() there.
 *
 * Return: The original value of @v.
 */
static __always_inline long
atomic_long_cmpxchg_acquire(atomic_long_t *v, long old, long new)
{
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_long_cmpxchg_acquire(v, old, new);
}

/**
 * atomic_long_cmpxchg_release() - atomic compare and exchange with release ordering
 * @v: pointer to atomic_long_t
 * @old: long value to compare with
 * @new: long value to assign
 *
 * If (@v == @old), atomically updates @v to @new with release ordering.
 * Otherwise, @v is not modified and relaxed ordering is provided.
 *
 * Unsafe to use in noinstr code; use raw_atomic_long_cmpxchg_release() there.
 *
 * Return: The original value of @v.
 */
static __always_inline long
atomic_long_cmpxchg_release(atomic_long_t *v, long old, long new)
{
        kcsan_release();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_long_cmpxchg_release(v, old, new);
}

/**
 * atomic_long_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering
 * @v: pointer to atomic_long_t
 * @old: long value to compare with
 * @new: long value to assign
 *
 * If (@v == @old), atomically updates @v to @new with relaxed ordering.
 * Otherwise, @v is not modified and relaxed ordering is provided.
 *
 * Unsafe to use in noinstr code; use raw_atomic_long_cmpxchg_relaxed() there.
 *
 * Return: The original value of @v.
 */
static __always_inline long
atomic_long_cmpxchg_relaxed(atomic_long_t *v, long old, long new)
{
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_long_cmpxchg_relaxed(v, old, new);
}

/**
 * atomic_long_try_cmpxchg() - atomic compare and exchange with full ordering
 * @v: pointer to atomic_long_t
 * @old: pointer to long value to compare with
 * @new: long value to assign
 *
 * If (@v == @old), atomically updates @v to @new with full ordering.
 * Otherwise, @v is not modified, @old is updated to the current value of @v,
 * and relaxed ordering is provided.
 *
 * Unsafe to use in noinstr code; use raw_atomic_long_try_cmpxchg() there.
 *
 * Return: @true if the exchange occured, @false otherwise.
 */
static __always_inline bool
atomic_long_try_cmpxchg(atomic_long_t *v, long *old, long new)
{
        kcsan_mb();
        instrument_atomic_read_write(v, sizeof(*v));
        instrument_atomic_read_write(old, sizeof(*old));
        return raw_atomic_long_try_cmpxchg(v, old, new);
}

/**
 * atomic_long_try_cmpxchg_acquire() - atomic compare and exchange with acquire ordering
 * @v: pointer to atomic_long_t
 * @old: pointer to long value to compare with
 * @new: long value to assign
 *
 * If (@v == @old), atomically updates @v to @new with acquire ordering.
 * Otherwise, @v is not modified, @old is updated to the current value of @v,
 * and relaxed ordering is provided.
 *
 * Unsafe to use in noinstr code; use raw_atomic_long_try_cmpxchg_acquire() there.
 *
 * Return: @true if the exchange occured, @false otherwise.
 */
static __always_inline bool
atomic_long_try_cmpxchg_acquire(atomic_long_t *v, long *old, long new)
{
        instrument_atomic_read_write(v, sizeof(*v));
        instrument_atomic_read_write(old, sizeof(*old));
        return raw_atomic_long_try_cmpxchg_acquire(v, old, new);
}

/**
 * atomic_long_try_cmpxchg_release() - atomic compare and exchange with release ordering
 * @v: pointer to atomic_long_t
 * @old: pointer to long value to compare with
 * @new: long value to assign
 *
 * If (@v == @old), atomically updates @v to @new with release ordering.
 * Otherwise, @v is not modified, @old is updated to the current value of @v,
 * and relaxed ordering is provided.
 *
 * Unsafe to use in noinstr code; use raw_atomic_long_try_cmpxchg_release() there.
 *
 * Return: @true if the exchange occured, @false otherwise.
 */
static __always_inline bool
atomic_long_try_cmpxchg_release(atomic_long_t *v, long *old, long new)
{
        kcsan_release();
        instrument_atomic_read_write(v, sizeof(*v));
        instrument_atomic_read_write(old, sizeof(*old));
        return raw_atomic_long_try_cmpxchg_release(v, old, new);
}

/**
 * atomic_long_try_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering
 * @v: pointer to atomic_long_t
 * @old: pointer to long value to compare with
 * @new: long value to assign
 *
 * If (@v == @old), atomically updates @v to @new with relaxed ordering.
 * Otherwise, @v is not modified, @old is updated to the current value of @v,
 * and relaxed ordering is provided.
 *
 * Unsafe to use in noinstr code; use raw_atomic_long_try_cmpxchg_relaxed() there.
 *
 * Return: @true if the exchange occured, @false otherwise.
 */
static __always_inline bool
atomic_long_try_cmpxchg_relaxed(atomic_long_t *v, long *old, long new)
{
        instrument_atomic_read_write(v, sizeof(*v));
        instrument_atomic_read_write(old, sizeof(*old));
        return raw_atomic_long_try_cmpxchg_relaxed(v, old, new);
}

/**
 * atomic_long_sub_and_test() - atomic subtract and test if zero with full ordering
 * @i: long value to subtract
 * @v: pointer to atomic_long_t
 *
 * Atomically updates @v to (@v - @i) with full ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_long_sub_and_test() there.
 *
 * Return: @true if the resulting value of @v is zero, @false otherwise.
 */
static __always_inline bool
atomic_long_sub_and_test(long i, atomic_long_t *v)
{
        kcsan_mb();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_long_sub_and_test(i, v);
}

/**
 * atomic_long_dec_and_test() - atomic decrement and test if zero with full ordering
 * @v: pointer to atomic_long_t
 *
 * Atomically updates @v to (@v - 1) with full ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_long_dec_and_test() there.
 *
 * Return: @true if the resulting value of @v is zero, @false otherwise.
 */
static __always_inline bool
atomic_long_dec_and_test(atomic_long_t *v)
{
        kcsan_mb();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_long_dec_and_test(v);
}

/**
 * atomic_long_inc_and_test() - atomic increment and test if zero with full ordering
 * @v: pointer to atomic_long_t
 *
 * Atomically updates @v to (@v + 1) with full ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_long_inc_and_test() there.
 *
 * Return: @true if the resulting value of @v is zero, @false otherwise.
 */
static __always_inline bool
atomic_long_inc_and_test(atomic_long_t *v)
{
        kcsan_mb();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_long_inc_and_test(v);
}

/**
 * atomic_long_add_negative() - atomic add and test if negative with full ordering
 * @i: long value to add
 * @v: pointer to atomic_long_t
 *
 * Atomically updates @v to (@v + @i) with full ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_long_add_negative() there.
 *
 * Return: @true if the resulting value of @v is negative, @false otherwise.
 */
static __always_inline bool
atomic_long_add_negative(long i, atomic_long_t *v)
{
        kcsan_mb();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_long_add_negative(i, v);
}

/**
 * atomic_long_add_negative_acquire() - atomic add and test if negative with acquire ordering
 * @i: long value to add
 * @v: pointer to atomic_long_t
 *
 * Atomically updates @v to (@v + @i) with acquire ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_long_add_negative_acquire() there.
 *
 * Return: @true if the resulting value of @v is negative, @false otherwise.
 */
static __always_inline bool
atomic_long_add_negative_acquire(long i, atomic_long_t *v)
{
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_long_add_negative_acquire(i, v);
}

/**
 * atomic_long_add_negative_release() - atomic add and test if negative with release ordering
 * @i: long value to add
 * @v: pointer to atomic_long_t
 *
 * Atomically updates @v to (@v + @i) with release ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_long_add_negative_release() there.
 *
 * Return: @true if the resulting value of @v is negative, @false otherwise.
 */
static __always_inline bool
atomic_long_add_negative_release(long i, atomic_long_t *v)
{
        kcsan_release();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_long_add_negative_release(i, v);
}

/**
 * atomic_long_add_negative_relaxed() - atomic add and test if negative with relaxed ordering
 * @i: long value to add
 * @v: pointer to atomic_long_t
 *
 * Atomically updates @v to (@v + @i) with relaxed ordering.
 *
 * Unsafe to use in noinstr code; use raw_atomic_long_add_negative_relaxed() there.
 *
 * Return: @true if the resulting value of @v is negative, @false otherwise.
 */
static __always_inline bool
atomic_long_add_negative_relaxed(long i, atomic_long_t *v)
{
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_long_add_negative_relaxed(i, v);
}

/**
 * atomic_long_fetch_add_unless() - atomic add unless value with full ordering
 * @v: pointer to atomic_long_t
 * @a: long value to add
 * @u: long value to compare with
 *
 * If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
 * Otherwise, @v is not modified and relaxed ordering is provided.
 *
 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_add_unless() there.
 *
 * Return: The original value of @v.
 */
static __always_inline long
atomic_long_fetch_add_unless(atomic_long_t *v, long a, long u)
{
        kcsan_mb();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_long_fetch_add_unless(v, a, u);
}

/**
 * atomic_long_add_unless() - atomic add unless value with full ordering
 * @v: pointer to atomic_long_t
 * @a: long value to add
 * @u: long value to compare with
 *
 * If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
 * Otherwise, @v is not modified and relaxed ordering is provided.
 *
 * Unsafe to use in noinstr code; use raw_atomic_long_add_unless() there.
 *
 * Return: @true if @v was updated, @false otherwise.
 */
static __always_inline bool
atomic_long_add_unless(atomic_long_t *v, long a, long u)
{
        kcsan_mb();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_long_add_unless(v, a, u);
}

/**
 * atomic_long_inc_not_zero() - atomic increment unless zero with full ordering
 * @v: pointer to atomic_long_t
 *
 * If (@v != 0), atomically updates @v to (@v + 1) with full ordering.
 * Otherwise, @v is not modified and relaxed ordering is provided.
 *
 * Unsafe to use in noinstr code; use raw_atomic_long_inc_not_zero() there.
 *
 * Return: @true if @v was updated, @false otherwise.
 */
static __always_inline bool
atomic_long_inc_not_zero(atomic_long_t *v)
{
        kcsan_mb();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_long_inc_not_zero(v);
}

/**
 * atomic_long_inc_unless_negative() - atomic increment unless negative with full ordering
 * @v: pointer to atomic_long_t
 *
 * If (@v >= 0), atomically updates @v to (@v + 1) with full ordering.
 * Otherwise, @v is not modified and relaxed ordering is provided.
 *
 * Unsafe to use in noinstr code; use raw_atomic_long_inc_unless_negative() there.
 *
 * Return: @true if @v was updated, @false otherwise.
 */
static __always_inline bool
atomic_long_inc_unless_negative(atomic_long_t *v)
{
        kcsan_mb();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_long_inc_unless_negative(v);
}

/**
 * atomic_long_dec_unless_positive() - atomic decrement unless positive with full ordering
 * @v: pointer to atomic_long_t
 *
 * If (@v <= 0), atomically updates @v to (@v - 1) with full ordering.
 * Otherwise, @v is not modified and relaxed ordering is provided.
 *
 * Unsafe to use in noinstr code; use raw_atomic_long_dec_unless_positive() there.
 *
 * Return: @true if @v was updated, @false otherwise.
 */
static __always_inline bool
atomic_long_dec_unless_positive(atomic_long_t *v)
{
        kcsan_mb();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_long_dec_unless_positive(v);
}

/**
 * atomic_long_dec_if_positive() - atomic decrement if positive with full ordering
 * @v: pointer to atomic_long_t
 *
 * If (@v > 0), atomically updates @v to (@v - 1) with full ordering.
 * Otherwise, @v is not modified and relaxed ordering is provided.
 *
 * Unsafe to use in noinstr code; use raw_atomic_long_dec_if_positive() there.
 *
 * Return: The old value of (@v - 1), regardless of whether @v was updated.
 */
static __always_inline long
atomic_long_dec_if_positive(atomic_long_t *v)
{
        kcsan_mb();
        instrument_atomic_read_write(v, sizeof(*v));
        return raw_atomic_long_dec_if_positive(v);
}

#define xchg(ptr, ...) \
({ \
        typeof(ptr) __ai_ptr = (ptr); \
        kcsan_mb(); \
        instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
        raw_xchg(__ai_ptr, __VA_ARGS__); \
})

#define xchg_acquire(ptr, ...) \
({ \
        typeof(ptr) __ai_ptr = (ptr); \
        instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
        raw_xchg_acquire(__ai_ptr, __VA_ARGS__); \
})

#define xchg_release(ptr, ...) \
({ \
        typeof(ptr) __ai_ptr = (ptr); \
        kcsan_release(); \
        instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
        raw_xchg_release(__ai_ptr, __VA_ARGS__); \
})

#define xchg_relaxed(ptr, ...) \
({ \
        typeof(ptr) __ai_ptr = (ptr); \
        instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
        raw_xchg_relaxed(__ai_ptr, __VA_ARGS__); \
})

#define cmpxchg(ptr, ...) \
({ \
        typeof(ptr) __ai_ptr = (ptr); \
        kcsan_mb(); \
        instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
        raw_cmpxchg(__ai_ptr, __VA_ARGS__); \
})

#define cmpxchg_acquire(ptr, ...) \
({ \
        typeof(ptr) __ai_ptr = (ptr); \
        instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
        raw_cmpxchg_acquire(__ai_ptr, __VA_ARGS__); \
})

#define cmpxchg_release(ptr, ...) \
({ \
        typeof(ptr) __ai_ptr = (ptr); \
        kcsan_release(); \
        instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
        raw_cmpxchg_release(__ai_ptr, __VA_ARGS__); \
})

#define cmpxchg_relaxed(ptr, ...) \
({ \
        typeof(ptr) __ai_ptr = (ptr); \
        instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
        raw_cmpxchg_relaxed(__ai_ptr, __VA_ARGS__); \
})

#define cmpxchg64(ptr, ...) \
({ \
        typeof(ptr) __ai_ptr = (ptr); \
        kcsan_mb(); \
        instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
        raw_cmpxchg64(__ai_ptr, __VA_ARGS__); \
})

#define cmpxchg64_acquire(ptr, ...) \
({ \
        typeof(ptr) __ai_ptr = (ptr); \
        instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
        raw_cmpxchg64_acquire(__ai_ptr, __VA_ARGS__); \
})

#define cmpxchg64_release(ptr, ...) \
({ \
        typeof(ptr) __ai_ptr = (ptr); \
        kcsan_release(); \
        instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
        raw_cmpxchg64_release(__ai_ptr, __VA_ARGS__); \
})

#define cmpxchg64_relaxed(ptr, ...) \
({ \
        typeof(ptr) __ai_ptr = (ptr); \
        instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
        raw_cmpxchg64_relaxed(__ai_ptr, __VA_ARGS__); \
})

#define cmpxchg128(ptr, ...) \
({ \
        typeof(ptr) __ai_ptr = (ptr); \
        kcsan_mb(); \
        instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
        raw_cmpxchg128(__ai_ptr, __VA_ARGS__); \
})

#define cmpxchg128_acquire(ptr, ...) \
({ \
        typeof(ptr) __ai_ptr = (ptr); \
        instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
        raw_cmpxchg128_acquire(__ai_ptr, __VA_ARGS__); \
})

#define cmpxchg128_release(ptr, ...) \
({ \
        typeof(ptr) __ai_ptr = (ptr); \
        kcsan_release(); \
        instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
        raw_cmpxchg128_release(__ai_ptr, __VA_ARGS__); \
})

#define cmpxchg128_relaxed(ptr, ...) \
({ \
        typeof(ptr) __ai_ptr = (ptr); \
        instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
        raw_cmpxchg128_relaxed(__ai_ptr, __VA_ARGS__); \
})

#define try_cmpxchg(ptr, oldp, ...) \
({ \
        typeof(ptr) __ai_ptr = (ptr); \
        typeof(oldp) __ai_oldp = (oldp); \
        kcsan_mb(); \
        instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
        instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
        raw_try_cmpxchg(__ai_ptr, __ai_oldp, __VA_ARGS__); \
})

#define try_cmpxchg_acquire(ptr, oldp, ...) \
({ \
        typeof(ptr) __ai_ptr = (ptr); \
        typeof(oldp) __ai_oldp = (oldp); \
        instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
        instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
        raw_try_cmpxchg_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \
})

#define try_cmpxchg_release(ptr, oldp, ...) \
({ \
        typeof(ptr) __ai_ptr = (ptr); \
        typeof(oldp) __ai_oldp = (oldp); \
        kcsan_release(); \
        instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
        instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
        raw_try_cmpxchg_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \
})

#define try_cmpxchg_relaxed(ptr, oldp, ...) \
({ \
        typeof(ptr) __ai_ptr = (ptr); \
        typeof(oldp) __ai_oldp = (oldp); \
        instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
        instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
        raw_try_cmpxchg_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \
})

#define try_cmpxchg64(ptr, oldp, ...) \
({ \
        typeof(ptr) __ai_ptr = (ptr); \
        typeof(oldp) __ai_oldp = (oldp); \
        kcsan_mb(); \
        instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
        instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
        raw_try_cmpxchg64(__ai_ptr, __ai_oldp, __VA_ARGS__); \
})

#define try_cmpxchg64_acquire(ptr, oldp, ...) \
({ \
        typeof(ptr) __ai_ptr = (ptr); \
        typeof(oldp) __ai_oldp = (oldp); \
        instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
        instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
        raw_try_cmpxchg64_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \
})

#define try_cmpxchg64_release(ptr, oldp, ...) \
({ \
        typeof(ptr) __ai_ptr = (ptr); \
        typeof(oldp) __ai_oldp = (oldp); \
        kcsan_release(); \
        instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
        instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
        raw_try_cmpxchg64_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \
})

#define try_cmpxchg64_relaxed(ptr, oldp, ...) \
({ \
        typeof(ptr) __ai_ptr = (ptr); \
        typeof(oldp) __ai_oldp = (oldp); \
        instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
        instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
        raw_try_cmpxchg64_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \
})

#define try_cmpxchg128(ptr, oldp, ...) \
({ \
        typeof(ptr) __ai_ptr = (ptr); \
        typeof(oldp) __ai_oldp = (oldp); \
        kcsan_mb(); \
        instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
        instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
        raw_try_cmpxchg128(__ai_ptr, __ai_oldp, __VA_ARGS__); \
})

#define try_cmpxchg128_acquire(ptr, oldp, ...) \
({ \
        typeof(ptr) __ai_ptr = (ptr); \
        typeof(oldp) __ai_oldp = (oldp); \
        instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
        instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
        raw_try_cmpxchg128_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \
})

#define try_cmpxchg128_release(ptr, oldp, ...) \
({ \
        typeof(ptr) __ai_ptr = (ptr); \
        typeof(oldp) __ai_oldp = (oldp); \
        kcsan_release(); \
        instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
        instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
        raw_try_cmpxchg128_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \
})

#define try_cmpxchg128_relaxed(ptr, oldp, ...) \
({ \
        typeof(ptr) __ai_ptr = (ptr); \
        typeof(oldp) __ai_oldp = (oldp); \
        instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
        instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
        raw_try_cmpxchg128_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \
})

#define cmpxchg_local(ptr, ...) \
({ \
        typeof(ptr) __ai_ptr = (ptr); \
        instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
        raw_cmpxchg_local(__ai_ptr, __VA_ARGS__); \
})

#define cmpxchg64_local(ptr, ...) \
({ \
        typeof(ptr) __ai_ptr = (ptr); \
        instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
        raw_cmpxchg64_local(__ai_ptr, __VA_ARGS__); \
})

#define cmpxchg128_local(ptr, ...) \
({ \
        typeof(ptr) __ai_ptr = (ptr); \
        instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
        raw_cmpxchg128_local(__ai_ptr, __VA_ARGS__); \
})

#define sync_cmpxchg(ptr, ...) \
({ \
        typeof(ptr) __ai_ptr = (ptr); \
        kcsan_mb(); \
        instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
        raw_sync_cmpxchg(__ai_ptr, __VA_ARGS__); \
})

#define try_cmpxchg_local(ptr, oldp, ...) \
({ \
        typeof(ptr) __ai_ptr = (ptr); \
        typeof(oldp) __ai_oldp = (oldp); \
        instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
        instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
        raw_try_cmpxchg_local(__ai_ptr, __ai_oldp, __VA_ARGS__); \
})

#define try_cmpxchg64_local(ptr, oldp, ...) \
({ \
        typeof(ptr) __ai_ptr = (ptr); \
        typeof(oldp) __ai_oldp = (oldp); \
        instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
        instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
        raw_try_cmpxchg64_local(__ai_ptr, __ai_oldp, __VA_ARGS__); \
})

#define try_cmpxchg128_local(ptr, oldp, ...) \
({ \
        typeof(ptr) __ai_ptr = (ptr); \
        typeof(oldp) __ai_oldp = (oldp); \
        instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
        instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
        raw_try_cmpxchg128_local(__ai_ptr, __ai_oldp, __VA_ARGS__); \
})

#define sync_try_cmpxchg(ptr, ...) \
({ \
        typeof(ptr) __ai_ptr = (ptr); \
        kcsan_mb(); \
        instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
        raw_sync_try_cmpxchg(__ai_ptr, __VA_ARGS__); \
})


#endif /* _LINUX_ATOMIC_INSTRUMENTED_H */
// 8829b337928e9508259079d32581775ececd415b
































































    1 






    1 






    1 



































































































































































    1 













    1 













    1 





















































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
 * Copyright (c) International Business Machines Corp., 2006
 *
 * Author: Artem Bityutskiy (Битюцкий Артём)
 */

#ifndef __UBI_DEBUG_H__
#define __UBI_DEBUG_H__

void ubi_dump_flash(struct ubi_device *ubi, int pnum, int offset, int len);
void ubi_dump_ec_hdr(const struct ubi_ec_hdr *ec_hdr);
void ubi_dump_vid_hdr(const struct ubi_vid_hdr *vid_hdr);

#include <linux/random.h>

#define ubi_assert(expr)  do {                                               \
        if (unlikely(!(expr))) {                                             \
                pr_crit("UBI assert failed in %s at %u (pid %d)\n",          \
                       __func__, __LINE__, current->pid);                    \
                dump_stack();                                                \
        }                                                                    \
} while (0)

#define ubi_dbg_print_hex_dump(l, ps, pt, r, g, b, len, a)                   \
                print_hex_dump(l, ps, pt, r, g, b, len, a)

#define ubi_dbg_msg(type, fmt, ...) \
        pr_debug("UBI DBG " type " (pid %d): " fmt "\n", current->pid,       \
                 ##__VA_ARGS__)

/* General debugging messages */
#define dbg_gen(fmt, ...) ubi_dbg_msg("gen", fmt, ##__VA_ARGS__)
/* Messages from the eraseblock association sub-system */
#define dbg_eba(fmt, ...) ubi_dbg_msg("eba", fmt, ##__VA_ARGS__)
/* Messages from the wear-leveling sub-system */
#define dbg_wl(fmt, ...)  ubi_dbg_msg("wl", fmt, ##__VA_ARGS__)
/* Messages from the input/output sub-system */
#define dbg_io(fmt, ...)  ubi_dbg_msg("io", fmt, ##__VA_ARGS__)
/* Initialization and build messages */
#define dbg_bld(fmt, ...) ubi_dbg_msg("bld", fmt, ##__VA_ARGS__)

void ubi_dump_vol_info(const struct ubi_volume *vol);
void ubi_dump_vtbl_record(const struct ubi_vtbl_record *r, int idx);
void ubi_dump_av(const struct ubi_ainf_volume *av);
void ubi_dump_aeb(const struct ubi_ainf_peb *aeb, int type);
void ubi_dump_mkvol_req(const struct ubi_mkvol_req *req);
int ubi_self_check_all_ff(struct ubi_device *ubi, int pnum, int offset,
                          int len);
int ubi_debugfs_init(void);
void ubi_debugfs_exit(void);
int ubi_debugfs_init_dev(struct ubi_device *ubi);
void ubi_debugfs_exit_dev(struct ubi_device *ubi);

/**
 * The following function is a legacy implementation of UBI fault-injection
 * hook. When using more powerful fault injection capabilities, the legacy
 * fault injection interface should be retained.
 */
int ubi_dbg_power_cut(struct ubi_device *ubi, int caller);

static inline int ubi_dbg_bitflip(const struct ubi_device *ubi)
{
        if (ubi->dbg.emulate_bitflips)
                return !get_random_u32_below(200);
        return 0;
}

static inline int ubi_dbg_write_failure(const struct ubi_device *ubi)
{
        if (ubi->dbg.emulate_io_failures)
                return !get_random_u32_below(500);
        return 0;
}

static inline int ubi_dbg_erase_failure(const struct ubi_device *ubi)
{
        if (ubi->dbg.emulate_io_failures)
                return !get_random_u32_below(400);
        return 0;
}

/**
 * MASK_XXX: Mask for emulate_failures in ubi_debug_info.The mask is used to
 * precisely control the type and process of fault injection.
 */
/* Emulate a power cut when writing EC/VID header */
#define MASK_POWER_CUT_EC                        (1 << 0)
#define MASK_POWER_CUT_VID                        (1 << 1)
/* Emulate a power cut when writing data*/
#define MASK_POWER_CUT_DATA                        (1 << 2)
/* Emulate bit-flips */
#define MASK_BITFLIPS                                (1 << 3)
/* Emulate ecc error */
#define MASK_ECCERR                                (1 << 4)
/* Emulates -EIO during data read */
#define MASK_READ_FAILURE                        (1 << 5)
#define MASK_READ_FAILURE_EC                        (1 << 6)
#define MASK_READ_FAILURE_VID                        (1 << 7)
/* Emulates -EIO during data write */
#define MASK_WRITE_FAILURE                        (1 << 8)
/* Emulates -EIO during erase a PEB*/
#define MASK_ERASE_FAILURE                        (1 << 9)
/* Return UBI_IO_FF when reading EC/VID header */
#define MASK_IO_FF_EC                                (1 << 10)
#define MASK_IO_FF_VID                                (1 << 11)
/* Return UBI_IO_FF_BITFLIPS when reading EC/VID header */
#define MASK_IO_FF_BITFLIPS_EC                        (1 << 12)
#define MASK_IO_FF_BITFLIPS_VID                        (1 << 13)
/* Return UBI_IO_BAD_HDR when reading EC/VID header */
#define MASK_BAD_HDR_EC                                (1 << 14)
#define MASK_BAD_HDR_VID                        (1 << 15)
/* Return UBI_IO_BAD_HDR_EBADMSG when reading EC/VID header */
#define MASK_BAD_HDR_EBADMSG_EC                        (1 << 16)
#define MASK_BAD_HDR_EBADMSG_VID                (1 << 17)

#ifdef CONFIG_MTD_UBI_FAULT_INJECTION

extern bool should_fail_eccerr(void);
extern bool should_fail_bitflips(void);
extern bool should_fail_read_failure(void);
extern bool should_fail_write_failure(void);
extern bool should_fail_erase_failure(void);
extern bool should_fail_power_cut(void);
extern bool should_fail_io_ff(void);
extern bool should_fail_io_ff_bitflips(void);
extern bool should_fail_bad_hdr(void);
extern bool should_fail_bad_hdr_ebadmsg(void);

static inline bool ubi_dbg_fail_bitflip(const struct ubi_device *ubi)
{
        if (ubi->dbg.emulate_failures & MASK_BITFLIPS)
                return should_fail_bitflips();
        return false;
}

static inline bool ubi_dbg_fail_write(const struct ubi_device *ubi)
{
        if (ubi->dbg.emulate_failures & MASK_WRITE_FAILURE)
                return should_fail_write_failure();
        return false;
}

static inline bool ubi_dbg_fail_erase(const struct ubi_device *ubi)
{
        if (ubi->dbg.emulate_failures & MASK_ERASE_FAILURE)
                return should_fail_erase_failure();
        return false;
}

static inline bool ubi_dbg_fail_power_cut(const struct ubi_device *ubi,
                                          unsigned int caller)
{
        if (ubi->dbg.emulate_failures & caller)
                return should_fail_power_cut();
        return false;
}

static inline bool ubi_dbg_fail_read(const struct ubi_device *ubi,
                                     unsigned int caller)
{
        if (ubi->dbg.emulate_failures & caller)
                return should_fail_read_failure();
        return false;
}

static inline bool ubi_dbg_fail_eccerr(const struct ubi_device *ubi)
{
        if (ubi->dbg.emulate_failures & MASK_ECCERR)
                return should_fail_eccerr();
        return false;
}

static inline bool ubi_dbg_fail_ff(const struct ubi_device *ubi,
                                   unsigned int caller)
{
        if (ubi->dbg.emulate_failures & caller)
                return should_fail_io_ff();
        return false;
}

static inline bool ubi_dbg_fail_ff_bitflips(const struct ubi_device *ubi,
                                            unsigned int caller)
{
        if (ubi->dbg.emulate_failures & caller)
                return should_fail_io_ff_bitflips();
        return false;
}

static inline bool ubi_dbg_fail_bad_hdr(const struct ubi_device *ubi,
                                         unsigned int caller)
{
        if (ubi->dbg.emulate_failures & caller)
                return should_fail_bad_hdr();
        return false;
}

static inline bool ubi_dbg_fail_bad_hdr_ebadmsg(const struct ubi_device *ubi,
                                                 unsigned int caller)
{
        if (ubi->dbg.emulate_failures & caller)
                return should_fail_bad_hdr_ebadmsg();
        return false;
}
#else /* CONFIG_MTD_UBI_FAULT_INJECTION */

#define ubi_dbg_fail_bitflip(u)             false
#define ubi_dbg_fail_write(u)               false
#define ubi_dbg_fail_erase(u)               false
#define ubi_dbg_fail_power_cut(u, c)        false
#define ubi_dbg_fail_read(u, c)             false
#define ubi_dbg_fail_eccerr(u)              false
#define ubi_dbg_fail_ff(u, c)               false
#define ubi_dbg_fail_ff_bitflips(u, v)      false
#define ubi_dbg_fail_bad_hdr(u, c)          false
#define ubi_dbg_fail_bad_hdr_ebadmsg(u, c)  false

#endif

/**
 * ubi_dbg_is_power_cut - if it is time to emulate power cut.
 * @ubi: UBI device description object
 *
 * Returns true if power cut should be emulated, otherwise returns false.
 */
static inline bool ubi_dbg_is_power_cut(struct ubi_device *ubi,
                                        unsigned int caller)
{
        if (ubi_dbg_power_cut(ubi, caller))
                return true;
        return ubi_dbg_fail_power_cut(ubi, caller);
}

/**
 * ubi_dbg_is_bitflip - if it is time to emulate a bit-flip.
 * @ubi: UBI device description object
 *
 * Returns true if a bit-flip should be emulated, otherwise returns false.
 */
static inline bool ubi_dbg_is_bitflip(const struct ubi_device *ubi)
{
        if (ubi_dbg_bitflip(ubi))
                return true;
        return ubi_dbg_fail_bitflip(ubi);
}

/**
 * ubi_dbg_is_write_failure - if it is time to emulate a write failure.
 * @ubi: UBI device description object
 *
 * Returns true if a write failure should be emulated, otherwise returns
 * false.
 */
static inline bool ubi_dbg_is_write_failure(const struct ubi_device *ubi)
{
        if (ubi_dbg_write_failure(ubi))
                return true;
        return ubi_dbg_fail_write(ubi);
}

/**
 * ubi_dbg_is_erase_failure - if its time to emulate an erase failure.
 * @ubi: UBI device description object
 *
 * Returns true if an erase failure should be emulated, otherwise returns
 * false.
 */
static inline bool ubi_dbg_is_erase_failure(const struct ubi_device *ubi)
{
        if (ubi_dbg_erase_failure(ubi))
                return true;
        return ubi_dbg_fail_erase(ubi);
}

/**
 * ubi_dbg_is_eccerr - if it is time to emulate ECC error.
 * @ubi: UBI device description object
 *
 * Returns true if a ECC error should be emulated, otherwise returns false.
 */
static inline bool ubi_dbg_is_eccerr(const struct ubi_device *ubi)
{
        return ubi_dbg_fail_eccerr(ubi);
}

/**
 * ubi_dbg_is_read_failure - if it is time to emulate a read failure.
 * @ubi: UBI device description object
 *
 * Returns true if a read failure should be emulated, otherwise returns
 * false.
 */
static inline bool ubi_dbg_is_read_failure(const struct ubi_device *ubi,
                                           unsigned int caller)
{
        return ubi_dbg_fail_read(ubi, caller);
}

/**
 * ubi_dbg_is_ff - if it is time to emulate that read region is only 0xFF.
 * @ubi: UBI device description object
 *
 * Returns true if read region should be emulated 0xFF, otherwise
 * returns false.
 */
static inline bool ubi_dbg_is_ff(const struct ubi_device *ubi,
                                 unsigned int caller)
{
        return ubi_dbg_fail_ff(ubi, caller);
}

/**
 * ubi_dbg_is_ff_bitflips - if it is time to emulate that read region is only 0xFF
 * with error reported by the MTD driver
 *
 * @ubi: UBI device description object
 *
 * Returns true if read region should be emulated 0xFF and error
 * reported by the MTD driver, otherwise returns false.
 */
static inline bool ubi_dbg_is_ff_bitflips(const struct ubi_device *ubi,
                                          unsigned int caller)
{
        return ubi_dbg_fail_ff_bitflips(ubi, caller);
}

/**
 * ubi_dbg_is_bad_hdr - if it is time to emulate a bad header
 * @ubi: UBI device description object
 *
 * Returns true if a bad header error should be emulated, otherwise
 * returns false.
 */
static inline bool ubi_dbg_is_bad_hdr(const struct ubi_device *ubi,
                                      unsigned int caller)
{
        return ubi_dbg_fail_bad_hdr(ubi, caller);
}

/**
 * ubi_dbg_is_bad_hdr_ebadmsg - if it is time to emulate a bad header with
 * ECC error.
 *
 * @ubi: UBI device description object
 *
 * Returns true if a bad header with ECC error should be emulated, otherwise
 * returns false.
 */
static inline bool ubi_dbg_is_bad_hdr_ebadmsg(const struct ubi_device *ubi,
                                              unsigned int caller)
{
        return ubi_dbg_fail_bad_hdr_ebadmsg(ubi, caller);
}

/**
 * ubi_dbg_is_bgt_disabled - if the background thread is disabled.
 * @ubi: UBI device description object
 *
 * Returns non-zero if the UBI background thread is disabled for testing
 * purposes.
 */
static inline int ubi_dbg_is_bgt_disabled(const struct ubi_device *ubi)
{
        return ubi->dbg.disable_bgt;
}

static inline int ubi_dbg_chk_io(const struct ubi_device *ubi)
{
        return ubi->dbg.chk_io;
}

static inline int ubi_dbg_chk_gen(const struct ubi_device *ubi)
{
        return ubi->dbg.chk_gen;
}

static inline int ubi_dbg_chk_fastmap(const struct ubi_device *ubi)
{
        return ubi->dbg.chk_fastmap;
}

static inline void ubi_enable_dbg_chk_fastmap(struct ubi_device *ubi)
{
        ubi->dbg.chk_fastmap = 1;
}

#endif /* !__UBI_DEBUG_H__ */











































    9 



    8 






    9 





    9 



















































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
// SPDX-License-Identifier: GPL-2.0

#include <linux/quotaops.h>
#include <linux/uuid.h>

#include "ext4.h"
#include "xattr.h"
#include "ext4_jbd2.h"

static void ext4_fname_from_fscrypt_name(struct ext4_filename *dst,
                                         const struct fscrypt_name *src)
{
        memset(dst, 0, sizeof(*dst));

        dst->usr_fname = src->usr_fname;
        dst->disk_name = src->disk_name;
        dst->hinfo.hash = src->hash;
        dst->hinfo.minor_hash = src->minor_hash;
        dst->crypto_buf = src->crypto_buf;
}

int ext4_fname_setup_filename(struct inode *dir, const struct qstr *iname,
                              int lookup, struct ext4_filename *fname)
{
        struct fscrypt_name name;
        int err;

        err = fscrypt_setup_filename(dir, iname, lookup, &name);
        if (err)
                return err;

        ext4_fname_from_fscrypt_name(fname, &name);

        err = ext4_fname_setup_ci_filename(dir, iname, fname);
        if (err)
                ext4_fname_free_filename(fname);

        return err;
}

int ext4_fname_prepare_lookup(struct inode *dir, struct dentry *dentry,
                              struct ext4_filename *fname)
{
        struct fscrypt_name name;
        int err;

        err = fscrypt_prepare_lookup(dir, dentry, &name);
        if (err)
                return err;

        ext4_fname_from_fscrypt_name(fname, &name);

        err = ext4_fname_setup_ci_filename(dir, &dentry->d_name, fname);
        if (err)
                ext4_fname_free_filename(fname);
        return err;
}

void ext4_fname_free_filename(struct ext4_filename *fname)
{
        struct fscrypt_name name;

        name.crypto_buf = fname->crypto_buf;
        fscrypt_free_filename(&name);

        fname->crypto_buf.name = NULL;
        fname->usr_fname = NULL;
        fname->disk_name.name = NULL;

        ext4_fname_free_ci_filename(fname);
}

static bool uuid_is_zero(__u8 u[16])
{
        int i;

        for (i = 0; i < 16; i++)
                if (u[i])
                        return false;
        return true;
}

int ext4_ioctl_get_encryption_pwsalt(struct file *filp, void __user *arg)
{
        struct super_block *sb = file_inode(filp)->i_sb;
        struct ext4_sb_info *sbi = EXT4_SB(sb);
        int err, err2;
        handle_t *handle;

        if (!ext4_has_feature_encrypt(sb))
                return -EOPNOTSUPP;

        if (uuid_is_zero(sbi->s_es->s_encrypt_pw_salt)) {
                err = mnt_want_write_file(filp);
                if (err)
                        return err;
                handle = ext4_journal_start_sb(sb, EXT4_HT_MISC, 1);
                if (IS_ERR(handle)) {
                        err = PTR_ERR(handle);
                        goto pwsalt_err_exit;
                }
                err = ext4_journal_get_write_access(handle, sb, sbi->s_sbh,
                                                    EXT4_JTR_NONE);
                if (err)
                        goto pwsalt_err_journal;
                lock_buffer(sbi->s_sbh);
                generate_random_uuid(sbi->s_es->s_encrypt_pw_salt);
                ext4_superblock_csum_set(sb);
                unlock_buffer(sbi->s_sbh);
                err = ext4_handle_dirty_metadata(handle, NULL, sbi->s_sbh);
pwsalt_err_journal:
                err2 = ext4_journal_stop(handle);
                if (err2 && !err)
                        err = err2;
pwsalt_err_exit:
                mnt_drop_write_file(filp);
                if (err)
                        return err;
        }

        if (copy_to_user(arg, sbi->s_es->s_encrypt_pw_salt, 16))
                return -EFAULT;
        return 0;
}

static int ext4_get_context(struct inode *inode, void *ctx, size_t len)
{
        return ext4_xattr_get(inode, EXT4_XATTR_INDEX_ENCRYPTION,
                                 EXT4_XATTR_NAME_ENCRYPTION_CONTEXT, ctx, len);
}

static int ext4_set_context(struct inode *inode, const void *ctx, size_t len,
                                                        void *fs_data)
{
        handle_t *handle = fs_data;
        int res, res2, credits, retries = 0;

        /*
         * Encrypting the root directory is not allowed because e2fsck expects
         * lost+found to exist and be unencrypted, and encrypting the root
         * directory would imply encrypting the lost+found directory as well as
         * the filename "lost+found" itself.
         */
        if (inode->i_ino == EXT4_ROOT_INO)
                return -EPERM;

        if (WARN_ON_ONCE(IS_DAX(inode) && i_size_read(inode)))
                return -EINVAL;

        if (ext4_test_inode_flag(inode, EXT4_INODE_DAX))
                return -EOPNOTSUPP;

        res = ext4_convert_inline_data(inode);
        if (res)
                return res;

        /*
         * If a journal handle was specified, then the encryption context is
         * being set on a new inode via inheritance and is part of a larger
         * transaction to create the inode.  Otherwise the encryption context is
         * being set on an existing inode in its own transaction.  Only in the
         * latter case should the "retry on ENOSPC" logic be used.
         */

        if (handle) {
                res = ext4_xattr_set_handle(handle, inode,
                                            EXT4_XATTR_INDEX_ENCRYPTION,
                                            EXT4_XATTR_NAME_ENCRYPTION_CONTEXT,
                                            ctx, len, 0);
                if (!res) {
                        ext4_set_inode_flag(inode, EXT4_INODE_ENCRYPT);
                        ext4_clear_inode_state(inode,
                                        EXT4_STATE_MAY_INLINE_DATA);
                        /*
                         * Update inode->i_flags - S_ENCRYPTED will be enabled,
                         * S_DAX may be disabled
                         */
                        ext4_set_inode_flags(inode, false);
                }
                return res;
        }

        res = dquot_initialize(inode);
        if (res)
                return res;
retry:
        res = ext4_xattr_set_credits(inode, len, false /* is_create */,
                                     &credits);
        if (res)
                return res;

        handle = ext4_journal_start(inode, EXT4_HT_MISC, credits);
        if (IS_ERR(handle))
                return PTR_ERR(handle);

        res = ext4_xattr_set_handle(handle, inode, EXT4_XATTR_INDEX_ENCRYPTION,
                                    EXT4_XATTR_NAME_ENCRYPTION_CONTEXT,
                                    ctx, len, 0);
        if (!res) {
                ext4_set_inode_flag(inode, EXT4_INODE_ENCRYPT);
                /*
                 * Update inode->i_flags - S_ENCRYPTED will be enabled,
                 * S_DAX may be disabled
                 */
                ext4_set_inode_flags(inode, false);
                res = ext4_mark_inode_dirty(handle, inode);
                if (res)
                        EXT4_ERROR_INODE(inode, "Failed to mark inode dirty");
        }
        res2 = ext4_journal_stop(handle);

        if (res == -ENOSPC && ext4_should_retry_alloc(inode->i_sb, &retries))
                goto retry;
        if (!res)
                res = res2;
        return res;
}

static const union fscrypt_policy *ext4_get_dummy_policy(struct super_block *sb)
{
        return EXT4_SB(sb)->s_dummy_enc_policy.policy;
}

static bool ext4_has_stable_inodes(struct super_block *sb)
{
        return ext4_has_feature_stable_inodes(sb);
}

const struct fscrypt_operations ext4_cryptops = {
        .needs_bounce_pages        = 1,
        .has_32bit_inodes        = 1,
        .supports_subblock_data_units = 1,
        .legacy_key_prefix        = "ext4:",
        .get_context                = ext4_get_context,
        .set_context                = ext4_set_context,
        .get_dummy_policy        = ext4_get_dummy_policy,
        .empty_dir                = ext4_empty_dir,
        .has_stable_inodes        = ext4_has_stable_inodes,
};
















































   15 




    1 




    1 





    1 









    7 














    5 



   12 














    3 













    2 
    1 














































    5 





    3 
    2 







    2 

    3 





    1 






    2 
    1 













    1 


















    1 









    1 






   16 





    1 
   14 



    1 



    1 








    1 




















    1 









    1 
















    1 








    2 














    1 



    1 









    1 





    1 













    1 







    1 












    1 








    1 
























































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
// SPDX-License-Identifier: GPL-2.0-only
/*
 * linux/kernel/power/user.c
 *
 * This file provides the user space interface for software suspend/resume.
 *
 * Copyright (C) 2006 Rafael J. Wysocki <rjw@sisk.pl>
 */

#include <linux/suspend.h>
#include <linux/reboot.h>
#include <linux/string.h>
#include <linux/device.h>
#include <linux/miscdevice.h>
#include <linux/mm.h>
#include <linux/swap.h>
#include <linux/swapops.h>
#include <linux/pm.h>
#include <linux/fs.h>
#include <linux/compat.h>
#include <linux/console.h>
#include <linux/cpu.h>
#include <linux/freezer.h>

#include <linux/uaccess.h>

#include "power.h"

static bool need_wait;

static struct snapshot_data {
        struct snapshot_handle handle;
        int swap;
        int mode;
        bool frozen;
        bool ready;
        bool platform_support;
        bool free_bitmaps;
        dev_t dev;
} snapshot_state;

int is_hibernate_resume_dev(dev_t dev)
{
        return hibernation_available() && snapshot_state.dev == dev;
}

static int snapshot_open(struct inode *inode, struct file *filp)
{
        struct snapshot_data *data;
        unsigned int sleep_flags;
        int error;

        if (!hibernation_available())
                return -EPERM;

        sleep_flags = lock_system_sleep();

        if (!hibernate_acquire()) {
                error = -EBUSY;
                goto Unlock;
        }

        if ((filp->f_flags & O_ACCMODE) == O_RDWR) {
                hibernate_release();
                error = -ENOSYS;
                goto Unlock;
        }
        nonseekable_open(inode, filp);
        data = &snapshot_state;
        filp->private_data = data;
        memset(&data->handle, 0, sizeof(struct snapshot_handle));
        if ((filp->f_flags & O_ACCMODE) == O_RDONLY) {
                /* Hibernating.  The image device should be accessible. */
                data->swap = swap_type_of(swsusp_resume_device, 0);
                data->mode = O_RDONLY;
                data->free_bitmaps = false;
                error = pm_notifier_call_chain_robust(PM_HIBERNATION_PREPARE, PM_POST_HIBERNATION);
        } else {
                /*
                 * Resuming.  We may need to wait for the image device to
                 * appear.
                 */
                need_wait = true;

                data->swap = -1;
                data->mode = O_WRONLY;
                error = pm_notifier_call_chain_robust(PM_RESTORE_PREPARE, PM_POST_RESTORE);
                if (!error) {
                        error = create_basic_memory_bitmaps();
                        data->free_bitmaps = !error;
                }
        }
        if (error)
                hibernate_release();

        data->frozen = false;
        data->ready = false;
        data->platform_support = false;
        data->dev = 0;

 Unlock:
        unlock_system_sleep(sleep_flags);

        return error;
}

static int snapshot_release(struct inode *inode, struct file *filp)
{
        struct snapshot_data *data;
        unsigned int sleep_flags;

        sleep_flags = lock_system_sleep();

        swsusp_free();
        data = filp->private_data;
        data->dev = 0;
        free_all_swap_pages(data->swap);
        if (data->frozen) {
                pm_restore_gfp_mask();
                free_basic_memory_bitmaps();
                thaw_processes();
        } else if (data->free_bitmaps) {
                free_basic_memory_bitmaps();
        }
        pm_notifier_call_chain(data->mode == O_RDONLY ?
                        PM_POST_HIBERNATION : PM_POST_RESTORE);
        hibernate_release();

        unlock_system_sleep(sleep_flags);

        return 0;
}

static ssize_t snapshot_read(struct file *filp, char __user *buf,
                             size_t count, loff_t *offp)
{
        loff_t pg_offp = *offp & ~PAGE_MASK;
        struct snapshot_data *data;
        unsigned int sleep_flags;
        ssize_t res;

        sleep_flags = lock_system_sleep();

        data = filp->private_data;
        if (!data->ready) {
                res = -ENODATA;
                goto Unlock;
        }
        if (!pg_offp) { /* on page boundary? */
                res = snapshot_read_next(&data->handle);
                if (res <= 0)
                        goto Unlock;
        } else {
                res = PAGE_SIZE - pg_offp;
        }

        res = simple_read_from_buffer(buf, count, &pg_offp,
                        data_of(data->handle), res);
        if (res > 0)
                *offp += res;

 Unlock:
        unlock_system_sleep(sleep_flags);

        return res;
}

static ssize_t snapshot_write(struct file *filp, const char __user *buf,
                              size_t count, loff_t *offp)
{
        loff_t pg_offp = *offp & ~PAGE_MASK;
        struct snapshot_data *data;
        unsigned long sleep_flags;
        ssize_t res;

        if (need_wait) {
                wait_for_device_probe();
                need_wait = false;
        }

        sleep_flags = lock_system_sleep();

        data = filp->private_data;

        if (!pg_offp) {
                res = snapshot_write_next(&data->handle);
                if (res <= 0)
                        goto unlock;
        } else {
                res = PAGE_SIZE;
        }

        if (!data_of(data->handle)) {
                res = -EINVAL;
                goto unlock;
        }

        res = simple_write_to_buffer(data_of(data->handle), res, &pg_offp,
                        buf, count);
        if (res > 0)
                *offp += res;
unlock:
        unlock_system_sleep(sleep_flags);

        return res;
}

struct compat_resume_swap_area {
        compat_loff_t offset;
        u32 dev;
} __packed;

static int snapshot_set_swap_area(struct snapshot_data *data,
                void __user *argp)
{
        sector_t offset;
        dev_t swdev;

        if (swsusp_swap_in_use())
                return -EPERM;

        if (in_compat_syscall()) {
                struct compat_resume_swap_area swap_area;

                if (copy_from_user(&swap_area, argp, sizeof(swap_area)))
                        return -EFAULT;
                swdev = new_decode_dev(swap_area.dev);
                offset = swap_area.offset;
        } else {
                struct resume_swap_area swap_area;

                if (copy_from_user(&swap_area, argp, sizeof(swap_area)))
                        return -EFAULT;
                swdev = new_decode_dev(swap_area.dev);
                offset = swap_area.offset;
        }

        /*
         * User space encodes device types as two-byte values,
         * so we need to recode them
         */
        data->swap = swap_type_of(swdev, offset);
        if (data->swap < 0)
                return swdev ? -ENODEV : -EINVAL;
        data->dev = swdev;
        return 0;
}

static long snapshot_ioctl(struct file *filp, unsigned int cmd,
                                                        unsigned long arg)
{
        int error = 0;
        struct snapshot_data *data;
        loff_t size;
        sector_t offset;

        if (need_wait) {
                wait_for_device_probe();
                need_wait = false;
        }

        if (_IOC_TYPE(cmd) != SNAPSHOT_IOC_MAGIC)
                return -ENOTTY;
        if (_IOC_NR(cmd) > SNAPSHOT_IOC_MAXNR)
                return -ENOTTY;
        if (!capable(CAP_SYS_ADMIN))
                return -EPERM;

        if (!mutex_trylock(&system_transition_mutex))
                return -EBUSY;

        lock_device_hotplug();
        data = filp->private_data;

        switch (cmd) {

        case SNAPSHOT_FREEZE:
                if (data->frozen)
                        break;

                ksys_sync_helper();

                error = freeze_processes();
                if (error)
                        break;

                error = create_basic_memory_bitmaps();
                if (error)
                        thaw_processes();
                else
                        data->frozen = true;

                break;

        case SNAPSHOT_UNFREEZE:
                if (!data->frozen || data->ready)
                        break;
                pm_restore_gfp_mask();
                free_basic_memory_bitmaps();
                data->free_bitmaps = false;
                thaw_processes();
                data->frozen = false;
                break;

        case SNAPSHOT_CREATE_IMAGE:
                if (data->mode != O_RDONLY || !data->frozen  || data->ready) {
                        error = -EPERM;
                        break;
                }
                pm_restore_gfp_mask();
                error = hibernation_snapshot(data->platform_support);
                if (!error) {
                        error = put_user(in_suspend, (int __user *)arg);
                        data->ready = !freezer_test_done && !error;
                        freezer_test_done = false;
                }
                break;

        case SNAPSHOT_ATOMIC_RESTORE:
                error = snapshot_write_finalize(&data->handle);
                if (error)
                        break;
                if (data->mode != O_WRONLY || !data->frozen ||
                    !snapshot_image_loaded(&data->handle)) {
                        error = -EPERM;
                        break;
                }
                error = hibernation_restore(data->platform_support);
                break;

        case SNAPSHOT_FREE:
                swsusp_free();
                memset(&data->handle, 0, sizeof(struct snapshot_handle));
                data->ready = false;
                /*
                 * It is necessary to thaw kernel threads here, because
                 * SNAPSHOT_CREATE_IMAGE may be invoked directly after
                 * SNAPSHOT_FREE.  In that case, if kernel threads were not
                 * thawed, the preallocation of memory carried out by
                 * hibernation_snapshot() might run into problems (i.e. it
                 * might fail or even deadlock).
                 */
                thaw_kernel_threads();
                break;

        case SNAPSHOT_PREF_IMAGE_SIZE:
                image_size = arg;
                break;

        case SNAPSHOT_GET_IMAGE_SIZE:
                if (!data->ready) {
                        error = -ENODATA;
                        break;
                }
                size = snapshot_get_image_size();
                size <<= PAGE_SHIFT;
                error = put_user(size, (loff_t __user *)arg);
                break;

        case SNAPSHOT_AVAIL_SWAP_SIZE:
                size = count_swap_pages(data->swap, 1);
                size <<= PAGE_SHIFT;
                error = put_user(size, (loff_t __user *)arg);
                break;

        case SNAPSHOT_ALLOC_SWAP_PAGE:
                if (data->swap < 0 || data->swap >= MAX_SWAPFILES) {
                        error = -ENODEV;
                        break;
                }
                offset = alloc_swapdev_block(data->swap);
                if (offset) {
                        offset <<= PAGE_SHIFT;
                        error = put_user(offset, (loff_t __user *)arg);
                } else {
                        error = -ENOSPC;
                }
                break;

        case SNAPSHOT_FREE_SWAP_PAGES:
                if (data->swap < 0 || data->swap >= MAX_SWAPFILES) {
                        error = -ENODEV;
                        break;
                }
                free_all_swap_pages(data->swap);
                break;

        case SNAPSHOT_S2RAM:
                if (!data->frozen) {
                        error = -EPERM;
                        break;
                }
                /*
                 * Tasks are frozen and the notifiers have been called with
                 * PM_HIBERNATION_PREPARE
                 */
                error = suspend_devices_and_enter(PM_SUSPEND_MEM);
                data->ready = false;
                break;

        case SNAPSHOT_PLATFORM_SUPPORT:
                data->platform_support = !!arg;
                break;

        case SNAPSHOT_POWER_OFF:
                if (data->platform_support)
                        error = hibernation_platform_enter();
                break;

        case SNAPSHOT_SET_SWAP_AREA:
                error = snapshot_set_swap_area(data, (void __user *)arg);
                break;

        default:
                error = -ENOTTY;

        }

        unlock_device_hotplug();
        mutex_unlock(&system_transition_mutex);

        return error;
}

#ifdef CONFIG_COMPAT
static long
snapshot_compat_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
{
        BUILD_BUG_ON(sizeof(loff_t) != sizeof(compat_loff_t));

        switch (cmd) {
        case SNAPSHOT_GET_IMAGE_SIZE:
        case SNAPSHOT_AVAIL_SWAP_SIZE:
        case SNAPSHOT_ALLOC_SWAP_PAGE:
        case SNAPSHOT_CREATE_IMAGE:
        case SNAPSHOT_SET_SWAP_AREA:
                return snapshot_ioctl(file, cmd,
                                      (unsigned long) compat_ptr(arg));
        default:
                return snapshot_ioctl(file, cmd, arg);
        }
}
#endif /* CONFIG_COMPAT */

static const struct file_operations snapshot_fops = {
        .open = snapshot_open,
        .release = snapshot_release,
        .read = snapshot_read,
        .write = snapshot_write,
        .unlocked_ioctl = snapshot_ioctl,
#ifdef CONFIG_COMPAT
        .compat_ioctl = snapshot_compat_ioctl,
#endif
};

static struct miscdevice snapshot_device = {
        .minor = SNAPSHOT_MINOR,
        .name = "snapshot",
        .fops = &snapshot_fops,
};

static int __init snapshot_device_init(void)
{
        return misc_register(&snapshot_device);
};

device_initcall(snapshot_device_init);



























    5 


    4 
















    8 




    8 

























































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
/* Protective Load Balancing (PLB)
 *
 * PLB was designed to reduce link load imbalance across datacenter
 * switches. PLB is a host-based optimization; it leverages congestion
 * signals from the transport layer to randomly change the path of the
 * connection experiencing sustained congestion. PLB prefers to repath
 * after idle periods to minimize packet reordering. It repaths by
 * changing the IPv6 Flow Label on the packets of a connection, which
 * datacenter switches include as part of ECMP/WCMP hashing.
 *
 * PLB is described in detail in:
 *
 *        Mubashir Adnan Qureshi, Yuchung Cheng, Qianwen Yin, Qiaobin Fu,
 *        Gautam Kumar, Masoud Moshref, Junhua Yan, Van Jacobson,
 *        David Wetherall,Abdul Kabbani:
 *        "PLB: Congestion Signals are Simple and Effective for
 *         Network Load Balancing"
 *        In ACM SIGCOMM 2022, Amsterdam Netherlands.
 *
 */

#include <net/tcp.h>

/* Called once per round-trip to update PLB state for a connection. */
void tcp_plb_update_state(const struct sock *sk, struct tcp_plb_state *plb,
                          const int cong_ratio)
{
        struct net *net = sock_net(sk);

        if (!READ_ONCE(net->ipv4.sysctl_tcp_plb_enabled))
                return;

        if (cong_ratio >= 0) {
                if (cong_ratio < READ_ONCE(net->ipv4.sysctl_tcp_plb_cong_thresh))
                        plb->consec_cong_rounds = 0;
                else if (plb->consec_cong_rounds <
                         READ_ONCE(net->ipv4.sysctl_tcp_plb_rehash_rounds))
                        plb->consec_cong_rounds++;
        }
}
EXPORT_SYMBOL_GPL(tcp_plb_update_state);

/* Check whether recent congestion has been persistent enough to warrant
 * a load balancing decision that switches the connection to another path.
 */
void tcp_plb_check_rehash(struct sock *sk, struct tcp_plb_state *plb)
{
        struct net *net = sock_net(sk);
        u32 max_suspend;
        bool forced_rehash = false, idle_rehash = false;

        if (!READ_ONCE(net->ipv4.sysctl_tcp_plb_enabled))
                return;

        forced_rehash = plb->consec_cong_rounds >=
                        READ_ONCE(net->ipv4.sysctl_tcp_plb_rehash_rounds);
        /* If sender goes idle then we check whether to rehash. */
        idle_rehash = READ_ONCE(net->ipv4.sysctl_tcp_plb_idle_rehash_rounds) &&
                      !tcp_sk(sk)->packets_out &&
                      plb->consec_cong_rounds >=
                      READ_ONCE(net->ipv4.sysctl_tcp_plb_idle_rehash_rounds);

        if (!forced_rehash && !idle_rehash)
                return;

        /* Note that tcp_jiffies32 can wrap; we detect wraps by checking for
         * cases where the max suspension end is before the actual suspension
         * end. We clear pause_until to 0 to indicate there is no recent
         * RTO event that constrains PLB rehashing.
         */
        max_suspend = 2 * READ_ONCE(net->ipv4.sysctl_tcp_plb_suspend_rto_sec) * HZ;
        if (plb->pause_until &&
            (!before(tcp_jiffies32, plb->pause_until) ||
             before(tcp_jiffies32 + max_suspend, plb->pause_until)))
                plb->pause_until = 0;

        if (plb->pause_until)
                return;

        sk_rethink_txhash(sk);
        plb->consec_cong_rounds = 0;
        tcp_sk(sk)->plb_rehash++;
        NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPPLBREHASH);
}
EXPORT_SYMBOL_GPL(tcp_plb_check_rehash);

/* Upon RTO, disallow load balancing for a while, to avoid having load
 * balancing decisions switch traffic to a black-holed path that was
 * previously avoided with a sk_rethink_txhash() call at RTO time.
 */
void tcp_plb_update_state_upon_rto(struct sock *sk, struct tcp_plb_state *plb)
{
        struct net *net = sock_net(sk);
        u32 pause;

        if (!READ_ONCE(net->ipv4.sysctl_tcp_plb_enabled))
                return;

        pause = READ_ONCE(net->ipv4.sysctl_tcp_plb_suspend_rto_sec) * HZ;
        pause += get_random_u32_below(pause);
        plb->pause_until = tcp_jiffies32 + pause;

        /* Reset PLB state upon RTO, since an RTO causes a sk_rethink_txhash() call
         * that may switch this connection to a path with completely different
         * congestion characteristics.
         */
        plb->consec_cong_rounds = 0;
}
EXPORT_SYMBOL_GPL(tcp_plb_update_state_upon_rto);



















































































































































































































































































































































































































































































































































































































































































































































































































    4 





























































































































































    2 



















































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
 * i2c.h - definitions for the Linux i2c bus interface
 * Copyright (C) 1995-2000 Simon G. Vogl
 * Copyright (C) 2013-2019 Wolfram Sang <wsa@kernel.org>
 *
 * With some changes from Kyösti Mälkki <kmalkki@cc.hut.fi> and
 * Frodo Looijaard <frodol@dds.nl>
 */
#ifndef _LINUX_I2C_H
#define _LINUX_I2C_H

#include <linux/acpi.h>                /* for acpi_handle */
#include <linux/bits.h>
#include <linux/mod_devicetable.h>
#include <linux/device.h>        /* for struct device */
#include <linux/sched.h>        /* for completion */
#include <linux/mutex.h>
#include <linux/regulator/consumer.h>
#include <linux/rtmutex.h>
#include <linux/irqdomain.h>                /* for Host Notify IRQ */
#include <linux/of.h>                /* for struct device_node */
#include <linux/swab.h>                /* for swab16 */
#include <uapi/linux/i2c.h>

extern const struct bus_type i2c_bus_type;
extern const struct device_type i2c_adapter_type;
extern const struct device_type i2c_client_type;

/* --- General options ------------------------------------------------        */

struct i2c_msg;
struct i2c_adapter;
struct i2c_client;
struct i2c_driver;
struct i2c_device_identity;
union i2c_smbus_data;
struct i2c_board_info;
enum i2c_slave_event;
typedef int (*i2c_slave_cb_t)(struct i2c_client *client,
                              enum i2c_slave_event event, u8 *val);

/* I2C Frequency Modes */
#define I2C_MAX_STANDARD_MODE_FREQ        100000
#define I2C_MAX_FAST_MODE_FREQ                400000
#define I2C_MAX_FAST_MODE_PLUS_FREQ        1000000
#define I2C_MAX_TURBO_MODE_FREQ                1400000
#define I2C_MAX_HIGH_SPEED_MODE_FREQ        3400000
#define I2C_MAX_ULTRA_FAST_MODE_FREQ        5000000

struct module;
struct property_entry;

#if IS_ENABLED(CONFIG_I2C)
/* Return the Frequency mode string based on the bus frequency */
const char *i2c_freq_mode_string(u32 bus_freq_hz);

/*
 * The master routines are the ones normally used to transmit data to devices
 * on a bus (or read from them). Apart from two basic transfer functions to
 * transmit one message at a time, a more complex version can be used to
 * transmit an arbitrary number of messages without interruption.
 * @count must be less than 64k since msg.len is u16.
 */
int i2c_transfer_buffer_flags(const struct i2c_client *client,
                              char *buf, int count, u16 flags);

/**
 * i2c_master_recv - issue a single I2C message in master receive mode
 * @client: Handle to slave device
 * @buf: Where to store data read from slave
 * @count: How many bytes to read, must be less than 64k since msg.len is u16
 *
 * Returns negative errno, or else the number of bytes read.
 */
static inline int i2c_master_recv(const struct i2c_client *client,
                                  char *buf, int count)
{
        return i2c_transfer_buffer_flags(client, buf, count, I2C_M_RD);
};

/**
 * i2c_master_recv_dmasafe - issue a single I2C message in master receive mode
 *                             using a DMA safe buffer
 * @client: Handle to slave device
 * @buf: Where to store data read from slave, must be safe to use with DMA
 * @count: How many bytes to read, must be less than 64k since msg.len is u16
 *
 * Returns negative errno, or else the number of bytes read.
 */
static inline int i2c_master_recv_dmasafe(const struct i2c_client *client,
                                          char *buf, int count)
{
        return i2c_transfer_buffer_flags(client, buf, count,
                                         I2C_M_RD | I2C_M_DMA_SAFE);
};

/**
 * i2c_master_send - issue a single I2C message in master transmit mode
 * @client: Handle to slave device
 * @buf: Data that will be written to the slave
 * @count: How many bytes to write, must be less than 64k since msg.len is u16
 *
 * Returns negative errno, or else the number of bytes written.
 */
static inline int i2c_master_send(const struct i2c_client *client,
                                  const char *buf, int count)
{
        return i2c_transfer_buffer_flags(client, (char *)buf, count, 0);
};

/**
 * i2c_master_send_dmasafe - issue a single I2C message in master transmit mode
 *                             using a DMA safe buffer
 * @client: Handle to slave device
 * @buf: Data that will be written to the slave, must be safe to use with DMA
 * @count: How many bytes to write, must be less than 64k since msg.len is u16
 *
 * Returns negative errno, or else the number of bytes written.
 */
static inline int i2c_master_send_dmasafe(const struct i2c_client *client,
                                          const char *buf, int count)
{
        return i2c_transfer_buffer_flags(client, (char *)buf, count,
                                         I2C_M_DMA_SAFE);
};

/* Transfer num messages.
 */
int i2c_transfer(struct i2c_adapter *adap, struct i2c_msg *msgs, int num);
/* Unlocked flavor */
int __i2c_transfer(struct i2c_adapter *adap, struct i2c_msg *msgs, int num);

/* This is the very generalized SMBus access routine. You probably do not
   want to use this, though; one of the functions below may be much easier,
   and probably just as fast.
   Note that we use i2c_adapter here, because you do not need a specific
   smbus adapter to call this function. */
s32 i2c_smbus_xfer(struct i2c_adapter *adapter, u16 addr,
                   unsigned short flags, char read_write, u8 command,
                   int protocol, union i2c_smbus_data *data);

/* Unlocked flavor */
s32 __i2c_smbus_xfer(struct i2c_adapter *adapter, u16 addr,
                     unsigned short flags, char read_write, u8 command,
                     int protocol, union i2c_smbus_data *data);

/* Now follow the 'nice' access routines. These also document the calling
   conventions of i2c_smbus_xfer. */

u8 i2c_smbus_pec(u8 crc, u8 *p, size_t count);
s32 i2c_smbus_read_byte(const struct i2c_client *client);
s32 i2c_smbus_write_byte(const struct i2c_client *client, u8 value);
s32 i2c_smbus_read_byte_data(const struct i2c_client *client, u8 command);
s32 i2c_smbus_write_byte_data(const struct i2c_client *client,
                              u8 command, u8 value);
s32 i2c_smbus_read_word_data(const struct i2c_client *client, u8 command);
s32 i2c_smbus_write_word_data(const struct i2c_client *client,
                              u8 command, u16 value);

static inline s32
i2c_smbus_read_word_swapped(const struct i2c_client *client, u8 command)
{
        s32 value = i2c_smbus_read_word_data(client, command);

        return (value < 0) ? value : swab16(value);
}

static inline s32
i2c_smbus_write_word_swapped(const struct i2c_client *client,
                             u8 command, u16 value)
{
        return i2c_smbus_write_word_data(client, command, swab16(value));
}

/* Returns the number of read bytes */
s32 i2c_smbus_read_block_data(const struct i2c_client *client,
                              u8 command, u8 *values);
s32 i2c_smbus_write_block_data(const struct i2c_client *client,
                               u8 command, u8 length, const u8 *values);
/* Returns the number of read bytes */
s32 i2c_smbus_read_i2c_block_data(const struct i2c_client *client,
                                  u8 command, u8 length, u8 *values);
s32 i2c_smbus_write_i2c_block_data(const struct i2c_client *client,
                                   u8 command, u8 length, const u8 *values);
s32 i2c_smbus_read_i2c_block_data_or_emulated(const struct i2c_client *client,
                                              u8 command, u8 length,
                                              u8 *values);
int i2c_get_device_id(const struct i2c_client *client,
                      struct i2c_device_identity *id);
const struct i2c_device_id *i2c_client_get_device_id(const struct i2c_client *client);
#endif /* I2C */

/**
 * struct i2c_device_identity - i2c client device identification
 * @manufacturer_id: 0 - 4095, database maintained by NXP
 * @part_id: 0 - 511, according to manufacturer
 * @die_revision: 0 - 7, according to manufacturer
 */
struct i2c_device_identity {
        u16 manufacturer_id;
#define I2C_DEVICE_ID_NXP_SEMICONDUCTORS                0
#define I2C_DEVICE_ID_NXP_SEMICONDUCTORS_1              1
#define I2C_DEVICE_ID_NXP_SEMICONDUCTORS_2              2
#define I2C_DEVICE_ID_NXP_SEMICONDUCTORS_3              3
#define I2C_DEVICE_ID_RAMTRON_INTERNATIONAL             4
#define I2C_DEVICE_ID_ANALOG_DEVICES                    5
#define I2C_DEVICE_ID_STMICROELECTRONICS                6
#define I2C_DEVICE_ID_ON_SEMICONDUCTOR                  7
#define I2C_DEVICE_ID_SPRINTEK_CORPORATION              8
#define I2C_DEVICE_ID_ESPROS_PHOTONICS_AG               9
#define I2C_DEVICE_ID_FUJITSU_SEMICONDUCTOR            10
#define I2C_DEVICE_ID_FLIR                             11
#define I2C_DEVICE_ID_O2MICRO                          12
#define I2C_DEVICE_ID_ATMEL                            13
#define I2C_DEVICE_ID_NONE                         0xffff
        u16 part_id;
        u8 die_revision;
};

enum i2c_alert_protocol {
        I2C_PROTOCOL_SMBUS_ALERT,
        I2C_PROTOCOL_SMBUS_HOST_NOTIFY,
};

/**
 * enum i2c_driver_flags - Flags for an I2C device driver
 *
 * @I2C_DRV_ACPI_WAIVE_D0_PROBE: Don't put the device in D0 state for probe
 */
enum i2c_driver_flags {
        I2C_DRV_ACPI_WAIVE_D0_PROBE = BIT(0),
};

/**
 * struct i2c_driver - represent an I2C device driver
 * @class: What kind of i2c device we instantiate (for detect)
 * @probe: Callback for device binding
 * @remove: Callback for device unbinding
 * @shutdown: Callback for device shutdown
 * @alert: Alert callback, for example for the SMBus alert protocol
 * @command: Callback for bus-wide signaling (optional)
 * @driver: Device driver model driver
 * @id_table: List of I2C devices supported by this driver
 * @detect: Callback for device detection
 * @address_list: The I2C addresses to probe (for detect)
 * @clients: List of detected clients we created (for i2c-core use only)
 * @flags: A bitmask of flags defined in &enum i2c_driver_flags
 *
 * The driver.owner field should be set to the module owner of this driver.
 * The driver.name field should be set to the name of this driver.
 *
 * For automatic device detection, both @detect and @address_list must
 * be defined. @class should also be set, otherwise only devices forced
 * with module parameters will be created. The detect function must
 * fill at least the name field of the i2c_board_info structure it is
 * handed upon successful detection, and possibly also the flags field.
 *
 * If @detect is missing, the driver will still work fine for enumerated
 * devices. Detected devices simply won't be supported. This is expected
 * for the many I2C/SMBus devices which can't be detected reliably, and
 * the ones which can always be enumerated in practice.
 *
 * The i2c_client structure which is handed to the @detect callback is
 * not a real i2c_client. It is initialized just enough so that you can
 * call i2c_smbus_read_byte_data and friends on it. Don't do anything
 * else with it. In particular, calling dev_dbg and friends on it is
 * not allowed.
 */
struct i2c_driver {
        unsigned int class;

        /* Standard driver model interfaces */
        int (*probe)(struct i2c_client *client);
        void (*remove)(struct i2c_client *client);


        /* driver model interfaces that don't relate to enumeration  */
        void (*shutdown)(struct i2c_client *client);

        /* Alert callback, for example for the SMBus alert protocol.
         * The format and meaning of the data value depends on the protocol.
         * For the SMBus alert protocol, there is a single bit of data passed
         * as the alert response's low bit ("event flag").
         * For the SMBus Host Notify protocol, the data corresponds to the
         * 16-bit payload data reported by the slave device acting as master.
         */
        void (*alert)(struct i2c_client *client, enum i2c_alert_protocol protocol,
                      unsigned int data);

        /* a ioctl like command that can be used to perform specific functions
         * with the device.
         */
        int (*command)(struct i2c_client *client, unsigned int cmd, void *arg);

        struct device_driver driver;
        const struct i2c_device_id *id_table;

        /* Device detection callback for automatic device creation */
        int (*detect)(struct i2c_client *client, struct i2c_board_info *info);
        const unsigned short *address_list;
        struct list_head clients;

        u32 flags;
};
#define to_i2c_driver(d) container_of_const(d, struct i2c_driver, driver)

/**
 * struct i2c_client - represent an I2C slave device
 * @flags: see I2C_CLIENT_* for possible flags
 * @addr: Address used on the I2C bus connected to the parent adapter.
 * @name: Indicates the type of the device, usually a chip name that's
 *        generic enough to hide second-sourcing and compatible revisions.
 * @adapter: manages the bus segment hosting this I2C device
 * @dev: Driver model device node for the slave.
 * @init_irq: IRQ that was set at initialization
 * @irq: indicates the IRQ generated by this device (if any)
 * @detected: member of an i2c_driver.clients list or i2c-core's
 *        userspace_devices list
 * @slave_cb: Callback when I2C slave mode of an adapter is used. The adapter
 *        calls it to pass on slave events to the slave driver.
 * @devres_group_id: id of the devres group that will be created for resources
 *        acquired when probing this device.
 * @debugfs: pointer to the debugfs subdirectory which the I2C core created
 *        for this client.
 *
 * An i2c_client identifies a single device (i.e. chip) connected to an
 * i2c bus. The behaviour exposed to Linux is defined by the driver
 * managing the device.
 */
struct i2c_client {
        unsigned short flags;                /* div., see below                */
#define I2C_CLIENT_PEC                0x04        /* Use Packet Error Checking */
#define I2C_CLIENT_TEN                0x10        /* we have a ten bit chip address */
                                        /* Must equal I2C_M_TEN below */
#define I2C_CLIENT_SLAVE        0x20        /* we are the slave */
#define I2C_CLIENT_HOST_NOTIFY        0x40        /* We want to use I2C host notify */
#define I2C_CLIENT_WAKE                0x80        /* for board_info; true iff can wake */
#define I2C_CLIENT_SCCB                0x9000        /* Use Omnivision SCCB protocol */
                                        /* Must match I2C_M_STOP|IGNORE_NAK */

        unsigned short addr;                /* chip address - NOTE: 7bit        */
                                        /* addresses are stored in the        */
                                        /* _LOWER_ 7 bits                */
        char name[I2C_NAME_SIZE];
        struct i2c_adapter *adapter;        /* the adapter we sit on        */
        struct device dev;                /* the device structure                */
        int init_irq;                        /* irq set at initialization        */
        int irq;                        /* irq issued by device                */
        struct list_head detected;
#if IS_ENABLED(CONFIG_I2C_SLAVE)
        i2c_slave_cb_t slave_cb;        /* callback for slave mode        */
#endif
        void *devres_group_id;                /* ID of probe devres group        */
        struct dentry *debugfs;                /* per-client debugfs dir        */
};
#define to_i2c_client(d) container_of(d, struct i2c_client, dev)

struct i2c_adapter *i2c_verify_adapter(struct device *dev);
const struct i2c_device_id *i2c_match_id(const struct i2c_device_id *id,
                                         const struct i2c_client *client);

const void *i2c_get_match_data(const struct i2c_client *client);

static inline struct i2c_client *kobj_to_i2c_client(struct kobject *kobj)
{
        struct device * const dev = kobj_to_dev(kobj);
        return to_i2c_client(dev);
}

static inline void *i2c_get_clientdata(const struct i2c_client *client)
{
        return dev_get_drvdata(&client->dev);
}

static inline void i2c_set_clientdata(struct i2c_client *client, void *data)
{
        dev_set_drvdata(&client->dev, data);
}

/* I2C slave support */

enum i2c_slave_event {
        I2C_SLAVE_READ_REQUESTED,
        I2C_SLAVE_WRITE_REQUESTED,
        I2C_SLAVE_READ_PROCESSED,
        I2C_SLAVE_WRITE_RECEIVED,
        I2C_SLAVE_STOP,
};

int i2c_slave_register(struct i2c_client *client, i2c_slave_cb_t slave_cb);
int i2c_slave_unregister(struct i2c_client *client);
int i2c_slave_event(struct i2c_client *client,
                    enum i2c_slave_event event, u8 *val);
#if IS_ENABLED(CONFIG_I2C_SLAVE)
bool i2c_detect_slave_mode(struct device *dev);
#else
static inline bool i2c_detect_slave_mode(struct device *dev) { return false; }
#endif

/**
 * struct i2c_board_info - template for device creation
 * @type: chip type, to initialize i2c_client.name
 * @flags: to initialize i2c_client.flags
 * @addr: stored in i2c_client.addr
 * @dev_name: Overrides the default <busnr>-<addr> dev_name if set
 * @platform_data: stored in i2c_client.dev.platform_data
 * @fwnode: device node supplied by the platform firmware
 * @swnode: software node for the device
 * @resources: resources associated with the device
 * @num_resources: number of resources in the @resources array
 * @irq: stored in i2c_client.irq
 *
 * I2C doesn't actually support hardware probing, although controllers and
 * devices may be able to use I2C_SMBUS_QUICK to tell whether or not there's
 * a device at a given address.  Drivers commonly need more information than
 * that, such as chip type, configuration, associated IRQ, and so on.
 *
 * i2c_board_info is used to build tables of information listing I2C devices
 * that are present.  This information is used to grow the driver model tree.
 * For mainboards this is done statically using i2c_register_board_info();
 * bus numbers identify adapters that aren't yet available.  For add-on boards,
 * i2c_new_client_device() does this dynamically with the adapter already known.
 */
struct i2c_board_info {
        char                type[I2C_NAME_SIZE];
        unsigned short        flags;
        unsigned short        addr;
        const char        *dev_name;
        void                *platform_data;
        struct fwnode_handle *fwnode;
        const struct software_node *swnode;
        const struct resource *resources;
        unsigned int        num_resources;
        int                irq;
};

/**
 * I2C_BOARD_INFO - macro used to list an i2c device and its address
 * @dev_type: identifies the device type
 * @dev_addr: the device's address on the bus.
 *
 * This macro initializes essential fields of a struct i2c_board_info,
 * declaring what has been provided on a particular board.  Optional
 * fields (such as associated irq, or device-specific platform_data)
 * are provided using conventional syntax.
 */
#define I2C_BOARD_INFO(dev_type, dev_addr) \
        .type = dev_type, .addr = (dev_addr)


#if IS_ENABLED(CONFIG_I2C)
/*
 * Add-on boards should register/unregister their devices; e.g. a board
 * with integrated I2C, a config eeprom, sensors, and a codec that's
 * used in conjunction with the primary hardware.
 */
struct i2c_client *
i2c_new_client_device(struct i2c_adapter *adap, struct i2c_board_info const *info);

/* If you don't know the exact address of an I2C device, use this variant
 * instead, which can probe for device presence in a list of possible
 * addresses. The "probe" callback function is optional. If it is provided,
 * it must return 1 on successful probe, 0 otherwise. If it is not provided,
 * a default probing method is used.
 */
struct i2c_client *
i2c_new_scanned_device(struct i2c_adapter *adap,
                       struct i2c_board_info *info,
                       unsigned short const *addr_list,
                       int (*probe)(struct i2c_adapter *adap, unsigned short addr));

/* Common custom probe functions */
int i2c_probe_func_quick_read(struct i2c_adapter *adap, unsigned short addr);

struct i2c_client *
i2c_new_dummy_device(struct i2c_adapter *adapter, u16 address);

struct i2c_client *
devm_i2c_new_dummy_device(struct device *dev, struct i2c_adapter *adap, u16 address);

struct i2c_client *
i2c_new_ancillary_device(struct i2c_client *client,
                         const char *name,
                         u16 default_addr);

void i2c_unregister_device(struct i2c_client *client);

struct i2c_client *i2c_verify_client(struct device *dev);
#else
static inline struct i2c_client *i2c_verify_client(struct device *dev)
{
        return NULL;
}
#endif /* I2C */

/* Mainboard arch_initcall() code should register all its I2C devices.
 * This is done at arch_initcall time, before declaring any i2c adapters.
 * Modules for add-on boards must use other calls.
 */
#ifdef CONFIG_I2C_BOARDINFO
int
i2c_register_board_info(int busnum, struct i2c_board_info const *info,
                        unsigned n);
#else
static inline int
i2c_register_board_info(int busnum, struct i2c_board_info const *info,
                        unsigned n)
{
        return 0;
}
#endif /* I2C_BOARDINFO */

/**
 * struct i2c_algorithm - represent I2C transfer methods
 * @xfer: Transfer a given number of messages defined by the msgs array via
 *   the specified adapter.
 * @xfer_atomic: Same as @xfer. Yet, only using atomic context so e.g. PMICs
 *   can be accessed very late before shutdown. Optional.
 * @smbus_xfer: Issue SMBus transactions to the given I2C adapter. If this
 *   is not present, then the bus layer will try and convert the SMBus calls
 *   into I2C transfers instead.
 * @smbus_xfer_atomic: Same as @smbus_xfer. Yet, only using atomic context
 *   so e.g. PMICs can be accessed very late before shutdown. Optional.
 * @functionality: Return the flags that this algorithm/adapter pair supports
 *   from the ``I2C_FUNC_*`` flags.
 * @reg_target: Register given client to local target mode of this adapter
 * @unreg_target: Unregister given client from local target mode of this adapter
 *
 * @master_xfer: deprecated, use @xfer
 * @master_xfer_atomic: deprecated, use @xfer_atomic
 * @reg_slave: deprecated, use @reg_target
 * @unreg_slave: deprecated, use @unreg_target
 *
 * i2c_algorithm is the interface to a class of hardware solutions which can
 * be addressed using the same bus algorithms - i.e. bit-banging or the PCF8584
 * to name two of the most common.
 *
 * The return codes from the ``xfer{_atomic}`` fields should indicate the
 * type of error code that occurred during the transfer, as documented in the
 * Kernel Documentation file Documentation/i2c/fault-codes.rst. Otherwise, the
 * number of messages executed should be returned.
 */
struct i2c_algorithm {
        /*
         * If an adapter algorithm can't do I2C-level access, set xfer
         * to NULL. If an adapter algorithm can do SMBus access, set
         * smbus_xfer. If set to NULL, the SMBus protocol is simulated
         * using common I2C messages.
         */
        union {
                int (*xfer)(struct i2c_adapter *adap, struct i2c_msg *msgs,
                            int num);
                int (*master_xfer)(struct i2c_adapter *adap, struct i2c_msg *msgs,
                                   int num);
        };
        union {
                int (*xfer_atomic)(struct i2c_adapter *adap,
                                   struct i2c_msg *msgs, int num);
                int (*master_xfer_atomic)(struct i2c_adapter *adap,
                                           struct i2c_msg *msgs, int num);
        };
        int (*smbus_xfer)(struct i2c_adapter *adap, u16 addr,
                          unsigned short flags, char read_write,
                          u8 command, int size, union i2c_smbus_data *data);
        int (*smbus_xfer_atomic)(struct i2c_adapter *adap, u16 addr,
                                 unsigned short flags, char read_write,
                                 u8 command, int size, union i2c_smbus_data *data);

        /* To determine what the adapter supports */
        u32 (*functionality)(struct i2c_adapter *adap);

#if IS_ENABLED(CONFIG_I2C_SLAVE)
        union {
                int (*reg_target)(struct i2c_client *client);
                int (*reg_slave)(struct i2c_client *client);
        };
        union {
                int (*unreg_target)(struct i2c_client *client);
                int (*unreg_slave)(struct i2c_client *client);
        };
#endif
};

/**
 * struct i2c_lock_operations - represent I2C locking operations
 * @lock_bus: Get exclusive access to an I2C bus segment
 * @trylock_bus: Try to get exclusive access to an I2C bus segment
 * @unlock_bus: Release exclusive access to an I2C bus segment
 *
 * The main operations are wrapped by i2c_lock_bus and i2c_unlock_bus.
 */
struct i2c_lock_operations {
        void (*lock_bus)(struct i2c_adapter *adapter, unsigned int flags);
        int (*trylock_bus)(struct i2c_adapter *adapter, unsigned int flags);
        void (*unlock_bus)(struct i2c_adapter *adapter, unsigned int flags);
};

/**
 * struct i2c_timings - I2C timing information
 * @bus_freq_hz: the bus frequency in Hz
 * @scl_rise_ns: time SCL signal takes to rise in ns; t(r) in the I2C specification
 * @scl_fall_ns: time SCL signal takes to fall in ns; t(f) in the I2C specification
 * @scl_int_delay_ns: time IP core additionally needs to setup SCL in ns
 * @sda_fall_ns: time SDA signal takes to fall in ns; t(f) in the I2C specification
 * @sda_hold_ns: time IP core additionally needs to hold SDA in ns
 * @digital_filter_width_ns: width in ns of spikes on i2c lines that the IP core
 *        digital filter can filter out
 * @analog_filter_cutoff_freq_hz: threshold frequency for the low pass IP core
 *        analog filter
 */
struct i2c_timings {
        u32 bus_freq_hz;
        u32 scl_rise_ns;
        u32 scl_fall_ns;
        u32 scl_int_delay_ns;
        u32 sda_fall_ns;
        u32 sda_hold_ns;
        u32 digital_filter_width_ns;
        u32 analog_filter_cutoff_freq_hz;
};

/**
 * struct i2c_bus_recovery_info - I2C bus recovery information
 * @recover_bus: Recover routine. Either pass driver's recover_bus() routine, or
 *        i2c_generic_scl_recovery().
 * @get_scl: This gets current value of SCL line. Mandatory for generic SCL
 *      recovery. Populated internally for generic GPIO recovery.
 * @set_scl: This sets/clears the SCL line. Mandatory for generic SCL recovery.
 *      Populated internally for generic GPIO recovery.
 * @get_sda: This gets current value of SDA line. This or set_sda() is mandatory
 *        for generic SCL recovery. Populated internally, if sda_gpio is a valid
 *        GPIO, for generic GPIO recovery.
 * @set_sda: This sets/clears the SDA line. This or get_sda() is mandatory for
 *        generic SCL recovery. Populated internally, if sda_gpio is a valid GPIO,
 *        for generic GPIO recovery.
 * @get_bus_free: Returns the bus free state as seen from the IP core in case it
 *        has a more complex internal logic than just reading SDA. Optional.
 * @prepare_recovery: This will be called before starting recovery. Platform may
 *        configure padmux here for SDA/SCL line or something else they want.
 * @unprepare_recovery: This will be called after completing recovery. Platform
 *        may configure padmux here for SDA/SCL line or something else they want.
 * @scl_gpiod: gpiod of the SCL line. Only required for GPIO recovery.
 * @sda_gpiod: gpiod of the SDA line. Only required for GPIO recovery.
 * @pinctrl: pinctrl used by GPIO recovery to change the state of the I2C pins.
 *      Optional.
 * @pins_default: default pinctrl state of SCL/SDA lines, when they are assigned
 *      to the I2C bus. Optional. Populated internally for GPIO recovery, if
 *      state with the name PINCTRL_STATE_DEFAULT is found and pinctrl is valid.
 * @pins_gpio: recovery pinctrl state of SCL/SDA lines, when they are used as
 *      GPIOs. Optional. Populated internally for GPIO recovery, if this state
 *      is called "gpio" or "recovery" and pinctrl is valid.
 */
struct i2c_bus_recovery_info {
        int (*recover_bus)(struct i2c_adapter *adap);

        int (*get_scl)(struct i2c_adapter *adap);
        void (*set_scl)(struct i2c_adapter *adap, int val);
        int (*get_sda)(struct i2c_adapter *adap);
        void (*set_sda)(struct i2c_adapter *adap, int val);
        int (*get_bus_free)(struct i2c_adapter *adap);

        void (*prepare_recovery)(struct i2c_adapter *adap);
        void (*unprepare_recovery)(struct i2c_adapter *adap);

        /* gpio recovery */
        struct gpio_desc *scl_gpiod;
        struct gpio_desc *sda_gpiod;
        struct pinctrl *pinctrl;
        struct pinctrl_state *pins_default;
        struct pinctrl_state *pins_gpio;
};

int i2c_recover_bus(struct i2c_adapter *adap);

/* Generic recovery routines */
int i2c_generic_scl_recovery(struct i2c_adapter *adap);

/**
 * struct i2c_adapter_quirks - describe flaws of an i2c adapter
 * @flags: see I2C_AQ_* for possible flags and read below
 * @max_num_msgs: maximum number of messages per transfer
 * @max_write_len: maximum length of a write message
 * @max_read_len: maximum length of a read message
 * @max_comb_1st_msg_len: maximum length of the first msg in a combined message
 * @max_comb_2nd_msg_len: maximum length of the second msg in a combined message
 *
 * Note about combined messages: Some I2C controllers can only send one message
 * per transfer, plus something called combined message or write-then-read.
 * This is (usually) a small write message followed by a read message and
 * barely enough to access register based devices like EEPROMs. There is a flag
 * to support this mode. It implies max_num_msg = 2 and does the length checks
 * with max_comb_*_len because combined message mode usually has its own
 * limitations. Because of HW implementations, some controllers can actually do
 * write-then-anything or other variants. To support that, write-then-read has
 * been broken out into smaller bits like write-first and read-second which can
 * be combined as needed.
 */

struct i2c_adapter_quirks {
        u64 flags;
        int max_num_msgs;
        u16 max_write_len;
        u16 max_read_len;
        u16 max_comb_1st_msg_len;
        u16 max_comb_2nd_msg_len;
};

/* enforce max_num_msgs = 2 and use max_comb_*_len for length checks */
#define I2C_AQ_COMB                        BIT(0)
/* first combined message must be write */
#define I2C_AQ_COMB_WRITE_FIRST                BIT(1)
/* second combined message must be read */
#define I2C_AQ_COMB_READ_SECOND                BIT(2)
/* both combined messages must have the same target address */
#define I2C_AQ_COMB_SAME_ADDR                BIT(3)
/* convenience macro for typical write-then read case */
#define I2C_AQ_COMB_WRITE_THEN_READ        (I2C_AQ_COMB | I2C_AQ_COMB_WRITE_FIRST | \
                                         I2C_AQ_COMB_READ_SECOND | I2C_AQ_COMB_SAME_ADDR)
/* clock stretching is not supported */
#define I2C_AQ_NO_CLK_STRETCH                BIT(4)
/* message cannot have length of 0 */
#define I2C_AQ_NO_ZERO_LEN_READ                BIT(5)
#define I2C_AQ_NO_ZERO_LEN_WRITE        BIT(6)
#define I2C_AQ_NO_ZERO_LEN                (I2C_AQ_NO_ZERO_LEN_READ | I2C_AQ_NO_ZERO_LEN_WRITE)
/* adapter cannot do repeated START */
#define I2C_AQ_NO_REP_START                BIT(7)

/*
 * i2c_adapter is the structure used to identify a physical i2c bus along
 * with the access algorithms necessary to access it.
 */
struct i2c_adapter {
        struct module *owner;
        unsigned int class;                  /* classes to allow probing for */
        const struct i2c_algorithm *algo; /* the algorithm to access the bus */
        void *algo_data;

        /* data fields that are valid for all devices        */
        const struct i2c_lock_operations *lock_ops;
        struct rt_mutex bus_lock;
        struct rt_mutex mux_lock;

        int timeout;                        /* in jiffies */
        int retries;
        struct device dev;                /* the adapter device */
        unsigned long locked_flags;        /* owned by the I2C core */
#define I2C_ALF_IS_SUSPENDED                0
#define I2C_ALF_SUSPEND_REPORTED        1

        int nr;
        char name[48];
        struct completion dev_released;

        struct mutex userspace_clients_lock;
        struct list_head userspace_clients;

        struct i2c_bus_recovery_info *bus_recovery_info;
        const struct i2c_adapter_quirks *quirks;

        struct irq_domain *host_notify_domain;
        struct regulator *bus_regulator;

        struct dentry *debugfs;

        /* 7bit address space */
        DECLARE_BITMAP(addrs_in_instantiation, 1 << 7);
};
#define to_i2c_adapter(d) container_of(d, struct i2c_adapter, dev)

static inline void *i2c_get_adapdata(const struct i2c_adapter *adap)
{
        return dev_get_drvdata(&adap->dev);
}

static inline void i2c_set_adapdata(struct i2c_adapter *adap, void *data)
{
        dev_set_drvdata(&adap->dev, data);
}

static inline struct i2c_adapter *
i2c_parent_is_i2c_adapter(const struct i2c_adapter *adapter)
{
#if IS_ENABLED(CONFIG_I2C_MUX)
        struct device *parent = adapter->dev.parent;

        if (parent != NULL && parent->type == &i2c_adapter_type)
                return to_i2c_adapter(parent);
        else
#endif
                return NULL;
}

int i2c_for_each_dev(void *data, int (*fn)(struct device *dev, void *data));

/* Adapter locking functions, exported for shared pin cases */
#define I2C_LOCK_ROOT_ADAPTER BIT(0)
#define I2C_LOCK_SEGMENT      BIT(1)

/**
 * i2c_lock_bus - Get exclusive access to an I2C bus segment
 * @adapter: Target I2C bus segment
 * @flags: I2C_LOCK_ROOT_ADAPTER locks the root i2c adapter, I2C_LOCK_SEGMENT
 *        locks only this branch in the adapter tree
 */
static inline void
i2c_lock_bus(struct i2c_adapter *adapter, unsigned int flags)
{
        adapter->lock_ops->lock_bus(adapter, flags);
}

/**
 * i2c_trylock_bus - Try to get exclusive access to an I2C bus segment
 * @adapter: Target I2C bus segment
 * @flags: I2C_LOCK_ROOT_ADAPTER tries to locks the root i2c adapter,
 *        I2C_LOCK_SEGMENT tries to lock only this branch in the adapter tree
 *
 * Return: true if the I2C bus segment is locked, false otherwise
 */
static inline int
i2c_trylock_bus(struct i2c_adapter *adapter, unsigned int flags)
{
        return adapter->lock_ops->trylock_bus(adapter, flags);
}

/**
 * i2c_unlock_bus - Release exclusive access to an I2C bus segment
 * @adapter: Target I2C bus segment
 * @flags: I2C_LOCK_ROOT_ADAPTER unlocks the root i2c adapter, I2C_LOCK_SEGMENT
 *        unlocks only this branch in the adapter tree
 */
static inline void
i2c_unlock_bus(struct i2c_adapter *adapter, unsigned int flags)
{
        adapter->lock_ops->unlock_bus(adapter, flags);
}

/**
 * i2c_mark_adapter_suspended - Report suspended state of the adapter to the core
 * @adap: Adapter to mark as suspended
 *
 * When using this helper to mark an adapter as suspended, the core will reject
 * further transfers to this adapter. The usage of this helper is optional but
 * recommended for devices having distinct handlers for system suspend and
 * runtime suspend. More complex devices are free to implement custom solutions
 * to reject transfers when suspended.
 */
static inline void i2c_mark_adapter_suspended(struct i2c_adapter *adap)
{
        i2c_lock_bus(adap, I2C_LOCK_ROOT_ADAPTER);
        set_bit(I2C_ALF_IS_SUSPENDED, &adap->locked_flags);
        i2c_unlock_bus(adap, I2C_LOCK_ROOT_ADAPTER);
}

/**
 * i2c_mark_adapter_resumed - Report resumed state of the adapter to the core
 * @adap: Adapter to mark as resumed
 *
 * When using this helper to mark an adapter as resumed, the core will allow
 * further transfers to this adapter. See also further notes to
 * @i2c_mark_adapter_suspended().
 */
static inline void i2c_mark_adapter_resumed(struct i2c_adapter *adap)
{
        i2c_lock_bus(adap, I2C_LOCK_ROOT_ADAPTER);
        clear_bit(I2C_ALF_IS_SUSPENDED, &adap->locked_flags);
        i2c_unlock_bus(adap, I2C_LOCK_ROOT_ADAPTER);
}

/* i2c adapter classes (bitmask) */
#define I2C_CLASS_HWMON                (1<<0)        /* lm_sensors, ... */
/* Warn users that the adapter doesn't support classes anymore */
#define I2C_CLASS_DEPRECATED        (1<<8)

/* Internal numbers to terminate lists */
#define I2C_CLIENT_END                0xfffeU

/* Construct an I2C_CLIENT_END-terminated array of i2c addresses */
#define I2C_ADDRS(addr, addrs...) \
        ((const unsigned short []){ addr, ## addrs, I2C_CLIENT_END })


/* ----- functions exported by i2c.o */

/* administration...
 */
#if IS_ENABLED(CONFIG_I2C)
int i2c_add_adapter(struct i2c_adapter *adap);
int devm_i2c_add_adapter(struct device *dev, struct i2c_adapter *adapter);
void i2c_del_adapter(struct i2c_adapter *adap);
int i2c_add_numbered_adapter(struct i2c_adapter *adap);

int i2c_register_driver(struct module *owner, struct i2c_driver *driver);
void i2c_del_driver(struct i2c_driver *driver);

/* use a define to avoid include chaining to get THIS_MODULE */
#define i2c_add_driver(driver) \
        i2c_register_driver(THIS_MODULE, driver)

static inline bool i2c_client_has_driver(struct i2c_client *client)
{
        return !IS_ERR_OR_NULL(client) && client->dev.driver;
}

/* call the i2c_client->command() of all attached clients with
 * the given arguments */
void i2c_clients_command(struct i2c_adapter *adap,
                         unsigned int cmd, void *arg);

struct i2c_adapter *i2c_get_adapter(int nr);
void i2c_put_adapter(struct i2c_adapter *adap);
unsigned int i2c_adapter_depth(struct i2c_adapter *adapter);

void i2c_parse_fw_timings(struct device *dev, struct i2c_timings *t, bool use_defaults);

/* Return the functionality mask */
static inline u32 i2c_get_functionality(struct i2c_adapter *adap)
{
        return adap->algo->functionality(adap);
}

/* Return 1 if adapter supports everything we need, 0 if not. */
static inline int i2c_check_functionality(struct i2c_adapter *adap, u32 func)
{
        return (func & i2c_get_functionality(adap)) == func;
}

/**
 * i2c_check_quirks() - Function for checking the quirk flags in an i2c adapter
 * @adap: i2c adapter
 * @quirks: quirk flags
 *
 * Return: true if the adapter has all the specified quirk flags, false if not
 */
static inline bool i2c_check_quirks(struct i2c_adapter *adap, u64 quirks)
{
        if (!adap->quirks)
                return false;
        return (adap->quirks->flags & quirks) == quirks;
}

/* Return the adapter number for a specific adapter */
static inline int i2c_adapter_id(struct i2c_adapter *adap)
{
        return adap->nr;
}

static inline u8 i2c_8bit_addr_from_msg(const struct i2c_msg *msg)
{
        return (msg->addr << 1) | (msg->flags & I2C_M_RD);
}

/*
 * 10-bit address
 *   addr_1: 5'b11110 | addr[9:8] | (R/nW)
 *   addr_2: addr[7:0]
 */
static inline u8 i2c_10bit_addr_hi_from_msg(const struct i2c_msg *msg)
{
        return 0xf0 | ((msg->addr & GENMASK(9, 8)) >> 7) | (msg->flags & I2C_M_RD);
}

static inline u8 i2c_10bit_addr_lo_from_msg(const struct i2c_msg *msg)
{
        return msg->addr & GENMASK(7, 0);
}

u8 *i2c_get_dma_safe_msg_buf(struct i2c_msg *msg, unsigned int threshold);
void i2c_put_dma_safe_msg_buf(u8 *buf, struct i2c_msg *msg, bool xferred);

int i2c_handle_smbus_host_notify(struct i2c_adapter *adap, unsigned short addr);
/**
 * module_i2c_driver() - Helper macro for registering a modular I2C driver
 * @__i2c_driver: i2c_driver struct
 *
 * Helper macro for I2C drivers which do not do anything special in module
 * init/exit. This eliminates a lot of boilerplate. Each module may only
 * use this macro once, and calling it replaces module_init() and module_exit()
 */
#define module_i2c_driver(__i2c_driver) \
        module_driver(__i2c_driver, i2c_add_driver, \
                        i2c_del_driver)

/**
 * builtin_i2c_driver() - Helper macro for registering a builtin I2C driver
 * @__i2c_driver: i2c_driver struct
 *
 * Helper macro for I2C drivers which do not do anything special in their
 * init. This eliminates a lot of boilerplate. Each driver may only
 * use this macro once, and calling it replaces device_initcall().
 */
#define builtin_i2c_driver(__i2c_driver) \
        builtin_driver(__i2c_driver, i2c_add_driver)

/* must call put_device() when done with returned i2c_client device */
struct i2c_client *i2c_find_device_by_fwnode(struct fwnode_handle *fwnode);

/* must call put_device() when done with returned i2c_adapter device */
struct i2c_adapter *i2c_find_adapter_by_fwnode(struct fwnode_handle *fwnode);

/* must call i2c_put_adapter() when done with returned i2c_adapter device */
struct i2c_adapter *i2c_get_adapter_by_fwnode(struct fwnode_handle *fwnode);

#else /* I2C */

static inline struct i2c_client *
i2c_find_device_by_fwnode(struct fwnode_handle *fwnode)
{
        return NULL;
}

static inline struct i2c_adapter *
i2c_find_adapter_by_fwnode(struct fwnode_handle *fwnode)
{
        return NULL;
}

static inline struct i2c_adapter *
i2c_get_adapter_by_fwnode(struct fwnode_handle *fwnode)
{
        return NULL;
}

#endif /* !I2C */

#if IS_ENABLED(CONFIG_OF)
/* must call put_device() when done with returned i2c_client device */
static inline struct i2c_client *of_find_i2c_device_by_node(struct device_node *node)
{
        return i2c_find_device_by_fwnode(of_fwnode_handle(node));
}

/* must call put_device() when done with returned i2c_adapter device */
static inline struct i2c_adapter *of_find_i2c_adapter_by_node(struct device_node *node)
{
        return i2c_find_adapter_by_fwnode(of_fwnode_handle(node));
}

/* must call i2c_put_adapter() when done with returned i2c_adapter device */
static inline struct i2c_adapter *of_get_i2c_adapter_by_node(struct device_node *node)
{
        return i2c_get_adapter_by_fwnode(of_fwnode_handle(node));
}

int of_i2c_get_board_info(struct device *dev, struct device_node *node,
                          struct i2c_board_info *info);

#else

static inline struct i2c_client *of_find_i2c_device_by_node(struct device_node *node)
{
        return NULL;
}

static inline struct i2c_adapter *of_find_i2c_adapter_by_node(struct device_node *node)
{
        return NULL;
}

static inline struct i2c_adapter *of_get_i2c_adapter_by_node(struct device_node *node)
{
        return NULL;
}

static inline int of_i2c_get_board_info(struct device *dev,
                                        struct device_node *node,
                                        struct i2c_board_info *info)
{
        return -ENOTSUPP;
}

#endif /* CONFIG_OF */

struct acpi_resource;
struct acpi_resource_i2c_serialbus;

#if IS_REACHABLE(CONFIG_ACPI) && IS_REACHABLE(CONFIG_I2C)
bool i2c_acpi_get_i2c_resource(struct acpi_resource *ares,
                               struct acpi_resource_i2c_serialbus **i2c);
int i2c_acpi_client_count(struct acpi_device *adev);
u32 i2c_acpi_find_bus_speed(struct device *dev);
struct i2c_client *i2c_acpi_new_device_by_fwnode(struct fwnode_handle *fwnode,
                                                 int index,
                                                 struct i2c_board_info *info);
struct i2c_adapter *i2c_acpi_find_adapter_by_handle(acpi_handle handle);
bool i2c_acpi_waive_d0_probe(struct device *dev);
#else
static inline bool i2c_acpi_get_i2c_resource(struct acpi_resource *ares,
                                             struct acpi_resource_i2c_serialbus **i2c)
{
        return false;
}
static inline int i2c_acpi_client_count(struct acpi_device *adev)
{
        return 0;
}
static inline u32 i2c_acpi_find_bus_speed(struct device *dev)
{
        return 0;
}
static inline struct i2c_client *i2c_acpi_new_device_by_fwnode(
                                        struct fwnode_handle *fwnode, int index,
                                        struct i2c_board_info *info)
{
        return ERR_PTR(-ENODEV);
}
static inline struct i2c_adapter *i2c_acpi_find_adapter_by_handle(acpi_handle handle)
{
        return NULL;
}
static inline bool i2c_acpi_waive_d0_probe(struct device *dev)
{
        return false;
}
#endif /* CONFIG_ACPI */

static inline struct i2c_client *i2c_acpi_new_device(struct device *dev,
                                                     int index,
                                                     struct i2c_board_info *info)
{
        return i2c_acpi_new_device_by_fwnode(dev_fwnode(dev), index, info);
}

#endif /* _LINUX_I2C_H */

























    5 

























    1 



    4 




    3 
    3 


    1 


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
// SPDX-License-Identifier: GPL-2.0
/* XDP user-space ring structure
 * Copyright(c) 2018 Intel Corporation.
 */

#include <linux/log2.h>
#include <linux/slab.h>
#include <linux/overflow.h>
#include <linux/vmalloc.h>
#include <net/xdp_sock_drv.h>

#include "xsk_queue.h"

static size_t xskq_get_ring_size(struct xsk_queue *q, bool umem_queue)
{
        struct xdp_umem_ring *umem_ring;
        struct xdp_rxtx_ring *rxtx_ring;

        if (umem_queue)
                return struct_size(umem_ring, desc, q->nentries);
        return struct_size(rxtx_ring, desc, q->nentries);
}

struct xsk_queue *xskq_create(u32 nentries, bool umem_queue)
{
        struct xsk_queue *q;
        size_t size;

        q = kzalloc(sizeof(*q), GFP_KERNEL);
        if (!q)
                return NULL;

        q->nentries = nentries;
        q->ring_mask = nentries - 1;

        size = xskq_get_ring_size(q, umem_queue);

        /* size which is overflowing or close to SIZE_MAX will become 0 in
         * PAGE_ALIGN(), checking SIZE_MAX is enough due to the previous
         * is_power_of_2(), the rest will be handled by vmalloc_user()
         */
        if (unlikely(size == SIZE_MAX)) {
                kfree(q);
                return NULL;
        }

        size = PAGE_ALIGN(size);

        q->ring = vmalloc_user(size);
        if (!q->ring) {
                kfree(q);
                return NULL;
        }

        q->ring_vmalloc_size = size;
        return q;
}

void xskq_destroy(struct xsk_queue *q)
{
        if (!q)
                return;

        vfree(q->ring);
        kfree(q);
}

























































    1 



    1 




















































































































































    2 







    2 




    2 
    1 
    1 































































































































































































    2 

















1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * Syntek DV4000 (STK014) subdriver
 *
 * Copyright (C) 2008 Jean-Francois Moine (http://moinejf.free.fr)
 */

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#define MODULE_NAME "stk014"

#include "gspca.h"
#include "jpeg.h"

MODULE_AUTHOR("Jean-Francois Moine <http://moinejf.free.fr>");
MODULE_DESCRIPTION("Syntek DV4000 (STK014) USB Camera Driver");
MODULE_LICENSE("GPL");

#define QUALITY 50

/* specific webcam descriptor */
struct sd {
        struct gspca_dev gspca_dev;        /* !! must be the first item */
        u8 jpeg_hdr[JPEG_HDR_SZ];
};

static const struct v4l2_pix_format vga_mode[] = {
        {320, 240, V4L2_PIX_FMT_JPEG, V4L2_FIELD_NONE,
                .bytesperline = 320,
                .sizeimage = 320 * 240 * 3 / 8 + 590,
                .colorspace = V4L2_COLORSPACE_JPEG,
                .priv = 1},
        {640, 480, V4L2_PIX_FMT_JPEG, V4L2_FIELD_NONE,
                .bytesperline = 640,
                .sizeimage = 640 * 480 * 3 / 8 + 590,
                .colorspace = V4L2_COLORSPACE_JPEG,
                .priv = 0},
};

/* -- read a register -- */
static u8 reg_r(struct gspca_dev *gspca_dev,
                        __u16 index)
{
        struct usb_device *dev = gspca_dev->dev;
        int ret;

        if (gspca_dev->usb_err < 0)
                return 0;
        ret = usb_control_msg(dev, usb_rcvctrlpipe(dev, 0),
                        0x00,
                        USB_DIR_IN | USB_TYPE_VENDOR | USB_RECIP_DEVICE,
                        0x00,
                        index,
                        gspca_dev->usb_buf, 1,
                        500);
        if (ret < 0) {
                pr_err("reg_r err %d\n", ret);
                gspca_dev->usb_err = ret;
                return 0;
        }
        return gspca_dev->usb_buf[0];
}

/* -- write a register -- */
static void reg_w(struct gspca_dev *gspca_dev,
                        __u16 index, __u16 value)
{
        struct usb_device *dev = gspca_dev->dev;
        int ret;

        if (gspca_dev->usb_err < 0)
                return;
        ret = usb_control_msg(dev, usb_sndctrlpipe(dev, 0),
                        0x01,
                        USB_DIR_OUT | USB_TYPE_VENDOR | USB_RECIP_DEVICE,
                        value,
                        index,
                        NULL,
                        0,
                        500);
        if (ret < 0) {
                pr_err("reg_w err %d\n", ret);
                gspca_dev->usb_err = ret;
        }
}

/* -- get a bulk value (4 bytes) -- */
static void rcv_val(struct gspca_dev *gspca_dev,
                        int ads)
{
        struct usb_device *dev = gspca_dev->dev;
        int alen, ret;

        reg_w(gspca_dev, 0x634, (ads >> 16) & 0xff);
        reg_w(gspca_dev, 0x635, (ads >> 8) & 0xff);
        reg_w(gspca_dev, 0x636, ads & 0xff);
        reg_w(gspca_dev, 0x637, 0);
        reg_w(gspca_dev, 0x638, 4);        /* len & 0xff */
        reg_w(gspca_dev, 0x639, 0);        /* len >> 8 */
        reg_w(gspca_dev, 0x63a, 0);
        reg_w(gspca_dev, 0x63b, 0);
        reg_w(gspca_dev, 0x630, 5);
        if (gspca_dev->usb_err < 0)
                return;
        ret = usb_bulk_msg(dev,
                        usb_rcvbulkpipe(dev, 0x05),
                        gspca_dev->usb_buf,
                        4,                /* length */
                        &alen,
                        500);                /* timeout in milliseconds */
        if (ret < 0) {
                pr_err("rcv_val err %d\n", ret);
                gspca_dev->usb_err = ret;
        }
}

/* -- send a bulk value -- */
static void snd_val(struct gspca_dev *gspca_dev,
                        int ads,
                        unsigned int val)
{
        struct usb_device *dev = gspca_dev->dev;
        int alen, ret;
        __u8 seq = 0;

        if (ads == 0x003f08) {
                reg_r(gspca_dev, 0x0704);
                seq = reg_r(gspca_dev, 0x0705);
                reg_r(gspca_dev, 0x0650);
                reg_w(gspca_dev, 0x654, seq);
        } else {
                reg_w(gspca_dev, 0x654, (ads >> 16) & 0xff);
        }
        reg_w(gspca_dev, 0x655, (ads >> 8) & 0xff);
        reg_w(gspca_dev, 0x656, ads & 0xff);
        reg_w(gspca_dev, 0x657, 0);
        reg_w(gspca_dev, 0x658, 0x04);        /* size */
        reg_w(gspca_dev, 0x659, 0);
        reg_w(gspca_dev, 0x65a, 0);
        reg_w(gspca_dev, 0x65b, 0);
        reg_w(gspca_dev, 0x650, 5);
        if (gspca_dev->usb_err < 0)
                return;
        gspca_dev->usb_buf[0] = val >> 24;
        gspca_dev->usb_buf[1] = val >> 16;
        gspca_dev->usb_buf[2] = val >> 8;
        gspca_dev->usb_buf[3] = val;
        ret = usb_bulk_msg(dev,
                        usb_sndbulkpipe(dev, 6),
                        gspca_dev->usb_buf,
                        4,
                        &alen,
                        500);        /* timeout in milliseconds */
        if (ret < 0) {
                pr_err("snd_val err %d\n", ret);
                gspca_dev->usb_err = ret;
        } else {
                if (ads == 0x003f08) {
                        seq += 4;
                        seq &= 0x3f;
                        reg_w(gspca_dev, 0x705, seq);
                }
        }
}

/* set a camera parameter */
static void set_par(struct gspca_dev *gspca_dev,
                   int parval)
{
        snd_val(gspca_dev, 0x003f08, parval);
}

static void setbrightness(struct gspca_dev *gspca_dev, s32 val)
{
        int parval;

        parval = 0x06000000                /* whiteness */
                + (val << 16);
        set_par(gspca_dev, parval);
}

static void setcontrast(struct gspca_dev *gspca_dev, s32 val)
{
        int parval;

        parval = 0x07000000                /* contrast */
                + (val << 16);
        set_par(gspca_dev, parval);
}

static void setcolors(struct gspca_dev *gspca_dev, s32 val)
{
        int parval;

        parval = 0x08000000                /* saturation */
                + (val << 16);
        set_par(gspca_dev, parval);
}

static void setlightfreq(struct gspca_dev *gspca_dev, s32 val)
{
        set_par(gspca_dev, val == 1
                        ? 0x33640000                /* 50 Hz */
                        : 0x33780000);                /* 60 Hz */
}

/* this function is called at probe time */
static int sd_config(struct gspca_dev *gspca_dev,
                        const struct usb_device_id *id)
{
        gspca_dev->cam.cam_mode = vga_mode;
        gspca_dev->cam.nmodes = ARRAY_SIZE(vga_mode);
        return 0;
}

/* this function is called at probe and resume time */
static int sd_init(struct gspca_dev *gspca_dev)
{
        u8 ret;

        /* check if the device responds */
        usb_set_interface(gspca_dev->dev, gspca_dev->iface, 1);
        ret = reg_r(gspca_dev, 0x0740);
        if (gspca_dev->usb_err >= 0) {
                if (ret != 0xff) {
                        pr_err("init reg: 0x%02x\n", ret);
                        gspca_dev->usb_err = -EIO;
                }
        }
        return gspca_dev->usb_err;
}

/* -- start the camera -- */
static int sd_start(struct gspca_dev *gspca_dev)
{
        struct sd *sd = (struct sd *) gspca_dev;
        int ret, value;

        /* create the JPEG header */
        jpeg_define(sd->jpeg_hdr, gspca_dev->pixfmt.height,
                        gspca_dev->pixfmt.width,
                        0x22);                /* JPEG 411 */
        jpeg_set_qual(sd->jpeg_hdr, QUALITY);

        /* work on alternate 1 */
        usb_set_interface(gspca_dev->dev, gspca_dev->iface, 1);

        set_par(gspca_dev, 0x10000000);
        set_par(gspca_dev, 0x00000000);
        set_par(gspca_dev, 0x8002e001);
        set_par(gspca_dev, 0x14000000);
        if (gspca_dev->pixfmt.width > 320)
                value = 0x8002e001;                /* 640x480 */
        else
                value = 0x4001f000;                /* 320x240 */
        set_par(gspca_dev, value);
        ret = usb_set_interface(gspca_dev->dev,
                                        gspca_dev->iface,
                                        gspca_dev->alt);
        if (ret < 0) {
                pr_err("set intf %d %d failed\n",
                       gspca_dev->iface, gspca_dev->alt);
                gspca_dev->usb_err = ret;
                goto out;
        }
        reg_r(gspca_dev, 0x0630);
        rcv_val(gspca_dev, 0x000020);        /* << (value ff ff ff ff) */
        reg_r(gspca_dev, 0x0650);
        snd_val(gspca_dev, 0x000020, 0xffffffff);
        reg_w(gspca_dev, 0x0620, 0);
        reg_w(gspca_dev, 0x0630, 0);
        reg_w(gspca_dev, 0x0640, 0);
        reg_w(gspca_dev, 0x0650, 0);
        reg_w(gspca_dev, 0x0660, 0);
        set_par(gspca_dev, 0x09800000);                /* Red ? */
        set_par(gspca_dev, 0x0a800000);                /* Green ? */
        set_par(gspca_dev, 0x0b800000);                /* Blue ? */
        set_par(gspca_dev, 0x0d030000);                /* Gamma ? */

        /* start the video flow */
        set_par(gspca_dev, 0x01000000);
        set_par(gspca_dev, 0x01000000);
        if (gspca_dev->usb_err >= 0)
                gspca_dbg(gspca_dev, D_STREAM, "camera started alt: 0x%02x\n",
                          gspca_dev->alt);
out:
        return gspca_dev->usb_err;
}

static void sd_stopN(struct gspca_dev *gspca_dev)
{
        struct usb_device *dev = gspca_dev->dev;

        set_par(gspca_dev, 0x02000000);
        set_par(gspca_dev, 0x02000000);
        usb_set_interface(dev, gspca_dev->iface, 1);
        reg_r(gspca_dev, 0x0630);
        rcv_val(gspca_dev, 0x000020);        /* << (value ff ff ff ff) */
        reg_r(gspca_dev, 0x0650);
        snd_val(gspca_dev, 0x000020, 0xffffffff);
        reg_w(gspca_dev, 0x0620, 0);
        reg_w(gspca_dev, 0x0630, 0);
        reg_w(gspca_dev, 0x0640, 0);
        reg_w(gspca_dev, 0x0650, 0);
        reg_w(gspca_dev, 0x0660, 0);
        gspca_dbg(gspca_dev, D_STREAM, "camera stopped\n");
}

static void sd_pkt_scan(struct gspca_dev *gspca_dev,
                        u8 *data,                        /* isoc packet */
                        int len)                        /* iso packet length */
{
        struct sd *sd = (struct sd *) gspca_dev;
        static unsigned char ffd9[] = {0xff, 0xd9};

        /* a frame starts with:
         *        - 0xff 0xfe
         *        - 0x08 0x00        - length (little endian ?!)
         *        - 4 bytes = size of whole frame (BE - including header)
         *        - 0x00 0x0c
         *        - 0xff 0xd8
         *        - ..        JPEG image with escape sequences (ff 00)
         *                (without ending - ff d9)
         */
        if (data[0] == 0xff && data[1] == 0xfe) {
                gspca_frame_add(gspca_dev, LAST_PACKET,
                                ffd9, 2);

                /* put the JPEG 411 header */
                gspca_frame_add(gspca_dev, FIRST_PACKET,
                        sd->jpeg_hdr, JPEG_HDR_SZ);

                /* beginning of the frame */
#define STKHDRSZ 12
                data += STKHDRSZ;
                len -= STKHDRSZ;
        }
        gspca_frame_add(gspca_dev, INTER_PACKET, data, len);
}

static int sd_s_ctrl(struct v4l2_ctrl *ctrl)
{
        struct gspca_dev *gspca_dev =
                container_of(ctrl->handler, struct gspca_dev, ctrl_handler);

        gspca_dev->usb_err = 0;

        if (!gspca_dev->streaming)
                return 0;

        switch (ctrl->id) {
        case V4L2_CID_BRIGHTNESS:
                setbrightness(gspca_dev, ctrl->val);
                break;
        case V4L2_CID_CONTRAST:
                setcontrast(gspca_dev, ctrl->val);
                break;
        case V4L2_CID_SATURATION:
                setcolors(gspca_dev, ctrl->val);
                break;
        case V4L2_CID_POWER_LINE_FREQUENCY:
                setlightfreq(gspca_dev, ctrl->val);
                break;
        }
        return gspca_dev->usb_err;
}

static const struct v4l2_ctrl_ops sd_ctrl_ops = {
        .s_ctrl = sd_s_ctrl,
};

static int sd_init_controls(struct gspca_dev *gspca_dev)
{
        struct v4l2_ctrl_handler *hdl = &gspca_dev->ctrl_handler;

        gspca_dev->vdev.ctrl_handler = hdl;
        v4l2_ctrl_handler_init(hdl, 4);
        v4l2_ctrl_new_std(hdl, &sd_ctrl_ops,
                        V4L2_CID_BRIGHTNESS, 0, 255, 1, 127);
        v4l2_ctrl_new_std(hdl, &sd_ctrl_ops,
                        V4L2_CID_CONTRAST, 0, 255, 1, 127);
        v4l2_ctrl_new_std(hdl, &sd_ctrl_ops,
                        V4L2_CID_SATURATION, 0, 255, 1, 127);
        v4l2_ctrl_new_std_menu(hdl, &sd_ctrl_ops,
                        V4L2_CID_POWER_LINE_FREQUENCY,
                        V4L2_CID_POWER_LINE_FREQUENCY_60HZ, 1,
                        V4L2_CID_POWER_LINE_FREQUENCY_50HZ);

        if (hdl->error) {
                pr_err("Could not initialize controls\n");
                return hdl->error;
        }
        return 0;
}

/* sub-driver description */
static const struct sd_desc sd_desc = {
        .name = MODULE_NAME,
        .config = sd_config,
        .init = sd_init,
        .init_controls = sd_init_controls,
        .start = sd_start,
        .stopN = sd_stopN,
        .pkt_scan = sd_pkt_scan,
};

/* -- module initialisation -- */
static const struct usb_device_id device_table[] = {
        {USB_DEVICE(0x05e1, 0x0893)},
        {}
};
MODULE_DEVICE_TABLE(usb, device_table);

/* -- device connect -- */
static int sd_probe(struct usb_interface *intf,
                        const struct usb_device_id *id)
{
        return gspca_dev_probe(intf, id, &sd_desc, sizeof(struct sd),
                                THIS_MODULE);
}

static struct usb_driver sd_driver = {
        .name = MODULE_NAME,
        .id_table = device_table,
        .probe = sd_probe,
        .disconnect = gspca_disconnect,
#ifdef CONFIG_PM
        .suspend = gspca_suspend,
        .resume = gspca_resume,
        .reset_resume = gspca_resume,
#endif
};

module_usb_driver(sd_driver);






































































































  434 



























  438 







  438 
  440 




   40 
   40 





    5 







    5 





    4 
















































    2 
    2 



    1 
    1 







    1 







    1 











    5 
    5 



    5 






    5 

    1 

    5 



    5 












    2 
    1 





    1 










   11 









    1 




    1 










    5 

    3 
    1 








    5 
    4 









    2 









    4 

    1 
    2 















    7 





    7 
















    7 




    7 









    8 




   17 















    1 













   29 





   18 


   11 
    1 


    1 





   11 

   15 
    1 


    1 


    1 











    1 






    1 





    2 




    1 




    2 





    1 





    1 





    1 





    1 






    1 
    1 





    1 


    1 







    3 




   14 











   61 
   13 

   80 




















   15 


















   14 














    1 
    1 

    1 








































    1 






    1 




    1 






    1 












   17 

















   62 










































    3 

    3 
    3 


    2 







    1 



    3 




    3 





    1 










    3 
    3 






















    5 





























   32 





























   86 



























   52 

   86 



   32 



   32 





   32 











    1 






    1 










    2 







    2 







    2 






    1 

    1 







































   32 











   32 



   31 






   31 






















1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
// SPDX-License-Identifier: GPL-2.0
#include <linux/anon_inodes.h>
#include <linux/exportfs.h>
#include <linux/file.h>
#include <linux/fs.h>
#include <linux/cgroup.h>
#include <linux/magic.h>
#include <linux/mount.h>
#include <linux/pid.h>
#include <linux/pidfs.h>
#include <linux/pid_namespace.h>
#include <linux/poll.h>
#include <linux/proc_fs.h>
#include <linux/proc_ns.h>
#include <linux/pseudo_fs.h>
#include <linux/ptrace.h>
#include <linux/seq_file.h>
#include <uapi/linux/pidfd.h>
#include <linux/ipc_namespace.h>
#include <linux/time_namespace.h>
#include <linux/utsname.h>
#include <net/net_namespace.h>
#include <linux/coredump.h>
#include <linux/xattr.h>

#include "internal.h"
#include "mount.h"

#define PIDFS_PID_DEAD ERR_PTR(-ESRCH)

static struct kmem_cache *pidfs_attr_cachep __ro_after_init;
static struct kmem_cache *pidfs_xattr_cachep __ro_after_init;

static struct path pidfs_root_path = {};

void pidfs_get_root(struct path *path)
{
        *path = pidfs_root_path;
        path_get(path);
}

/*
 * Stashes information that userspace needs to access even after the
 * process has been reaped.
 */
struct pidfs_exit_info {
        __u64 cgroupid;
        __s32 exit_code;
        __u32 coredump_mask;
};

struct pidfs_attr {
        struct simple_xattrs *xattrs;
        struct pidfs_exit_info __pei;
        struct pidfs_exit_info *exit_info;
};

static struct rb_root pidfs_ino_tree = RB_ROOT;

#if BITS_PER_LONG == 32
static inline unsigned long pidfs_ino(u64 ino)
{
        return lower_32_bits(ino);
}

/* On 32 bit the generation number are the upper 32 bits. */
static inline u32 pidfs_gen(u64 ino)
{
        return upper_32_bits(ino);
}

#else

/* On 64 bit simply return ino. */
static inline unsigned long pidfs_ino(u64 ino)
{
        return ino;
}

/* On 64 bit the generation number is 0. */
static inline u32 pidfs_gen(u64 ino)
{
        return 0;
}
#endif

static int pidfs_ino_cmp(struct rb_node *a, const struct rb_node *b)
{
        struct pid *pid_a = rb_entry(a, struct pid, pidfs_node);
        struct pid *pid_b = rb_entry(b, struct pid, pidfs_node);
        u64 pid_ino_a = pid_a->ino;
        u64 pid_ino_b = pid_b->ino;

        if (pid_ino_a < pid_ino_b)
                return -1;
        if (pid_ino_a > pid_ino_b)
                return 1;
        return 0;
}

void pidfs_add_pid(struct pid *pid)
{
        static u64 pidfs_ino_nr = 2;

        /*
         * On 64 bit nothing special happens. The 64bit number assigned
         * to struct pid is the inode number.
         *
         * On 32 bit the 64 bit number assigned to struct pid is split
         * into two 32 bit numbers. The lower 32 bits are used as the
         * inode number and the upper 32 bits are used as the inode
         * generation number.
         *
         * On 32 bit pidfs_ino() will return the lower 32 bit. When
         * pidfs_ino() returns zero a wrap around happened. When a
         * wraparound happens the 64 bit number will be incremented by 2
         * so inode numbering starts at 2 again.
         *
         * On 64 bit comparing two pidfds is as simple as comparing
         * inode numbers.
         *
         * When a wraparound happens on 32 bit multiple pidfds with the
         * same inode number are likely to exist (This isn't a problem
         * since before pidfs pidfds used the anonymous inode meaning
         * all pidfds had the same inode number.). Userspace can
         * reconstruct the 64 bit identifier by retrieving both the
         * inode number and the inode generation number to compare or
         * use file handles.
         */
        if (pidfs_ino(pidfs_ino_nr) == 0)
                pidfs_ino_nr += 2;

        pid->ino = pidfs_ino_nr;
        pid->stashed = NULL;
        pid->attr = NULL;
        pidfs_ino_nr++;

        write_seqcount_begin(&pidmap_lock_seq);
        rb_find_add_rcu(&pid->pidfs_node, &pidfs_ino_tree, pidfs_ino_cmp);
        write_seqcount_end(&pidmap_lock_seq);
}

void pidfs_remove_pid(struct pid *pid)
{
        write_seqcount_begin(&pidmap_lock_seq);
        rb_erase(&pid->pidfs_node, &pidfs_ino_tree);
        write_seqcount_end(&pidmap_lock_seq);
}

void pidfs_free_pid(struct pid *pid)
{
        struct pidfs_attr *attr __free(kfree) = no_free_ptr(pid->attr);
        struct simple_xattrs *xattrs __free(kfree) = NULL;

        /*
         * Any dentry must've been wiped from the pid by now.
         * Otherwise there's a reference count bug.
         */
        VFS_WARN_ON_ONCE(pid->stashed);

        /*
         * This if an error occurred during e.g., task creation that
         * causes us to never go through the exit path.
         */
        if (unlikely(!attr))
                return;

        /* This never had a pidfd created. */
        if (IS_ERR(attr))
                return;

        xattrs = no_free_ptr(attr->xattrs);
        if (xattrs)
                simple_xattrs_free(xattrs, NULL);
}

#ifdef CONFIG_PROC_FS
/**
 * pidfd_show_fdinfo - print information about a pidfd
 * @m: proc fdinfo file
 * @f: file referencing a pidfd
 *
 * Pid:
 * This function will print the pid that a given pidfd refers to in the
 * pid namespace of the procfs instance.
 * If the pid namespace of the process is not a descendant of the pid
 * namespace of the procfs instance 0 will be shown as its pid. This is
 * similar to calling getppid() on a process whose parent is outside of
 * its pid namespace.
 *
 * NSpid:
 * If pid namespaces are supported then this function will also print
 * the pid of a given pidfd refers to for all descendant pid namespaces
 * starting from the current pid namespace of the instance, i.e. the
 * Pid field and the first entry in the NSpid field will be identical.
 * If the pid namespace of the process is not a descendant of the pid
 * namespace of the procfs instance 0 will be shown as its first NSpid
 * entry and no others will be shown.
 * Note that this differs from the Pid and NSpid fields in
 * /proc/<pid>/status where Pid and NSpid are always shown relative to
 * the  pid namespace of the procfs instance. The difference becomes
 * obvious when sending around a pidfd between pid namespaces from a
 * different branch of the tree, i.e. where no ancestral relation is
 * present between the pid namespaces:
 * - create two new pid namespaces ns1 and ns2 in the initial pid
 *   namespace (also take care to create new mount namespaces in the
 *   new pid namespace and mount procfs)
 * - create a process with a pidfd in ns1
 * - send pidfd from ns1 to ns2
 * - read /proc/self/fdinfo/<pidfd> and observe that both Pid and NSpid
 *   have exactly one entry, which is 0
 */
static void pidfd_show_fdinfo(struct seq_file *m, struct file *f)
{
        struct pid *pid = pidfd_pid(f);
        struct pid_namespace *ns;
        pid_t nr = -1;

        if (likely(pid_has_task(pid, PIDTYPE_PID))) {
                ns = proc_pid_ns(file_inode(m->file)->i_sb);
                nr = pid_nr_ns(pid, ns);
        }

        seq_put_decimal_ll(m, "Pid:\t", nr);

#ifdef CONFIG_PID_NS
        seq_put_decimal_ll(m, "\nNSpid:\t", nr);
        if (nr > 0) {
                int i;

                /* If nr is non-zero it means that 'pid' is valid and that
                 * ns, i.e. the pid namespace associated with the procfs
                 * instance, is in the pid namespace hierarchy of pid.
                 * Start at one below the already printed level.
                 */
                for (i = ns->level + 1; i <= pid->level; i++)
                        seq_put_decimal_ll(m, "\t", pid->numbers[i].nr);
        }
#endif
        seq_putc(m, '\n');
}
#endif

/*
 * Poll support for process exit notification.
 */
static __poll_t pidfd_poll(struct file *file, struct poll_table_struct *pts)
{
        struct pid *pid = pidfd_pid(file);
        struct task_struct *task;
        __poll_t poll_flags = 0;

        poll_wait(file, &pid->wait_pidfd, pts);
        /*
         * Don't wake waiters if the thread-group leader exited
         * prematurely. They either get notified when the last subthread
         * exits or not at all if one of the remaining subthreads execs
         * and assumes the struct pid of the old thread-group leader.
         */
        guard(rcu)();
        task = pid_task(pid, PIDTYPE_PID);
        if (!task)
                poll_flags = EPOLLIN | EPOLLRDNORM | EPOLLHUP;
        else if (task->exit_state && !delay_group_leader(task))
                poll_flags = EPOLLIN | EPOLLRDNORM;

        return poll_flags;
}

static inline bool pid_in_current_pidns(const struct pid *pid)
{
        const struct pid_namespace *ns = task_active_pid_ns(current);

        if (ns->level <= pid->level)
                return pid->numbers[ns->level].ns == ns;

        return false;
}

static __u32 pidfs_coredump_mask(unsigned long mm_flags)
{
        switch (__get_dumpable(mm_flags)) {
        case SUID_DUMP_USER:
                return PIDFD_COREDUMP_USER;
        case SUID_DUMP_ROOT:
                return PIDFD_COREDUMP_ROOT;
        case SUID_DUMP_DISABLE:
                return PIDFD_COREDUMP_SKIP;
        default:
                WARN_ON_ONCE(true);
        }

        return 0;
}

static long pidfd_info(struct file *file, unsigned int cmd, unsigned long arg)
{
        struct pidfd_info __user *uinfo = (struct pidfd_info __user *)arg;
        struct pid *pid = pidfd_pid(file);
        size_t usize = _IOC_SIZE(cmd);
        struct pidfd_info kinfo = {};
        struct pidfs_exit_info *exit_info;
        struct user_namespace *user_ns;
        struct task_struct *task;
        struct pidfs_attr *attr;
        const struct cred *c;
        __u64 mask;

        if (!uinfo)
                return -EINVAL;
        if (usize < PIDFD_INFO_SIZE_VER0)
                return -EINVAL; /* First version, no smaller struct possible */

        if (copy_from_user(&mask, &uinfo->mask, sizeof(mask)))
                return -EFAULT;

        /*
         * Restrict information retrieval to tasks within the caller's pid
         * namespace hierarchy.
         */
        if (!pid_in_current_pidns(pid))
                return -ESRCH;

        attr = READ_ONCE(pid->attr);
        if (mask & PIDFD_INFO_EXIT) {
                exit_info = READ_ONCE(attr->exit_info);
                if (exit_info) {
                        kinfo.mask |= PIDFD_INFO_EXIT;
#ifdef CONFIG_CGROUPS
                        kinfo.cgroupid = exit_info->cgroupid;
                        kinfo.mask |= PIDFD_INFO_CGROUPID;
#endif
                        kinfo.exit_code = exit_info->exit_code;
                }
        }

        if (mask & PIDFD_INFO_COREDUMP) {
                kinfo.mask |= PIDFD_INFO_COREDUMP;
                kinfo.coredump_mask = READ_ONCE(attr->__pei.coredump_mask);
        }

        task = get_pid_task(pid, PIDTYPE_PID);
        if (!task) {
                /*
                 * If the task has already been reaped, only exit
                 * information is available
                 */
                if (!(mask & PIDFD_INFO_EXIT))
                        return -ESRCH;

                goto copy_out;
        }

        c = get_task_cred(task);
        if (!c)
                return -ESRCH;

        if ((kinfo.mask & PIDFD_INFO_COREDUMP) && !(kinfo.coredump_mask)) {
                task_lock(task);
                if (task->mm)
                        kinfo.coredump_mask = pidfs_coredump_mask(task->mm->flags);
                task_unlock(task);
        }

        /* Unconditionally return identifiers and credentials, the rest only on request */

        user_ns = current_user_ns();
        kinfo.ruid = from_kuid_munged(user_ns, c->uid);
        kinfo.rgid = from_kgid_munged(user_ns, c->gid);
        kinfo.euid = from_kuid_munged(user_ns, c->euid);
        kinfo.egid = from_kgid_munged(user_ns, c->egid);
        kinfo.suid = from_kuid_munged(user_ns, c->suid);
        kinfo.sgid = from_kgid_munged(user_ns, c->sgid);
        kinfo.fsuid = from_kuid_munged(user_ns, c->fsuid);
        kinfo.fsgid = from_kgid_munged(user_ns, c->fsgid);
        kinfo.mask |= PIDFD_INFO_CREDS;
        put_cred(c);

#ifdef CONFIG_CGROUPS
        if (!kinfo.cgroupid) {
                struct cgroup *cgrp;

                rcu_read_lock();
                cgrp = task_dfl_cgroup(task);
                kinfo.cgroupid = cgroup_id(cgrp);
                kinfo.mask |= PIDFD_INFO_CGROUPID;
                rcu_read_unlock();
        }
#endif

        /*
         * Copy pid/tgid last, to reduce the chances the information might be
         * stale. Note that it is not possible to ensure it will be valid as the
         * task might return as soon as the copy_to_user finishes, but that's ok
         * and userspace expects that might happen and can act accordingly, so
         * this is just best-effort. What we can do however is checking that all
         * the fields are set correctly, or return ESRCH to avoid providing
         * incomplete information. */

        kinfo.ppid = task_ppid_nr_ns(task, NULL);
        kinfo.tgid = task_tgid_vnr(task);
        kinfo.pid = task_pid_vnr(task);
        kinfo.mask |= PIDFD_INFO_PID;

        if (kinfo.pid == 0 || kinfo.tgid == 0)
                return -ESRCH;

copy_out:
        /*
         * If userspace and the kernel have the same struct size it can just
         * be copied. If userspace provides an older struct, only the bits that
         * userspace knows about will be copied. If userspace provides a new
         * struct, only the bits that the kernel knows about will be copied.
         */
        return copy_struct_to_user(uinfo, usize, &kinfo, sizeof(kinfo), NULL);
}

static bool pidfs_ioctl_valid(unsigned int cmd)
{
        switch (cmd) {
        case FS_IOC_GETVERSION:
        case PIDFD_GET_CGROUP_NAMESPACE:
        case PIDFD_GET_IPC_NAMESPACE:
        case PIDFD_GET_MNT_NAMESPACE:
        case PIDFD_GET_NET_NAMESPACE:
        case PIDFD_GET_PID_FOR_CHILDREN_NAMESPACE:
        case PIDFD_GET_TIME_NAMESPACE:
        case PIDFD_GET_TIME_FOR_CHILDREN_NAMESPACE:
        case PIDFD_GET_UTS_NAMESPACE:
        case PIDFD_GET_USER_NAMESPACE:
        case PIDFD_GET_PID_NAMESPACE:
                return true;
        }

        /* Extensible ioctls require some more careful checks. */
        switch (_IOC_NR(cmd)) {
        case _IOC_NR(PIDFD_GET_INFO):
                /*
                 * Try to prevent performing a pidfd ioctl when someone
                 * erronously mistook the file descriptor for a pidfd.
                 * This is not perfect but will catch most cases.
                 */
                return (_IOC_TYPE(cmd) == _IOC_TYPE(PIDFD_GET_INFO));
        }

        return false;
}

static long pidfd_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
{
        struct task_struct *task __free(put_task) = NULL;
        struct nsproxy *nsp __free(put_nsproxy) = NULL;
        struct ns_common *ns_common = NULL;
        struct pid_namespace *pid_ns;

        if (!pidfs_ioctl_valid(cmd))
                return -ENOIOCTLCMD;

        if (cmd == FS_IOC_GETVERSION) {
                if (!arg)
                        return -EINVAL;

                __u32 __user *argp = (__u32 __user *)arg;
                return put_user(file_inode(file)->i_generation, argp);
        }

        /* Extensible IOCTL that does not open namespace FDs, take a shortcut */
        if (_IOC_NR(cmd) == _IOC_NR(PIDFD_GET_INFO))
                return pidfd_info(file, cmd, arg);

        task = get_pid_task(pidfd_pid(file), PIDTYPE_PID);
        if (!task)
                return -ESRCH;

        if (arg)
                return -EINVAL;

        scoped_guard(task_lock, task) {
                nsp = task->nsproxy;
                if (nsp)
                        get_nsproxy(nsp);
        }
        if (!nsp)
                return -ESRCH; /* just pretend it didn't exist */

        /*
         * We're trying to open a file descriptor to the namespace so perform a
         * filesystem cred ptrace check. Also, we mirror nsfs behavior.
         */
        if (!ptrace_may_access(task, PTRACE_MODE_READ_FSCREDS))
                return -EACCES;

        switch (cmd) {
        /* Namespaces that hang of nsproxy. */
        case PIDFD_GET_CGROUP_NAMESPACE:
                if (IS_ENABLED(CONFIG_CGROUPS)) {
                        get_cgroup_ns(nsp->cgroup_ns);
                        ns_common = to_ns_common(nsp->cgroup_ns);
                }
                break;
        case PIDFD_GET_IPC_NAMESPACE:
                if (IS_ENABLED(CONFIG_IPC_NS)) {
                        get_ipc_ns(nsp->ipc_ns);
                        ns_common = to_ns_common(nsp->ipc_ns);
                }
                break;
        case PIDFD_GET_MNT_NAMESPACE:
                get_mnt_ns(nsp->mnt_ns);
                ns_common = to_ns_common(nsp->mnt_ns);
                break;
        case PIDFD_GET_NET_NAMESPACE:
                if (IS_ENABLED(CONFIG_NET_NS)) {
                        ns_common = to_ns_common(nsp->net_ns);
                        get_net_ns(ns_common);
                }
                break;
        case PIDFD_GET_PID_FOR_CHILDREN_NAMESPACE:
                if (IS_ENABLED(CONFIG_PID_NS)) {
                        get_pid_ns(nsp->pid_ns_for_children);
                        ns_common = to_ns_common(nsp->pid_ns_for_children);
                }
                break;
        case PIDFD_GET_TIME_NAMESPACE:
                if (IS_ENABLED(CONFIG_TIME_NS)) {
                        get_time_ns(nsp->time_ns);
                        ns_common = to_ns_common(nsp->time_ns);
                }
                break;
        case PIDFD_GET_TIME_FOR_CHILDREN_NAMESPACE:
                if (IS_ENABLED(CONFIG_TIME_NS)) {
                        get_time_ns(nsp->time_ns_for_children);
                        ns_common = to_ns_common(nsp->time_ns_for_children);
                }
                break;
        case PIDFD_GET_UTS_NAMESPACE:
                if (IS_ENABLED(CONFIG_UTS_NS)) {
                        get_uts_ns(nsp->uts_ns);
                        ns_common = to_ns_common(nsp->uts_ns);
                }
                break;
        /* Namespaces that don't hang of nsproxy. */
        case PIDFD_GET_USER_NAMESPACE:
                if (IS_ENABLED(CONFIG_USER_NS)) {
                        rcu_read_lock();
                        ns_common = to_ns_common(get_user_ns(task_cred_xxx(task, user_ns)));
                        rcu_read_unlock();
                }
                break;
        case PIDFD_GET_PID_NAMESPACE:
                if (IS_ENABLED(CONFIG_PID_NS)) {
                        rcu_read_lock();
                        pid_ns = task_active_pid_ns(task);
                        if (pid_ns)
                                ns_common = to_ns_common(get_pid_ns(pid_ns));
                        rcu_read_unlock();
                }
                break;
        default:
                return -ENOIOCTLCMD;
        }

        if (!ns_common)
                return -EOPNOTSUPP;

        /* open_namespace() unconditionally consumes the reference */
        return open_namespace(ns_common);
}

static const struct file_operations pidfs_file_operations = {
        .poll                = pidfd_poll,
#ifdef CONFIG_PROC_FS
        .show_fdinfo        = pidfd_show_fdinfo,
#endif
        .unlocked_ioctl        = pidfd_ioctl,
        .compat_ioctl   = compat_ptr_ioctl,
};

struct pid *pidfd_pid(const struct file *file)
{
        if (file->f_op != &pidfs_file_operations)
                return ERR_PTR(-EBADF);
        return file_inode(file)->i_private;
}

/*
 * We're called from release_task(). We know there's at least one
 * reference to struct pid being held that won't be released until the
 * task has been reaped which cannot happen until we're out of
 * release_task().
 *
 * If this struct pid has at least once been referred to by a pidfd then
 * pid->attr will be allocated. If not we mark the struct pid as dead so
 * anyone who is trying to register it with pidfs will fail to do so.
 * Otherwise we would hand out pidfs for reaped tasks without having
 * exit information available.
 *
 * Worst case is that we've filled in the info and the pid gets freed
 * right away in free_pid() when no one holds a pidfd anymore. Since
 * pidfs_exit() currently is placed after exit_task_work() we know that
 * it cannot be us aka the exiting task holding a pidfd to itself.
 */
void pidfs_exit(struct task_struct *tsk)
{
        struct pid *pid = task_pid(tsk);
        struct pidfs_attr *attr;
        struct pidfs_exit_info *exit_info;
#ifdef CONFIG_CGROUPS
        struct cgroup *cgrp;
#endif

        might_sleep();

        guard(spinlock_irq)(&pid->wait_pidfd.lock);
        attr = pid->attr;
        if (!attr) {
                /*
                 * No one ever held a pidfd for this struct pid.
                 * Mark it as dead so no one can add a pidfs
                 * entry anymore. We're about to be reaped and
                 * so no exit information would be available.
                 */
                pid->attr = PIDFS_PID_DEAD;
                return;
        }

        /*
         * If @pid->attr is set someone might still legitimately hold a
         * pidfd to @pid or someone might concurrently still be getting
         * a reference to an already stashed dentry from @pid->stashed.
         * So defer cleaning @pid->attr until the last reference to @pid
         * is put
         */

        exit_info = &attr->__pei;

#ifdef CONFIG_CGROUPS
        rcu_read_lock();
        cgrp = task_dfl_cgroup(tsk);
        exit_info->cgroupid = cgroup_id(cgrp);
        rcu_read_unlock();
#endif
        exit_info->exit_code = tsk->exit_code;

        /* Ensure that PIDFD_GET_INFO sees either all or nothing. */
        smp_store_release(&attr->exit_info, &attr->__pei);
}

#ifdef CONFIG_COREDUMP
void pidfs_coredump(const struct coredump_params *cprm)
{
        struct pid *pid = cprm->pid;
        struct pidfs_exit_info *exit_info;
        struct pidfs_attr *attr;
        __u32 coredump_mask = 0;

        attr = READ_ONCE(pid->attr);

        VFS_WARN_ON_ONCE(!attr);
        VFS_WARN_ON_ONCE(attr == PIDFS_PID_DEAD);

        exit_info = &attr->__pei;
        /* Note how we were coredumped. */
        coredump_mask = pidfs_coredump_mask(cprm->mm_flags);
        /* Note that we actually did coredump. */
        coredump_mask |= PIDFD_COREDUMPED;
        /* If coredumping is set to skip we should never end up here. */
        VFS_WARN_ON_ONCE(coredump_mask & PIDFD_COREDUMP_SKIP);
        smp_store_release(&exit_info->coredump_mask, coredump_mask);
}
#endif

static struct vfsmount *pidfs_mnt __ro_after_init;

/*
 * The vfs falls back to simple_setattr() if i_op->setattr() isn't
 * implemented. Let's reject it completely until we have a clean
 * permission concept for pidfds.
 */
static int pidfs_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
                         struct iattr *attr)
{
        return anon_inode_setattr(idmap, dentry, attr);
}

static int pidfs_getattr(struct mnt_idmap *idmap, const struct path *path,
                         struct kstat *stat, u32 request_mask,
                         unsigned int query_flags)
{
        return anon_inode_getattr(idmap, path, stat, request_mask, query_flags);
}

static ssize_t pidfs_listxattr(struct dentry *dentry, char *buf, size_t size)
{
        struct inode *inode = d_inode(dentry);
        struct pid *pid = inode->i_private;
        struct pidfs_attr *attr = pid->attr;
        struct simple_xattrs *xattrs;

        xattrs = READ_ONCE(attr->xattrs);
        if (!xattrs)
                return 0;

        return simple_xattr_list(inode, xattrs, buf, size);
}

static const struct inode_operations pidfs_inode_operations = {
        .getattr        = pidfs_getattr,
        .setattr        = pidfs_setattr,
        .listxattr        = pidfs_listxattr,
};

static void pidfs_evict_inode(struct inode *inode)
{
        struct pid *pid = inode->i_private;

        clear_inode(inode);
        put_pid(pid);
}

static const struct super_operations pidfs_sops = {
        .drop_inode        = generic_delete_inode,
        .evict_inode        = pidfs_evict_inode,
        .statfs                = simple_statfs,
};

/*
 * 'lsof' has knowledge of out historical anon_inode use, and expects
 * the pidfs dentry name to start with 'anon_inode'.
 */
static char *pidfs_dname(struct dentry *dentry, char *buffer, int buflen)
{
        return dynamic_dname(buffer, buflen, "anon_inode:[pidfd]");
}

const struct dentry_operations pidfs_dentry_operations = {
        .d_dname        = pidfs_dname,
        .d_prune        = stashed_dentry_prune,
};

static int pidfs_encode_fh(struct inode *inode, u32 *fh, int *max_len,
                           struct inode *parent)
{
        const struct pid *pid = inode->i_private;

        if (*max_len < 2) {
                *max_len = 2;
                return FILEID_INVALID;
        }

        *max_len = 2;
        *(u64 *)fh = pid->ino;
        return FILEID_KERNFS;
}

static int pidfs_ino_find(const void *key, const struct rb_node *node)
{
        const u64 pid_ino = *(u64 *)key;
        const struct pid *pid = rb_entry(node, struct pid, pidfs_node);

        if (pid_ino < pid->ino)
                return -1;
        if (pid_ino > pid->ino)
                return 1;
        return 0;
}

/* Find a struct pid based on the inode number. */
static struct pid *pidfs_ino_get_pid(u64 ino)
{
        struct pid *pid;
        struct rb_node *node;
        unsigned int seq;

        guard(rcu)();
        do {
                seq = read_seqcount_begin(&pidmap_lock_seq);
                node = rb_find_rcu(&ino, &pidfs_ino_tree, pidfs_ino_find);
                if (node)
                        break;
        } while (read_seqcount_retry(&pidmap_lock_seq, seq));

        if (!node)
                return NULL;

        pid = rb_entry(node, struct pid, pidfs_node);

        /* Within our pid namespace hierarchy? */
        if (pid_vnr(pid) == 0)
                return NULL;

        return get_pid(pid);
}

static struct dentry *pidfs_fh_to_dentry(struct super_block *sb,
                                         struct fid *fid, int fh_len,
                                         int fh_type)
{
        int ret;
        u64 pid_ino;
        struct path path;
        struct pid *pid;

        if (fh_len < 2)
                return NULL;

        switch (fh_type) {
        case FILEID_KERNFS:
                pid_ino = *(u64 *)fid;
                break;
        default:
                return NULL;
        }

        pid = pidfs_ino_get_pid(pid_ino);
        if (!pid)
                return NULL;

        ret = path_from_stashed(&pid->stashed, pidfs_mnt, pid, &path);
        if (ret < 0)
                return ERR_PTR(ret);

        VFS_WARN_ON_ONCE(!pid->attr);

        mntput(path.mnt);
        return path.dentry;
}

/*
 * Make sure that we reject any nonsensical flags that users pass via
 * open_by_handle_at(). Note that PIDFD_THREAD is defined as O_EXCL, and
 * PIDFD_NONBLOCK as O_NONBLOCK.
 */
#define VALID_FILE_HANDLE_OPEN_FLAGS \
        (O_RDONLY | O_WRONLY | O_RDWR | O_NONBLOCK | O_CLOEXEC | O_EXCL)

static int pidfs_export_permission(struct handle_to_path_ctx *ctx,
                                   unsigned int oflags)
{
        if (oflags & ~(VALID_FILE_HANDLE_OPEN_FLAGS | O_LARGEFILE))
                return -EINVAL;

        /*
         * pidfd_ino_get_pid() will verify that the struct pid is part
         * of the caller's pid namespace hierarchy. No further
         * permission checks are needed.
         */
        return 0;
}

static struct file *pidfs_export_open(struct path *path, unsigned int oflags)
{
        /*
         * Clear O_LARGEFILE as open_by_handle_at() forces it and raise
         * O_RDWR as pidfds always are.
         */
        oflags &= ~O_LARGEFILE;
        return dentry_open(path, oflags | O_RDWR, current_cred());
}

static const struct export_operations pidfs_export_operations = {
        .encode_fh        = pidfs_encode_fh,
        .fh_to_dentry        = pidfs_fh_to_dentry,
        .open                = pidfs_export_open,
        .permission        = pidfs_export_permission,
};

static int pidfs_init_inode(struct inode *inode, void *data)
{
        const struct pid *pid = data;

        inode->i_private = data;
        inode->i_flags |= S_PRIVATE | S_ANON_INODE;
        /* We allow to set xattrs. */
        inode->i_flags &= ~S_IMMUTABLE;
        inode->i_mode |= S_IRWXU;
        inode->i_op = &pidfs_inode_operations;
        inode->i_fop = &pidfs_file_operations;
        inode->i_ino = pidfs_ino(pid->ino);
        inode->i_generation = pidfs_gen(pid->ino);
        return 0;
}

static void pidfs_put_data(void *data)
{
        struct pid *pid = data;
        put_pid(pid);
}

/**
 * pidfs_register_pid - register a struct pid in pidfs
 * @pid: pid to pin
 *
 * Register a struct pid in pidfs.
 *
 * Return: On success zero, on error a negative error code is returned.
 */
int pidfs_register_pid(struct pid *pid)
{
        struct pidfs_attr *new_attr __free(kfree) = NULL;
        struct pidfs_attr *attr;

        might_sleep();

        if (!pid)
                return 0;

        attr = READ_ONCE(pid->attr);
        if (unlikely(attr == PIDFS_PID_DEAD))
                return PTR_ERR(PIDFS_PID_DEAD);
        if (attr)
                return 0;

        new_attr = kmem_cache_zalloc(pidfs_attr_cachep, GFP_KERNEL);
        if (!new_attr)
                return -ENOMEM;

        /* Synchronize with pidfs_exit(). */
        guard(spinlock_irq)(&pid->wait_pidfd.lock);

        attr = pid->attr;
        if (unlikely(attr == PIDFS_PID_DEAD))
                return PTR_ERR(PIDFS_PID_DEAD);
        if (unlikely(attr))
                return 0;

        pid->attr = no_free_ptr(new_attr);
        return 0;
}

static struct dentry *pidfs_stash_dentry(struct dentry **stashed,
                                         struct dentry *dentry)
{
        int ret;
        struct pid *pid = d_inode(dentry)->i_private;

        VFS_WARN_ON_ONCE(stashed != &pid->stashed);

        ret = pidfs_register_pid(pid);
        if (ret)
                return ERR_PTR(ret);

        return stash_dentry(stashed, dentry);
}

static const struct stashed_operations pidfs_stashed_ops = {
        .stash_dentry        = pidfs_stash_dentry,
        .init_inode        = pidfs_init_inode,
        .put_data        = pidfs_put_data,
};

static int pidfs_xattr_get(const struct xattr_handler *handler,
                           struct dentry *unused, struct inode *inode,
                           const char *suffix, void *value, size_t size)
{
        struct pid *pid = inode->i_private;
        struct pidfs_attr *attr = pid->attr;
        const char *name;
        struct simple_xattrs *xattrs;

        xattrs = READ_ONCE(attr->xattrs);
        if (!xattrs)
                return 0;

        name = xattr_full_name(handler, suffix);
        return simple_xattr_get(xattrs, name, value, size);
}

static int pidfs_xattr_set(const struct xattr_handler *handler,
                           struct mnt_idmap *idmap, struct dentry *unused,
                           struct inode *inode, const char *suffix,
                           const void *value, size_t size, int flags)
{
        struct pid *pid = inode->i_private;
        struct pidfs_attr *attr = pid->attr;
        const char *name;
        struct simple_xattrs *xattrs;
        struct simple_xattr *old_xattr;

        /* Ensure we're the only one to set @attr->xattrs. */
        WARN_ON_ONCE(!inode_is_locked(inode));

        xattrs = READ_ONCE(attr->xattrs);
        if (!xattrs) {
                xattrs = kmem_cache_zalloc(pidfs_xattr_cachep, GFP_KERNEL);
                if (!xattrs)
                        return -ENOMEM;

                simple_xattrs_init(xattrs);
                smp_store_release(&pid->attr->xattrs, xattrs);
        }

        name = xattr_full_name(handler, suffix);
        old_xattr = simple_xattr_set(xattrs, name, value, size, flags);
        if (IS_ERR(old_xattr))
                return PTR_ERR(old_xattr);

        simple_xattr_free(old_xattr);
        return 0;
}

static const struct xattr_handler pidfs_trusted_xattr_handler = {
        .prefix = XATTR_TRUSTED_PREFIX,
        .get        = pidfs_xattr_get,
        .set        = pidfs_xattr_set,
};

static const struct xattr_handler *const pidfs_xattr_handlers[] = {
        &pidfs_trusted_xattr_handler,
        NULL
};

static int pidfs_init_fs_context(struct fs_context *fc)
{
        struct pseudo_fs_context *ctx;

        ctx = init_pseudo(fc, PID_FS_MAGIC);
        if (!ctx)
                return -ENOMEM;

        fc->s_iflags |= SB_I_NOEXEC;
        fc->s_iflags |= SB_I_NODEV;
        ctx->ops = &pidfs_sops;
        ctx->eops = &pidfs_export_operations;
        ctx->dops = &pidfs_dentry_operations;
        ctx->xattr = pidfs_xattr_handlers;
        fc->s_fs_info = (void *)&pidfs_stashed_ops;
        return 0;
}

static struct file_system_type pidfs_type = {
        .name                        = "pidfs",
        .init_fs_context        = pidfs_init_fs_context,
        .kill_sb                = kill_anon_super,
};

struct file *pidfs_alloc_file(struct pid *pid, unsigned int flags)
{
        struct file *pidfd_file;
        struct path path __free(path_put) = {};
        int ret;

        /*
         * Ensure that PIDFD_STALE can be passed as a flag without
         * overloading other uapi pidfd flags.
         */
        BUILD_BUG_ON(PIDFD_STALE == PIDFD_THREAD);
        BUILD_BUG_ON(PIDFD_STALE == PIDFD_NONBLOCK);

        ret = path_from_stashed(&pid->stashed, pidfs_mnt, get_pid(pid), &path);
        if (ret < 0)
                return ERR_PTR(ret);

        VFS_WARN_ON_ONCE(!pid->attr);

        flags &= ~PIDFD_STALE;
        flags |= O_RDWR;
        pidfd_file = dentry_open(&path, flags, current_cred());
        /* Raise PIDFD_THREAD explicitly as do_dentry_open() strips it. */
        if (!IS_ERR(pidfd_file))
                pidfd_file->f_flags |= (flags & PIDFD_THREAD);

        return pidfd_file;
}

void __init pidfs_init(void)
{
        pidfs_attr_cachep = kmem_cache_create("pidfs_attr_cache", sizeof(struct pidfs_attr), 0,
                                         (SLAB_HWCACHE_ALIGN | SLAB_RECLAIM_ACCOUNT |
                                          SLAB_ACCOUNT | SLAB_PANIC), NULL);

        pidfs_xattr_cachep = kmem_cache_create("pidfs_xattr_cache",
                                               sizeof(struct simple_xattrs), 0,
                                               (SLAB_HWCACHE_ALIGN | SLAB_RECLAIM_ACCOUNT |
                                                SLAB_ACCOUNT | SLAB_PANIC), NULL);

        pidfs_mnt = kern_mount(&pidfs_type);
        if (IS_ERR(pidfs_mnt))
                panic("Failed to mount pidfs pseudo filesystem");

        pidfs_root_path.mnt = pidfs_mnt;
        pidfs_root_path.dentry = pidfs_mnt->mnt_root;
}





















































































































































































































































































































































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
// SPDX-License-Identifier: GPL-2.0-only
/* Copyright (C) 2003-2013 Jozsef Kadlecsik <kadlec@netfilter.org> */

/* Kernel module implementing an IP set type: the hash:ip,port,net type */

#include <linux/jhash.h>
#include <linux/module.h>
#include <linux/ip.h>
#include <linux/skbuff.h>
#include <linux/errno.h>
#include <linux/random.h>
#include <net/ip.h>
#include <net/ipv6.h>
#include <net/netlink.h>
#include <net/tcp.h>

#include <linux/netfilter.h>
#include <linux/netfilter/ipset/pfxlen.h>
#include <linux/netfilter/ipset/ip_set.h>
#include <linux/netfilter/ipset/ip_set_getport.h>
#include <linux/netfilter/ipset/ip_set_hash.h>

#define IPSET_TYPE_REV_MIN        0
/*                                0    Comments support added */
/*                                1    Forceadd support added */
/*                                2    skbinfo support added */
#define IPSET_TYPE_REV_MAX        3 /* bucketsize, initval support added */

MODULE_LICENSE("GPL");
MODULE_AUTHOR("Oliver Smith <oliver@8.c.9.b.0.7.4.0.1.0.0.2.ip6.arpa>");
IP_SET_MODULE_DESC("hash:net,port,net", IPSET_TYPE_REV_MIN, IPSET_TYPE_REV_MAX);
MODULE_ALIAS("ip_set_hash:net,port,net");

/* Type specific function prefix */
#define HTYPE                hash_netportnet
#define IP_SET_HASH_WITH_PROTO
#define IP_SET_HASH_WITH_NETS
#define IPSET_NET_COUNT 2
#define IP_SET_HASH_WITH_NET0

/* IPv4 variant */

/* Member elements */
struct hash_netportnet4_elem {
        union {
                __be32 ip[2];
                __be64 ipcmp;
        };
        __be16 port;
        union {
                u8 cidr[2];
                u16 ccmp;
        };
        u16 padding;
        u8 nomatch;
        u8 proto;
};

/* Common functions */

static bool
hash_netportnet4_data_equal(const struct hash_netportnet4_elem *ip1,
                            const struct hash_netportnet4_elem *ip2,
                            u32 *multi)
{
        return ip1->ipcmp == ip2->ipcmp &&
               ip1->ccmp == ip2->ccmp &&
               ip1->port == ip2->port &&
               ip1->proto == ip2->proto;
}

static int
hash_netportnet4_do_data_match(const struct hash_netportnet4_elem *elem)
{
        return elem->nomatch ? -ENOTEMPTY : 1;
}

static void
hash_netportnet4_data_set_flags(struct hash_netportnet4_elem *elem, u32 flags)
{
        elem->nomatch = !!((flags >> 16) & IPSET_FLAG_NOMATCH);
}

static void
hash_netportnet4_data_reset_flags(struct hash_netportnet4_elem *elem, u8 *flags)
{
        swap(*flags, elem->nomatch);
}

static void
hash_netportnet4_data_reset_elem(struct hash_netportnet4_elem *elem,
                                 struct hash_netportnet4_elem *orig)
{
        elem->ip[1] = orig->ip[1];
}

static void
hash_netportnet4_data_netmask(struct hash_netportnet4_elem *elem,
                              u8 cidr, bool inner)
{
        if (inner) {
                elem->ip[1] &= ip_set_netmask(cidr);
                elem->cidr[1] = cidr;
        } else {
                elem->ip[0] &= ip_set_netmask(cidr);
                elem->cidr[0] = cidr;
        }
}

static bool
hash_netportnet4_data_list(struct sk_buff *skb,
                           const struct hash_netportnet4_elem *data)
{
        u32 flags = data->nomatch ? IPSET_FLAG_NOMATCH : 0;

        if (nla_put_ipaddr4(skb, IPSET_ATTR_IP, data->ip[0]) ||
            nla_put_ipaddr4(skb, IPSET_ATTR_IP2, data->ip[1]) ||
            nla_put_net16(skb, IPSET_ATTR_PORT, data->port) ||
            nla_put_u8(skb, IPSET_ATTR_CIDR, data->cidr[0]) ||
            nla_put_u8(skb, IPSET_ATTR_CIDR2, data->cidr[1]) ||
            nla_put_u8(skb, IPSET_ATTR_PROTO, data->proto) ||
            (flags &&
             nla_put_net32(skb, IPSET_ATTR_CADT_FLAGS, htonl(flags))))
                goto nla_put_failure;
        return false;

nla_put_failure:
        return true;
}

static void
hash_netportnet4_data_next(struct hash_netportnet4_elem *next,
                           const struct hash_netportnet4_elem *d)
{
        next->ipcmp = d->ipcmp;
        next->port = d->port;
}

#define MTYPE                hash_netportnet4
#define HOST_MASK        32
#include "ip_set_hash_gen.h"

static void
hash_netportnet4_init(struct hash_netportnet4_elem *e)
{
        e->cidr[0] = HOST_MASK;
        e->cidr[1] = HOST_MASK;
}

static int
hash_netportnet4_kadt(struct ip_set *set, const struct sk_buff *skb,
                      const struct xt_action_param *par,
                      enum ipset_adt adt, struct ip_set_adt_opt *opt)
{
        const struct hash_netportnet4 *h = set->data;
        ipset_adtfn adtfn = set->variant->adt[adt];
        struct hash_netportnet4_elem e = { };
        struct ip_set_ext ext = IP_SET_INIT_KEXT(skb, opt, set);

        e.cidr[0] = INIT_CIDR(h->nets[0].cidr[0], HOST_MASK);
        e.cidr[1] = INIT_CIDR(h->nets[0].cidr[1], HOST_MASK);
        if (adt == IPSET_TEST)
                e.ccmp = (HOST_MASK << (sizeof(e.cidr[0]) * 8)) | HOST_MASK;

        if (!ip_set_get_ip4_port(skb, opt->flags & IPSET_DIM_TWO_SRC,
                                 &e.port, &e.proto))
                return -EINVAL;

        ip4addrptr(skb, opt->flags & IPSET_DIM_ONE_SRC, &e.ip[0]);
        ip4addrptr(skb, opt->flags & IPSET_DIM_THREE_SRC, &e.ip[1]);
        e.ip[0] &= ip_set_netmask(e.cidr[0]);
        e.ip[1] &= ip_set_netmask(e.cidr[1]);

        return adtfn(set, &e, &ext, &opt->ext, opt->cmdflags);
}

static u32
hash_netportnet4_range_to_cidr(u32 from, u32 to, u8 *cidr)
{
        if (from == 0 && to == UINT_MAX) {
                *cidr = 0;
                return to;
        }
        return ip_set_range_to_cidr(from, to, cidr);
}

static int
hash_netportnet4_uadt(struct ip_set *set, struct nlattr *tb[],
                      enum ipset_adt adt, u32 *lineno, u32 flags, bool retried)
{
        struct hash_netportnet4 *h = set->data;
        ipset_adtfn adtfn = set->variant->adt[adt];
        struct hash_netportnet4_elem e = { };
        struct ip_set_ext ext = IP_SET_INIT_UEXT(set);
        u32 ip = 0, ip_to = 0, p = 0, port, port_to;
        u32 ip2_from = 0, ip2_to = 0, ip2, i = 0;
        bool with_ports = false;
        int ret;

        if (tb[IPSET_ATTR_LINENO])
                *lineno = nla_get_u32(tb[IPSET_ATTR_LINENO]);

        hash_netportnet4_init(&e);
        if (unlikely(!tb[IPSET_ATTR_IP] || !tb[IPSET_ATTR_IP2] ||
                     !ip_set_attr_netorder(tb, IPSET_ATTR_PORT) ||
                     !ip_set_optattr_netorder(tb, IPSET_ATTR_PORT_TO) ||
                     !ip_set_optattr_netorder(tb, IPSET_ATTR_CADT_FLAGS)))
                return -IPSET_ERR_PROTOCOL;

        ret = ip_set_get_hostipaddr4(tb[IPSET_ATTR_IP], &ip);
        if (ret)
                return ret;

        ret = ip_set_get_hostipaddr4(tb[IPSET_ATTR_IP2], &ip2_from);
        if (ret)
                return ret;

        ret = ip_set_get_extensions(set, tb, &ext);
        if (ret)
                return ret;

        if (tb[IPSET_ATTR_CIDR]) {
                e.cidr[0] = nla_get_u8(tb[IPSET_ATTR_CIDR]);
                if (e.cidr[0] > HOST_MASK)
                        return -IPSET_ERR_INVALID_CIDR;
        }

        if (tb[IPSET_ATTR_CIDR2]) {
                e.cidr[1] = nla_get_u8(tb[IPSET_ATTR_CIDR2]);
                if (e.cidr[1] > HOST_MASK)
                        return -IPSET_ERR_INVALID_CIDR;
        }

        e.port = nla_get_be16(tb[IPSET_ATTR_PORT]);

        if (tb[IPSET_ATTR_PROTO]) {
                e.proto = nla_get_u8(tb[IPSET_ATTR_PROTO]);
                with_ports = ip_set_proto_with_ports(e.proto);

                if (e.proto == 0)
                        return -IPSET_ERR_INVALID_PROTO;
        } else {
                return -IPSET_ERR_MISSING_PROTO;
        }

        if (!(with_ports || e.proto == IPPROTO_ICMP))
                e.port = 0;

        if (tb[IPSET_ATTR_CADT_FLAGS]) {
                u32 cadt_flags = ip_set_get_h32(tb[IPSET_ATTR_CADT_FLAGS]);

                if (cadt_flags & IPSET_FLAG_NOMATCH)
                        flags |= (IPSET_FLAG_NOMATCH << 16);
        }

        with_ports = with_ports && tb[IPSET_ATTR_PORT_TO];
        if (adt == IPSET_TEST ||
            !(tb[IPSET_ATTR_IP_TO] || with_ports || tb[IPSET_ATTR_IP2_TO])) {
                e.ip[0] = htonl(ip & ip_set_hostmask(e.cidr[0]));
                e.ip[1] = htonl(ip2_from & ip_set_hostmask(e.cidr[1]));
                ret = adtfn(set, &e, &ext, &ext, flags);
                return ip_set_enomatch(ret, flags, adt, set) ? -ret :
                       ip_set_eexist(ret, flags) ? 0 : ret;
        }

        ip_to = ip;
        if (tb[IPSET_ATTR_IP_TO]) {
                ret = ip_set_get_hostipaddr4(tb[IPSET_ATTR_IP_TO], &ip_to);
                if (ret)
                        return ret;
                if (ip > ip_to)
                        swap(ip, ip_to);
                if (unlikely(ip + UINT_MAX == ip_to))
                        return -IPSET_ERR_HASH_RANGE;
        } else {
                ip_set_mask_from_to(ip, ip_to, e.cidr[0]);
        }

        port_to = port = ntohs(e.port);
        if (tb[IPSET_ATTR_PORT_TO]) {
                port_to = ip_set_get_h16(tb[IPSET_ATTR_PORT_TO]);
                if (port > port_to)
                        swap(port, port_to);
        }

        ip2_to = ip2_from;
        if (tb[IPSET_ATTR_IP2_TO]) {
                ret = ip_set_get_hostipaddr4(tb[IPSET_ATTR_IP2_TO], &ip2_to);
                if (ret)
                        return ret;
                if (ip2_from > ip2_to)
                        swap(ip2_from, ip2_to);
                if (unlikely(ip2_from + UINT_MAX == ip2_to))
                        return -IPSET_ERR_HASH_RANGE;
        } else {
                ip_set_mask_from_to(ip2_from, ip2_to, e.cidr[1]);
        }

        if (retried) {
                ip = ntohl(h->next.ip[0]);
                p = ntohs(h->next.port);
                ip2 = ntohl(h->next.ip[1]);
        } else {
                p = port;
                ip2 = ip2_from;
        }

        do {
                e.ip[0] = htonl(ip);
                ip = hash_netportnet4_range_to_cidr(ip, ip_to, &e.cidr[0]);
                for (; p <= port_to; p++) {
                        e.port = htons(p);
                        do {
                                i++;
                                e.ip[1] = htonl(ip2);
                                if (i > IPSET_MAX_RANGE) {
                                        hash_netportnet4_data_next(&h->next,
                                                                   &e);
                                        return -ERANGE;
                                }
                                ip2 = hash_netportnet4_range_to_cidr(ip2,
                                                        ip2_to, &e.cidr[1]);
                                ret = adtfn(set, &e, &ext, &ext, flags);
                                if (ret && !ip_set_eexist(ret, flags))
                                        return ret;

                                ret = 0;
                        } while (ip2++ < ip2_to);
                        ip2 = ip2_from;
                }
                p = port;
        } while (ip++ < ip_to);
        return ret;
}

/* IPv6 variant */

struct hash_netportnet6_elem {
        union nf_inet_addr ip[2];
        __be16 port;
        union {
                u8 cidr[2];
                u16 ccmp;
        };
        u16 padding;
        u8 nomatch;
        u8 proto;
};

/* Common functions */

static bool
hash_netportnet6_data_equal(const struct hash_netportnet6_elem *ip1,
                            const struct hash_netportnet6_elem *ip2,
                            u32 *multi)
{
        return ipv6_addr_equal(&ip1->ip[0].in6, &ip2->ip[0].in6) &&
               ipv6_addr_equal(&ip1->ip[1].in6, &ip2->ip[1].in6) &&
               ip1->ccmp == ip2->ccmp &&
               ip1->port == ip2->port &&
               ip1->proto == ip2->proto;
}

static int
hash_netportnet6_do_data_match(const struct hash_netportnet6_elem *elem)
{
        return elem->nomatch ? -ENOTEMPTY : 1;
}

static void
hash_netportnet6_data_set_flags(struct hash_netportnet6_elem *elem, u32 flags)
{
        elem->nomatch = !!((flags >> 16) & IPSET_FLAG_NOMATCH);
}

static void
hash_netportnet6_data_reset_flags(struct hash_netportnet6_elem *elem, u8 *flags)
{
        swap(*flags, elem->nomatch);
}

static void
hash_netportnet6_data_reset_elem(struct hash_netportnet6_elem *elem,
                                 struct hash_netportnet6_elem *orig)
{
        elem->ip[1] = orig->ip[1];
}

static void
hash_netportnet6_data_netmask(struct hash_netportnet6_elem *elem,
                              u8 cidr, bool inner)
{
        if (inner) {
                ip6_netmask(&elem->ip[1], cidr);
                elem->cidr[1] = cidr;
        } else {
                ip6_netmask(&elem->ip[0], cidr);
                elem->cidr[0] = cidr;
        }
}

static bool
hash_netportnet6_data_list(struct sk_buff *skb,
                           const struct hash_netportnet6_elem *data)
{
        u32 flags = data->nomatch ? IPSET_FLAG_NOMATCH : 0;

        if (nla_put_ipaddr6(skb, IPSET_ATTR_IP, &data->ip[0].in6) ||
            nla_put_ipaddr6(skb, IPSET_ATTR_IP2, &data->ip[1].in6) ||
            nla_put_net16(skb, IPSET_ATTR_PORT, data->port) ||
            nla_put_u8(skb, IPSET_ATTR_CIDR, data->cidr[0]) ||
            nla_put_u8(skb, IPSET_ATTR_CIDR2, data->cidr[1]) ||
            nla_put_u8(skb, IPSET_ATTR_PROTO, data->proto) ||
            (flags &&
             nla_put_net32(skb, IPSET_ATTR_CADT_FLAGS, htonl(flags))))
                goto nla_put_failure;
        return false;

nla_put_failure:
        return true;
}

static void
hash_netportnet6_data_next(struct hash_netportnet6_elem *next,
                           const struct hash_netportnet6_elem *d)
{
        next->port = d->port;
}

#undef MTYPE
#undef HOST_MASK

#define MTYPE                hash_netportnet6
#define HOST_MASK        128
#define IP_SET_EMIT_CREATE
#include "ip_set_hash_gen.h"

static void
hash_netportnet6_init(struct hash_netportnet6_elem *e)
{
        e->cidr[0] = HOST_MASK;
        e->cidr[1] = HOST_MASK;
}

static int
hash_netportnet6_kadt(struct ip_set *set, const struct sk_buff *skb,
                      const struct xt_action_param *par,
                      enum ipset_adt adt, struct ip_set_adt_opt *opt)
{
        const struct hash_netportnet6 *h = set->data;
        ipset_adtfn adtfn = set->variant->adt[adt];
        struct hash_netportnet6_elem e = { };
        struct ip_set_ext ext = IP_SET_INIT_KEXT(skb, opt, set);

        e.cidr[0] = INIT_CIDR(h->nets[0].cidr[0], HOST_MASK);
        e.cidr[1] = INIT_CIDR(h->nets[0].cidr[1], HOST_MASK);
        if (adt == IPSET_TEST)
                e.ccmp = (HOST_MASK << (sizeof(u8) * 8)) | HOST_MASK;

        if (!ip_set_get_ip6_port(skb, opt->flags & IPSET_DIM_TWO_SRC,
                                 &e.port, &e.proto))
                return -EINVAL;

        ip6addrptr(skb, opt->flags & IPSET_DIM_ONE_SRC, &e.ip[0].in6);
        ip6addrptr(skb, opt->flags & IPSET_DIM_THREE_SRC, &e.ip[1].in6);
        ip6_netmask(&e.ip[0], e.cidr[0]);
        ip6_netmask(&e.ip[1], e.cidr[1]);

        return adtfn(set, &e, &ext, &opt->ext, opt->cmdflags);
}

static int
hash_netportnet6_uadt(struct ip_set *set, struct nlattr *tb[],
                      enum ipset_adt adt, u32 *lineno, u32 flags, bool retried)
{
        const struct hash_netportnet6 *h = set->data;
        ipset_adtfn adtfn = set->variant->adt[adt];
        struct hash_netportnet6_elem e = { };
        struct ip_set_ext ext = IP_SET_INIT_UEXT(set);
        u32 port, port_to;
        bool with_ports = false;
        int ret;

        if (tb[IPSET_ATTR_LINENO])
                *lineno = nla_get_u32(tb[IPSET_ATTR_LINENO]);

        hash_netportnet6_init(&e);
        if (unlikely(!tb[IPSET_ATTR_IP] || !tb[IPSET_ATTR_IP2] ||
                     !ip_set_attr_netorder(tb, IPSET_ATTR_PORT) ||
                     !ip_set_optattr_netorder(tb, IPSET_ATTR_PORT_TO) ||
                     !ip_set_optattr_netorder(tb, IPSET_ATTR_CADT_FLAGS)))
                return -IPSET_ERR_PROTOCOL;
        if (unlikely(tb[IPSET_ATTR_IP_TO] || tb[IPSET_ATTR_IP2_TO]))
                return -IPSET_ERR_HASH_RANGE_UNSUPPORTED;

        ret = ip_set_get_ipaddr6(tb[IPSET_ATTR_IP], &e.ip[0]);
        if (ret)
                return ret;

        ret = ip_set_get_ipaddr6(tb[IPSET_ATTR_IP2], &e.ip[1]);
        if (ret)
                return ret;

        ret = ip_set_get_extensions(set, tb, &ext);
        if (ret)
                return ret;

        if (tb[IPSET_ATTR_CIDR]) {
                e.cidr[0] = nla_get_u8(tb[IPSET_ATTR_CIDR]);
                if (e.cidr[0] > HOST_MASK)
                        return -IPSET_ERR_INVALID_CIDR;
        }

        if (tb[IPSET_ATTR_CIDR2]) {
                e.cidr[1] = nla_get_u8(tb[IPSET_ATTR_CIDR2]);
                if (e.cidr[1] > HOST_MASK)
                        return -IPSET_ERR_INVALID_CIDR;
        }

        ip6_netmask(&e.ip[0], e.cidr[0]);
        ip6_netmask(&e.ip[1], e.cidr[1]);

        e.port = nla_get_be16(tb[IPSET_ATTR_PORT]);

        if (tb[IPSET_ATTR_PROTO]) {
                e.proto = nla_get_u8(tb[IPSET_ATTR_PROTO]);
                with_ports = ip_set_proto_with_ports(e.proto);

                if (e.proto == 0)
                        return -IPSET_ERR_INVALID_PROTO;
        } else {
                return -IPSET_ERR_MISSING_PROTO;
        }

        if (!(with_ports || e.proto == IPPROTO_ICMPV6))
                e.port = 0;

        if (tb[IPSET_ATTR_CADT_FLAGS]) {
                u32 cadt_flags = ip_set_get_h32(tb[IPSET_ATTR_CADT_FLAGS]);

                if (cadt_flags & IPSET_FLAG_NOMATCH)
                        flags |= (IPSET_FLAG_NOMATCH << 16);
        }

        if (adt == IPSET_TEST || !with_ports || !tb[IPSET_ATTR_PORT_TO]) {
                ret = adtfn(set, &e, &ext, &ext, flags);
                return ip_set_enomatch(ret, flags, adt, set) ? -ret :
                       ip_set_eexist(ret, flags) ? 0 : ret;
        }

        port = ntohs(e.port);
        port_to = ip_set_get_h16(tb[IPSET_ATTR_PORT_TO]);
        if (port > port_to)
                swap(port, port_to);

        if (retried)
                port = ntohs(h->next.port);
        for (; port <= port_to; port++) {
                e.port = htons(port);
                ret = adtfn(set, &e, &ext, &ext, flags);

                if (ret && !ip_set_eexist(ret, flags))
                        return ret;

                ret = 0;
        }
        return ret;
}

static struct ip_set_type hash_netportnet_type __read_mostly = {
        .name                = "hash:net,port,net",
        .protocol        = IPSET_PROTOCOL,
        .features        = IPSET_TYPE_IP | IPSET_TYPE_PORT | IPSET_TYPE_IP2 |
                          IPSET_TYPE_NOMATCH,
        .dimension        = IPSET_DIM_THREE,
        .family                = NFPROTO_UNSPEC,
        .revision_min        = IPSET_TYPE_REV_MIN,
        .revision_max        = IPSET_TYPE_REV_MAX,
        .create_flags[IPSET_TYPE_REV_MAX] = IPSET_CREATE_FLAG_BUCKETSIZE,
        .create                = hash_netportnet_create,
        .create_policy        = {
                [IPSET_ATTR_HASHSIZE]        = { .type = NLA_U32 },
                [IPSET_ATTR_MAXELEM]        = { .type = NLA_U32 },
                [IPSET_ATTR_INITVAL]        = { .type = NLA_U32 },
                [IPSET_ATTR_BUCKETSIZE]        = { .type = NLA_U8 },
                [IPSET_ATTR_RESIZE]        = { .type = NLA_U8  },
                [IPSET_ATTR_TIMEOUT]        = { .type = NLA_U32 },
                [IPSET_ATTR_CADT_FLAGS]        = { .type = NLA_U32 },
        },
        .adt_policy        = {
                [IPSET_ATTR_IP]                = { .type = NLA_NESTED },
                [IPSET_ATTR_IP_TO]        = { .type = NLA_NESTED },
                [IPSET_ATTR_IP2]        = { .type = NLA_NESTED },
                [IPSET_ATTR_IP2_TO]        = { .type = NLA_NESTED },
                [IPSET_ATTR_PORT]        = { .type = NLA_U16 },
                [IPSET_ATTR_PORT_TO]        = { .type = NLA_U16 },
                [IPSET_ATTR_CIDR]        = { .type = NLA_U8 },
                [IPSET_ATTR_CIDR2]        = { .type = NLA_U8 },
                [IPSET_ATTR_PROTO]        = { .type = NLA_U8 },
                [IPSET_ATTR_CADT_FLAGS]        = { .type = NLA_U32 },
                [IPSET_ATTR_TIMEOUT]        = { .type = NLA_U32 },
                [IPSET_ATTR_LINENO]        = { .type = NLA_U32 },
                [IPSET_ATTR_BYTES]        = { .type = NLA_U64 },
                [IPSET_ATTR_PACKETS]        = { .type = NLA_U64 },
                [IPSET_ATTR_COMMENT]        = { .type = NLA_NUL_STRING,
                                            .len  = IPSET_MAX_COMMENT_SIZE },
                [IPSET_ATTR_SKBMARK]        = { .type = NLA_U64 },
                [IPSET_ATTR_SKBPRIO]        = { .type = NLA_U32 },
                [IPSET_ATTR_SKBQUEUE]        = { .type = NLA_U16 },
        },
        .me                = THIS_MODULE,
};

static int __init
hash_netportnet_init(void)
{
        return ip_set_type_register(&hash_netportnet_type);
}

static void __exit
hash_netportnet_fini(void)
{
        rcu_barrier();
        ip_set_type_unregister(&hash_netportnet_type);
}

module_init(hash_netportnet_init);
module_exit(hash_netportnet_fini);















































































































































































































    7 






    7 















1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
// SPDX-License-Identifier: GPL-2.0
/*
 * Block stat tracking code
 *
 * Copyright (C) 2016 Jens Axboe
 */
#include <linux/kernel.h>
#include <linux/rculist.h>

#include "blk-stat.h"
#include "blk-mq.h"
#include "blk.h"

struct blk_queue_stats {
        struct list_head callbacks;
        spinlock_t lock;
        int accounting;
};

void blk_rq_stat_init(struct blk_rq_stat *stat)
{
        stat->min = -1ULL;
        stat->max = stat->nr_samples = stat->mean = 0;
        stat->batch = 0;
}

/* src is a per-cpu stat, mean isn't initialized */
void blk_rq_stat_sum(struct blk_rq_stat *dst, struct blk_rq_stat *src)
{
        if (dst->nr_samples + src->nr_samples <= dst->nr_samples)
                return;

        dst->min = min(dst->min, src->min);
        dst->max = max(dst->max, src->max);

        dst->mean = div_u64(src->batch + dst->mean * dst->nr_samples,
                                dst->nr_samples + src->nr_samples);

        dst->nr_samples += src->nr_samples;
}

void blk_rq_stat_add(struct blk_rq_stat *stat, u64 value)
{
        stat->min = min(stat->min, value);
        stat->max = max(stat->max, value);
        stat->batch += value;
        stat->nr_samples++;
}

void blk_stat_add(struct request *rq, u64 now)
{
        struct request_queue *q = rq->q;
        struct blk_stat_callback *cb;
        struct blk_rq_stat *stat;
        int bucket, cpu;
        u64 value;

        value = (now >= rq->io_start_time_ns) ? now - rq->io_start_time_ns : 0;

        rcu_read_lock();
        cpu = get_cpu();
        list_for_each_entry_rcu(cb, &q->stats->callbacks, list) {
                if (!blk_stat_is_active(cb))
                        continue;

                bucket = cb->bucket_fn(rq);
                if (bucket < 0)
                        continue;

                stat = &per_cpu_ptr(cb->cpu_stat, cpu)[bucket];
                blk_rq_stat_add(stat, value);
        }
        put_cpu();
        rcu_read_unlock();
}

static void blk_stat_timer_fn(struct timer_list *t)
{
        struct blk_stat_callback *cb = timer_container_of(cb, t, timer);
        unsigned int bucket;
        int cpu;

        for (bucket = 0; bucket < cb->buckets; bucket++)
                blk_rq_stat_init(&cb->stat[bucket]);

        for_each_online_cpu(cpu) {
                struct blk_rq_stat *cpu_stat;

                cpu_stat = per_cpu_ptr(cb->cpu_stat, cpu);
                for (bucket = 0; bucket < cb->buckets; bucket++) {
                        blk_rq_stat_sum(&cb->stat[bucket], &cpu_stat[bucket]);
                        blk_rq_stat_init(&cpu_stat[bucket]);
                }
        }

        cb->timer_fn(cb);
}

struct blk_stat_callback *
blk_stat_alloc_callback(void (*timer_fn)(struct blk_stat_callback *),
                        int (*bucket_fn)(const struct request *),
                        unsigned int buckets, void *data)
{
        struct blk_stat_callback *cb;

        cb = kmalloc(sizeof(*cb), GFP_KERNEL);
        if (!cb)
                return NULL;

        cb->stat = kmalloc_array(buckets, sizeof(struct blk_rq_stat),
                                 GFP_KERNEL);
        if (!cb->stat) {
                kfree(cb);
                return NULL;
        }
        cb->cpu_stat = __alloc_percpu(buckets * sizeof(struct blk_rq_stat),
                                      __alignof__(struct blk_rq_stat));
        if (!cb->cpu_stat) {
                kfree(cb->stat);
                kfree(cb);
                return NULL;
        }

        cb->timer_fn = timer_fn;
        cb->bucket_fn = bucket_fn;
        cb->data = data;
        cb->buckets = buckets;
        timer_setup(&cb->timer, blk_stat_timer_fn, 0);

        return cb;
}

void blk_stat_add_callback(struct request_queue *q,
                           struct blk_stat_callback *cb)
{
        unsigned int bucket;
        unsigned long flags;
        int cpu;

        for_each_possible_cpu(cpu) {
                struct blk_rq_stat *cpu_stat;

                cpu_stat = per_cpu_ptr(cb->cpu_stat, cpu);
                for (bucket = 0; bucket < cb->buckets; bucket++)
                        blk_rq_stat_init(&cpu_stat[bucket]);
        }

        spin_lock_irqsave(&q->stats->lock, flags);
        list_add_tail_rcu(&cb->list, &q->stats->callbacks);
        blk_queue_flag_set(QUEUE_FLAG_STATS, q);
        spin_unlock_irqrestore(&q->stats->lock, flags);
}

void blk_stat_remove_callback(struct request_queue *q,
                              struct blk_stat_callback *cb)
{
        unsigned long flags;

        spin_lock_irqsave(&q->stats->lock, flags);
        list_del_rcu(&cb->list);
        if (list_empty(&q->stats->callbacks) && !q->stats->accounting)
                blk_queue_flag_clear(QUEUE_FLAG_STATS, q);
        spin_unlock_irqrestore(&q->stats->lock, flags);

        timer_delete_sync(&cb->timer);
}

static void blk_stat_free_callback_rcu(struct rcu_head *head)
{
        struct blk_stat_callback *cb;

        cb = container_of(head, struct blk_stat_callback, rcu);
        free_percpu(cb->cpu_stat);
        kfree(cb->stat);
        kfree(cb);
}

void blk_stat_free_callback(struct blk_stat_callback *cb)
{
        if (cb)
                call_rcu(&cb->rcu, blk_stat_free_callback_rcu);
}

void blk_stat_disable_accounting(struct request_queue *q)
{
        unsigned long flags;

        spin_lock_irqsave(&q->stats->lock, flags);
        if (!--q->stats->accounting && list_empty(&q->stats->callbacks))
                blk_queue_flag_clear(QUEUE_FLAG_STATS, q);
        spin_unlock_irqrestore(&q->stats->lock, flags);
}
EXPORT_SYMBOL_GPL(blk_stat_disable_accounting);

void blk_stat_enable_accounting(struct request_queue *q)
{
        unsigned long flags;

        spin_lock_irqsave(&q->stats->lock, flags);
        if (!q->stats->accounting++ && list_empty(&q->stats->callbacks))
                blk_queue_flag_set(QUEUE_FLAG_STATS, q);
        spin_unlock_irqrestore(&q->stats->lock, flags);
}
EXPORT_SYMBOL_GPL(blk_stat_enable_accounting);

struct blk_queue_stats *blk_alloc_queue_stats(void)
{
        struct blk_queue_stats *stats;

        stats = kmalloc(sizeof(*stats), GFP_KERNEL);
        if (!stats)
                return NULL;

        INIT_LIST_HEAD(&stats->callbacks);
        spin_lock_init(&stats->lock);
        stats->accounting = 0;

        return stats;
}

void blk_free_queue_stats(struct blk_queue_stats *stats)
{
        if (!stats)
                return;

        WARN_ON(!list_empty(&stats->callbacks));

        kfree(stats);
}





















































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
/* SPDX-License-Identifier: GPL-2.0-only */
#ifndef __KVM_X86_VMX_COMMON_H
#define __KVM_X86_VMX_COMMON_H

#include <linux/kvm_host.h>
#include <asm/posted_intr.h>

#include "mmu.h"

union vmx_exit_reason {
        struct {
                u32        basic                        : 16;
                u32        reserved16                : 1;
                u32        reserved17                : 1;
                u32        reserved18                : 1;
                u32        reserved19                : 1;
                u32        reserved20                : 1;
                u32        reserved21                : 1;
                u32        reserved22                : 1;
                u32        reserved23                : 1;
                u32        reserved24                : 1;
                u32        reserved25                : 1;
                u32        bus_lock_detected        : 1;
                u32        enclave_mode                : 1;
                u32        smi_pending_mtf                : 1;
                u32        smi_from_vmx_root        : 1;
                u32        reserved30                : 1;
                u32        failed_vmentry                : 1;
        };
        u32 full;
};

struct vcpu_vt {
        /* Posted interrupt descriptor */
        struct pi_desc pi_desc;

        /* Used if this vCPU is waiting for PI notification wakeup. */
        struct list_head pi_wakeup_list;

        union vmx_exit_reason exit_reason;

        unsigned long        exit_qualification;
        u32                exit_intr_info;

        /*
         * If true, guest state has been loaded into hardware, and host state
         * saved into vcpu_{vt,vmx,tdx}.  If false, host state is loaded into
         * hardware.
         */
        bool                guest_state_loaded;
        bool                emulation_required;

#ifdef CONFIG_X86_64
        u64                msr_host_kernel_gs_base;
#endif
};

#ifdef CONFIG_KVM_INTEL_TDX

static __always_inline bool is_td(struct kvm *kvm)
{
        return kvm->arch.vm_type == KVM_X86_TDX_VM;
}

static __always_inline bool is_td_vcpu(struct kvm_vcpu *vcpu)
{
        return is_td(vcpu->kvm);
}

#else

static __always_inline bool is_td(struct kvm *kvm) { return false; }
static __always_inline bool is_td_vcpu(struct kvm_vcpu *vcpu) { return false; }

#endif

static inline bool vt_is_tdx_private_gpa(struct kvm *kvm, gpa_t gpa)
{
        /* For TDX the direct mask is the shared mask. */
        return !kvm_is_addr_direct(kvm, gpa);
}

static inline int __vmx_handle_ept_violation(struct kvm_vcpu *vcpu, gpa_t gpa,
                                             unsigned long exit_qualification)
{
        u64 error_code;

        /* Is it a read fault? */
        error_code = (exit_qualification & EPT_VIOLATION_ACC_READ)
                     ? PFERR_USER_MASK : 0;
        /* Is it a write fault? */
        error_code |= (exit_qualification & EPT_VIOLATION_ACC_WRITE)
                      ? PFERR_WRITE_MASK : 0;
        /* Is it a fetch fault? */
        error_code |= (exit_qualification & EPT_VIOLATION_ACC_INSTR)
                      ? PFERR_FETCH_MASK : 0;
        /* ept page table entry is present? */
        error_code |= (exit_qualification & EPT_VIOLATION_PROT_MASK)
                      ? PFERR_PRESENT_MASK : 0;

        if (error_code & EPT_VIOLATION_GVA_IS_VALID)
                error_code |= (exit_qualification & EPT_VIOLATION_GVA_TRANSLATED) ?
                              PFERR_GUEST_FINAL_MASK : PFERR_GUEST_PAGE_MASK;

        if (vt_is_tdx_private_gpa(vcpu->kvm, gpa))
                error_code |= PFERR_PRIVATE_ACCESS;

        return kvm_mmu_page_fault(vcpu, gpa, error_code, NULL, 0);
}

static inline void kvm_vcpu_trigger_posted_interrupt(struct kvm_vcpu *vcpu,
                                                     int pi_vec)
{
#ifdef CONFIG_SMP
        if (vcpu->mode == IN_GUEST_MODE) {
                /*
                 * The vector of the virtual has already been set in the PIR.
                 * Send a notification event to deliver the virtual interrupt
                 * unless the vCPU is the currently running vCPU, i.e. the
                 * event is being sent from a fastpath VM-Exit handler, in
                 * which case the PIR will be synced to the vIRR before
                 * re-entering the guest.
                 *
                 * When the target is not the running vCPU, the following
                 * possibilities emerge:
                 *
                 * Case 1: vCPU stays in non-root mode. Sending a notification
                 * event posts the interrupt to the vCPU.
                 *
                 * Case 2: vCPU exits to root mode and is still runnable. The
                 * PIR will be synced to the vIRR before re-entering the guest.
                 * Sending a notification event is ok as the host IRQ handler
                 * will ignore the spurious event.
                 *
                 * Case 3: vCPU exits to root mode and is blocked. vcpu_block()
                 * has already synced PIR to vIRR and never blocks the vCPU if
                 * the vIRR is not empty. Therefore, a blocked vCPU here does
                 * not wait for any requested interrupts in PIR, and sending a
                 * notification event also results in a benign, spurious event.
                 */

                if (vcpu != kvm_get_running_vcpu())
                        __apic_send_IPI_mask(get_cpu_mask(vcpu->cpu), pi_vec);
                return;
        }
#endif
        /*
         * The vCPU isn't in the guest; wake the vCPU in case it is blocking,
         * otherwise do nothing as KVM will grab the highest priority pending
         * IRQ via ->sync_pir_to_irr() in vcpu_enter_guest().
         */
        kvm_vcpu_wake_up(vcpu);
}

/*
 * Post an interrupt to a vCPU's PIR and trigger the vCPU to process the
 * interrupt if necessary.
 */
static inline void __vmx_deliver_posted_interrupt(struct kvm_vcpu *vcpu,
                                                  struct pi_desc *pi_desc, int vector)
{
        if (pi_test_and_set_pir(vector, pi_desc))
                return;

        /* If a previous notification has sent the IPI, nothing to do.  */
        if (pi_test_and_set_on(pi_desc))
                return;

        /*
         * The implied barrier in pi_test_and_set_on() pairs with the smp_mb_*()
         * after setting vcpu->mode in vcpu_enter_guest(), thus the vCPU is
         * guaranteed to see PID.ON=1 and sync the PIR to IRR if triggering a
         * posted interrupt "fails" because vcpu->mode != IN_GUEST_MODE.
         */
        kvm_vcpu_trigger_posted_interrupt(vcpu, POSTED_INTR_VECTOR);
}

noinstr void vmx_handle_nmi(struct kvm_vcpu *vcpu);

#endif /* __KVM_X86_VMX_COMMON_H */

































































    1 









    1 





    1 























    1 






    1 






    1 
    1 

































































































































    1 































































































































    1 






    1 



    1 















    1 









































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
// SPDX-License-Identifier: GPL-2.0-only
 /*
 * USB Driver for ALi m5602 based webcams
 *
 * Copyright (C) 2008 Erik Andrén
 * Copyright (C) 2007 Ilyes Gouta. Based on the m5603x Linux Driver Project.
 * Copyright (C) 2005 m5603x Linux Driver Project <m5602@x3ng.com.br>
 *
 * Portions of code to USB interface and ALi driver software,
 * Copyright (c) 2006 Willem Duinker
 * v4l2 interface modeled after the V4L2 driver
 * for SN9C10x PC Camera Controllers
 */

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#include "m5602_ov9650.h"
#include "m5602_ov7660.h"
#include "m5602_mt9m111.h"
#include "m5602_po1030.h"
#include "m5602_s5k83a.h"
#include "m5602_s5k4aa.h"

/* Kernel module parameters */
int force_sensor;
static bool dump_bridge;
bool dump_sensor;

static const struct usb_device_id m5602_table[] = {
        {USB_DEVICE(0x0402, 0x5602)},
        {}
};

MODULE_DEVICE_TABLE(usb, m5602_table);

/* A skeleton used for sending messages to the sensor */
static const unsigned char sensor_urb_skeleton[] = {
        0x23, M5602_XB_GPIO_EN_H, 0x81, 0x06,
        0x23, M5602_XB_MISC_CTRL, 0x81, 0x80,
        0x13, M5602_XB_I2C_DEV_ADDR, 0x81, 0x00,
        0x13, M5602_XB_I2C_REG_ADDR, 0x81, 0x00,
        0x13, M5602_XB_I2C_DATA, 0x81, 0x00,
        0x13, M5602_XB_I2C_CTRL, 0x81, 0x11
};

/* A skeleton used for sending messages to the m5602 bridge */
static const unsigned char bridge_urb_skeleton[] = {
        0x13, 0x00, 0x81, 0x00
};

/* Reads a byte from the m5602 */
int m5602_read_bridge(struct sd *sd, const u8 address, u8 *i2c_data)
{
        int err;
        struct gspca_dev *gspca_dev = (struct gspca_dev *) sd;
        struct usb_device *udev = sd->gspca_dev.dev;
        __u8 *buf = sd->gspca_dev.usb_buf;

        err = usb_control_msg(udev, usb_rcvctrlpipe(udev, 0),
                              0x04, 0xc0, 0x14,
                              0x8100 + address, buf,
                              1, M5602_URB_MSG_TIMEOUT);
        *i2c_data = buf[0];

        gspca_dbg(gspca_dev, D_CONF, "Reading bridge register 0x%x containing 0x%x\n",
                  address, *i2c_data);

        /* usb_control_msg(...) returns the number of bytes sent upon success,
        mask that and return zero instead*/
        return (err < 0) ? err : 0;
}

/* Writes a byte to the m5602 */
int m5602_write_bridge(struct sd *sd, const u8 address, const u8 i2c_data)
{
        int err;
        struct gspca_dev *gspca_dev = (struct gspca_dev *) sd;
        struct usb_device *udev = sd->gspca_dev.dev;
        __u8 *buf = sd->gspca_dev.usb_buf;

        gspca_dbg(gspca_dev, D_CONF, "Writing bridge register 0x%x with 0x%x\n",
                  address, i2c_data);

        memcpy(buf, bridge_urb_skeleton,
               sizeof(bridge_urb_skeleton));
        buf[1] = address;
        buf[3] = i2c_data;

        err = usb_control_msg(udev, usb_sndctrlpipe(udev, 0),
                                0x04, 0x40, 0x19,
                                0x0000, buf,
                                4, M5602_URB_MSG_TIMEOUT);

        /* usb_control_msg(...) returns the number of bytes sent upon success,
           mask that and return zero instead */
        return (err < 0) ? err : 0;
}

static int m5602_wait_for_i2c(struct sd *sd)
{
        int err;
        u8 data;

        do {
                err = m5602_read_bridge(sd, M5602_XB_I2C_STATUS, &data);
        } while ((data & I2C_BUSY) && !err);
        return err;
}

int m5602_read_sensor(struct sd *sd, const u8 address,
                       u8 *i2c_data, const u8 len)
{
        int err, i;
        struct gspca_dev *gspca_dev = (struct gspca_dev *) sd;

        if (!len || len > sd->sensor->i2c_regW)
                return -EINVAL;

        err = m5602_wait_for_i2c(sd);
        if (err < 0)
                return err;

        err = m5602_write_bridge(sd, M5602_XB_I2C_DEV_ADDR,
                                 sd->sensor->i2c_slave_id);
        if (err < 0)
                return err;

        err = m5602_write_bridge(sd, M5602_XB_I2C_REG_ADDR, address);
        if (err < 0)
                return err;

        /* Sensors with registers that are of only
           one byte width are differently read */

        /* FIXME: This works with the ov9650, but has issues with the po1030 */
        if (sd->sensor->i2c_regW == 1) {
                err = m5602_write_bridge(sd, M5602_XB_I2C_CTRL, 1);
                if (err < 0)
                        return err;

                err = m5602_write_bridge(sd, M5602_XB_I2C_CTRL, 0x08);
        } else {
                err = m5602_write_bridge(sd, M5602_XB_I2C_CTRL, 0x18 + len);
        }

        for (i = 0; (i < len) && !err; i++) {
                err = m5602_wait_for_i2c(sd);
                if (err < 0)
                        return err;

                err = m5602_read_bridge(sd, M5602_XB_I2C_DATA, &(i2c_data[i]));

                gspca_dbg(gspca_dev, D_CONF, "Reading sensor register 0x%x containing 0x%x\n",
                          address, *i2c_data);
        }
        return err;
}

int m5602_write_sensor(struct sd *sd, const u8 address,
                        u8 *i2c_data, const u8 len)
{
        int err, i;
        u8 *p;
        struct gspca_dev *gspca_dev = (struct gspca_dev *) sd;
        struct usb_device *udev = sd->gspca_dev.dev;
        __u8 *buf = sd->gspca_dev.usb_buf;

        /* No sensor with a data width larger than 16 bits has yet been seen */
        if (len > sd->sensor->i2c_regW || !len)
                return -EINVAL;

        memcpy(buf, sensor_urb_skeleton,
               sizeof(sensor_urb_skeleton));

        buf[11] = sd->sensor->i2c_slave_id;
        buf[15] = address;

        /* Special case larger sensor writes */
        p = buf + 16;

        /* Copy a four byte write sequence for each byte to be written to */
        for (i = 0; i < len; i++) {
                memcpy(p, sensor_urb_skeleton + 16, 4);
                p[3] = i2c_data[i];
                p += 4;
                gspca_dbg(gspca_dev, D_CONF, "Writing sensor register 0x%x with 0x%x\n",
                          address, i2c_data[i]);
        }

        /* Copy the tailer */
        memcpy(p, sensor_urb_skeleton + 20, 4);

        /* Set the total length */
        p[3] = 0x10 + len;

        err = usb_control_msg(udev, usb_sndctrlpipe(udev, 0),
                              0x04, 0x40, 0x19,
                              0x0000, buf,
                              20 + len * 4, M5602_URB_MSG_TIMEOUT);

        return (err < 0) ? err : 0;
}

/* Dump all the registers of the m5602 bridge,
   unfortunately this breaks the camera until it's power cycled */
static void m5602_dump_bridge(struct sd *sd)
{
        int i;
        for (i = 0; i < 0x80; i++) {
                unsigned char val = 0;
                m5602_read_bridge(sd, i, &val);
                pr_info("ALi m5602 address 0x%x contains 0x%x\n", i, val);
        }
        pr_info("Warning: The ALi m5602 webcam probably won't work until it's power cycled\n");
}

static int m5602_probe_sensor(struct sd *sd)
{
        /* Try the po1030 */
        sd->sensor = &po1030;
        if (!sd->sensor->probe(sd))
                return 0;

        /* Try the mt9m111 sensor */
        sd->sensor = &mt9m111;
        if (!sd->sensor->probe(sd))
                return 0;

        /* Try the s5k4aa */
        sd->sensor = &s5k4aa;
        if (!sd->sensor->probe(sd))
                return 0;

        /* Try the ov9650 */
        sd->sensor = &ov9650;
        if (!sd->sensor->probe(sd))
                return 0;

        /* Try the ov7660 */
        sd->sensor = &ov7660;
        if (!sd->sensor->probe(sd))
                return 0;

        /* Try the s5k83a */
        sd->sensor = &s5k83a;
        if (!sd->sensor->probe(sd))
                return 0;

        /* More sensor probe function goes here */
        pr_info("Failed to find a sensor\n");
        sd->sensor = NULL;
        return -ENODEV;
}

static int m5602_configure(struct gspca_dev *gspca_dev,
                           const struct usb_device_id *id);

static int m5602_init(struct gspca_dev *gspca_dev)
{
        struct sd *sd = (struct sd *) gspca_dev;
        int err;

        gspca_dbg(gspca_dev, D_CONF, "Initializing ALi m5602 webcam\n");
        /* Run the init sequence */
        err = sd->sensor->init(sd);

        return err;
}

static int m5602_init_controls(struct gspca_dev *gspca_dev)
{
        struct sd *sd = (struct sd *) gspca_dev;

        if (!sd->sensor->init_controls)
                return 0;

        return sd->sensor->init_controls(sd);
}

static int m5602_start_transfer(struct gspca_dev *gspca_dev)
{
        struct sd *sd = (struct sd *) gspca_dev;
        __u8 *buf = sd->gspca_dev.usb_buf;
        int err;

        /* Send start command to the camera */
        const u8 buffer[4] = {0x13, 0xf9, 0x0f, 0x01};

        if (sd->sensor->start)
                sd->sensor->start(sd);

        memcpy(buf, buffer, sizeof(buffer));
        err = usb_control_msg(gspca_dev->dev,
                              usb_sndctrlpipe(gspca_dev->dev, 0),
                              0x04, 0x40, 0x19, 0x0000, buf,
                              sizeof(buffer), M5602_URB_MSG_TIMEOUT);

        gspca_dbg(gspca_dev, D_STREAM, "Transfer started\n");
        return (err < 0) ? err : 0;
}

static void m5602_urb_complete(struct gspca_dev *gspca_dev,
                                u8 *data, int len)
{
        struct sd *sd = (struct sd *) gspca_dev;

        if (len < 6) {
                gspca_dbg(gspca_dev, D_PACK, "Packet is less than 6 bytes\n");
                return;
        }

        /* Frame delimiter: ff xx xx xx ff ff */
        if (data[0] == 0xff && data[4] == 0xff && data[5] == 0xff &&
            data[2] != sd->frame_id) {
                gspca_dbg(gspca_dev, D_FRAM, "Frame delimiter detected\n");
                sd->frame_id = data[2];

                /* Remove the extra fluff appended on each header */
                data += 6;
                len -= 6;

                /* Complete the last frame (if any) */
                gspca_frame_add(gspca_dev, LAST_PACKET,
                                NULL, 0);
                sd->frame_count++;

                /* Create a new frame */
                gspca_frame_add(gspca_dev, FIRST_PACKET, data, len);

                gspca_dbg(gspca_dev, D_FRAM, "Starting new frame %d\n",
                          sd->frame_count);

        } else {
                int cur_frame_len;

                cur_frame_len = gspca_dev->image_len;
                /* Remove urb header */
                data += 4;
                len -= 4;

                if (cur_frame_len + len <= gspca_dev->pixfmt.sizeimage) {
                        gspca_dbg(gspca_dev, D_FRAM, "Continuing frame %d copying %d bytes\n",
                                  sd->frame_count, len);

                        gspca_frame_add(gspca_dev, INTER_PACKET,
                                        data, len);
                } else {
                        /* Add the remaining data up to frame size */
                        gspca_frame_add(gspca_dev, INTER_PACKET, data,
                                gspca_dev->pixfmt.sizeimage - cur_frame_len);
                }
        }
}

static void m5602_stop_transfer(struct gspca_dev *gspca_dev)
{
        struct sd *sd = (struct sd *) gspca_dev;

        /* Run the sensor specific end transfer sequence */
        if (sd->sensor->stop)
                sd->sensor->stop(sd);
}

/* sub-driver description */
static const struct sd_desc sd_desc = {
        .name                = MODULE_NAME,
        .config                = m5602_configure,
        .init                = m5602_init,
        .init_controls        = m5602_init_controls,
        .start                = m5602_start_transfer,
        .stopN                = m5602_stop_transfer,
        .pkt_scan        = m5602_urb_complete
};

/* this function is called at probe time */
static int m5602_configure(struct gspca_dev *gspca_dev,
                           const struct usb_device_id *id)
{
        struct sd *sd = (struct sd *) gspca_dev;
        struct cam *cam;
        int err;

        cam = &gspca_dev->cam;

        if (dump_bridge)
                m5602_dump_bridge(sd);

        /* Probe sensor */
        err = m5602_probe_sensor(sd);
        if (err)
                goto fail;

        return 0;

fail:
        gspca_err(gspca_dev, "ALi m5602 webcam failed\n");
        cam->cam_mode = NULL;
        cam->nmodes = 0;

        return err;
}

static int m5602_probe(struct usb_interface *intf,
                       const struct usb_device_id *id)
{
        return gspca_dev_probe(intf, id, &sd_desc, sizeof(struct sd),
                               THIS_MODULE);
}

static void m5602_disconnect(struct usb_interface *intf)
{
        struct gspca_dev *gspca_dev = usb_get_intfdata(intf);
        struct sd *sd = (struct sd *) gspca_dev;

        if (sd->sensor->disconnect)
                sd->sensor->disconnect(sd);

        gspca_disconnect(intf);
}

static struct usb_driver sd_driver = {
        .name = MODULE_NAME,
        .id_table = m5602_table,
        .probe = m5602_probe,
#ifdef CONFIG_PM
        .suspend = gspca_suspend,
        .resume = gspca_resume,
        .reset_resume = gspca_resume,
#endif
        .disconnect = m5602_disconnect
};

module_usb_driver(sd_driver);

MODULE_AUTHOR(DRIVER_AUTHOR);
MODULE_DESCRIPTION(DRIVER_DESC);
MODULE_LICENSE("GPL");
module_param(force_sensor, int, S_IRUGO | S_IWUSR);
MODULE_PARM_DESC(force_sensor,
                "forces detection of a sensor, 1 = OV9650, 2 = S5K83A, 3 = S5K4AA, 4 = MT9M111, 5 = PO1030, 6 = OV7660");

module_param(dump_bridge, bool, S_IRUGO | S_IWUSR);
MODULE_PARM_DESC(dump_bridge, "Dumps all usb bridge registers at startup");

module_param(dump_sensor, bool, S_IRUGO | S_IWUSR);
MODULE_PARM_DESC(dump_sensor, "Dumps all usb sensor registers at startup providing a sensor is found");























































































    3 









    3 












































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_KCOV_H
#define _LINUX_KCOV_H

#include <linux/sched.h>
#include <uapi/linux/kcov.h>

struct task_struct;

#ifdef CONFIG_KCOV

enum kcov_mode {
        /* Coverage collection is not enabled yet. */
        KCOV_MODE_DISABLED = 0,
        /* KCOV was initialized, but tracing mode hasn't been chosen yet. */
        KCOV_MODE_INIT = 1,
        /*
         * Tracing coverage collection mode.
         * Covered PCs are collected in a per-task buffer.
         */
        KCOV_MODE_TRACE_PC = 2,
        /* Collecting comparison operands mode. */
        KCOV_MODE_TRACE_CMP = 3,
        /* The process owns a KCOV remote reference. */
        KCOV_MODE_REMOTE = 4,
};

#define KCOV_IN_CTXSW        (1 << 30)

void kcov_task_init(struct task_struct *t);
void kcov_task_exit(struct task_struct *t);

#define kcov_prepare_switch(t)                        \
do {                                                \
        (t)->kcov_mode |= KCOV_IN_CTXSW;        \
} while (0)

#define kcov_finish_switch(t)                        \
do {                                                \
        (t)->kcov_mode &= ~KCOV_IN_CTXSW;        \
} while (0)

/* See Documentation/dev-tools/kcov.rst for usage details. */
void kcov_remote_start(u64 handle);
void kcov_remote_stop(void);
u64 kcov_common_handle(void);

static inline void kcov_remote_start_common(u64 id)
{
        kcov_remote_start(kcov_remote_handle(KCOV_SUBSYSTEM_COMMON, id));
}

static inline void kcov_remote_start_usb(u64 id)
{
        kcov_remote_start(kcov_remote_handle(KCOV_SUBSYSTEM_USB, id));
}

/*
 * The softirq flavor of kcov_remote_*() functions is introduced as a temporary
 * workaround for KCOV's lack of nested remote coverage sections support.
 *
 * Adding support is tracked in https://bugzilla.kernel.org/show_bug.cgi?id=210337.
 *
 * kcov_remote_start_usb_softirq():
 *
 * 1. Only collects coverage when called in the softirq context. This allows
 *    avoiding nested remote coverage collection sections in the task context.
 *    For example, USB/IP calls usb_hcd_giveback_urb() in the task context
 *    within an existing remote coverage collection section. Thus, KCOV should
 *    not attempt to start collecting coverage within the coverage collection
 *    section in __usb_hcd_giveback_urb() in this case.
 *
 * 2. Disables interrupts for the duration of the coverage collection section.
 *    This allows avoiding nested remote coverage collection sections in the
 *    softirq context (a softirq might occur during the execution of a work in
 *    the BH workqueue, which runs with in_serving_softirq() > 0).
 *    For example, usb_giveback_urb_bh() runs in the BH workqueue with
 *    interrupts enabled, so __usb_hcd_giveback_urb() might be interrupted in
 *    the middle of its remote coverage collection section, and the interrupt
 *    handler might invoke __usb_hcd_giveback_urb() again.
 */

static inline unsigned long kcov_remote_start_usb_softirq(u64 id)
{
        unsigned long flags = 0;

        if (in_serving_softirq()) {
                local_irq_save(flags);
                kcov_remote_start_usb(id);
        }

        return flags;
}

static inline void kcov_remote_stop_softirq(unsigned long flags)
{
        if (in_serving_softirq()) {
                kcov_remote_stop();
                local_irq_restore(flags);
        }
}

#ifdef CONFIG_64BIT
typedef unsigned long kcov_u64;
#else
typedef unsigned long long kcov_u64;
#endif

void __sanitizer_cov_trace_pc(void);
void __sanitizer_cov_trace_cmp1(u8 arg1, u8 arg2);
void __sanitizer_cov_trace_cmp2(u16 arg1, u16 arg2);
void __sanitizer_cov_trace_cmp4(u32 arg1, u32 arg2);
void __sanitizer_cov_trace_cmp8(kcov_u64 arg1, kcov_u64 arg2);
void __sanitizer_cov_trace_const_cmp1(u8 arg1, u8 arg2);
void __sanitizer_cov_trace_const_cmp2(u16 arg1, u16 arg2);
void __sanitizer_cov_trace_const_cmp4(u32 arg1, u32 arg2);
void __sanitizer_cov_trace_const_cmp8(kcov_u64 arg1, kcov_u64 arg2);
void __sanitizer_cov_trace_switch(kcov_u64 val, void *cases);

#else

static inline void kcov_task_init(struct task_struct *t) {}
static inline void kcov_task_exit(struct task_struct *t) {}
static inline void kcov_prepare_switch(struct task_struct *t) {}
static inline void kcov_finish_switch(struct task_struct *t) {}
static inline void kcov_remote_start(u64 handle) {}
static inline void kcov_remote_stop(void) {}
static inline u64 kcov_common_handle(void)
{
        return 0;
}
static inline void kcov_remote_start_common(u64 id) {}
static inline void kcov_remote_start_usb(u64 id) {}
static inline unsigned long kcov_remote_start_usb_softirq(u64 id)
{
        return 0;
}
static inline void kcov_remote_stop_softirq(unsigned long flags) {}

#endif /* CONFIG_KCOV */
#endif /* _LINUX_KCOV_H */




































































































































    1 






1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
// SPDX-License-Identifier: GPL-2.0-only
/***************************************************************************
 *   Copyright (C) 2010-2012 by Bruno Prémont <bonbons@linux-vserver.org>  *
 *                                                                         *
 *   Based on Logitech G13 driver (v0.4)                                   *
 *     Copyright (C) 2009 by Rick L. Vinyard, Jr. <rvinyard@cs.nmsu.edu>   *
 *                                                                         *
 ***************************************************************************/

#include <linux/hid.h>
#include <linux/hid-debug.h>
#include <linux/input.h>
#include "hid-ids.h"

#include <linux/fb.h>
#include <linux/vmalloc.h>
#include <linux/backlight.h>
#include <linux/lcd.h>

#include <linux/leds.h>

#include <linux/seq_file.h>
#include <linux/debugfs.h>

#include <linux/completion.h>
#include <linux/uaccess.h>
#include <linux/module.h>
#include <media/rc-core.h>

#include "hid-picolcd.h"


int picolcd_raw_cir(struct picolcd_data *data,
                struct hid_report *report, u8 *raw_data, int size)
{
        unsigned long flags;
        int i, w, sz;
        struct ir_raw_event rawir = {};

        /* ignore if rc_dev is NULL or status is shunned */
        spin_lock_irqsave(&data->lock, flags);
        if (!data->rc_dev || (data->status & PICOLCD_CIR_SHUN)) {
                spin_unlock_irqrestore(&data->lock, flags);
                return 1;
        }
        spin_unlock_irqrestore(&data->lock, flags);

        /* PicoLCD USB packets contain 16-bit intervals in network order,
         * with value negated for pulse. Intervals are in microseconds.
         *
         * Note: some userspace LIRC code for PicoLCD says negated values
         * for space - is it a matter of IR chip? (pulse for my TSOP2236)
         *
         * In addition, the first interval seems to be around 15000 + base
         * interval for non-first report of IR data - thus the quirk below
         * to get RC_CODE to understand Sony and JVC remotes I have at hand
         */
        sz = size > 0 ? min((int)raw_data[0], size-1) : 0;
        for (i = 0; i+1 < sz; i += 2) {
                w = (raw_data[i] << 8) | (raw_data[i+1]);
                rawir.pulse = !!(w & 0x8000);
                rawir.duration = rawir.pulse ? (65536 - w) : w;
                /* Quirk!! - see above */
                if (i == 0 && rawir.duration > 15000)
                        rawir.duration -= 15000;
                ir_raw_event_store(data->rc_dev, &rawir);
        }
        ir_raw_event_handle(data->rc_dev);

        return 1;
}

static int picolcd_cir_open(struct rc_dev *dev)
{
        struct picolcd_data *data = dev->priv;
        unsigned long flags;

        spin_lock_irqsave(&data->lock, flags);
        data->status &= ~PICOLCD_CIR_SHUN;
        spin_unlock_irqrestore(&data->lock, flags);
        return 0;
}

static void picolcd_cir_close(struct rc_dev *dev)
{
        struct picolcd_data *data = dev->priv;
        unsigned long flags;

        spin_lock_irqsave(&data->lock, flags);
        data->status |= PICOLCD_CIR_SHUN;
        spin_unlock_irqrestore(&data->lock, flags);
}

/* initialize CIR input device */
int picolcd_init_cir(struct picolcd_data *data, struct hid_report *report)
{
        struct rc_dev *rdev;
        int ret = 0;

        rdev = rc_allocate_device(RC_DRIVER_IR_RAW);
        if (!rdev)
                return -ENOMEM;

        rdev->priv             = data;
        rdev->allowed_protocols = RC_PROTO_BIT_ALL_IR_DECODER;
        rdev->open             = picolcd_cir_open;
        rdev->close            = picolcd_cir_close;
        rdev->device_name      = data->hdev->name;
        rdev->input_phys       = data->hdev->phys;
        rdev->input_id.bustype = data->hdev->bus;
        rdev->input_id.vendor  = data->hdev->vendor;
        rdev->input_id.product = data->hdev->product;
        rdev->input_id.version = data->hdev->version;
        rdev->dev.parent       = &data->hdev->dev;
        rdev->driver_name      = PICOLCD_NAME;
        rdev->map_name         = RC_MAP_RC6_MCE;
        rdev->timeout          = MS_TO_US(100);
        rdev->rx_resolution    = 1;

        ret = rc_register_device(rdev);
        if (ret)
                goto err;
        data->rc_dev = rdev;
        return 0;

err:
        rc_free_device(rdev);
        return ret;
}

void picolcd_exit_cir(struct picolcd_data *data)
{
        struct rc_dev *rdev = data->rc_dev;

        data->rc_dev = NULL;
        rc_unregister_device(rdev);
}

























    1 









    1 


































    1 


























































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
// SPDX-License-Identifier: GPL-2.0-only
/* DVB USB framework compliant Linux driver for the AVerMedia AverTV DVB-T
 * USB2.0 (A800) DVB-T receiver.
 *
 * Copyright (C) 2005 Patrick Boettcher (patrick.boettcher@posteo.de)
 *
 * Thanks to
 *   - AVerMedia who kindly provided information and
 *   - Glen Harris who suffered from my mistakes during development.
 *
 * see Documentation/driver-api/media/drivers/dvb-usb.rst for more information
 */
#include "dibusb.h"

static int debug;
module_param(debug, int, 0644);
MODULE_PARM_DESC(debug, "set debugging level (rc=1 (or-able))." DVB_USB_DEBUG_STATUS);

DVB_DEFINE_MOD_OPT_ADAPTER_NR(adapter_nr);

#define deb_rc(args...)   dprintk(debug,0x01,args)

static int a800_power_ctrl(struct dvb_usb_device *d, int onoff)
{
        /* do nothing for the AVerMedia */
        return 0;
}

/* assure to put cold to 0 for iManufacturer == 1 */
static int a800_identify_state(struct usb_device *udev,
                               const struct dvb_usb_device_properties *props,
                               const struct dvb_usb_device_description **desc,
                               int *cold)
{
        *cold = udev->descriptor.iManufacturer != 1;
        return 0;
}

static int a800_rc_query(struct dvb_usb_device *d)
{
        int ret = 0;
        u8 *key = kmalloc(5, GFP_KERNEL);
        if (!key)
                return -ENOMEM;

        if (usb_control_msg(d->udev,usb_rcvctrlpipe(d->udev,0),
                                0x04, USB_TYPE_VENDOR | USB_DIR_IN, 0, 0, key, 5,
                                2000) != 5) {
                ret = -ENODEV;
                goto out;
        }

        /* Note that extended nec and nec32 are dropped */
        if (key[0] == 1)
                rc_keydown(d->rc_dev, RC_PROTO_NEC,
                           RC_SCANCODE_NEC(key[1], key[3]), 0);
        else if (key[0] == 2)
                rc_repeat(d->rc_dev);
out:
        kfree(key);
        return ret;
}

/* USB Driver stuff */
static struct dvb_usb_device_properties a800_properties;

static int a800_probe(struct usb_interface *intf,
                const struct usb_device_id *id)
{
        return dvb_usb_device_init(intf, &a800_properties,
                                   THIS_MODULE, NULL, adapter_nr);
}

/* do not change the order of the ID table */
enum {
        AVERMEDIA_DVBT_USB2_COLD,
        AVERMEDIA_DVBT_USB2_WARM,
};

static const struct usb_device_id a800_table[] = {
        DVB_USB_DEV(AVERMEDIA, AVERMEDIA_DVBT_USB2_COLD),
        DVB_USB_DEV(AVERMEDIA, AVERMEDIA_DVBT_USB2_WARM),
        { }
};

MODULE_DEVICE_TABLE (usb, a800_table);

static struct dvb_usb_device_properties a800_properties = {
        .caps = DVB_USB_IS_AN_I2C_ADAPTER,

        .usb_ctrl = CYPRESS_FX2,
        .firmware = "dvb-usb-avertv-a800-02.fw",

        .num_adapters = 1,
        .adapter = {
                {
                .num_frontends = 1,
                .fe = {{
                        .caps = DVB_USB_ADAP_HAS_PID_FILTER | DVB_USB_ADAP_PID_FILTER_CAN_BE_TURNED_OFF,
                        .pid_filter_count = 32,
                        .streaming_ctrl   = dibusb2_0_streaming_ctrl,
                        .pid_filter       = dibusb_pid_filter,
                        .pid_filter_ctrl  = dibusb_pid_filter_ctrl,

                        .frontend_attach  = dibusb_dib3000mc_frontend_attach,
                        .tuner_attach     = dibusb_dib3000mc_tuner_attach,

                        /* parameter for the MPEG2-data transfer */
                                        .stream = {
                                                .type = USB_BULK,
                                .count = 7,
                                .endpoint = 0x06,
                                .u = {
                                        .bulk = {
                                                .buffersize = 4096,
                                        }
                                }
                        },
                }},
                        .size_of_priv     = sizeof(struct dibusb_state),
                },
        },

        .power_ctrl       = a800_power_ctrl,
        .identify_state   = a800_identify_state,

        .rc.core = {
                .rc_interval        = DEFAULT_RC_INTERVAL,
                .rc_codes        = RC_MAP_AVERMEDIA_M135A,
                .module_name        = KBUILD_MODNAME,
                .rc_query        = a800_rc_query,
                .allowed_protos = RC_PROTO_BIT_NEC,
        },

        .i2c_algo         = &dibusb_i2c_algo,

        .generic_bulk_ctrl_endpoint = 0x01,
        .num_device_descs = 1,
        .devices = {
                {   "AVerMedia AverTV DVB-T USB 2.0 (A800)",
                        { &a800_table[AVERMEDIA_DVBT_USB2_COLD], NULL },
                        { &a800_table[AVERMEDIA_DVBT_USB2_WARM], NULL },
                },
        }
};

static struct usb_driver a800_driver = {
        .name                = "dvb_usb_a800",
        .probe                = a800_probe,
        .disconnect = dvb_usb_device_exit,
        .id_table        = a800_table,
};

module_usb_driver(a800_driver);

MODULE_AUTHOR("Patrick Boettcher <patrick.boettcher@posteo.de>");
MODULE_DESCRIPTION("AVerMedia AverTV DVB-T USB 2.0 (A800)");
MODULE_VERSION("1.0");
MODULE_LICENSE("GPL");































    4 






    4 






    6 






    2 






    2 




















    4 





















    6 















































































    7 






    6 

















1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
#undef TRACE_SYSTEM
#define TRACE_SYSTEM rtc

#if !defined(_TRACE_RTC_H) || defined(TRACE_HEADER_MULTI_READ)
#define _TRACE_RTC_H

#include <linux/rtc.h>
#include <linux/tracepoint.h>

DECLARE_EVENT_CLASS(rtc_time_alarm_class,

        TP_PROTO(time64_t secs, int err),

        TP_ARGS(secs, err),

        TP_STRUCT__entry(
                __field(time64_t, secs)
                __field(int, err)
        ),

        TP_fast_assign(
                __entry->secs = secs;
                __entry->err = err;
        ),

        TP_printk("UTC (%lld) (%d)",
                  __entry->secs, __entry->err
        )
);

DEFINE_EVENT(rtc_time_alarm_class, rtc_set_time,

        TP_PROTO(time64_t secs, int err),

        TP_ARGS(secs, err)
);

DEFINE_EVENT(rtc_time_alarm_class, rtc_read_time,

        TP_PROTO(time64_t secs, int err),

        TP_ARGS(secs, err)
);

DEFINE_EVENT(rtc_time_alarm_class, rtc_set_alarm,

        TP_PROTO(time64_t secs, int err),

        TP_ARGS(secs, err)
);

DEFINE_EVENT(rtc_time_alarm_class, rtc_read_alarm,

        TP_PROTO(time64_t secs, int err),

        TP_ARGS(secs, err)
);

TRACE_EVENT(rtc_irq_set_freq,

        TP_PROTO(int freq, int err),

        TP_ARGS(freq, err),

        TP_STRUCT__entry(
                __field(int, freq)
                __field(int, err)
        ),

        TP_fast_assign(
                __entry->freq = freq;
                __entry->err = err;
        ),

        TP_printk("set RTC periodic IRQ frequency:%u (%d)",
                  __entry->freq, __entry->err
        )
);

TRACE_EVENT(rtc_irq_set_state,

        TP_PROTO(int enabled, int err),

        TP_ARGS(enabled, err),

        TP_STRUCT__entry(
                __field(int, enabled)
                __field(int, err)
        ),

        TP_fast_assign(
                __entry->enabled = enabled;
                __entry->err = err;
        ),

        TP_printk("%s RTC 2^N Hz periodic IRQs (%d)",
                  __entry->enabled ? "enable" : "disable",
                  __entry->err
        )
);

TRACE_EVENT(rtc_alarm_irq_enable,

        TP_PROTO(unsigned int enabled, int err),

        TP_ARGS(enabled, err),

        TP_STRUCT__entry(
                __field(unsigned int, enabled)
                __field(int, err)
        ),

        TP_fast_assign(
                __entry->enabled = enabled;
                __entry->err = err;
        ),

        TP_printk("%s RTC alarm IRQ (%d)",
                  __entry->enabled ? "enable" : "disable",
                  __entry->err
        )
);

DECLARE_EVENT_CLASS(rtc_offset_class,

        TP_PROTO(long offset, int err),

        TP_ARGS(offset, err),

        TP_STRUCT__entry(
                __field(long, offset)
                __field(int, err)
        ),

        TP_fast_assign(
                __entry->offset = offset;
                __entry->err = err;
        ),

        TP_printk("RTC offset: %ld (%d)",
                  __entry->offset, __entry->err
        )
);

DEFINE_EVENT(rtc_offset_class, rtc_set_offset,

        TP_PROTO(long offset, int err),

        TP_ARGS(offset, err)
);

DEFINE_EVENT(rtc_offset_class, rtc_read_offset,

        TP_PROTO(long offset, int err),

        TP_ARGS(offset, err)
);

DECLARE_EVENT_CLASS(rtc_timer_class,

        TP_PROTO(struct rtc_timer *timer),

        TP_ARGS(timer),

        TP_STRUCT__entry(
                __field(struct rtc_timer *, timer)
                __field(ktime_t, expires)
                __field(ktime_t, period)
        ),

        TP_fast_assign(
                __entry->timer = timer;
                __entry->expires = timer->node.expires;
                __entry->period = timer->period;
        ),

        TP_printk("RTC timer:(%p) expires:%lld period:%lld",
                  __entry->timer, __entry->expires, __entry->period
        )
);

DEFINE_EVENT(rtc_timer_class, rtc_timer_enqueue,

        TP_PROTO(struct rtc_timer *timer),

        TP_ARGS(timer)
);

DEFINE_EVENT(rtc_timer_class, rtc_timer_dequeue,

        TP_PROTO(struct rtc_timer *timer),

        TP_ARGS(timer)
);

DEFINE_EVENT(rtc_timer_class, rtc_timer_fired,

        TP_PROTO(struct rtc_timer *timer),

        TP_ARGS(timer)
);

#endif /* _TRACE_RTC_H */

/* This part must be outside protection */
#include <trace/define_trace.h>



























































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_DAX_H
#define _LINUX_DAX_H

#include <linux/fs.h>
#include <linux/mm.h>
#include <linux/radix-tree.h>

typedef unsigned long dax_entry_t;

struct dax_device;
struct gendisk;
struct iomap_ops;
struct iomap_iter;
struct iomap;

enum dax_access_mode {
        DAX_ACCESS,
        DAX_RECOVERY_WRITE,
};

struct dax_operations {
        /*
         * direct_access: translate a device-relative
         * logical-page-offset into an absolute physical pfn. Return the
         * number of pages available for DAX at that pfn.
         */
        long (*direct_access)(struct dax_device *, pgoff_t, long,
                        enum dax_access_mode, void **, unsigned long *);
        /* zero_page_range: required operation. Zero page range   */
        int (*zero_page_range)(struct dax_device *, pgoff_t, size_t);
        /*
         * recovery_write: recover a poisoned range by DAX device driver
         * capable of clearing poison.
         */
        size_t (*recovery_write)(struct dax_device *dax_dev, pgoff_t pgoff,
                        void *addr, size_t bytes, struct iov_iter *iter);
};

struct dax_holder_operations {
        /*
         * notify_failure - notify memory failure into inner holder device
         * @dax_dev: the dax device which contains the holder
         * @offset: offset on this dax device where memory failure occurs
         * @len: length of this memory failure event
         * @flags: action flags for memory failure handler
         */
        int (*notify_failure)(struct dax_device *dax_dev, u64 offset,
                        u64 len, int mf_flags);
};

#if IS_ENABLED(CONFIG_DAX)
struct dax_device *alloc_dax(void *private, const struct dax_operations *ops);
void *dax_holder(struct dax_device *dax_dev);
void put_dax(struct dax_device *dax_dev);
void kill_dax(struct dax_device *dax_dev);
void dax_write_cache(struct dax_device *dax_dev, bool wc);
bool dax_write_cache_enabled(struct dax_device *dax_dev);
bool dax_synchronous(struct dax_device *dax_dev);
void set_dax_nocache(struct dax_device *dax_dev);
void set_dax_nomc(struct dax_device *dax_dev);
void set_dax_synchronous(struct dax_device *dax_dev);
size_t dax_recovery_write(struct dax_device *dax_dev, pgoff_t pgoff,
                void *addr, size_t bytes, struct iov_iter *i);
/*
 * Check if given mapping is supported by the file / underlying device.
 */
static inline bool daxdev_mapping_supported(vm_flags_t vm_flags,
                                            const struct inode *inode,
                                            struct dax_device *dax_dev)
{
        if (!(vm_flags & VM_SYNC))
                return true;
        if (!IS_DAX(inode))
                return false;
        return dax_synchronous(dax_dev);
}
#else
static inline void *dax_holder(struct dax_device *dax_dev)
{
        return NULL;
}
static inline struct dax_device *alloc_dax(void *private,
                const struct dax_operations *ops)
{
        return ERR_PTR(-EOPNOTSUPP);
}
static inline void put_dax(struct dax_device *dax_dev)
{
}
static inline void kill_dax(struct dax_device *dax_dev)
{
}
static inline void dax_write_cache(struct dax_device *dax_dev, bool wc)
{
}
static inline bool dax_write_cache_enabled(struct dax_device *dax_dev)
{
        return false;
}
static inline bool dax_synchronous(struct dax_device *dax_dev)
{
        return true;
}
static inline void set_dax_nocache(struct dax_device *dax_dev)
{
}
static inline void set_dax_nomc(struct dax_device *dax_dev)
{
}
static inline void set_dax_synchronous(struct dax_device *dax_dev)
{
}
static inline bool daxdev_mapping_supported(vm_flags_t vm_flags,
                                            const struct inode *inode,
                                            struct dax_device *dax_dev)
{
        return !(vm_flags & VM_SYNC);
}
static inline size_t dax_recovery_write(struct dax_device *dax_dev,
                pgoff_t pgoff, void *addr, size_t bytes, struct iov_iter *i)
{
        return 0;
}
#endif

struct writeback_control;
#if defined(CONFIG_BLOCK) && defined(CONFIG_FS_DAX)
int dax_add_host(struct dax_device *dax_dev, struct gendisk *disk);
void dax_remove_host(struct gendisk *disk);
struct dax_device *fs_dax_get_by_bdev(struct block_device *bdev, u64 *start_off,
                void *holder, const struct dax_holder_operations *ops);
void fs_put_dax(struct dax_device *dax_dev, void *holder);
#else
static inline int dax_add_host(struct dax_device *dax_dev, struct gendisk *disk)
{
        return 0;
}
static inline void dax_remove_host(struct gendisk *disk)
{
}
static inline struct dax_device *fs_dax_get_by_bdev(struct block_device *bdev,
                u64 *start_off, void *holder,
                const struct dax_holder_operations *ops)
{
        return NULL;
}
static inline void fs_put_dax(struct dax_device *dax_dev, void *holder)
{
}
#endif /* CONFIG_BLOCK && CONFIG_FS_DAX */

#if IS_ENABLED(CONFIG_FS_DAX)
int dax_writeback_mapping_range(struct address_space *mapping,
                struct dax_device *dax_dev, struct writeback_control *wbc);

struct page *dax_layout_busy_page(struct address_space *mapping);
struct page *dax_layout_busy_page_range(struct address_space *mapping, loff_t start, loff_t end);
dax_entry_t dax_lock_folio(struct folio *folio);
void dax_unlock_folio(struct folio *folio, dax_entry_t cookie);
dax_entry_t dax_lock_mapping_entry(struct address_space *mapping,
                unsigned long index, struct page **page);
void dax_unlock_mapping_entry(struct address_space *mapping,
                unsigned long index, dax_entry_t cookie);
#else
static inline struct page *dax_layout_busy_page(struct address_space *mapping)
{
        return NULL;
}

static inline struct page *dax_layout_busy_page_range(struct address_space *mapping, pgoff_t start, pgoff_t nr_pages)
{
        return NULL;
}

static inline int dax_writeback_mapping_range(struct address_space *mapping,
                struct dax_device *dax_dev, struct writeback_control *wbc)
{
        return -EOPNOTSUPP;
}

static inline dax_entry_t dax_lock_folio(struct folio *folio)
{
        if (IS_DAX(folio->mapping->host))
                return ~0UL;
        return 0;
}

static inline void dax_unlock_folio(struct folio *folio, dax_entry_t cookie)
{
}

static inline dax_entry_t dax_lock_mapping_entry(struct address_space *mapping,
                unsigned long index, struct page **page)
{
        return 0;
}

static inline void dax_unlock_mapping_entry(struct address_space *mapping,
                unsigned long index, dax_entry_t cookie)
{
}
#endif

int dax_file_unshare(struct inode *inode, loff_t pos, loff_t len,
                const struct iomap_ops *ops);
int dax_zero_range(struct inode *inode, loff_t pos, loff_t len, bool *did_zero,
                const struct iomap_ops *ops);
int dax_truncate_page(struct inode *inode, loff_t pos, bool *did_zero,
                const struct iomap_ops *ops);

static inline bool dax_page_is_idle(struct page *page)
{
        return page && page_ref_count(page) == 0;
}

#if IS_ENABLED(CONFIG_DAX)
int dax_read_lock(void);
void dax_read_unlock(int id);
#else
static inline int dax_read_lock(void)
{
        return 0;
}

static inline void dax_read_unlock(int id)
{
}
#endif /* CONFIG_DAX */

#if !IS_ENABLED(CONFIG_FS_DAX)
static inline int __must_check dax_break_layout(struct inode *inode,
                            loff_t start, loff_t end, void (cb)(struct inode *))
{
        return 0;
}

static inline void dax_break_layout_final(struct inode *inode)
{
}
#endif

bool dax_alive(struct dax_device *dax_dev);
void *dax_get_private(struct dax_device *dax_dev);
long dax_direct_access(struct dax_device *dax_dev, pgoff_t pgoff, long nr_pages,
                enum dax_access_mode mode, void **kaddr, unsigned long *pfn);
size_t dax_copy_from_iter(struct dax_device *dax_dev, pgoff_t pgoff, void *addr,
                size_t bytes, struct iov_iter *i);
size_t dax_copy_to_iter(struct dax_device *dax_dev, pgoff_t pgoff, void *addr,
                size_t bytes, struct iov_iter *i);
int dax_zero_page_range(struct dax_device *dax_dev, pgoff_t pgoff,
                        size_t nr_pages);
int dax_holder_notify_failure(struct dax_device *dax_dev, u64 off, u64 len,
                int mf_flags);
void dax_flush(struct dax_device *dax_dev, void *addr, size_t size);

ssize_t dax_iomap_rw(struct kiocb *iocb, struct iov_iter *iter,
                const struct iomap_ops *ops);
vm_fault_t dax_iomap_fault(struct vm_fault *vmf, unsigned int order,
                        unsigned long *pfnp, int *errp,
                        const struct iomap_ops *ops);
vm_fault_t dax_finish_sync_fault(struct vm_fault *vmf,
                unsigned int order, unsigned long pfn);
int dax_delete_mapping_entry(struct address_space *mapping, pgoff_t index);
void dax_delete_mapping_range(struct address_space *mapping,
                                loff_t start, loff_t end);
int dax_invalidate_mapping_entry_sync(struct address_space *mapping,
                                      pgoff_t index);
int __must_check dax_break_layout(struct inode *inode, loff_t start,
                                loff_t end, void (cb)(struct inode *));
static inline int __must_check dax_break_layout_inode(struct inode *inode,
                                                void (cb)(struct inode *))
{
        return dax_break_layout(inode, 0, LLONG_MAX, cb);
}
void dax_break_layout_final(struct inode *inode);
int dax_dedupe_file_range_compare(struct inode *src, loff_t srcoff,
                                  struct inode *dest, loff_t destoff,
                                  loff_t len, bool *is_same,
                                  const struct iomap_ops *ops);
int dax_remap_file_range_prep(struct file *file_in, loff_t pos_in,
                              struct file *file_out, loff_t pos_out,
                              loff_t *len, unsigned int remap_flags,
                              const struct iomap_ops *ops);
static inline bool dax_mapping(struct address_space *mapping)
{
        return mapping->host && IS_DAX(mapping->host);
}

/*
 * Due to dax's memory and block duo personalities, hwpoison reporting
 * takes into consideration which personality is presently visible.
 * When dax acts like a block device, such as in block IO, an encounter of
 * dax hwpoison is reported as -EIO.
 * When dax acts like memory, such as in page fault, a detection of hwpoison
 * is reported as -EHWPOISON which leads to VM_FAULT_HWPOISON.
 */
static inline int dax_mem2blk_err(int err)
{
        return (err == -EHWPOISON) ? -EIO : err;
}

#ifdef CONFIG_DEV_DAX_HMEM_DEVICES
void hmem_register_resource(int target_nid, struct resource *r);
#else
static inline void hmem_register_resource(int target_nid, struct resource *r)
{
}
#endif

typedef int (*walk_hmem_fn)(struct device *dev, int target_nid,
                            const struct resource *res);
int walk_hmem_resources(struct device *dev, walk_hmem_fn fn);
#endif


































































    3 

















    3 
















































































   68 







   68 



   66 



   68 














































































































































































   15 
   19 







   19 






   19 






   19 
   19 

   19 










   19 








   19 


























































   19 



   19 













   19 

   19 








































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 *        Video for Linux Two
 *
 *        A generic video device interface for the LINUX operating system
 *        using a set of device structures/vectors for low level operations.
 *
 *        This file replaces the videodev.c file that comes with the
 *        regular kernel distribution.
 *
 * Author:        Bill Dirks <bill@thedirks.org>
 *                based on code by Alan Cox, <alan@cymru.net>
 */

/*
 * Video capture interface for Linux
 *
 *        A generic video device interface for the LINUX operating system
 *        using a set of device structures/vectors for low level operations.
 *
 * Author:        Alan Cox, <alan@lxorguk.ukuu.org.uk>
 *
 * Fixes:
 */

/*
 * Video4linux 1/2 integration by Justin Schoeman
 * <justin@suntiger.ee.up.ac.za>
 * 2.4 PROCFS support ported from 2.4 kernels by
 *  Iñaki García Etxebarria <garetxe@euskalnet.net>
 * Makefile fix by "W. Michael Petullo" <mike@flyn.org>
 * 2.4 devfs support ported from 2.4 kernels by
 *  Dan Merillat <dan@merillat.org>
 * Added Gerd Knorrs v4l1 enhancements (Justin Schoeman)
 */

#include <linux/module.h>
#include <linux/types.h>
#include <linux/kernel.h>
#include <linux/mm.h>
#include <linux/string.h>
#include <linux/errno.h>
#include <linux/uaccess.h>
#include <asm/io.h>
#include <asm/div64.h>
#include <media/v4l2-common.h>
#include <media/v4l2-device.h>
#include <media/v4l2-ctrls.h>

#include <linux/videodev2.h>

/*
 *
 *        V 4 L 2   D R I V E R   H E L P E R   A P I
 *
 */

/*
 *  Video Standard Operations (contributed by Michael Schimek)
 */

/* Helper functions for control handling                             */

/* Fill in a struct v4l2_queryctrl */
int v4l2_ctrl_query_fill(struct v4l2_queryctrl *qctrl, s32 _min, s32 _max, s32 _step, s32 _def)
{
        const char *name;
        s64 min = _min;
        s64 max = _max;
        u64 step = _step;
        s64 def = _def;

        v4l2_ctrl_fill(qctrl->id, &name, &qctrl->type,
                       &min, &max, &step, &def, &qctrl->flags);

        if (name == NULL)
                return -EINVAL;

        qctrl->minimum = min;
        qctrl->maximum = max;
        qctrl->step = step;
        qctrl->default_value = def;
        qctrl->reserved[0] = qctrl->reserved[1] = 0;
        strscpy(qctrl->name, name, sizeof(qctrl->name));
        return 0;
}
EXPORT_SYMBOL(v4l2_ctrl_query_fill);

/* Clamp x to be between min and max, aligned to a multiple of 2^align.  min
 * and max don't have to be aligned, but there must be at least one valid
 * value.  E.g., min=17,max=31,align=4 is not allowed as there are no multiples
 * of 16 between 17 and 31.  */
static unsigned int clamp_align(unsigned int x, unsigned int min,
                                unsigned int max, unsigned int align)
{
        /* Bits that must be zero to be aligned */
        unsigned int mask = ~((1 << align) - 1);

        /* Clamp to aligned min and max */
        x = clamp(x, (min + ~mask) & mask, max & mask);

        /* Round to nearest aligned value */
        if (align)
                x = (x + (1 << (align - 1))) & mask;

        return x;
}

static unsigned int clamp_roundup(unsigned int x, unsigned int min,
                                   unsigned int max, unsigned int alignment)
{
        x = clamp(x, min, max);
        if (alignment)
                x = round_up(x, alignment);

        return x;
}

void v4l_bound_align_image(u32 *w, unsigned int wmin, unsigned int wmax,
                           unsigned int walign,
                           u32 *h, unsigned int hmin, unsigned int hmax,
                           unsigned int halign, unsigned int salign)
{
        *w = clamp_align(*w, wmin, wmax, walign);
        *h = clamp_align(*h, hmin, hmax, halign);

        /* Usually we don't need to align the size and are done now. */
        if (!salign)
                return;

        /* How much alignment do we have? */
        walign = __ffs(*w);
        halign = __ffs(*h);
        /* Enough to satisfy the image alignment? */
        if (walign + halign < salign) {
                /* Max walign where there is still a valid width */
                unsigned int wmaxa = __fls(wmax ^ (wmin - 1));
                /* Max halign where there is still a valid height */
                unsigned int hmaxa = __fls(hmax ^ (hmin - 1));

                /* up the smaller alignment until we have enough */
                do {
                        if (halign >= hmaxa ||
                            (walign <= halign && walign < wmaxa)) {
                                *w = clamp_align(*w, wmin, wmax, walign + 1);
                                walign = __ffs(*w);
                        } else {
                                *h = clamp_align(*h, hmin, hmax, halign + 1);
                                halign = __ffs(*h);
                        }
                } while (halign + walign < salign);
        }
}
EXPORT_SYMBOL_GPL(v4l_bound_align_image);

const void *
__v4l2_find_nearest_size_conditional(const void *array, size_t array_size,
                                     size_t entry_size, size_t width_offset,
                                     size_t height_offset, s32 width,
                                     s32 height,
                                     bool (*func)(const void *array,
                                                  size_t index,
                                                  const void *context),
                                     const void *context)
{
        u32 error, min_error = U32_MAX;
        const void *best = NULL;
        size_t i;

        if (!array)
                return NULL;

        for (i = 0; i < array_size; i++, array += entry_size) {
                const u32 *entry_width = array + width_offset;
                const u32 *entry_height = array + height_offset;

                if (func && !func(array, i, context))
                        continue;

                error = abs(*entry_width - width) + abs(*entry_height - height);
                if (error > min_error)
                        continue;

                min_error = error;
                best = array;
                if (!error)
                        break;
        }

        return best;
}
EXPORT_SYMBOL_GPL(__v4l2_find_nearest_size_conditional);

int v4l2_g_parm_cap(struct video_device *vdev,
                    struct v4l2_subdev *sd, struct v4l2_streamparm *a)
{
        struct v4l2_subdev_frame_interval ival = { 0 };
        int ret;

        if (a->type != V4L2_BUF_TYPE_VIDEO_CAPTURE &&
            a->type != V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE)
                return -EINVAL;

        if (vdev->device_caps & V4L2_CAP_READWRITE)
                a->parm.capture.readbuffers = 2;
        if (v4l2_subdev_has_op(sd, pad, get_frame_interval))
                a->parm.capture.capability = V4L2_CAP_TIMEPERFRAME;
        ret = v4l2_subdev_call_state_active(sd, pad, get_frame_interval, &ival);
        if (!ret)
                a->parm.capture.timeperframe = ival.interval;
        return ret;
}
EXPORT_SYMBOL_GPL(v4l2_g_parm_cap);

int v4l2_s_parm_cap(struct video_device *vdev,
                    struct v4l2_subdev *sd, struct v4l2_streamparm *a)
{
        struct v4l2_subdev_frame_interval ival = {
                .interval = a->parm.capture.timeperframe
        };
        int ret;

        if (a->type != V4L2_BUF_TYPE_VIDEO_CAPTURE &&
            a->type != V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE)
                return -EINVAL;

        memset(&a->parm, 0, sizeof(a->parm));
        if (vdev->device_caps & V4L2_CAP_READWRITE)
                a->parm.capture.readbuffers = 2;
        else
                a->parm.capture.readbuffers = 0;

        if (v4l2_subdev_has_op(sd, pad, get_frame_interval))
                a->parm.capture.capability = V4L2_CAP_TIMEPERFRAME;
        ret = v4l2_subdev_call_state_active(sd, pad, set_frame_interval, &ival);
        if (!ret)
                a->parm.capture.timeperframe = ival.interval;
        return ret;
}
EXPORT_SYMBOL_GPL(v4l2_s_parm_cap);

const struct v4l2_format_info *v4l2_format_info(u32 format)
{
        static const struct v4l2_format_info formats[] = {
                /* RGB formats */
                { .format = V4L2_PIX_FMT_BGR24,   .pixel_enc = V4L2_PIXEL_ENC_RGB, .mem_planes = 1, .comp_planes = 1, .bpp = { 3, 0, 0, 0 }, .bpp_div = { 1, 1, 1, 1 }, .hdiv = 1, .vdiv = 1 },
                { .format = V4L2_PIX_FMT_RGB24,   .pixel_enc = V4L2_PIXEL_ENC_RGB, .mem_planes = 1, .comp_planes = 1, .bpp = { 3, 0, 0, 0 }, .bpp_div = { 1, 1, 1, 1 }, .hdiv = 1, .vdiv = 1 },
                { .format = V4L2_PIX_FMT_HSV24,   .pixel_enc = V4L2_PIXEL_ENC_RGB, .mem_planes = 1, .comp_planes = 1, .bpp = { 3, 0, 0, 0 }, .bpp_div = { 1, 1, 1, 1 }, .hdiv = 1, .vdiv = 1 },
                { .format = V4L2_PIX_FMT_BGR32,   .pixel_enc = V4L2_PIXEL_ENC_RGB, .mem_planes = 1, .comp_planes = 1, .bpp = { 4, 0, 0, 0 }, .bpp_div = { 1, 1, 1, 1 }, .hdiv = 1, .vdiv = 1 },
                { .format = V4L2_PIX_FMT_XBGR32,  .pixel_enc = V4L2_PIXEL_ENC_RGB, .mem_planes = 1, .comp_planes = 1, .bpp = { 4, 0, 0, 0 }, .bpp_div = { 1, 1, 1, 1 }, .hdiv = 1, .vdiv = 1 },
                { .format = V4L2_PIX_FMT_BGRX32,  .pixel_enc = V4L2_PIXEL_ENC_RGB, .mem_planes = 1, .comp_planes = 1, .bpp = { 4, 0, 0, 0 }, .bpp_div = { 1, 1, 1, 1 }, .hdiv = 1, .vdiv = 1 },
                { .format = V4L2_PIX_FMT_RGB32,   .pixel_enc = V4L2_PIXEL_ENC_RGB, .mem_planes = 1, .comp_planes = 1, .bpp = { 4, 0, 0, 0 }, .bpp_div = { 1, 1, 1, 1 }, .hdiv = 1, .vdiv = 1 },
                { .format = V4L2_PIX_FMT_XRGB32,  .pixel_enc = V4L2_PIXEL_ENC_RGB, .mem_planes = 1, .comp_planes = 1, .bpp = { 4, 0, 0, 0 }, .bpp_div = { 1, 1, 1, 1 }, .hdiv = 1, .vdiv = 1 },
                { .format = V4L2_PIX_FMT_RGBX32,  .pixel_enc = V4L2_PIXEL_ENC_RGB, .mem_planes = 1, .comp_planes = 1, .bpp = { 4, 0, 0, 0 }, .bpp_div = { 1, 1, 1, 1 }, .hdiv = 1, .vdiv = 1 },
                { .format = V4L2_PIX_FMT_HSV32,   .pixel_enc = V4L2_PIXEL_ENC_RGB, .mem_planes = 1, .comp_planes = 1, .bpp = { 4, 0, 0, 0 }, .bpp_div = { 1, 1, 1, 1 }, .hdiv = 1, .vdiv = 1 },
                { .format = V4L2_PIX_FMT_ARGB32,  .pixel_enc = V4L2_PIXEL_ENC_RGB, .mem_planes = 1, .comp_planes = 1, .bpp = { 4, 0, 0, 0 }, .bpp_div = { 1, 1, 1, 1 }, .hdiv = 1, .vdiv = 1 },
                { .format = V4L2_PIX_FMT_RGBA32,  .pixel_enc = V4L2_PIXEL_ENC_RGB, .mem_planes = 1, .comp_planes = 1, .bpp = { 4, 0, 0, 0 }, .bpp_div = { 1, 1, 1, 1 }, .hdiv = 1, .vdiv = 1 },
                { .format = V4L2_PIX_FMT_ABGR32,  .pixel_enc = V4L2_PIXEL_ENC_RGB, .mem_planes = 1, .comp_planes = 1, .bpp = { 4, 0, 0, 0 }, .bpp_div = { 1, 1, 1, 1 }, .hdiv = 1, .vdiv = 1 },
                { .format = V4L2_PIX_FMT_BGRA32,  .pixel_enc = V4L2_PIXEL_ENC_RGB, .mem_planes = 1, .comp_planes = 1, .bpp = { 4, 0, 0, 0 }, .bpp_div = { 1, 1, 1, 1 }, .hdiv = 1, .vdiv = 1 },
                { .format = V4L2_PIX_FMT_RGB565,  .pixel_enc = V4L2_PIXEL_ENC_RGB, .mem_planes = 1, .comp_planes = 1, .bpp = { 2, 0, 0, 0 }, .bpp_div = { 1, 1, 1, 1 }, .hdiv = 1, .vdiv = 1 },
                { .format = V4L2_PIX_FMT_RGB565X, .pixel_enc = V4L2_PIXEL_ENC_RGB, .mem_planes = 1, .comp_planes = 1, .bpp = { 2, 0, 0, 0 }, .bpp_div = { 1, 1, 1, 1 }, .hdiv = 1, .vdiv = 1 },
                { .format = V4L2_PIX_FMT_RGB555,  .pixel_enc = V4L2_PIXEL_ENC_RGB, .mem_planes = 1, .comp_planes = 1, .bpp = { 2, 0, 0, 0 }, .bpp_div = { 1, 1, 1, 1 }, .hdiv = 1, .vdiv = 1 },
                { .format = V4L2_PIX_FMT_BGR666,  .pixel_enc = V4L2_PIXEL_ENC_RGB, .mem_planes = 1, .comp_planes = 1, .bpp = { 4, 0, 0, 0 }, .bpp_div = { 1, 1, 1, 1 }, .hdiv = 1, .vdiv = 1 },
                { .format = V4L2_PIX_FMT_BGR48_12, .pixel_enc = V4L2_PIXEL_ENC_RGB, .mem_planes = 1, .comp_planes = 1, .bpp = { 6, 0, 0, 0 }, .bpp_div = { 1, 1, 1, 1 }, .hdiv = 1, .vdiv = 1 },
                { .format = V4L2_PIX_FMT_BGR48, .pixel_enc = V4L2_PIXEL_ENC_RGB, .mem_planes = 1, .comp_planes = 1, .bpp = { 6, 0, 0, 0 }, .bpp_div = { 1, 1, 1, 1 }, .hdiv = 1, .vdiv = 1 },
                { .format = V4L2_PIX_FMT_RGB48, .pixel_enc = V4L2_PIXEL_ENC_RGB, .mem_planes = 1, .comp_planes = 1, .bpp = { 6, 0, 0, 0 }, .bpp_div = { 1, 1, 1, 1 }, .hdiv = 1, .vdiv = 1 },
                { .format = V4L2_PIX_FMT_ABGR64_12, .pixel_enc = V4L2_PIXEL_ENC_RGB, .mem_planes = 1, .comp_planes = 1, .bpp = { 8, 0, 0, 0 }, .bpp_div = { 1, 1, 1, 1 }, .hdiv = 1, .vdiv = 1 },
                { .format = V4L2_PIX_FMT_RGBA1010102, .pixel_enc = V4L2_PIXEL_ENC_RGB, .mem_planes = 1, .comp_planes = 1, .bpp = { 4, 0, 0, 0 }, .bpp_div = { 1, 1, 1, 1 }, .hdiv = 1, .vdiv = 1 },
                { .format = V4L2_PIX_FMT_RGBX1010102, .pixel_enc = V4L2_PIXEL_ENC_RGB, .mem_planes = 1, .comp_planes = 1, .bpp = { 4, 0, 0, 0 }, .bpp_div = { 1, 1, 1, 1 }, .hdiv = 1, .vdiv = 1 },
                { .format = V4L2_PIX_FMT_ARGB2101010, .pixel_enc = V4L2_PIXEL_ENC_RGB, .mem_planes = 1, .comp_planes = 1, .bpp = { 4, 0, 0, 0 }, .bpp_div = { 1, 1, 1, 1 }, .hdiv = 1, .vdiv = 1 },

                /* YUV packed formats */
                { .format = V4L2_PIX_FMT_YUYV,    .pixel_enc = V4L2_PIXEL_ENC_YUV, .mem_planes = 1, .comp_planes = 1, .bpp = { 2, 0, 0, 0 }, .bpp_div = { 1, 1, 1, 1 }, .hdiv = 2, .vdiv = 1 },
                { .format = V4L2_PIX_FMT_YVYU,    .pixel_enc = V4L2_PIXEL_ENC_YUV, .mem_planes = 1, .comp_planes = 1, .bpp = { 2, 0, 0, 0 }, .bpp_div = { 1, 1, 1, 1 }, .hdiv = 2, .vdiv = 1 },
                { .format = V4L2_PIX_FMT_UYVY,    .pixel_enc = V4L2_PIXEL_ENC_YUV, .mem_planes = 1, .comp_planes = 1, .bpp = { 2, 0, 0, 0 }, .bpp_div = { 1, 1, 1, 1 }, .hdiv = 2, .vdiv = 1 },
                { .format = V4L2_PIX_FMT_VYUY,    .pixel_enc = V4L2_PIXEL_ENC_YUV, .mem_planes = 1, .comp_planes = 1, .bpp = { 2, 0, 0, 0 }, .bpp_div = { 1, 1, 1, 1 }, .hdiv = 2, .vdiv = 1 },
                { .format = V4L2_PIX_FMT_Y210,    .pixel_enc = V4L2_PIXEL_ENC_YUV, .mem_planes = 1, .comp_planes = 1, .bpp = { 4, 0, 0, 0 }, .bpp_div = { 1, 1, 1, 1 }, .hdiv = 2, .vdiv = 1 },
                { .format = V4L2_PIX_FMT_Y212,    .pixel_enc = V4L2_PIXEL_ENC_YUV, .mem_planes = 1, .comp_planes = 1, .bpp = { 4, 0, 0, 0 }, .bpp_div = { 1, 1, 1, 1 }, .hdiv = 2, .vdiv = 1 },
                { .format = V4L2_PIX_FMT_Y216,    .pixel_enc = V4L2_PIXEL_ENC_YUV, .mem_planes = 1, .comp_planes = 1, .bpp = { 4, 0, 0, 0 }, .bpp_div = { 1, 1, 1, 1 }, .hdiv = 2, .vdiv = 1 },
                { .format = V4L2_PIX_FMT_YUV48_12, .pixel_enc = V4L2_PIXEL_ENC_YUV, .mem_planes = 1, .comp_planes = 1, .bpp = { 6, 0, 0, 0 }, .bpp_div = { 1, 1, 1, 1 }, .hdiv = 1, .vdiv = 1 },
                { .format = V4L2_PIX_FMT_MT2110T, .pixel_enc = V4L2_PIXEL_ENC_YUV, .mem_planes = 2, .comp_planes = 2, .bpp = { 5, 10, 0, 0 }, .bpp_div = { 4, 4, 1, 1 }, .hdiv = 2, .vdiv = 2,
                  .block_w = { 16, 8, 0, 0 }, .block_h = { 32, 16, 0, 0 }},
                { .format = V4L2_PIX_FMT_MT2110R, .pixel_enc = V4L2_PIXEL_ENC_YUV, .mem_planes = 2, .comp_planes = 2, .bpp = { 5, 10, 0, 0 }, .bpp_div = { 4, 4, 1, 1 }, .hdiv = 2, .vdiv = 2,
                  .block_w = { 16, 8, 0, 0 }, .block_h = { 32, 16, 0, 0 }},

                /* YUV planar formats */
                { .format = V4L2_PIX_FMT_NV12,    .pixel_enc = V4L2_PIXEL_ENC_YUV, .mem_planes = 1, .comp_planes = 2, .bpp = { 1, 2, 0, 0 }, .bpp_div = { 1, 1, 1, 1 }, .hdiv = 2, .vdiv = 2 },
                { .format = V4L2_PIX_FMT_NV21,    .pixel_enc = V4L2_PIXEL_ENC_YUV, .mem_planes = 1, .comp_planes = 2, .bpp = { 1, 2, 0, 0 }, .bpp_div = { 1, 1, 1, 1 }, .hdiv = 2, .vdiv = 2 },
                { .format = V4L2_PIX_FMT_NV15,    .pixel_enc = V4L2_PIXEL_ENC_YUV, .mem_planes = 1, .comp_planes = 2, .bpp = { 5, 10, 0, 0 }, .bpp_div = { 4, 4, 1, 1 }, .hdiv = 2, .vdiv = 2 },
                { .format = V4L2_PIX_FMT_NV16,    .pixel_enc = V4L2_PIXEL_ENC_YUV, .mem_planes = 1, .comp_planes = 2, .bpp = { 1, 2, 0, 0 }, .bpp_div = { 1, 1, 1, 1 }, .hdiv = 2, .vdiv = 1 },
                { .format = V4L2_PIX_FMT_NV61,    .pixel_enc = V4L2_PIXEL_ENC_YUV, .mem_planes = 1, .comp_planes = 2, .bpp = { 1, 2, 0, 0 }, .bpp_div = { 1, 1, 1, 1 }, .hdiv = 2, .vdiv = 1 },
                { .format = V4L2_PIX_FMT_NV20,    .pixel_enc = V4L2_PIXEL_ENC_YUV, .mem_planes = 1, .comp_planes = 2, .bpp = { 5, 10, 0, 0 }, .bpp_div = { 4, 4, 1, 1 }, .hdiv = 2, .vdiv = 1 },
                { .format = V4L2_PIX_FMT_NV24,    .pixel_enc = V4L2_PIXEL_ENC_YUV, .mem_planes = 1, .comp_planes = 2, .bpp = { 1, 2, 0, 0 }, .bpp_div = { 1, 1, 1, 1 }, .hdiv = 1, .vdiv = 1 },
                { .format = V4L2_PIX_FMT_NV42,    .pixel_enc = V4L2_PIXEL_ENC_YUV, .mem_planes = 1, .comp_planes = 2, .bpp = { 1, 2, 0, 0 }, .bpp_div = { 1, 1, 1, 1 }, .hdiv = 1, .vdiv = 1 },
                { .format = V4L2_PIX_FMT_P010,    .pixel_enc = V4L2_PIXEL_ENC_YUV, .mem_planes = 1, .comp_planes = 2, .bpp = { 2, 2, 0, 0 }, .bpp_div = { 1, 1, 1, 1 }, .hdiv = 2, .vdiv = 1 },
                { .format = V4L2_PIX_FMT_P012,    .pixel_enc = V4L2_PIXEL_ENC_YUV, .mem_planes = 1, .comp_planes = 2, .bpp = { 2, 4, 0, 0 }, .bpp_div = { 1, 1, 1, 1 }, .hdiv = 2, .vdiv = 2 },

                { .format = V4L2_PIX_FMT_YUV410,  .pixel_enc = V4L2_PIXEL_ENC_YUV, .mem_planes = 1, .comp_planes = 3, .bpp = { 1, 1, 1, 0 }, .bpp_div = { 1, 1, 1, 1 }, .hdiv = 4, .vdiv = 4 },
                { .format = V4L2_PIX_FMT_YVU410,  .pixel_enc = V4L2_PIXEL_ENC_YUV, .mem_planes = 1, .comp_planes = 3, .bpp = { 1, 1, 1, 0 }, .bpp_div = { 1, 1, 1, 1 }, .hdiv = 4, .vdiv = 4 },
                { .format = V4L2_PIX_FMT_YUV411P, .pixel_enc = V4L2_PIXEL_ENC_YUV, .mem_planes = 1, .comp_planes = 3, .bpp = { 1, 1, 1, 0 }, .bpp_div = { 1, 1, 1, 1 }, .hdiv = 4, .vdiv = 1 },
                { .format = V4L2_PIX_FMT_YUV420,  .pixel_enc = V4L2_PIXEL_ENC_YUV, .mem_planes = 1, .comp_planes = 3, .bpp = { 1, 1, 1, 0 }, .bpp_div = { 1, 1, 1, 1 }, .hdiv = 2, .vdiv = 2 },
                { .format = V4L2_PIX_FMT_YVU420,  .pixel_enc = V4L2_PIXEL_ENC_YUV, .mem_planes = 1, .comp_planes = 3, .bpp = { 1, 1, 1, 0 }, .bpp_div = { 1, 1, 1, 1 }, .hdiv = 2, .vdiv = 2 },
                { .format = V4L2_PIX_FMT_YUV422P, .pixel_enc = V4L2_PIXEL_ENC_YUV, .mem_planes = 1, .comp_planes = 3, .bpp = { 1, 1, 1, 0 }, .bpp_div = { 1, 1, 1, 1 }, .hdiv = 2, .vdiv = 1 },
                { .format = V4L2_PIX_FMT_GREY,    .pixel_enc = V4L2_PIXEL_ENC_YUV, .mem_planes = 1, .comp_planes = 1, .bpp = { 1, 0, 0, 0 }, .bpp_div = { 1, 1, 1, 1 }, .hdiv = 1, .vdiv = 1 },

                /* Tiled YUV formats */
                { .format = V4L2_PIX_FMT_NV12_4L4, .pixel_enc = V4L2_PIXEL_ENC_YUV, .mem_planes = 1, .comp_planes = 2, .bpp = { 1, 2, 0, 0 }, .bpp_div = { 1, 1, 1, 1 }, .hdiv = 2, .vdiv = 2 },
                { .format = V4L2_PIX_FMT_NV15_4L4, .pixel_enc = V4L2_PIXEL_ENC_YUV, .mem_planes = 1, .comp_planes = 2, .bpp = { 5, 10, 0, 0 }, .bpp_div = { 4, 4, 1, 1 }, .hdiv = 2, .vdiv = 2,
                  .block_w = { 4, 2, 0, 0 }, .block_h = { 1, 1, 0, 0 }},
                { .format = V4L2_PIX_FMT_P010_4L4, .pixel_enc = V4L2_PIXEL_ENC_YUV, .mem_planes = 1, .comp_planes = 2, .bpp = { 2, 4, 0, 0 }, .bpp_div = { 1, 1, 1, 1 }, .hdiv = 2, .vdiv = 2 },

                /* YUV planar formats, non contiguous variant */
                { .format = V4L2_PIX_FMT_YUV420M, .pixel_enc = V4L2_PIXEL_ENC_YUV, .mem_planes = 3, .comp_planes = 3, .bpp = { 1, 1, 1, 0 }, .bpp_div = { 1, 1, 1, 1 }, .hdiv = 2, .vdiv = 2 },
                { .format = V4L2_PIX_FMT_YVU420M, .pixel_enc = V4L2_PIXEL_ENC_YUV, .mem_planes = 3, .comp_planes = 3, .bpp = { 1, 1, 1, 0 }, .bpp_div = { 1, 1, 1, 1 }, .hdiv = 2, .vdiv = 2 },
                { .format = V4L2_PIX_FMT_YUV422M, .pixel_enc = V4L2_PIXEL_ENC_YUV, .mem_planes = 3, .comp_planes = 3, .bpp = { 1, 1, 1, 0 }, .bpp_div = { 1, 1, 1, 1 }, .hdiv = 2, .vdiv = 1 },
                { .format = V4L2_PIX_FMT_YVU422M, .pixel_enc = V4L2_PIXEL_ENC_YUV, .mem_planes = 3, .comp_planes = 3, .bpp = { 1, 1, 1, 0 }, .bpp_div = { 1, 1, 1, 1 }, .hdiv = 2, .vdiv = 1 },
                { .format = V4L2_PIX_FMT_YUV444M, .pixel_enc = V4L2_PIXEL_ENC_YUV, .mem_planes = 3, .comp_planes = 3, .bpp = { 1, 1, 1, 0 }, .bpp_div = { 1, 1, 1, 1 }, .hdiv = 1, .vdiv = 1 },
                { .format = V4L2_PIX_FMT_YVU444M, .pixel_enc = V4L2_PIXEL_ENC_YUV, .mem_planes = 3, .comp_planes = 3, .bpp = { 1, 1, 1, 0 }, .bpp_div = { 1, 1, 1, 1 }, .hdiv = 1, .vdiv = 1 },

                { .format = V4L2_PIX_FMT_NV12M,   .pixel_enc = V4L2_PIXEL_ENC_YUV, .mem_planes = 2, .comp_planes = 2, .bpp = { 1, 2, 0, 0 }, .bpp_div = { 1, 1, 1, 1 }, .hdiv = 2, .vdiv = 2 },
                { .format = V4L2_PIX_FMT_NV21M,   .pixel_enc = V4L2_PIXEL_ENC_YUV, .mem_planes = 2, .comp_planes = 2, .bpp = { 1, 2, 0, 0 }, .bpp_div = { 1, 1, 1, 1 }, .hdiv = 2, .vdiv = 2 },
                { .format = V4L2_PIX_FMT_NV16M,   .pixel_enc = V4L2_PIXEL_ENC_YUV, .mem_planes = 2, .comp_planes = 2, .bpp = { 1, 2, 0, 0 }, .bpp_div = { 1, 1, 1, 1 }, .hdiv = 2, .vdiv = 1 },
                { .format = V4L2_PIX_FMT_NV61M,   .pixel_enc = V4L2_PIXEL_ENC_YUV, .mem_planes = 2, .comp_planes = 2, .bpp = { 1, 2, 0, 0 }, .bpp_div = { 1, 1, 1, 1 }, .hdiv = 2, .vdiv = 1 },
                { .format = V4L2_PIX_FMT_P012M,   .pixel_enc = V4L2_PIXEL_ENC_YUV, .mem_planes = 2, .comp_planes = 2, .bpp = { 2, 4, 0, 0 }, .bpp_div = { 1, 1, 1, 1 }, .hdiv = 2, .vdiv = 2 },

                /* Bayer RGB formats */
                { .format = V4L2_PIX_FMT_SBGGR8,        .pixel_enc = V4L2_PIXEL_ENC_BAYER, .mem_planes = 1, .comp_planes = 1, .bpp = { 1, 0, 0, 0 }, .bpp_div = { 1, 1, 1, 1 }, .hdiv = 1, .vdiv = 1 },
                { .format = V4L2_PIX_FMT_SGBRG8,        .pixel_enc = V4L2_PIXEL_ENC_BAYER, .mem_planes = 1, .comp_planes = 1, .bpp = { 1, 0, 0, 0 }, .bpp_div = { 1, 1, 1, 1 }, .hdiv = 1, .vdiv = 1 },
                { .format = V4L2_PIX_FMT_SGRBG8,        .pixel_enc = V4L2_PIXEL_ENC_BAYER, .mem_planes = 1, .comp_planes = 1, .bpp = { 1, 0, 0, 0 }, .bpp_div = { 1, 1, 1, 1 }, .hdiv = 1, .vdiv = 1 },
                { .format = V4L2_PIX_FMT_SRGGB8,        .pixel_enc = V4L2_PIXEL_ENC_BAYER, .mem_planes = 1, .comp_planes = 1, .bpp = { 1, 0, 0, 0 }, .bpp_div = { 1, 1, 1, 1 }, .hdiv = 1, .vdiv = 1 },
                { .format = V4L2_PIX_FMT_SBGGR10,        .pixel_enc = V4L2_PIXEL_ENC_BAYER, .mem_planes = 1, .comp_planes = 1, .bpp = { 2, 0, 0, 0 }, .bpp_div = { 1, 1, 1, 1 }, .hdiv = 1, .vdiv = 1 },
                { .format = V4L2_PIX_FMT_SGBRG10,        .pixel_enc = V4L2_PIXEL_ENC_BAYER, .mem_planes = 1, .comp_planes = 1, .bpp = { 2, 0, 0, 0 }, .bpp_div = { 1, 1, 1, 1 }, .hdiv = 1, .vdiv = 1 },
                { .format = V4L2_PIX_FMT_SGRBG10,        .pixel_enc = V4L2_PIXEL_ENC_BAYER, .mem_planes = 1, .comp_planes = 1, .bpp = { 2, 0, 0, 0 }, .bpp_div = { 1, 1, 1, 1 }, .hdiv = 1, .vdiv = 1 },
                { .format = V4L2_PIX_FMT_SRGGB10,        .pixel_enc = V4L2_PIXEL_ENC_BAYER, .mem_planes = 1, .comp_planes = 1, .bpp = { 2, 0, 0, 0 }, .bpp_div = { 1, 1, 1, 1 }, .hdiv = 1, .vdiv = 1 },
                { .format = V4L2_PIX_FMT_SBGGR10ALAW8,        .pixel_enc = V4L2_PIXEL_ENC_BAYER, .mem_planes = 1, .comp_planes = 1, .bpp = { 1, 0, 0, 0 }, .bpp_div = { 1, 1, 1, 1 }, .hdiv = 1, .vdiv = 1 },
                { .format = V4L2_PIX_FMT_SGBRG10ALAW8,        .pixel_enc = V4L2_PIXEL_ENC_BAYER, .mem_planes = 1, .comp_planes = 1, .bpp = { 1, 0, 0, 0 }, .bpp_div = { 1, 1, 1, 1 }, .hdiv = 1, .vdiv = 1 },
                { .format = V4L2_PIX_FMT_SGRBG10ALAW8,        .pixel_enc = V4L2_PIXEL_ENC_BAYER, .mem_planes = 1, .comp_planes = 1, .bpp = { 1, 0, 0, 0 }, .bpp_div = { 1, 1, 1, 1 }, .hdiv = 1, .vdiv = 1 },
                { .format = V4L2_PIX_FMT_SRGGB10ALAW8,        .pixel_enc = V4L2_PIXEL_ENC_BAYER, .mem_planes = 1, .comp_planes = 1, .bpp = { 1, 0, 0, 0 }, .bpp_div = { 1, 1, 1, 1 }, .hdiv = 1, .vdiv = 1 },
                { .format = V4L2_PIX_FMT_SBGGR10DPCM8,        .pixel_enc = V4L2_PIXEL_ENC_BAYER, .mem_planes = 1, .comp_planes = 1, .bpp = { 1, 0, 0, 0 }, .bpp_div = { 1, 1, 1, 1 }, .hdiv = 1, .vdiv = 1 },
                { .format = V4L2_PIX_FMT_SGBRG10DPCM8,        .pixel_enc = V4L2_PIXEL_ENC_BAYER, .mem_planes = 1, .comp_planes = 1, .bpp = { 1, 0, 0, 0 }, .bpp_div = { 1, 1, 1, 1 }, .hdiv = 1, .vdiv = 1 },
                { .format = V4L2_PIX_FMT_SGRBG10DPCM8,        .pixel_enc = V4L2_PIXEL_ENC_BAYER, .mem_planes = 1, .comp_planes = 1, .bpp = { 1, 0, 0, 0 }, .bpp_div = { 1, 1, 1, 1 }, .hdiv = 1, .vdiv = 1 },
                { .format = V4L2_PIX_FMT_SRGGB10DPCM8,        .pixel_enc = V4L2_PIXEL_ENC_BAYER, .mem_planes = 1, .comp_planes = 1, .bpp = { 1, 0, 0, 0 }, .bpp_div = { 1, 1, 1, 1 }, .hdiv = 1, .vdiv = 1 },
                { .format = V4L2_PIX_FMT_SBGGR12,        .pixel_enc = V4L2_PIXEL_ENC_BAYER, .mem_planes = 1, .comp_planes = 1, .bpp = { 2, 0, 0, 0 }, .bpp_div = { 1, 1, 1, 1 }, .hdiv = 1, .vdiv = 1 },
                { .format = V4L2_PIX_FMT_SGBRG12,        .pixel_enc = V4L2_PIXEL_ENC_BAYER, .mem_planes = 1, .comp_planes = 1, .bpp = { 2, 0, 0, 0 }, .bpp_div = { 1, 1, 1, 1 }, .hdiv = 1, .vdiv = 1 },
                { .format = V4L2_PIX_FMT_SGRBG12,        .pixel_enc = V4L2_PIXEL_ENC_BAYER, .mem_planes = 1, .comp_planes = 1, .bpp = { 2, 0, 0, 0 }, .bpp_div = { 1, 1, 1, 1 }, .hdiv = 1, .vdiv = 1 },
                { .format = V4L2_PIX_FMT_SRGGB12,        .pixel_enc = V4L2_PIXEL_ENC_BAYER, .mem_planes = 1, .comp_planes = 1, .bpp = { 2, 0, 0, 0 }, .bpp_div = { 1, 1, 1, 1 }, .hdiv = 1, .vdiv = 1 },

                /* Renesas Camera Data Receiver Unit formats, bayer order agnostic */
                { .format = V4L2_PIX_FMT_RAW_CRU10,        .pixel_enc = V4L2_PIXEL_ENC_BAYER, .mem_planes = 1, .comp_planes = 1, .bpp = { 8, 0, 0, 0 }, .bpp_div = { 6, 1, 1, 1 }, .hdiv = 1, .vdiv = 1 },
                { .format = V4L2_PIX_FMT_RAW_CRU12,        .pixel_enc = V4L2_PIXEL_ENC_BAYER, .mem_planes = 1, .comp_planes = 1, .bpp = { 8, 0, 0, 0 }, .bpp_div = { 5, 1, 1, 1 }, .hdiv = 1, .vdiv = 1 },
                { .format = V4L2_PIX_FMT_RAW_CRU14,        .pixel_enc = V4L2_PIXEL_ENC_BAYER, .mem_planes = 1, .comp_planes = 1, .bpp = { 8, 0, 0, 0 }, .bpp_div = { 4, 1, 1, 1 }, .hdiv = 1, .vdiv = 1 },
                { .format = V4L2_PIX_FMT_RAW_CRU20,        .pixel_enc = V4L2_PIXEL_ENC_BAYER, .mem_planes = 1, .comp_planes = 1, .bpp = { 8, 0, 0, 0 }, .bpp_div = { 3, 1, 1, 1 }, .hdiv = 1, .vdiv = 1 },
        };
        unsigned int i;

        for (i = 0; i < ARRAY_SIZE(formats); ++i)
                if (formats[i].format == format)
                        return &formats[i];
        return NULL;
}
EXPORT_SYMBOL(v4l2_format_info);

static inline unsigned int v4l2_format_block_width(const struct v4l2_format_info *info, int plane)
{
        if (!info->block_w[plane])
                return 1;
        return info->block_w[plane];
}

static inline unsigned int v4l2_format_block_height(const struct v4l2_format_info *info, int plane)
{
        if (!info->block_h[plane])
                return 1;
        return info->block_h[plane];
}

static inline unsigned int v4l2_format_plane_stride(const struct v4l2_format_info *info, int plane,
                                                    unsigned int width)
{
        unsigned int hdiv = plane ? info->hdiv : 1;
        unsigned int aligned_width =
                ALIGN(width, v4l2_format_block_width(info, plane));

        return DIV_ROUND_UP(aligned_width, hdiv) *
               info->bpp[plane] / info->bpp_div[plane];
}

static inline unsigned int v4l2_format_plane_height(const struct v4l2_format_info *info, int plane,
                                                    unsigned int height)
{
        unsigned int vdiv = plane ? info->vdiv : 1;
        unsigned int aligned_height =
                ALIGN(height, v4l2_format_block_height(info, plane));

        return DIV_ROUND_UP(aligned_height, vdiv);
}

static inline unsigned int v4l2_format_plane_size(const struct v4l2_format_info *info, int plane,
                                                  unsigned int width, unsigned int height)
{
        return v4l2_format_plane_stride(info, plane, width) *
               v4l2_format_plane_height(info, plane, height);
}

void v4l2_apply_frmsize_constraints(u32 *width, u32 *height,
                                    const struct v4l2_frmsize_stepwise *frmsize)
{
        if (!frmsize)
                return;

        /*
         * Clamp width/height to meet min/max constraints and round it up to
         * macroblock alignment.
         */
        *width = clamp_roundup(*width, frmsize->min_width, frmsize->max_width,
                               frmsize->step_width);
        *height = clamp_roundup(*height, frmsize->min_height, frmsize->max_height,
                                frmsize->step_height);
}
EXPORT_SYMBOL_GPL(v4l2_apply_frmsize_constraints);

int v4l2_fill_pixfmt_mp(struct v4l2_pix_format_mplane *pixfmt,
                        u32 pixelformat, u32 width, u32 height)
{
        const struct v4l2_format_info *info;
        struct v4l2_plane_pix_format *plane;
        int i;

        info = v4l2_format_info(pixelformat);
        if (!info)
                return -EINVAL;

        pixfmt->width = width;
        pixfmt->height = height;
        pixfmt->pixelformat = pixelformat;
        pixfmt->num_planes = info->mem_planes;

        if (info->mem_planes == 1) {
                plane = &pixfmt->plane_fmt[0];
                plane->bytesperline = v4l2_format_plane_stride(info, 0, width);
                plane->sizeimage = 0;

                for (i = 0; i < info->comp_planes; i++)
                        plane->sizeimage +=
                                v4l2_format_plane_size(info, i, width, height);
        } else {
                for (i = 0; i < info->comp_planes; i++) {
                        plane = &pixfmt->plane_fmt[i];
                        plane->bytesperline =
                                v4l2_format_plane_stride(info, i, width);
                        plane->sizeimage = plane->bytesperline *
                                v4l2_format_plane_height(info, i, height);
                }
        }
        return 0;
}
EXPORT_SYMBOL_GPL(v4l2_fill_pixfmt_mp);

int v4l2_fill_pixfmt(struct v4l2_pix_format *pixfmt, u32 pixelformat,
                     u32 width, u32 height)
{
        const struct v4l2_format_info *info;
        int i;

        info = v4l2_format_info(pixelformat);
        if (!info)
                return -EINVAL;

        /* Single planar API cannot be used for multi plane formats. */
        if (info->mem_planes > 1)
                return -EINVAL;

        pixfmt->width = width;
        pixfmt->height = height;
        pixfmt->pixelformat = pixelformat;
        pixfmt->bytesperline = v4l2_format_plane_stride(info, 0, width);
        pixfmt->sizeimage = 0;

        for (i = 0; i < info->comp_planes; i++)
                pixfmt->sizeimage +=
                        v4l2_format_plane_size(info, i, width, height);
        return 0;
}
EXPORT_SYMBOL_GPL(v4l2_fill_pixfmt);

s64 __v4l2_get_link_freq_ctrl(struct v4l2_ctrl_handler *handler,
                              unsigned int mul, unsigned int div)
{
        struct v4l2_ctrl *ctrl;
        s64 freq;

        ctrl = v4l2_ctrl_find(handler, V4L2_CID_LINK_FREQ);
        if (ctrl) {
                struct v4l2_querymenu qm = { .id = V4L2_CID_LINK_FREQ };
                int ret;

                qm.index = v4l2_ctrl_g_ctrl(ctrl);

                ret = v4l2_querymenu(handler, &qm);
                if (ret)
                        return -ENOENT;

                freq = qm.value;
        } else {
                if (!mul || !div)
                        return -ENOENT;

                ctrl = v4l2_ctrl_find(handler, V4L2_CID_PIXEL_RATE);
                if (!ctrl)
                        return -ENOENT;

                freq = div_u64(v4l2_ctrl_g_ctrl_int64(ctrl) * mul, div);

                pr_warn_once("%s: Link frequency estimated using pixel rate: result might be inaccurate\n",
                             __func__);
                pr_warn_once("%s: Consider implementing support for V4L2_CID_LINK_FREQ in the transmitter driver\n",
                             __func__);
        }

        return freq > 0 ? freq : -EINVAL;
}
EXPORT_SYMBOL_GPL(__v4l2_get_link_freq_ctrl);

#ifdef CONFIG_MEDIA_CONTROLLER
s64 __v4l2_get_link_freq_pad(struct media_pad *pad, unsigned int mul,
                             unsigned int div)
{
        struct v4l2_mbus_config mbus_config = {};
        struct v4l2_subdev *sd;
        int ret;

        sd = media_entity_to_v4l2_subdev(pad->entity);
        ret = v4l2_subdev_call(sd, pad, get_mbus_config, pad->index,
                               &mbus_config);
        if (ret < 0 && ret != -ENOIOCTLCMD)
                return ret;

        if (mbus_config.link_freq)
                return mbus_config.link_freq;

        /*
         * Fall back to using the link frequency control if the media bus config
         * doesn't provide a link frequency.
         */
        return __v4l2_get_link_freq_ctrl(sd->ctrl_handler, mul, div);
}
EXPORT_SYMBOL_GPL(__v4l2_get_link_freq_pad);
#endif /* CONFIG_MEDIA_CONTROLLER */

/*
 * Simplify a fraction using a simple continued fraction decomposition. The
 * idea here is to convert fractions such as 333333/10000000 to 1/30 using
 * 32 bit arithmetic only. The algorithm is not perfect and relies upon two
 * arbitrary parameters to remove non-significative terms from the simple
 * continued fraction decomposition. Using 8 and 333 for n_terms and threshold
 * respectively seems to give nice results.
 */
void v4l2_simplify_fraction(u32 *numerator, u32 *denominator,
                unsigned int n_terms, unsigned int threshold)
{
        u32 *an;
        u32 x, y, r;
        unsigned int i, n;

        an = kmalloc_array(n_terms, sizeof(*an), GFP_KERNEL);
        if (an == NULL)
                return;

        /*
         * Convert the fraction to a simple continued fraction. See
         * https://en.wikipedia.org/wiki/Continued_fraction
         * Stop if the current term is bigger than or equal to the given
         * threshold.
         */
        x = *numerator;
        y = *denominator;

        for (n = 0; n < n_terms && y != 0; ++n) {
                an[n] = x / y;
                if (an[n] >= threshold) {
                        if (n < 2)
                                n++;
                        break;
                }

                r = x - an[n] * y;
                x = y;
                y = r;
        }

        /* Expand the simple continued fraction back to an integer fraction. */
        x = 0;
        y = 1;

        for (i = n; i > 0; --i) {
                r = y;
                y = an[i-1] * y + x;
                x = r;
        }

        *numerator = y;
        *denominator = x;
        kfree(an);
}
EXPORT_SYMBOL_GPL(v4l2_simplify_fraction);

/*
 * Convert a fraction to a frame interval in 100ns multiples. The idea here is
 * to compute numerator / denominator * 10000000 using 32 bit fixed point
 * arithmetic only.
 */
u32 v4l2_fraction_to_interval(u32 numerator, u32 denominator)
{
        u32 multiplier;

        /* Saturate the result if the operation would overflow. */
        if (denominator == 0 ||
            numerator/denominator >= ((u32)-1)/10000000)
                return (u32)-1;

        /*
         * Divide both the denominator and the multiplier by two until
         * numerator * multiplier doesn't overflow. If anyone knows a better
         * algorithm please let me know.
         */
        multiplier = 10000000;
        while (numerator > ((u32)-1)/multiplier) {
                multiplier /= 2;
                denominator /= 2;
        }

        return denominator ? numerator * multiplier / denominator : 0;
}
EXPORT_SYMBOL_GPL(v4l2_fraction_to_interval);

int v4l2_link_freq_to_bitmap(struct device *dev, const u64 *fw_link_freqs,
                             unsigned int num_of_fw_link_freqs,
                             const s64 *driver_link_freqs,
                             unsigned int num_of_driver_link_freqs,
                             unsigned long *bitmap)
{
        unsigned int i;

        *bitmap = 0;

        if (!num_of_fw_link_freqs) {
                dev_err(dev, "no link frequencies in firmware\n");
                return -ENODATA;
        }

        for (i = 0; i < num_of_fw_link_freqs; i++) {
                unsigned int j;

                for (j = 0; j < num_of_driver_link_freqs; j++) {
                        if (fw_link_freqs[i] != driver_link_freqs[j])
                                continue;

                        dev_dbg(dev, "enabling link frequency %lld Hz\n",
                                driver_link_freqs[j]);
                        *bitmap |= BIT(j);
                        break;
                }
        }

        if (!*bitmap) {
                dev_err(dev, "no matching link frequencies found\n");

                dev_dbg(dev, "specified in firmware:\n");
                for (i = 0; i < num_of_fw_link_freqs; i++)
                        dev_dbg(dev, "\t%llu Hz\n", fw_link_freqs[i]);

                dev_dbg(dev, "driver supported:\n");
                for (i = 0; i < num_of_driver_link_freqs; i++)
                        dev_dbg(dev, "\t%lld Hz\n", driver_link_freqs[i]);

                return -ENOENT;
        }

        return 0;
}
EXPORT_SYMBOL_GPL(v4l2_link_freq_to_bitmap);








































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __FS_CEPH_BUFFER_H
#define __FS_CEPH_BUFFER_H

#include <linux/kref.h>
#include <linux/mm.h>
#include <linux/vmalloc.h>
#include <linux/types.h>
#include <linux/uio.h>

/*
 * a simple reference counted buffer.
 *
 * use kmalloc for smaller sizes, vmalloc for larger sizes.
 */
struct ceph_buffer {
        struct kref kref;
        struct kvec vec;
        size_t alloc_len;
};

extern struct ceph_buffer *ceph_buffer_new(size_t len, gfp_t gfp);
extern void ceph_buffer_release(struct kref *kref);

static inline struct ceph_buffer *ceph_buffer_get(struct ceph_buffer *b)
{
        kref_get(&b->kref);
        return b;
}

static inline void ceph_buffer_put(struct ceph_buffer *b)
{
        if (b)
                kref_put(&b->kref, ceph_buffer_release);
}

extern int ceph_decode_buffer(struct ceph_buffer **b, void **p, void *end);

#endif














































































































































































































































































































































































































































































































































































































    1 
























    1 





    1 




    1 

    1 

    1 


    1 













    1 















    1 




    1 



    1 



















































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































    7 
















    1 









    6 










    5 

    1 







    3 
















    2 




    1 







    1 









    1 

















    1 

    1 













































    2 





    2 


    1 


























    1 




















































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
// SPDX-License-Identifier: GPL-2.0-only
/*
 *  HID driver for Logitech receivers
 *
 *  Copyright (c) 2011 Logitech
 */



#include <linux/device.h>
#include <linux/hid.h>
#include <linux/module.h>
#include <linux/kfifo.h>
#include <linux/delay.h>
#include <linux/usb.h> /* For to_usb_interface for kvm extra intf check */
#include <linux/unaligned.h>
#include "hid-ids.h"

#define DJ_MAX_PAIRED_DEVICES                        7
#define DJ_MAX_NUMBER_NOTIFS                        8
#define DJ_RECEIVER_INDEX                        0
#define DJ_DEVICE_INDEX_MIN                        1
#define DJ_DEVICE_INDEX_MAX                        7

#define DJREPORT_SHORT_LENGTH                        15
#define DJREPORT_LONG_LENGTH                        32

#define REPORT_ID_DJ_SHORT                        0x20
#define REPORT_ID_DJ_LONG                        0x21

#define REPORT_ID_HIDPP_SHORT                        0x10
#define REPORT_ID_HIDPP_LONG                        0x11
#define REPORT_ID_HIDPP_VERY_LONG                0x12

#define HIDPP_REPORT_SHORT_LENGTH                7
#define HIDPP_REPORT_LONG_LENGTH                20

#define HIDPP_RECEIVER_INDEX                        0xff

#define REPORT_TYPE_RFREPORT_FIRST                0x01
#define REPORT_TYPE_RFREPORT_LAST                0x1F

/* Command Switch to DJ mode */
#define REPORT_TYPE_CMD_SWITCH                        0x80
#define CMD_SWITCH_PARAM_DEVBITFIELD                0x00
#define CMD_SWITCH_PARAM_TIMEOUT_SECONDS        0x01
#define TIMEOUT_NO_KEEPALIVE                        0x00

/* Command to Get the list of Paired devices */
#define REPORT_TYPE_CMD_GET_PAIRED_DEVICES        0x81

/* Device Paired Notification */
#define REPORT_TYPE_NOTIF_DEVICE_PAIRED                0x41
#define SPFUNCTION_MORE_NOTIF_EXPECTED                0x01
#define SPFUNCTION_DEVICE_LIST_EMPTY                0x02
#define DEVICE_PAIRED_PARAM_SPFUNCTION                0x00
#define DEVICE_PAIRED_PARAM_EQUAD_ID_LSB        0x01
#define DEVICE_PAIRED_PARAM_EQUAD_ID_MSB        0x02
#define DEVICE_PAIRED_RF_REPORT_TYPE                0x03

/* Device Un-Paired Notification */
#define REPORT_TYPE_NOTIF_DEVICE_UNPAIRED        0x40

/* Connection Status Notification */
#define REPORT_TYPE_NOTIF_CONNECTION_STATUS        0x42
#define CONNECTION_STATUS_PARAM_STATUS                0x00
#define STATUS_LINKLOSS                                0x01

/* Error Notification */
#define REPORT_TYPE_NOTIF_ERROR                        0x7F
#define NOTIF_ERROR_PARAM_ETYPE                        0x00
#define ETYPE_KEEPALIVE_TIMEOUT                        0x01

/* supported DJ HID && RF report types */
#define REPORT_TYPE_KEYBOARD                        0x01
#define REPORT_TYPE_MOUSE                        0x02
#define REPORT_TYPE_CONSUMER_CONTROL                0x03
#define REPORT_TYPE_SYSTEM_CONTROL                0x04
#define REPORT_TYPE_MEDIA_CENTER                0x08
#define REPORT_TYPE_LEDS                        0x0E

/* RF Report types bitfield */
#define STD_KEYBOARD                                BIT(1)
#define STD_MOUSE                                BIT(2)
#define MULTIMEDIA                                BIT(3)
#define POWER_KEYS                                BIT(4)
#define KBD_MOUSE                                BIT(5)
#define MEDIA_CENTER                                BIT(8)
#define KBD_LEDS                                BIT(14)
/* Fake (bitnr > NUMBER_OF_HID_REPORTS) bit to track HID++ capability */
#define HIDPP                                        BIT_ULL(63)

/* HID++ Device Connected Notification */
#define REPORT_TYPE_NOTIF_DEVICE_CONNECTED        0x41
#define HIDPP_PARAM_PROTO_TYPE                        0x00
#define HIDPP_PARAM_DEVICE_INFO                        0x01
#define HIDPP_PARAM_EQUAD_LSB                        0x02
#define HIDPP_PARAM_EQUAD_MSB                        0x03
#define HIDPP_PARAM_27MHZ_DEVID                        0x03
#define HIDPP_DEVICE_TYPE_MASK                        GENMASK(3, 0)
#define HIDPP_LINK_STATUS_MASK                        BIT(6)
#define HIDPP_MANUFACTURER_MASK                        BIT(7)
#define HIDPP_27MHZ_SECURE_MASK                        BIT(7)

#define HIDPP_DEVICE_TYPE_KEYBOARD                1
#define HIDPP_DEVICE_TYPE_MOUSE                        2

#define HIDPP_SET_REGISTER                        0x80
#define HIDPP_GET_LONG_REGISTER                        0x83
#define HIDPP_REG_CONNECTION_STATE                0x02
#define HIDPP_REG_PAIRING_INFORMATION                0xB5
#define HIDPP_PAIRING_INFORMATION                0x20
#define HIDPP_FAKE_DEVICE_ARRIVAL                0x02

enum recvr_type {
        recvr_type_dj,
        recvr_type_hidpp,
        recvr_type_gaming_hidpp,
        recvr_type_mouse_only,
        recvr_type_27mhz,
        recvr_type_bluetooth,
        recvr_type_dinovo,
};

struct dj_report {
        u8 report_id;
        u8 device_index;
        u8 report_type;
        u8 report_params[DJREPORT_SHORT_LENGTH - 3];
};

struct hidpp_event {
        u8 report_id;
        u8 device_index;
        u8 sub_id;
        u8 params[HIDPP_REPORT_LONG_LENGTH - 3U];
} __packed;

struct dj_receiver_dev {
        struct hid_device *mouse;
        struct hid_device *keyboard;
        struct hid_device *hidpp;
        struct dj_device *paired_dj_devices[DJ_MAX_PAIRED_DEVICES +
                                            DJ_DEVICE_INDEX_MIN];
        struct list_head list;
        struct kref kref;
        struct work_struct work;
        struct kfifo notif_fifo;
        unsigned long last_query; /* in jiffies */
        bool ready;
        enum recvr_type type;
        unsigned int unnumbered_application;
        spinlock_t lock;
};

struct dj_device {
        struct hid_device *hdev;
        struct dj_receiver_dev *dj_receiver_dev;
        u64 reports_supported;
        u8 device_index;
};

#define WORKITEM_TYPE_EMPTY        0
#define WORKITEM_TYPE_PAIRED        1
#define WORKITEM_TYPE_UNPAIRED        2
#define WORKITEM_TYPE_UNKNOWN        255

struct dj_workitem {
        u8 type;                /* WORKITEM_TYPE_* */
        u8 device_index;
        u8 device_type;
        u8 quad_id_msb;
        u8 quad_id_lsb;
        u64 reports_supported;
};

/* Keyboard descriptor (1) */
static const char kbd_descriptor[] = {
        0x05, 0x01,                /* USAGE_PAGE (generic Desktop)     */
        0x09, 0x06,                /* USAGE (Keyboard)         */
        0xA1, 0x01,                /* COLLECTION (Application)     */
        0x85, 0x01,                /* REPORT_ID (1)            */
        0x95, 0x08,                /*   REPORT_COUNT (8)           */
        0x75, 0x01,                /*   REPORT_SIZE (1)            */
        0x15, 0x00,                /*   LOGICAL_MINIMUM (0)        */
        0x25, 0x01,                /*   LOGICAL_MAXIMUM (1)        */
        0x05, 0x07,                /*   USAGE_PAGE (Keyboard)      */
        0x19, 0xE0,                /*   USAGE_MINIMUM (Left Control)   */
        0x29, 0xE7,                /*   USAGE_MAXIMUM (Right GUI)      */
        0x81, 0x02,                /*   INPUT (Data,Var,Abs)       */
        0x95, 0x06,                /*   REPORT_COUNT (6)           */
        0x75, 0x08,                /*   REPORT_SIZE (8)            */
        0x15, 0x00,                /*   LOGICAL_MINIMUM (0)        */
        0x26, 0xFF, 0x00,        /*   LOGICAL_MAXIMUM (255)      */
        0x05, 0x07,                /*   USAGE_PAGE (Keyboard)      */
        0x19, 0x00,                /*   USAGE_MINIMUM (no event)       */
        0x2A, 0xFF, 0x00,        /*   USAGE_MAXIMUM (reserved)       */
        0x81, 0x00,                /*   INPUT (Data,Ary,Abs)       */
        0x85, 0x0e,                /* REPORT_ID (14)               */
        0x05, 0x08,                /*   USAGE PAGE (LED page)      */
        0x95, 0x05,                /*   REPORT COUNT (5)           */
        0x75, 0x01,                /*   REPORT SIZE (1)            */
        0x15, 0x00,                /*   LOGICAL_MINIMUM (0)        */
        0x25, 0x01,                /*   LOGICAL_MAXIMUM (1)        */
        0x19, 0x01,                /*   USAGE MINIMUM (1)          */
        0x29, 0x05,                /*   USAGE MAXIMUM (5)          */
        0x91, 0x02,                /*   OUTPUT (Data, Variable, Absolute)  */
        0x95, 0x01,                /*   REPORT COUNT (1)           */
        0x75, 0x03,                /*   REPORT SIZE (3)            */
        0x91, 0x01,                /*   OUTPUT (Constant)          */
        0xC0
};

/* Mouse descriptor (2)     */
static const char mse_descriptor[] = {
        0x05, 0x01,                /*  USAGE_PAGE (Generic Desktop)        */
        0x09, 0x02,                /*  USAGE (Mouse)                       */
        0xA1, 0x01,                /*  COLLECTION (Application)            */
        0x85, 0x02,                /*    REPORT_ID = 2                     */
        0x09, 0x01,                /*    USAGE (pointer)                   */
        0xA1, 0x00,                /*    COLLECTION (physical)             */
        0x05, 0x09,                /*      USAGE_PAGE (buttons)            */
        0x19, 0x01,                /*      USAGE_MIN (1)                   */
        0x29, 0x10,                /*      USAGE_MAX (16)                  */
        0x15, 0x00,                /*      LOGICAL_MIN (0)                 */
        0x25, 0x01,                /*      LOGICAL_MAX (1)                 */
        0x95, 0x10,                /*      REPORT_COUNT (16)               */
        0x75, 0x01,                /*      REPORT_SIZE (1)                 */
        0x81, 0x02,                /*      INPUT (data var abs)            */
        0x05, 0x01,                /*      USAGE_PAGE (generic desktop)    */
        0x16, 0x01, 0xF8,        /*      LOGICAL_MIN (-2047)             */
        0x26, 0xFF, 0x07,        /*      LOGICAL_MAX (2047)              */
        0x75, 0x0C,                /*      REPORT_SIZE (12)                */
        0x95, 0x02,                /*      REPORT_COUNT (2)                */
        0x09, 0x30,                /*      USAGE (X)                       */
        0x09, 0x31,                /*      USAGE (Y)                       */
        0x81, 0x06,                /*      INPUT                           */
        0x15, 0x81,                /*      LOGICAL_MIN (-127)              */
        0x25, 0x7F,                /*      LOGICAL_MAX (127)               */
        0x75, 0x08,                /*      REPORT_SIZE (8)                 */
        0x95, 0x01,                /*      REPORT_COUNT (1)                */
        0x09, 0x38,                /*      USAGE (wheel)                   */
        0x81, 0x06,                /*      INPUT                           */
        0x05, 0x0C,                /*      USAGE_PAGE(consumer)            */
        0x0A, 0x38, 0x02,        /*      USAGE(AC Pan)                   */
        0x95, 0x01,                /*      REPORT_COUNT (1)                */
        0x81, 0x06,                /*      INPUT                           */
        0xC0,                        /*    END_COLLECTION                    */
        0xC0,                        /*  END_COLLECTION                      */
};

/* Mouse descriptor (2) for 27 MHz receiver, only 8 buttons */
static const char mse_27mhz_descriptor[] = {
        0x05, 0x01,                /*  USAGE_PAGE (Generic Desktop)        */
        0x09, 0x02,                /*  USAGE (Mouse)                       */
        0xA1, 0x01,                /*  COLLECTION (Application)            */
        0x85, 0x02,                /*    REPORT_ID = 2                     */
        0x09, 0x01,                /*    USAGE (pointer)                   */
        0xA1, 0x00,                /*    COLLECTION (physical)             */
        0x05, 0x09,                /*      USAGE_PAGE (buttons)            */
        0x19, 0x01,                /*      USAGE_MIN (1)                   */
        0x29, 0x08,                /*      USAGE_MAX (8)                   */
        0x15, 0x00,                /*      LOGICAL_MIN (0)                 */
        0x25, 0x01,                /*      LOGICAL_MAX (1)                 */
        0x95, 0x08,                /*      REPORT_COUNT (8)                */
        0x75, 0x01,                /*      REPORT_SIZE (1)                 */
        0x81, 0x02,                /*      INPUT (data var abs)            */
        0x05, 0x01,                /*      USAGE_PAGE (generic desktop)    */
        0x16, 0x01, 0xF8,        /*      LOGICAL_MIN (-2047)             */
        0x26, 0xFF, 0x07,        /*      LOGICAL_MAX (2047)              */
        0x75, 0x0C,                /*      REPORT_SIZE (12)                */
        0x95, 0x02,                /*      REPORT_COUNT (2)                */
        0x09, 0x30,                /*      USAGE (X)                       */
        0x09, 0x31,                /*      USAGE (Y)                       */
        0x81, 0x06,                /*      INPUT                           */
        0x15, 0x81,                /*      LOGICAL_MIN (-127)              */
        0x25, 0x7F,                /*      LOGICAL_MAX (127)               */
        0x75, 0x08,                /*      REPORT_SIZE (8)                 */
        0x95, 0x01,                /*      REPORT_COUNT (1)                */
        0x09, 0x38,                /*      USAGE (wheel)                   */
        0x81, 0x06,                /*      INPUT                           */
        0x05, 0x0C,                /*      USAGE_PAGE(consumer)            */
        0x0A, 0x38, 0x02,        /*      USAGE(AC Pan)                   */
        0x95, 0x01,                /*      REPORT_COUNT (1)                */
        0x81, 0x06,                /*      INPUT                           */
        0xC0,                        /*    END_COLLECTION                    */
        0xC0,                        /*  END_COLLECTION                      */
};

/* Mouse descriptor (2) for Bluetooth receiver, low-res hwheel, 12 buttons */
static const char mse_bluetooth_descriptor[] = {
        0x05, 0x01,                /*  USAGE_PAGE (Generic Desktop)        */
        0x09, 0x02,                /*  USAGE (Mouse)                       */
        0xA1, 0x01,                /*  COLLECTION (Application)            */
        0x85, 0x02,                /*    REPORT_ID = 2                     */
        0x09, 0x01,                /*    USAGE (pointer)                   */
        0xA1, 0x00,                /*    COLLECTION (physical)             */
        0x05, 0x09,                /*      USAGE_PAGE (buttons)            */
        0x19, 0x01,                /*      USAGE_MIN (1)                   */
        0x29, 0x08,                /*      USAGE_MAX (8)                   */
        0x15, 0x00,                /*      LOGICAL_MIN (0)                 */
        0x25, 0x01,                /*      LOGICAL_MAX (1)                 */
        0x95, 0x08,                /*      REPORT_COUNT (8)                */
        0x75, 0x01,                /*      REPORT_SIZE (1)                 */
        0x81, 0x02,                /*      INPUT (data var abs)            */
        0x05, 0x01,                /*      USAGE_PAGE (generic desktop)    */
        0x16, 0x01, 0xF8,        /*      LOGICAL_MIN (-2047)             */
        0x26, 0xFF, 0x07,        /*      LOGICAL_MAX (2047)              */
        0x75, 0x0C,                /*      REPORT_SIZE (12)                */
        0x95, 0x02,                /*      REPORT_COUNT (2)                */
        0x09, 0x30,                /*      USAGE (X)                       */
        0x09, 0x31,                /*      USAGE (Y)                       */
        0x81, 0x06,                /*      INPUT                           */
        0x15, 0x81,                /*      LOGICAL_MIN (-127)              */
        0x25, 0x7F,                /*      LOGICAL_MAX (127)               */
        0x75, 0x08,                /*      REPORT_SIZE (8)                 */
        0x95, 0x01,                /*      REPORT_COUNT (1)                */
        0x09, 0x38,                /*      USAGE (wheel)                   */
        0x81, 0x06,                /*      INPUT                           */
        0x05, 0x0C,                /*      USAGE_PAGE(consumer)            */
        0x0A, 0x38, 0x02,        /*      USAGE(AC Pan)                   */
        0x15, 0xF9,                /*      LOGICAL_MIN (-7)                */
        0x25, 0x07,                /*      LOGICAL_MAX (7)                 */
        0x75, 0x04,                /*      REPORT_SIZE (4)                 */
        0x95, 0x01,                /*      REPORT_COUNT (1)                */
        0x81, 0x06,                /*      INPUT                           */
        0x05, 0x09,                /*      USAGE_PAGE (buttons)            */
        0x19, 0x09,                /*      USAGE_MIN (9)                   */
        0x29, 0x0C,                /*      USAGE_MAX (12)                  */
        0x15, 0x00,                /*      LOGICAL_MIN (0)                 */
        0x25, 0x01,                /*      LOGICAL_MAX (1)                 */
        0x75, 0x01,                /*      REPORT_SIZE (1)                 */
        0x95, 0x04,                /*      REPORT_COUNT (4)                */
        0x81, 0x02,                /*      INPUT (Data,Var,Abs)            */
        0xC0,                        /*    END_COLLECTION                    */
        0xC0,                        /*  END_COLLECTION                      */
};

/* Mouse descriptor (5) for Bluetooth receiver, normal-res hwheel, 8 buttons */
static const char mse5_bluetooth_descriptor[] = {
        0x05, 0x01,                /*  USAGE_PAGE (Generic Desktop)        */
        0x09, 0x02,                /*  Usage (Mouse)                       */
        0xa1, 0x01,                /*  Collection (Application)            */
        0x85, 0x05,                /*   Report ID (5)                      */
        0x09, 0x01,                /*   Usage (Pointer)                    */
        0xa1, 0x00,                /*   Collection (Physical)              */
        0x05, 0x09,                /*    Usage Page (Button)               */
        0x19, 0x01,                /*    Usage Minimum (1)                 */
        0x29, 0x08,                /*    Usage Maximum (8)                 */
        0x15, 0x00,                /*    Logical Minimum (0)               */
        0x25, 0x01,                /*    Logical Maximum (1)               */
        0x95, 0x08,                /*    Report Count (8)                  */
        0x75, 0x01,                /*    Report Size (1)                   */
        0x81, 0x02,                /*    Input (Data,Var,Abs)              */
        0x05, 0x01,                /*    Usage Page (Generic Desktop)      */
        0x16, 0x01, 0xf8,        /*    Logical Minimum (-2047)           */
        0x26, 0xff, 0x07,        /*    Logical Maximum (2047)            */
        0x75, 0x0c,                /*    Report Size (12)                  */
        0x95, 0x02,                /*    Report Count (2)                  */
        0x09, 0x30,                /*    Usage (X)                         */
        0x09, 0x31,                /*    Usage (Y)                         */
        0x81, 0x06,                /*    Input (Data,Var,Rel)              */
        0x15, 0x81,                /*    Logical Minimum (-127)            */
        0x25, 0x7f,                /*    Logical Maximum (127)             */
        0x75, 0x08,                /*    Report Size (8)                   */
        0x95, 0x01,                /*    Report Count (1)                  */
        0x09, 0x38,                /*    Usage (Wheel)                     */
        0x81, 0x06,                /*    Input (Data,Var,Rel)              */
        0x05, 0x0c,                /*    Usage Page (Consumer Devices)     */
        0x0a, 0x38, 0x02,        /*    Usage (AC Pan)                    */
        0x15, 0x81,                /*    Logical Minimum (-127)            */
        0x25, 0x7f,                /*    Logical Maximum (127)             */
        0x75, 0x08,                /*    Report Size (8)                   */
        0x95, 0x01,                /*    Report Count (1)                  */
        0x81, 0x06,                /*    Input (Data,Var,Rel)              */
        0xc0,                        /*   End Collection                     */
        0xc0,                        /*  End Collection                      */
};

/* Gaming Mouse descriptor (2) */
static const char mse_high_res_descriptor[] = {
        0x05, 0x01,                /*  USAGE_PAGE (Generic Desktop)        */
        0x09, 0x02,                /*  USAGE (Mouse)                       */
        0xA1, 0x01,                /*  COLLECTION (Application)            */
        0x85, 0x02,                /*    REPORT_ID = 2                     */
        0x09, 0x01,                /*    USAGE (pointer)                   */
        0xA1, 0x00,                /*    COLLECTION (physical)             */
        0x05, 0x09,                /*      USAGE_PAGE (buttons)            */
        0x19, 0x01,                /*      USAGE_MIN (1)                   */
        0x29, 0x10,                /*      USAGE_MAX (16)                  */
        0x15, 0x00,                /*      LOGICAL_MIN (0)                 */
        0x25, 0x01,                /*      LOGICAL_MAX (1)                 */
        0x95, 0x10,                /*      REPORT_COUNT (16)               */
        0x75, 0x01,                /*      REPORT_SIZE (1)                 */
        0x81, 0x02,                /*      INPUT (data var abs)            */
        0x05, 0x01,                /*      USAGE_PAGE (generic desktop)    */
        0x16, 0x01, 0x80,        /*      LOGICAL_MIN (-32767)            */
        0x26, 0xFF, 0x7F,        /*      LOGICAL_MAX (32767)             */
        0x75, 0x10,                /*      REPORT_SIZE (16)                */
        0x95, 0x02,                /*      REPORT_COUNT (2)                */
        0x09, 0x30,                /*      USAGE (X)                       */
        0x09, 0x31,                /*      USAGE (Y)                       */
        0x81, 0x06,                /*      INPUT                           */
        0x15, 0x81,                /*      LOGICAL_MIN (-127)              */
        0x25, 0x7F,                /*      LOGICAL_MAX (127)               */
        0x75, 0x08,                /*      REPORT_SIZE (8)                 */
        0x95, 0x01,                /*      REPORT_COUNT (1)                */
        0x09, 0x38,                /*      USAGE (wheel)                   */
        0x81, 0x06,                /*      INPUT                           */
        0x05, 0x0C,                /*      USAGE_PAGE(consumer)            */
        0x0A, 0x38, 0x02,        /*      USAGE(AC Pan)                   */
        0x95, 0x01,                /*      REPORT_COUNT (1)                */
        0x81, 0x06,                /*      INPUT                           */
        0xC0,                        /*    END_COLLECTION                    */
        0xC0,                        /*  END_COLLECTION                      */
};

/* Consumer Control descriptor (3) */
static const char consumer_descriptor[] = {
        0x05, 0x0C,                /* USAGE_PAGE (Consumer Devices)       */
        0x09, 0x01,                /* USAGE (Consumer Control)            */
        0xA1, 0x01,                /* COLLECTION (Application)            */
        0x85, 0x03,                /* REPORT_ID = 3                       */
        0x75, 0x10,                /* REPORT_SIZE (16)                    */
        0x95, 0x02,                /* REPORT_COUNT (2)                    */
        0x15, 0x01,                /* LOGICAL_MIN (1)                     */
        0x26, 0xFF, 0x02,        /* LOGICAL_MAX (767)                   */
        0x19, 0x01,                /* USAGE_MIN (1)                       */
        0x2A, 0xFF, 0x02,        /* USAGE_MAX (767)                     */
        0x81, 0x00,                /* INPUT (Data Ary Abs)                */
        0xC0,                        /* END_COLLECTION                      */
};                                /*                                     */

/* System control descriptor (4) */
static const char syscontrol_descriptor[] = {
        0x05, 0x01,                /*   USAGE_PAGE (Generic Desktop)      */
        0x09, 0x80,                /*   USAGE (System Control)            */
        0xA1, 0x01,                /*   COLLECTION (Application)          */
        0x85, 0x04,                /*   REPORT_ID = 4                     */
        0x75, 0x02,                /*   REPORT_SIZE (2)                   */
        0x95, 0x01,                /*   REPORT_COUNT (1)                  */
        0x15, 0x01,                /*   LOGICAL_MIN (1)                   */
        0x25, 0x03,                /*   LOGICAL_MAX (3)                   */
        0x09, 0x82,                /*   USAGE (System Sleep)              */
        0x09, 0x81,                /*   USAGE (System Power Down)         */
        0x09, 0x83,                /*   USAGE (System Wake Up)            */
        0x81, 0x60,                /*   INPUT (Data Ary Abs NPrf Null)    */
        0x75, 0x06,                /*   REPORT_SIZE (6)                   */
        0x81, 0x03,                /*   INPUT (Cnst Var Abs)              */
        0xC0,                        /*   END_COLLECTION                    */
};

/* Media descriptor (8) */
static const char media_descriptor[] = {
        0x06, 0xbc, 0xff,        /* Usage Page 0xffbc                   */
        0x09, 0x88,                /* Usage 0x0088                        */
        0xa1, 0x01,                /* BeginCollection                     */
        0x85, 0x08,                /*   Report ID 8                       */
        0x19, 0x01,                /*   Usage Min 0x0001                  */
        0x29, 0xff,                /*   Usage Max 0x00ff                  */
        0x15, 0x01,                /*   Logical Min 1                     */
        0x26, 0xff, 0x00,        /*   Logical Max 255                   */
        0x75, 0x08,                /*   Report Size 8                     */
        0x95, 0x01,                /*   Report Count 1                    */
        0x81, 0x00,                /*   Input                             */
        0xc0,                        /* EndCollection                       */
};                                /*                                     */

/* HIDPP descriptor */
static const char hidpp_descriptor[] = {
        0x06, 0x00, 0xff,        /* Usage Page (Vendor Defined Page 1)  */
        0x09, 0x01,                /* Usage (Vendor Usage 1)              */
        0xa1, 0x01,                /* Collection (Application)            */
        0x85, 0x10,                /*   Report ID (16)                    */
        0x75, 0x08,                /*   Report Size (8)                   */
        0x95, 0x06,                /*   Report Count (6)                  */
        0x15, 0x00,                /*   Logical Minimum (0)               */
        0x26, 0xff, 0x00,        /*   Logical Maximum (255)             */
        0x09, 0x01,                /*   Usage (Vendor Usage 1)            */
        0x81, 0x00,                /*   Input (Data,Arr,Abs)              */
        0x09, 0x01,                /*   Usage (Vendor Usage 1)            */
        0x91, 0x00,                /*   Output (Data,Arr,Abs)             */
        0xc0,                        /* End Collection                      */
        0x06, 0x00, 0xff,        /* Usage Page (Vendor Defined Page 1)  */
        0x09, 0x02,                /* Usage (Vendor Usage 2)              */
        0xa1, 0x01,                /* Collection (Application)            */
        0x85, 0x11,                /*   Report ID (17)                    */
        0x75, 0x08,                /*   Report Size (8)                   */
        0x95, 0x13,                /*   Report Count (19)                 */
        0x15, 0x00,                /*   Logical Minimum (0)               */
        0x26, 0xff, 0x00,        /*   Logical Maximum (255)             */
        0x09, 0x02,                /*   Usage (Vendor Usage 2)            */
        0x81, 0x00,                /*   Input (Data,Arr,Abs)              */
        0x09, 0x02,                /*   Usage (Vendor Usage 2)            */
        0x91, 0x00,                /*   Output (Data,Arr,Abs)             */
        0xc0,                        /* End Collection                      */
        0x06, 0x00, 0xff,        /* Usage Page (Vendor Defined Page 1)  */
        0x09, 0x04,                /* Usage (Vendor Usage 0x04)           */
        0xa1, 0x01,                /* Collection (Application)            */
        0x85, 0x20,                /*   Report ID (32)                    */
        0x75, 0x08,                /*   Report Size (8)                   */
        0x95, 0x0e,                /*   Report Count (14)                 */
        0x15, 0x00,                /*   Logical Minimum (0)               */
        0x26, 0xff, 0x00,        /*   Logical Maximum (255)             */
        0x09, 0x41,                /*   Usage (Vendor Usage 0x41)         */
        0x81, 0x00,                /*   Input (Data,Arr,Abs)              */
        0x09, 0x41,                /*   Usage (Vendor Usage 0x41)         */
        0x91, 0x00,                /*   Output (Data,Arr,Abs)             */
        0x85, 0x21,                /*   Report ID (33)                    */
        0x95, 0x1f,                /*   Report Count (31)                 */
        0x15, 0x00,                /*   Logical Minimum (0)               */
        0x26, 0xff, 0x00,        /*   Logical Maximum (255)             */
        0x09, 0x42,                /*   Usage (Vendor Usage 0x42)         */
        0x81, 0x00,                /*   Input (Data,Arr,Abs)              */
        0x09, 0x42,                /*   Usage (Vendor Usage 0x42)         */
        0x91, 0x00,                /*   Output (Data,Arr,Abs)             */
        0xc0,                        /* End Collection                      */
};

/* Maximum size of all defined hid reports in bytes (including report id) */
#define MAX_REPORT_SIZE 8

/* Make sure all descriptors are present here */
#define MAX_RDESC_SIZE                                \
        (sizeof(kbd_descriptor) +                \
         sizeof(mse_bluetooth_descriptor) +        \
         sizeof(mse5_bluetooth_descriptor) +        \
         sizeof(consumer_descriptor) +                \
         sizeof(syscontrol_descriptor) +        \
         sizeof(media_descriptor) +        \
         sizeof(hidpp_descriptor))

/* Number of possible hid report types that can be created by this driver.
 *
 * Right now, RF report types have the same report types (or report id's)
 * than the hid report created from those RF reports. In the future
 * this doesnt have to be true.
 *
 * For instance, RF report type 0x01 which has a size of 8 bytes, corresponds
 * to hid report id 0x01, this is standard keyboard. Same thing applies to mice
 * reports and consumer control, etc. If a new RF report is created, it doesn't
 * has to have the same report id as its corresponding hid report, so an
 * translation may have to take place for future report types.
 */
#define NUMBER_OF_HID_REPORTS 32
static const u8 hid_reportid_size_map[NUMBER_OF_HID_REPORTS] = {
        [1] = 8,                /* Standard keyboard */
        [2] = 8,                /* Standard mouse */
        [3] = 5,                /* Consumer control */
        [4] = 2,                /* System control */
        [8] = 2,                /* Media Center */
};


#define LOGITECH_DJ_INTERFACE_NUMBER 0x02

static const struct hid_ll_driver logi_dj_ll_driver;

static int logi_dj_recv_query_paired_devices(struct dj_receiver_dev *djrcv_dev);
static void delayedwork_callback(struct work_struct *work);

static LIST_HEAD(dj_hdev_list);
static DEFINE_MUTEX(dj_hdev_list_lock);

static bool recvr_type_is_bluetooth(enum recvr_type type)
{
        return type == recvr_type_bluetooth || type == recvr_type_dinovo;
}

/*
 * dj/HID++ receivers are really a single logical entity, but for BIOS/Windows
 * compatibility they have multiple USB interfaces. On HID++ receivers we need
 * to listen for input reports on both interfaces. The functions below are used
 * to create a single struct dj_receiver_dev for all interfaces belonging to
 * a single USB-device / receiver.
 */
static struct dj_receiver_dev *dj_find_receiver_dev(struct hid_device *hdev,
                                                    enum recvr_type type)
{
        struct dj_receiver_dev *djrcv_dev;
        char sep;

        /*
         * The bluetooth receiver contains a built-in hub and has separate
         * USB-devices for the keyboard and mouse interfaces.
         */
        sep = recvr_type_is_bluetooth(type) ? '.' : '/';

        /* Try to find an already-probed interface from the same device */
        list_for_each_entry(djrcv_dev, &dj_hdev_list, list) {
                if (djrcv_dev->mouse &&
                    hid_compare_device_paths(hdev, djrcv_dev->mouse, sep)) {
                        kref_get(&djrcv_dev->kref);
                        return djrcv_dev;
                }
                if (djrcv_dev->keyboard &&
                    hid_compare_device_paths(hdev, djrcv_dev->keyboard, sep)) {
                        kref_get(&djrcv_dev->kref);
                        return djrcv_dev;
                }
                if (djrcv_dev->hidpp &&
                    hid_compare_device_paths(hdev, djrcv_dev->hidpp, sep)) {
                        kref_get(&djrcv_dev->kref);
                        return djrcv_dev;
                }
        }

        return NULL;
}

static void dj_release_receiver_dev(struct kref *kref)
{
        struct dj_receiver_dev *djrcv_dev = container_of(kref, struct dj_receiver_dev, kref);

        list_del(&djrcv_dev->list);
        kfifo_free(&djrcv_dev->notif_fifo);
        kfree(djrcv_dev);
}

static void dj_put_receiver_dev(struct hid_device *hdev)
{
        struct dj_receiver_dev *djrcv_dev = hid_get_drvdata(hdev);

        mutex_lock(&dj_hdev_list_lock);

        if (djrcv_dev->mouse == hdev)
                djrcv_dev->mouse = NULL;
        if (djrcv_dev->keyboard == hdev)
                djrcv_dev->keyboard = NULL;
        if (djrcv_dev->hidpp == hdev)
                djrcv_dev->hidpp = NULL;

        kref_put(&djrcv_dev->kref, dj_release_receiver_dev);

        mutex_unlock(&dj_hdev_list_lock);
}

static struct dj_receiver_dev *dj_get_receiver_dev(struct hid_device *hdev,
                                                   enum recvr_type type,
                                                   unsigned int application,
                                                   bool is_hidpp)
{
        struct dj_receiver_dev *djrcv_dev;

        mutex_lock(&dj_hdev_list_lock);

        djrcv_dev = dj_find_receiver_dev(hdev, type);
        if (!djrcv_dev) {
                djrcv_dev = kzalloc(sizeof(*djrcv_dev), GFP_KERNEL);
                if (!djrcv_dev)
                        goto out;

                INIT_WORK(&djrcv_dev->work, delayedwork_callback);
                spin_lock_init(&djrcv_dev->lock);
                if (kfifo_alloc(&djrcv_dev->notif_fifo,
                            DJ_MAX_NUMBER_NOTIFS * sizeof(struct dj_workitem),
                            GFP_KERNEL)) {
                        kfree(djrcv_dev);
                        djrcv_dev = NULL;
                        goto out;
                }
                kref_init(&djrcv_dev->kref);
                list_add_tail(&djrcv_dev->list, &dj_hdev_list);
                djrcv_dev->last_query = jiffies;
                djrcv_dev->type = type;
        }

        if (application == HID_GD_KEYBOARD)
                djrcv_dev->keyboard = hdev;
        if (application == HID_GD_MOUSE)
                djrcv_dev->mouse = hdev;
        if (is_hidpp)
                djrcv_dev->hidpp = hdev;

        hid_set_drvdata(hdev, djrcv_dev);
out:
        mutex_unlock(&dj_hdev_list_lock);
        return djrcv_dev;
}

static void logi_dj_recv_destroy_djhid_device(struct dj_receiver_dev *djrcv_dev,
                                              struct dj_workitem *workitem)
{
        /* Called in delayed work context */
        struct dj_device *dj_dev;
        unsigned long flags;

        spin_lock_irqsave(&djrcv_dev->lock, flags);
        dj_dev = djrcv_dev->paired_dj_devices[workitem->device_index];
        djrcv_dev->paired_dj_devices[workitem->device_index] = NULL;
        spin_unlock_irqrestore(&djrcv_dev->lock, flags);

        if (dj_dev != NULL) {
                hid_destroy_device(dj_dev->hdev);
                kfree(dj_dev);
        } else {
                hid_err(djrcv_dev->hidpp, "%s: can't destroy a NULL device\n",
                        __func__);
        }
}

static void logi_dj_recv_add_djhid_device(struct dj_receiver_dev *djrcv_dev,
                                          struct dj_workitem *workitem)
{
        /* Called in delayed work context */
        struct hid_device *djrcv_hdev = djrcv_dev->hidpp;
        struct hid_device *dj_hiddev;
        struct dj_device *dj_dev;
        u8 device_index = workitem->device_index;
        unsigned long flags;

        /* Device index goes from 1 to 6, we need 3 bytes to store the
         * semicolon, the index, and a null terminator
         */
        unsigned char tmpstr[3];

        /* We are the only one ever adding a device, no need to lock */
        if (djrcv_dev->paired_dj_devices[device_index]) {
                /* The device is already known. No need to reallocate it. */
                dbg_hid("%s: device is already known\n", __func__);
                return;
        }

        dj_hiddev = hid_allocate_device();
        if (IS_ERR(dj_hiddev)) {
                hid_err(djrcv_hdev, "%s: hid_allocate_dev failed\n", __func__);
                return;
        }

        dj_hiddev->ll_driver = &logi_dj_ll_driver;

        dj_hiddev->dev.parent = &djrcv_hdev->dev;
        dj_hiddev->bus = BUS_USB;
        dj_hiddev->vendor = djrcv_hdev->vendor;
        dj_hiddev->product = (workitem->quad_id_msb << 8) |
                              workitem->quad_id_lsb;
        if (workitem->device_type) {
                const char *type_str = "Device";

                switch (workitem->device_type) {
                case 0x01: type_str = "Keyboard";        break;
                case 0x02: type_str = "Mouse";                break;
                case 0x03: type_str = "Numpad";                break;
                case 0x04: type_str = "Presenter";        break;
                case 0x07: type_str = "Remote Control";        break;
                case 0x08: type_str = "Trackball";        break;
                case 0x09: type_str = "Touchpad";        break;
                }
                snprintf(dj_hiddev->name, sizeof(dj_hiddev->name),
                        "Logitech Wireless %s PID:%04x",
                        type_str, dj_hiddev->product);
        } else {
                snprintf(dj_hiddev->name, sizeof(dj_hiddev->name),
                        "Logitech Wireless Device PID:%04x",
                        dj_hiddev->product);
        }

        if (djrcv_dev->type == recvr_type_27mhz)
                dj_hiddev->group = HID_GROUP_LOGITECH_27MHZ_DEVICE;
        else
                dj_hiddev->group = HID_GROUP_LOGITECH_DJ_DEVICE;

        memcpy(dj_hiddev->phys, djrcv_hdev->phys, sizeof(djrcv_hdev->phys));
        snprintf(tmpstr, sizeof(tmpstr), ":%d", device_index);
        strlcat(dj_hiddev->phys, tmpstr, sizeof(dj_hiddev->phys));

        dj_dev = kzalloc(sizeof(struct dj_device), GFP_KERNEL);

        if (!dj_dev) {
                hid_err(djrcv_hdev, "%s: failed allocating dj_dev\n", __func__);
                goto dj_device_allocate_fail;
        }

        dj_dev->reports_supported = workitem->reports_supported;
        dj_dev->hdev = dj_hiddev;
        dj_dev->dj_receiver_dev = djrcv_dev;
        dj_dev->device_index = device_index;
        dj_hiddev->driver_data = dj_dev;

        spin_lock_irqsave(&djrcv_dev->lock, flags);
        djrcv_dev->paired_dj_devices[device_index] = dj_dev;
        spin_unlock_irqrestore(&djrcv_dev->lock, flags);

        if (hid_add_device(dj_hiddev)) {
                hid_err(djrcv_hdev, "%s: failed adding dj_device\n", __func__);
                goto hid_add_device_fail;
        }

        return;

hid_add_device_fail:
        spin_lock_irqsave(&djrcv_dev->lock, flags);
        djrcv_dev->paired_dj_devices[device_index] = NULL;
        spin_unlock_irqrestore(&djrcv_dev->lock, flags);
        kfree(dj_dev);
dj_device_allocate_fail:
        hid_destroy_device(dj_hiddev);
}

static void delayedwork_callback(struct work_struct *work)
{
        struct dj_receiver_dev *djrcv_dev =
                container_of(work, struct dj_receiver_dev, work);

        struct dj_workitem workitem;
        unsigned long flags;
        int count;
        int retval;

        dbg_hid("%s\n", __func__);

        spin_lock_irqsave(&djrcv_dev->lock, flags);

        /*
         * Since we attach to multiple interfaces, we may get scheduled before
         * we are bound to the HID++ interface, catch this.
         */
        if (!djrcv_dev->ready) {
                pr_warn("%s: delayedwork queued before hidpp interface was enumerated\n",
                        __func__);
                spin_unlock_irqrestore(&djrcv_dev->lock, flags);
                return;
        }

        count = kfifo_out(&djrcv_dev->notif_fifo, &workitem, sizeof(workitem));

        if (count != sizeof(workitem)) {
                spin_unlock_irqrestore(&djrcv_dev->lock, flags);
                return;
        }

        if (!kfifo_is_empty(&djrcv_dev->notif_fifo))
                schedule_work(&djrcv_dev->work);

        spin_unlock_irqrestore(&djrcv_dev->lock, flags);

        switch (workitem.type) {
        case WORKITEM_TYPE_PAIRED:
                logi_dj_recv_add_djhid_device(djrcv_dev, &workitem);
                break;
        case WORKITEM_TYPE_UNPAIRED:
                logi_dj_recv_destroy_djhid_device(djrcv_dev, &workitem);
                break;
        case WORKITEM_TYPE_UNKNOWN:
                retval = logi_dj_recv_query_paired_devices(djrcv_dev);
                if (retval) {
                        hid_err(djrcv_dev->hidpp, "%s: logi_dj_recv_query_paired_devices error: %d\n",
                                __func__, retval);
                }
                break;
        case WORKITEM_TYPE_EMPTY:
                dbg_hid("%s: device list is empty\n", __func__);
                break;
        }
}

/*
 * Sometimes we receive reports for which we do not have a paired dj_device
 * associated with the device_index or report-type to forward the report to.
 * This means that the original "device paired" notification corresponding
 * to the dj_device never arrived to this driver. Possible reasons for this are:
 * 1) hid-core discards all packets coming from a device during probe().
 * 2) if the receiver is plugged into a KVM switch then the pairing reports
 * are only forwarded to it if the focus is on this PC.
 * This function deals with this by re-asking the receiver for the list of
 * connected devices in the delayed work callback.
 * This function MUST be called with djrcv->lock held.
 */
static void logi_dj_recv_queue_unknown_work(struct dj_receiver_dev *djrcv_dev)
{
        struct dj_workitem workitem = { .type = WORKITEM_TYPE_UNKNOWN };

        /* Rate limit queries done because of unhandled reports to 2/sec */
        if (time_before(jiffies, djrcv_dev->last_query + HZ / 2))
                return;

        kfifo_in(&djrcv_dev->notif_fifo, &workitem, sizeof(workitem));
        schedule_work(&djrcv_dev->work);
}

static void logi_dj_recv_queue_notification(struct dj_receiver_dev *djrcv_dev,
                                           struct dj_report *dj_report)
{
        /* We are called from atomic context (tasklet && djrcv->lock held) */
        struct dj_workitem workitem = {
                .device_index = dj_report->device_index,
        };

        switch (dj_report->report_type) {
        case REPORT_TYPE_NOTIF_DEVICE_PAIRED:
                workitem.type = WORKITEM_TYPE_PAIRED;
                if (dj_report->report_params[DEVICE_PAIRED_PARAM_SPFUNCTION] &
                    SPFUNCTION_DEVICE_LIST_EMPTY) {
                        workitem.type = WORKITEM_TYPE_EMPTY;
                        break;
                }
                fallthrough;
        case REPORT_TYPE_NOTIF_DEVICE_UNPAIRED:
                workitem.quad_id_msb =
                        dj_report->report_params[DEVICE_PAIRED_PARAM_EQUAD_ID_MSB];
                workitem.quad_id_lsb =
                        dj_report->report_params[DEVICE_PAIRED_PARAM_EQUAD_ID_LSB];
                workitem.reports_supported = get_unaligned_le32(
                                                dj_report->report_params +
                                                DEVICE_PAIRED_RF_REPORT_TYPE);
                workitem.reports_supported |= HIDPP;
                if (dj_report->report_type == REPORT_TYPE_NOTIF_DEVICE_UNPAIRED)
                        workitem.type = WORKITEM_TYPE_UNPAIRED;
                break;
        default:
                logi_dj_recv_queue_unknown_work(djrcv_dev);
                return;
        }

        kfifo_in(&djrcv_dev->notif_fifo, &workitem, sizeof(workitem));
        schedule_work(&djrcv_dev->work);
}

/*
 * Some quad/bluetooth keyboards have a builtin touchpad in this case we see
 * only 1 paired device with a device_type of REPORT_TYPE_KEYBOARD. For the
 * touchpad to work we must also forward mouse input reports to the dj_hiddev
 * created for the keyboard (instead of forwarding them to a second paired
 * device with a device_type of REPORT_TYPE_MOUSE as we normally would).
 *
 * On Dinovo receivers the keyboard's touchpad and an optional paired actual
 * mouse send separate input reports, INPUT(2) aka STD_MOUSE for the mouse
 * and INPUT(5) aka KBD_MOUSE for the keyboard's touchpad.
 *
 * On MX5x00 receivers (which can also be paired with a Dinovo keyboard)
 * INPUT(2) is used for both an optional paired actual mouse and for the
 * keyboard's touchpad.
 */
static const u16 kbd_builtin_touchpad_ids[] = {
        0xb309, /* Dinovo Edge */
        0xb30c, /* Dinovo Mini */
};

static void logi_hidpp_dev_conn_notif_equad(struct hid_device *hdev,
                                            struct hidpp_event *hidpp_report,
                                            struct dj_workitem *workitem)
{
        struct dj_receiver_dev *djrcv_dev = hid_get_drvdata(hdev);
        int i, id;

        workitem->type = WORKITEM_TYPE_PAIRED;
        workitem->device_type = hidpp_report->params[HIDPP_PARAM_DEVICE_INFO] &
                                HIDPP_DEVICE_TYPE_MASK;
        workitem->quad_id_msb = hidpp_report->params[HIDPP_PARAM_EQUAD_MSB];
        workitem->quad_id_lsb = hidpp_report->params[HIDPP_PARAM_EQUAD_LSB];
        switch (workitem->device_type) {
        case REPORT_TYPE_KEYBOARD:
                workitem->reports_supported |= STD_KEYBOARD | MULTIMEDIA |
                                               POWER_KEYS | MEDIA_CENTER |
                                               HIDPP;
                id = (workitem->quad_id_msb << 8) | workitem->quad_id_lsb;
                for (i = 0; i < ARRAY_SIZE(kbd_builtin_touchpad_ids); i++) {
                        if (id == kbd_builtin_touchpad_ids[i]) {
                                if (djrcv_dev->type == recvr_type_dinovo)
                                        workitem->reports_supported |= KBD_MOUSE;
                                else
                                        workitem->reports_supported |= STD_MOUSE;
                                break;
                        }
                }
                break;
        case REPORT_TYPE_MOUSE:
                workitem->reports_supported |= STD_MOUSE | HIDPP | MULTIMEDIA;
                break;
        }
}

static void logi_hidpp_dev_conn_notif_27mhz(struct hid_device *hdev,
                                            struct hidpp_event *hidpp_report,
                                            struct dj_workitem *workitem)
{
        workitem->type = WORKITEM_TYPE_PAIRED;
        workitem->quad_id_lsb = hidpp_report->params[HIDPP_PARAM_27MHZ_DEVID];
        switch (hidpp_report->device_index) {
        case 1: /* Index 1 is always a mouse */
        case 2: /* Index 2 is always a mouse */
                workitem->device_type = HIDPP_DEVICE_TYPE_MOUSE;
                workitem->reports_supported |= STD_MOUSE | HIDPP;
                break;
        case 3: /* Index 3 is always the keyboard */
                if (hidpp_report->params[HIDPP_PARAM_DEVICE_INFO] & HIDPP_27MHZ_SECURE_MASK) {
                        hid_info(hdev, "Keyboard connection is encrypted\n");
                } else {
                        hid_warn(hdev, "Keyboard events are send over the air in plain-text / unencrypted\n");
                        hid_warn(hdev, "See: https://gitlab.freedesktop.org/jwrdegoede/logitech-27mhz-keyboard-encryption-setup/\n");
                }
                fallthrough;
        case 4: /* Index 4 is used for an optional separate numpad */
                workitem->device_type = HIDPP_DEVICE_TYPE_KEYBOARD;
                workitem->reports_supported |= STD_KEYBOARD | MULTIMEDIA |
                                               POWER_KEYS | HIDPP;
                break;
        default:
                hid_warn(hdev, "%s: unexpected device-index %d", __func__,
                         hidpp_report->device_index);
        }
}

static void logi_hidpp_recv_queue_notif(struct hid_device *hdev,
                                        struct hidpp_event *hidpp_report)
{
        /* We are called from atomic context (tasklet && djrcv->lock held) */
        struct dj_receiver_dev *djrcv_dev = hid_get_drvdata(hdev);
        const char *device_type = "UNKNOWN";
        struct dj_workitem workitem = {
                .type = WORKITEM_TYPE_EMPTY,
                .device_index = hidpp_report->device_index,
        };

        switch (hidpp_report->params[HIDPP_PARAM_PROTO_TYPE]) {
        case 0x01:
                device_type = "Bluetooth";
                /* Bluetooth connect packet contents is the same as (e)QUAD */
                logi_hidpp_dev_conn_notif_equad(hdev, hidpp_report, &workitem);
                if (!(hidpp_report->params[HIDPP_PARAM_DEVICE_INFO] &
                                                HIDPP_MANUFACTURER_MASK)) {
                        hid_info(hdev, "Non Logitech device connected on slot %d\n",
                                 hidpp_report->device_index);
                        workitem.reports_supported &= ~HIDPP;
                }
                break;
        case 0x02:
                device_type = "27 Mhz";
                logi_hidpp_dev_conn_notif_27mhz(hdev, hidpp_report, &workitem);
                break;
        case 0x03:
                device_type = "QUAD or eQUAD";
                logi_hidpp_dev_conn_notif_equad(hdev, hidpp_report, &workitem);
                break;
        case 0x04:
                device_type = "eQUAD step 4 DJ";
                logi_hidpp_dev_conn_notif_equad(hdev, hidpp_report, &workitem);
                break;
        case 0x05:
                device_type = "DFU Lite";
                break;
        case 0x06:
                device_type = "eQUAD step 4 Lite";
                logi_hidpp_dev_conn_notif_equad(hdev, hidpp_report, &workitem);
                break;
        case 0x07:
                device_type = "eQUAD step 4 Gaming";
                logi_hidpp_dev_conn_notif_equad(hdev, hidpp_report, &workitem);
                workitem.reports_supported |= STD_KEYBOARD;
                break;
        case 0x08:
                device_type = "eQUAD step 4 for gamepads";
                break;
        case 0x0a:
                device_type = "eQUAD nano Lite";
                logi_hidpp_dev_conn_notif_equad(hdev, hidpp_report, &workitem);
                break;
        case 0x0c:
                device_type = "eQUAD Lightspeed 1";
                logi_hidpp_dev_conn_notif_equad(hdev, hidpp_report, &workitem);
                workitem.reports_supported |= STD_KEYBOARD;
                break;
        case 0x0d:
                device_type = "eQUAD Lightspeed 1.1";
                logi_hidpp_dev_conn_notif_equad(hdev, hidpp_report, &workitem);
                workitem.reports_supported |= STD_KEYBOARD;
                break;
        case 0x0f:
        case 0x11:
                device_type = "eQUAD Lightspeed 1.2";
                logi_hidpp_dev_conn_notif_equad(hdev, hidpp_report, &workitem);
                workitem.reports_supported |= STD_KEYBOARD;
                break;
        }

        /* custom receiver device (eg. powerplay) */
        if (hidpp_report->device_index == 7) {
                workitem.reports_supported |= HIDPP;
        }

        if (workitem.type == WORKITEM_TYPE_EMPTY) {
                hid_warn(hdev,
                         "unusable device of type %s (0x%02x) connected on slot %d",
                         device_type,
                         hidpp_report->params[HIDPP_PARAM_PROTO_TYPE],
                         hidpp_report->device_index);
                return;
        }

        hid_info(hdev, "device of type %s (0x%02x) connected on slot %d",
                 device_type, hidpp_report->params[HIDPP_PARAM_PROTO_TYPE],
                 hidpp_report->device_index);

        kfifo_in(&djrcv_dev->notif_fifo, &workitem, sizeof(workitem));
        schedule_work(&djrcv_dev->work);
}

static void logi_dj_recv_forward_null_report(struct dj_receiver_dev *djrcv_dev,
                                             struct dj_report *dj_report)
{
        /* We are called from atomic context (tasklet && djrcv->lock held) */
        unsigned int i;
        u8 reportbuffer[MAX_REPORT_SIZE];
        struct dj_device *djdev;

        djdev = djrcv_dev->paired_dj_devices[dj_report->device_index];

        memset(reportbuffer, 0, sizeof(reportbuffer));

        for (i = 0; i < NUMBER_OF_HID_REPORTS; i++) {
                if (djdev->reports_supported & (1 << i)) {
                        reportbuffer[0] = i;
                        if (hid_input_report(djdev->hdev,
                                             HID_INPUT_REPORT,
                                             reportbuffer,
                                             hid_reportid_size_map[i], 1)) {
                                dbg_hid("hid_input_report error sending null "
                                        "report\n");
                        }
                }
        }
}

static void logi_dj_recv_forward_dj(struct dj_receiver_dev *djrcv_dev,
                                    struct dj_report *dj_report)
{
        /* We are called from atomic context (tasklet && djrcv->lock held) */
        struct dj_device *dj_device;

        dj_device = djrcv_dev->paired_dj_devices[dj_report->device_index];

        if ((dj_report->report_type > ARRAY_SIZE(hid_reportid_size_map) - 1) ||
            (hid_reportid_size_map[dj_report->report_type] == 0)) {
                dbg_hid("invalid report type:%x\n", dj_report->report_type);
                return;
        }

        if (hid_input_report(dj_device->hdev,
                        HID_INPUT_REPORT, &dj_report->report_type,
                        hid_reportid_size_map[dj_report->report_type], 1)) {
                dbg_hid("hid_input_report error\n");
        }
}

static void logi_dj_recv_forward_report(struct dj_device *dj_dev, u8 *data,
                                        int size)
{
        /* We are called from atomic context (tasklet && djrcv->lock held) */
        if (hid_input_report(dj_dev->hdev, HID_INPUT_REPORT, data, size, 1))
                dbg_hid("hid_input_report error\n");
}

static void logi_dj_recv_forward_input_report(struct hid_device *hdev,
                                              u8 *data, int size)
{
        struct dj_receiver_dev *djrcv_dev = hid_get_drvdata(hdev);
        struct dj_device *dj_dev;
        unsigned long flags;
        u8 report = data[0];
        int i;

        if (report > REPORT_TYPE_RFREPORT_LAST) {
                hid_err(hdev, "Unexpected input report number %d\n", report);
                return;
        }

        spin_lock_irqsave(&djrcv_dev->lock, flags);
        for (i = 0; i < (DJ_MAX_PAIRED_DEVICES + DJ_DEVICE_INDEX_MIN); i++) {
                dj_dev = djrcv_dev->paired_dj_devices[i];
                if (dj_dev && (dj_dev->reports_supported & BIT(report))) {
                        logi_dj_recv_forward_report(dj_dev, data, size);
                        spin_unlock_irqrestore(&djrcv_dev->lock, flags);
                        return;
                }
        }

        logi_dj_recv_queue_unknown_work(djrcv_dev);
        spin_unlock_irqrestore(&djrcv_dev->lock, flags);

        dbg_hid("No dj-devs handling input report number %d\n", report);
}

static int logi_dj_recv_send_report(struct dj_receiver_dev *djrcv_dev,
                                    struct dj_report *dj_report)
{
        struct hid_device *hdev = djrcv_dev->hidpp;
        struct hid_report *report;
        struct hid_report_enum *output_report_enum;
        u8 *data = (u8 *)(&dj_report->device_index);
        unsigned int i;

        output_report_enum = &hdev->report_enum[HID_OUTPUT_REPORT];
        report = output_report_enum->report_id_hash[REPORT_ID_DJ_SHORT];

        if (!report) {
                hid_err(hdev, "%s: unable to find dj report\n", __func__);
                return -ENODEV;
        }

        for (i = 0; i < DJREPORT_SHORT_LENGTH - 1; i++)
                report->field[0]->value[i] = data[i];

        hid_hw_request(hdev, report, HID_REQ_SET_REPORT);

        return 0;
}

static int logi_dj_recv_query_hidpp_devices(struct dj_receiver_dev *djrcv_dev)
{
        static const u8 template[] = {
                REPORT_ID_HIDPP_SHORT,
                HIDPP_RECEIVER_INDEX,
                HIDPP_SET_REGISTER,
                HIDPP_REG_CONNECTION_STATE,
                HIDPP_FAKE_DEVICE_ARRIVAL,
                0x00, 0x00
        };
        u8 *hidpp_report;
        int retval;

        hidpp_report = kmemdup(template, sizeof(template), GFP_KERNEL);
        if (!hidpp_report)
                return -ENOMEM;

        retval = hid_hw_raw_request(djrcv_dev->hidpp,
                                    REPORT_ID_HIDPP_SHORT,
                                    hidpp_report, sizeof(template),
                                    HID_OUTPUT_REPORT,
                                    HID_REQ_SET_REPORT);

        kfree(hidpp_report);
        return (retval < 0) ? retval : 0;
}

static int logi_dj_recv_query_paired_devices(struct dj_receiver_dev *djrcv_dev)
{
        struct dj_report *dj_report;
        int retval;

        djrcv_dev->last_query = jiffies;

        if (djrcv_dev->type != recvr_type_dj)
                return logi_dj_recv_query_hidpp_devices(djrcv_dev);

        dj_report = kzalloc(sizeof(struct dj_report), GFP_KERNEL);
        if (!dj_report)
                return -ENOMEM;
        dj_report->report_id = REPORT_ID_DJ_SHORT;
        dj_report->device_index = HIDPP_RECEIVER_INDEX;
        dj_report->report_type = REPORT_TYPE_CMD_GET_PAIRED_DEVICES;
        retval = logi_dj_recv_send_report(djrcv_dev, dj_report);
        kfree(dj_report);
        return retval;
}


static int logi_dj_recv_switch_to_dj_mode(struct dj_receiver_dev *djrcv_dev,
                                          unsigned timeout)
{
        struct hid_device *hdev = djrcv_dev->hidpp;
        struct dj_report *dj_report;
        u8 *buf;
        int retval = 0;

        dj_report = kzalloc(sizeof(struct dj_report), GFP_KERNEL);
        if (!dj_report)
                return -ENOMEM;

        if (djrcv_dev->type == recvr_type_dj) {
                dj_report->report_id = REPORT_ID_DJ_SHORT;
                dj_report->device_index = HIDPP_RECEIVER_INDEX;
                dj_report->report_type = REPORT_TYPE_CMD_SWITCH;
                dj_report->report_params[CMD_SWITCH_PARAM_DEVBITFIELD] = 0x3F;
                dj_report->report_params[CMD_SWITCH_PARAM_TIMEOUT_SECONDS] =
                                                                (u8)timeout;

                retval = logi_dj_recv_send_report(djrcv_dev, dj_report);

                /*
                 * Ugly sleep to work around a USB 3.0 bug when the receiver is
                 * still processing the "switch-to-dj" command while we send an
                 * other command.
                 * 50 msec should gives enough time to the receiver to be ready.
                 */
                msleep(50);

                if (retval) {
                        kfree(dj_report);
                        return retval;
                }
        }

        /*
         * Magical bits to set up hidpp notifications when the dj devices
         * are connected/disconnected.
         *
         * We can reuse dj_report because HIDPP_REPORT_SHORT_LENGTH is smaller
         * than DJREPORT_SHORT_LENGTH.
         */
        buf = (u8 *)dj_report;

        memset(buf, 0, HIDPP_REPORT_SHORT_LENGTH);

        buf[0] = REPORT_ID_HIDPP_SHORT;
        buf[1] = HIDPP_RECEIVER_INDEX;
        buf[2] = 0x80;
        buf[3] = 0x00;
        buf[4] = 0x00;
        buf[5] = 0x09;
        buf[6] = 0x00;

        retval = hid_hw_raw_request(hdev, REPORT_ID_HIDPP_SHORT, buf,
                        HIDPP_REPORT_SHORT_LENGTH, HID_OUTPUT_REPORT,
                        HID_REQ_SET_REPORT);

        kfree(dj_report);
        return retval;
}


static int logi_dj_ll_open(struct hid_device *hid)
{
        dbg_hid("%s: %s\n", __func__, hid->phys);
        return 0;

}

static void logi_dj_ll_close(struct hid_device *hid)
{
        dbg_hid("%s: %s\n", __func__, hid->phys);
}

/*
 * Register 0xB5 is "pairing information". It is solely intended for the
 * receiver, so do not overwrite the device index.
 */
static u8 unifying_pairing_query[]  = { REPORT_ID_HIDPP_SHORT,
                                        HIDPP_RECEIVER_INDEX,
                                        HIDPP_GET_LONG_REGISTER,
                                        HIDPP_REG_PAIRING_INFORMATION };
static u8 unifying_pairing_answer[] = { REPORT_ID_HIDPP_LONG,
                                        HIDPP_RECEIVER_INDEX,
                                        HIDPP_GET_LONG_REGISTER,
                                        HIDPP_REG_PAIRING_INFORMATION };

static int logi_dj_ll_raw_request(struct hid_device *hid,
                                  unsigned char reportnum, __u8 *buf,
                                  size_t count, unsigned char report_type,
                                  int reqtype)
{
        struct dj_device *djdev = hid->driver_data;
        struct dj_receiver_dev *djrcv_dev = djdev->dj_receiver_dev;
        u8 *out_buf;
        int ret;

        if ((buf[0] == REPORT_ID_HIDPP_SHORT) ||
            (buf[0] == REPORT_ID_HIDPP_LONG) ||
            (buf[0] == REPORT_ID_HIDPP_VERY_LONG)) {
                if (count < 2)
                        return -EINVAL;

                /* special case where we should not overwrite
                 * the device_index */
                if (count == 7 && !memcmp(buf, unifying_pairing_query,
                                          sizeof(unifying_pairing_query)))
                        buf[4] = (buf[4] & 0xf0) | (djdev->device_index - 1);
                else
                        buf[1] = djdev->device_index;
                return hid_hw_raw_request(djrcv_dev->hidpp, reportnum, buf,
                                count, report_type, reqtype);
        }

        if (buf[0] != REPORT_TYPE_LEDS)
                return -EINVAL;

        if (djrcv_dev->type != recvr_type_dj && count >= 2) {
                if (!djrcv_dev->keyboard) {
                        hid_warn(hid, "Received REPORT_TYPE_LEDS request before the keyboard interface was enumerated\n");
                        return 0;
                }
                /* usbhid overrides the report ID and ignores the first byte */
                return hid_hw_raw_request(djrcv_dev->keyboard, 0, buf, count,
                                          report_type, reqtype);
        }

        out_buf = kzalloc(DJREPORT_SHORT_LENGTH, GFP_ATOMIC);
        if (!out_buf)
                return -ENOMEM;

        if (count > DJREPORT_SHORT_LENGTH - 2)
                count = DJREPORT_SHORT_LENGTH - 2;

        out_buf[0] = REPORT_ID_DJ_SHORT;
        out_buf[1] = djdev->device_index;
        memcpy(out_buf + 2, buf, count);

        ret = hid_hw_raw_request(djrcv_dev->hidpp, out_buf[0], out_buf,
                DJREPORT_SHORT_LENGTH, report_type, reqtype);

        kfree(out_buf);
        return ret;
}

static void rdcat(char *rdesc, unsigned int *rsize, const char *data, unsigned int size)
{
        memcpy(rdesc + *rsize, data, size);
        *rsize += size;
}

static int logi_dj_ll_parse(struct hid_device *hid)
{
        struct dj_device *djdev = hid->driver_data;
        unsigned int rsize = 0;
        char *rdesc;
        int retval;

        dbg_hid("%s\n", __func__);

        djdev->hdev->version = 0x0111;
        djdev->hdev->country = 0x00;

        rdesc = kmalloc(MAX_RDESC_SIZE, GFP_KERNEL);
        if (!rdesc)
                return -ENOMEM;

        if (djdev->reports_supported & STD_KEYBOARD) {
                dbg_hid("%s: sending a kbd descriptor, reports_supported: %llx\n",
                        __func__, djdev->reports_supported);
                rdcat(rdesc, &rsize, kbd_descriptor, sizeof(kbd_descriptor));
        }

        if (djdev->reports_supported & STD_MOUSE) {
                dbg_hid("%s: sending a mouse descriptor, reports_supported: %llx\n",
                        __func__, djdev->reports_supported);
                if (djdev->dj_receiver_dev->type == recvr_type_gaming_hidpp ||
                    djdev->dj_receiver_dev->type == recvr_type_mouse_only)
                        rdcat(rdesc, &rsize, mse_high_res_descriptor,
                              sizeof(mse_high_res_descriptor));
                else if (djdev->dj_receiver_dev->type == recvr_type_27mhz)
                        rdcat(rdesc, &rsize, mse_27mhz_descriptor,
                              sizeof(mse_27mhz_descriptor));
                else if (recvr_type_is_bluetooth(djdev->dj_receiver_dev->type))
                        rdcat(rdesc, &rsize, mse_bluetooth_descriptor,
                              sizeof(mse_bluetooth_descriptor));
                else
                        rdcat(rdesc, &rsize, mse_descriptor,
                              sizeof(mse_descriptor));
        }

        if (djdev->reports_supported & KBD_MOUSE) {
                dbg_hid("%s: sending a kbd-mouse descriptor, reports_supported: %llx\n",
                        __func__, djdev->reports_supported);
                rdcat(rdesc, &rsize, mse5_bluetooth_descriptor,
                      sizeof(mse5_bluetooth_descriptor));
        }

        if (djdev->reports_supported & MULTIMEDIA) {
                dbg_hid("%s: sending a multimedia report descriptor: %llx\n",
                        __func__, djdev->reports_supported);
                rdcat(rdesc, &rsize, consumer_descriptor, sizeof(consumer_descriptor));
        }

        if (djdev->reports_supported & POWER_KEYS) {
                dbg_hid("%s: sending a power keys report descriptor: %llx\n",
                        __func__, djdev->reports_supported);
                rdcat(rdesc, &rsize, syscontrol_descriptor, sizeof(syscontrol_descriptor));
        }

        if (djdev->reports_supported & MEDIA_CENTER) {
                dbg_hid("%s: sending a media center report descriptor: %llx\n",
                        __func__, djdev->reports_supported);
                rdcat(rdesc, &rsize, media_descriptor, sizeof(media_descriptor));
        }

        if (djdev->reports_supported & KBD_LEDS) {
                dbg_hid("%s: need to send kbd leds report descriptor: %llx\n",
                        __func__, djdev->reports_supported);
        }

        if (djdev->reports_supported & HIDPP) {
                dbg_hid("%s: sending a HID++ descriptor, reports_supported: %llx\n",
                        __func__, djdev->reports_supported);
                rdcat(rdesc, &rsize, hidpp_descriptor,
                      sizeof(hidpp_descriptor));
        }

        retval = hid_parse_report(hid, rdesc, rsize);
        kfree(rdesc);

        return retval;
}

static int logi_dj_ll_start(struct hid_device *hid)
{
        dbg_hid("%s\n", __func__);
        return 0;
}

static void logi_dj_ll_stop(struct hid_device *hid)
{
        dbg_hid("%s\n", __func__);
}

static bool logi_dj_ll_may_wakeup(struct hid_device *hid)
{
        struct dj_device *djdev = hid->driver_data;
        struct dj_receiver_dev *djrcv_dev = djdev->dj_receiver_dev;

        return hid_hw_may_wakeup(djrcv_dev->hidpp);
}

static const struct hid_ll_driver logi_dj_ll_driver = {
        .parse = logi_dj_ll_parse,
        .start = logi_dj_ll_start,
        .stop = logi_dj_ll_stop,
        .open = logi_dj_ll_open,
        .close = logi_dj_ll_close,
        .raw_request = logi_dj_ll_raw_request,
        .may_wakeup = logi_dj_ll_may_wakeup,
};

static int logi_dj_dj_event(struct hid_device *hdev,
                             struct hid_report *report, u8 *data,
                             int size)
{
        struct dj_receiver_dev *djrcv_dev = hid_get_drvdata(hdev);
        struct dj_report *dj_report = (struct dj_report *) data;
        unsigned long flags;

        /*
         * Here we receive all data coming from iface 2, there are 3 cases:
         *
         * 1) Data is intended for this driver i. e. data contains arrival,
         * departure, etc notifications, in which case we queue them for delayed
         * processing by the work queue. We return 1 to hid-core as no further
         * processing is required from it.
         *
         * 2) Data informs a connection change, if the change means rf link
         * loss, then we must send a null report to the upper layer to discard
         * potentially pressed keys that may be repeated forever by the input
         * layer. Return 1 to hid-core as no further processing is required.
         *
         * 3) Data is an actual input event from a paired DJ device in which
         * case we forward it to the correct hid device (via hid_input_report()
         * ) and return 1 so hid-core does not anything else with it.
         */

        if ((dj_report->device_index < DJ_DEVICE_INDEX_MIN) ||
            (dj_report->device_index > DJ_DEVICE_INDEX_MAX)) {
                /*
                 * Device index is wrong, bail out.
                 * This driver can ignore safely the receiver notifications,
                 * so ignore those reports too.
                 */
                if (dj_report->device_index != DJ_RECEIVER_INDEX)
                        hid_err(hdev, "%s: invalid device index:%d\n",
                                __func__, dj_report->device_index);
                return false;
        }

        spin_lock_irqsave(&djrcv_dev->lock, flags);

        if (!djrcv_dev->paired_dj_devices[dj_report->device_index]) {
                /* received an event for an unknown device, bail out */
                logi_dj_recv_queue_notification(djrcv_dev, dj_report);
                goto out;
        }

        switch (dj_report->report_type) {
        case REPORT_TYPE_NOTIF_DEVICE_PAIRED:
                /* pairing notifications are handled above the switch */
                break;
        case REPORT_TYPE_NOTIF_DEVICE_UNPAIRED:
                logi_dj_recv_queue_notification(djrcv_dev, dj_report);
                break;
        case REPORT_TYPE_NOTIF_CONNECTION_STATUS:
                if (dj_report->report_params[CONNECTION_STATUS_PARAM_STATUS] ==
                    STATUS_LINKLOSS) {
                        logi_dj_recv_forward_null_report(djrcv_dev, dj_report);
                }
                break;
        default:
                logi_dj_recv_forward_dj(djrcv_dev, dj_report);
        }

out:
        spin_unlock_irqrestore(&djrcv_dev->lock, flags);

        return true;
}

static int logi_dj_hidpp_event(struct hid_device *hdev,
                             struct hid_report *report, u8 *data,
                             int size)
{
        struct dj_receiver_dev *djrcv_dev = hid_get_drvdata(hdev);
        struct hidpp_event *hidpp_report = (struct hidpp_event *) data;
        struct dj_device *dj_dev;
        unsigned long flags;
        u8 device_index = hidpp_report->device_index;

        if (device_index == HIDPP_RECEIVER_INDEX) {
                /* special case were the device wants to know its unifying
                 * name */
                if (size == HIDPP_REPORT_LONG_LENGTH &&
                    !memcmp(data, unifying_pairing_answer,
                            sizeof(unifying_pairing_answer)))
                        device_index = (data[4] & 0x0F) + 1;
                else
                        return false;
        }

        /*
         * Data is from the HID++ collection, in this case, we forward the
         * data to the corresponding child dj device and return 0 to hid-core
         * so he data also goes to the hidraw device of the receiver. This
         * allows a user space application to implement the full HID++ routing
         * via the receiver.
         */

        if ((device_index < DJ_DEVICE_INDEX_MIN) ||
            (device_index > DJ_DEVICE_INDEX_MAX)) {
                /*
                 * Device index is wrong, bail out.
                 * This driver can ignore safely the receiver notifications,
                 * so ignore those reports too.
                 */
                hid_err(hdev, "%s: invalid device index:%d\n", __func__,
                        hidpp_report->device_index);
                return false;
        }

        spin_lock_irqsave(&djrcv_dev->lock, flags);

        dj_dev = djrcv_dev->paired_dj_devices[device_index];

        /*
         * With 27 MHz receivers, we do not get an explicit unpair event,
         * remove the old device if the user has paired a *different* device.
         */
        if (djrcv_dev->type == recvr_type_27mhz && dj_dev &&
            hidpp_report->sub_id == REPORT_TYPE_NOTIF_DEVICE_CONNECTED &&
            hidpp_report->params[HIDPP_PARAM_PROTO_TYPE] == 0x02 &&
            hidpp_report->params[HIDPP_PARAM_27MHZ_DEVID] !=
                                                dj_dev->hdev->product) {
                struct dj_workitem workitem = {
                        .device_index = hidpp_report->device_index,
                        .type = WORKITEM_TYPE_UNPAIRED,
                };
                kfifo_in(&djrcv_dev->notif_fifo, &workitem, sizeof(workitem));
                /* logi_hidpp_recv_queue_notif will queue the work */
                dj_dev = NULL;
        }

        if (dj_dev) {
                logi_dj_recv_forward_report(dj_dev, data, size);
        } else {
                if (hidpp_report->sub_id == REPORT_TYPE_NOTIF_DEVICE_CONNECTED)
                        logi_hidpp_recv_queue_notif(hdev, hidpp_report);
                else
                        logi_dj_recv_queue_unknown_work(djrcv_dev);
        }

        spin_unlock_irqrestore(&djrcv_dev->lock, flags);

        return false;
}

static int logi_dj_raw_event(struct hid_device *hdev,
                             struct hid_report *report, u8 *data,
                             int size)
{
        struct dj_receiver_dev *djrcv_dev = hid_get_drvdata(hdev);
        dbg_hid("%s, size:%d\n", __func__, size);

        if (!djrcv_dev)
                return 0;

        if (!hdev->report_enum[HID_INPUT_REPORT].numbered) {

                if (djrcv_dev->unnumbered_application == HID_GD_KEYBOARD) {
                        /*
                         * For the keyboard, we can reuse the same report by
                         * using the second byte which is constant in the USB
                         * HID report descriptor.
                         */
                        data[1] = data[0];
                        data[0] = REPORT_TYPE_KEYBOARD;

                        logi_dj_recv_forward_input_report(hdev, data, size);

                        /* restore previous state */
                        data[0] = data[1];
                        data[1] = 0;
                }
                /*
                 * Mouse-only receivers send unnumbered mouse data. The 27 MHz
                 * receiver uses 6 byte packets, the nano receiver 8 bytes.
                 */
                if (djrcv_dev->unnumbered_application == HID_GD_MOUSE &&
                    size <= 8) {
                        u8 mouse_report[9];

                        /* Prepend report id */
                        mouse_report[0] = REPORT_TYPE_MOUSE;
                        memcpy(mouse_report + 1, data, size);
                        logi_dj_recv_forward_input_report(hdev, mouse_report,
                                                          size + 1);
                }

                return false;
        }

        switch (data[0]) {
        case REPORT_ID_DJ_SHORT:
                if (size != DJREPORT_SHORT_LENGTH) {
                        hid_err(hdev, "Short DJ report bad size (%d)", size);
                        return false;
                }
                return logi_dj_dj_event(hdev, report, data, size);
        case REPORT_ID_DJ_LONG:
                if (size != DJREPORT_LONG_LENGTH) {
                        hid_err(hdev, "Long DJ report bad size (%d)", size);
                        return false;
                }
                return logi_dj_dj_event(hdev, report, data, size);
        case REPORT_ID_HIDPP_SHORT:
                if (size != HIDPP_REPORT_SHORT_LENGTH) {
                        hid_err(hdev, "Short HID++ report bad size (%d)", size);
                        return false;
                }
                return logi_dj_hidpp_event(hdev, report, data, size);
        case REPORT_ID_HIDPP_LONG:
                if (size != HIDPP_REPORT_LONG_LENGTH) {
                        hid_err(hdev, "Long HID++ report bad size (%d)", size);
                        return false;
                }
                return logi_dj_hidpp_event(hdev, report, data, size);
        }

        logi_dj_recv_forward_input_report(hdev, data, size);

        return false;
}

static int logi_dj_probe(struct hid_device *hdev,
                         const struct hid_device_id *id)
{
        struct hid_report_enum *rep_enum;
        struct hid_report *rep;
        struct dj_receiver_dev *djrcv_dev;
        struct usb_interface *intf;
        unsigned int no_dj_interfaces = 0;
        bool has_hidpp = false;
        unsigned long flags;
        int retval;

        /*
         * Call to usbhid to fetch the HID descriptors of the current
         * interface subsequently call to the hid/hid-core to parse the
         * fetched descriptors.
         */
        retval = hid_parse(hdev);
        if (retval) {
                hid_err(hdev, "%s: parse failed\n", __func__);
                return retval;
        }

        /*
         * Some KVMs add an extra interface for e.g. mouse emulation. If we
         * treat these as logitech-dj interfaces then this causes input events
         * reported through this extra interface to not be reported correctly.
         * To avoid this, we treat these as generic-hid devices.
         */
        switch (id->driver_data) {
        case recvr_type_dj:                no_dj_interfaces = 3; break;
        case recvr_type_hidpp:                no_dj_interfaces = 2; break;
        case recvr_type_gaming_hidpp:        no_dj_interfaces = 3; break;
        case recvr_type_mouse_only:        no_dj_interfaces = 2; break;
        case recvr_type_27mhz:                no_dj_interfaces = 2; break;
        case recvr_type_bluetooth:        no_dj_interfaces = 2; break;
        case recvr_type_dinovo:                no_dj_interfaces = 2; break;
        }
        if (hid_is_usb(hdev)) {
                intf = to_usb_interface(hdev->dev.parent);
                if (intf && intf->altsetting->desc.bInterfaceNumber >=
                                                        no_dj_interfaces) {
                        hdev->quirks |= HID_QUIRK_INPUT_PER_APP;
                        return hid_hw_start(hdev, HID_CONNECT_DEFAULT);
                }
        }

        rep_enum = &hdev->report_enum[HID_INPUT_REPORT];

        /* no input reports, bail out */
        if (list_empty(&rep_enum->report_list))
                return -ENODEV;

        /*
         * Check for the HID++ application.
         * Note: we should theoretically check for HID++ and DJ
         * collections, but this will do.
         */
        list_for_each_entry(rep, &rep_enum->report_list, list) {
                if (rep->application == 0xff000001)
                        has_hidpp = true;
        }

        /*
         * Ignore interfaces without DJ/HID++ collection, they will not carry
         * any data, dont create any hid_device for them.
         */
        if (!has_hidpp && id->driver_data == recvr_type_dj)
                return -ENODEV;

        /* get the current application attached to the node */
        rep = list_first_entry(&rep_enum->report_list, struct hid_report, list);
        djrcv_dev = dj_get_receiver_dev(hdev, id->driver_data,
                                        rep->application, has_hidpp);
        if (!djrcv_dev) {
                hid_err(hdev, "%s: dj_get_receiver_dev failed\n", __func__);
                return -ENOMEM;
        }

        if (!rep_enum->numbered)
                djrcv_dev->unnumbered_application = rep->application;

        /* Starts the usb device and connects to upper interfaces hiddev and
         * hidraw */
        retval = hid_hw_start(hdev, HID_CONNECT_HIDRAW|HID_CONNECT_HIDDEV);
        if (retval) {
                hid_err(hdev, "%s: hid_hw_start returned error\n", __func__);
                goto hid_hw_start_fail;
        }

        if (has_hidpp) {
                retval = logi_dj_recv_switch_to_dj_mode(djrcv_dev, 0);
                if (retval < 0) {
                        hid_err(hdev, "%s: logi_dj_recv_switch_to_dj_mode returned error:%d\n",
                                __func__, retval);
                        goto switch_to_dj_mode_fail;
                }
        }

        /* This is enabling the polling urb on the IN endpoint */
        retval = hid_hw_open(hdev);
        if (retval < 0) {
                hid_err(hdev, "%s: hid_hw_open returned error:%d\n",
                        __func__, retval);
                goto llopen_failed;
        }

        /* Allow incoming packets to arrive: */
        hid_device_io_start(hdev);

        if (has_hidpp) {
                spin_lock_irqsave(&djrcv_dev->lock, flags);
                djrcv_dev->ready = true;
                spin_unlock_irqrestore(&djrcv_dev->lock, flags);
                retval = logi_dj_recv_query_paired_devices(djrcv_dev);
                if (retval < 0) {
                        hid_err(hdev, "%s: logi_dj_recv_query_paired_devices error:%d\n",
                                __func__, retval);
                        /*
                         * This can happen with a KVM, let the probe succeed,
                         * logi_dj_recv_queue_unknown_work will retry later.
                         */
                }
        }

        return 0;

llopen_failed:
switch_to_dj_mode_fail:
        hid_hw_stop(hdev);

hid_hw_start_fail:
        dj_put_receiver_dev(hdev);
        return retval;
}

#ifdef CONFIG_PM
static int logi_dj_reset_resume(struct hid_device *hdev)
{
        int retval;
        struct dj_receiver_dev *djrcv_dev = hid_get_drvdata(hdev);

        if (!djrcv_dev || djrcv_dev->hidpp != hdev)
                return 0;

        retval = logi_dj_recv_switch_to_dj_mode(djrcv_dev, 0);
        if (retval < 0) {
                hid_err(hdev, "%s: logi_dj_recv_switch_to_dj_mode returned error:%d\n",
                        __func__, retval);
        }

        return 0;
}
#endif

static void logi_dj_remove(struct hid_device *hdev)
{
        struct dj_receiver_dev *djrcv_dev = hid_get_drvdata(hdev);
        struct dj_device *dj_dev;
        unsigned long flags;
        int i;

        dbg_hid("%s\n", __func__);

        if (!djrcv_dev)
                return hid_hw_stop(hdev);

        /*
         * This ensures that if the work gets requeued from another
         * interface of the same receiver it will be a no-op.
         */
        spin_lock_irqsave(&djrcv_dev->lock, flags);
        djrcv_dev->ready = false;
        spin_unlock_irqrestore(&djrcv_dev->lock, flags);

        cancel_work_sync(&djrcv_dev->work);

        hid_hw_close(hdev);
        hid_hw_stop(hdev);

        /*
         * For proper operation we need access to all interfaces, so we destroy
         * the paired devices when we're unbound from any interface.
         *
         * Note we may still be bound to other interfaces, sharing the same
         * djrcv_dev, so we need locking here.
         */
        for (i = 0; i < (DJ_MAX_PAIRED_DEVICES + DJ_DEVICE_INDEX_MIN); i++) {
                spin_lock_irqsave(&djrcv_dev->lock, flags);
                dj_dev = djrcv_dev->paired_dj_devices[i];
                djrcv_dev->paired_dj_devices[i] = NULL;
                spin_unlock_irqrestore(&djrcv_dev->lock, flags);
                if (dj_dev != NULL) {
                        hid_destroy_device(dj_dev->hdev);
                        kfree(dj_dev);
                }
        }

        dj_put_receiver_dev(hdev);
}

static const struct hid_device_id logi_dj_receivers[] = {
        { /* Logitech unifying receiver (0xc52b) */
          HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH,
                USB_DEVICE_ID_LOGITECH_UNIFYING_RECEIVER),
         .driver_data = recvr_type_dj},
        { /* Logitech unifying receiver (0xc532) */
          HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH,
                USB_DEVICE_ID_LOGITECH_UNIFYING_RECEIVER_2),
         .driver_data = recvr_type_dj},

        { /* Logitech Nano mouse only receiver (0xc52f) */
          HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH,
                         USB_DEVICE_ID_LOGITECH_NANO_RECEIVER),
         .driver_data = recvr_type_mouse_only},
        { /* Logitech Nano (non DJ) receiver (0xc534) */
          HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH,
                         USB_DEVICE_ID_LOGITECH_NANO_RECEIVER_2),
         .driver_data = recvr_type_hidpp},

        { /* Logitech G700(s) receiver (0xc531) */
          HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH,
                         USB_DEVICE_ID_LOGITECH_G700_RECEIVER),
         .driver_data = recvr_type_gaming_hidpp},
        { /* Logitech G602 receiver (0xc537) */
          HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH,
                0xc537),
         .driver_data = recvr_type_gaming_hidpp},
        { /* Logitech lightspeed receiver (0xc539) */
          HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH,
                USB_DEVICE_ID_LOGITECH_NANO_RECEIVER_LIGHTSPEED_1),
         .driver_data = recvr_type_gaming_hidpp},
        { /* Logitech powerplay receiver (0xc53a) */
          HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH,
                USB_DEVICE_ID_LOGITECH_NANO_RECEIVER_POWERPLAY),
         .driver_data = recvr_type_gaming_hidpp},
        { /* Logitech lightspeed receiver (0xc53f) */
          HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH,
                USB_DEVICE_ID_LOGITECH_NANO_RECEIVER_LIGHTSPEED_1_1),
         .driver_data = recvr_type_gaming_hidpp},

        { /* Logitech 27 MHz HID++ 1.0 receiver (0xc513) */
          HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_MX3000_RECEIVER),
         .driver_data = recvr_type_27mhz},
        { /* Logitech 27 MHz HID++ 1.0 receiver (0xc517) */
          HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH,
                USB_DEVICE_ID_S510_RECEIVER_2),
         .driver_data = recvr_type_27mhz},
        { /* Logitech 27 MHz HID++ 1.0 mouse-only receiver (0xc51b) */
          HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH,
                USB_DEVICE_ID_LOGITECH_27MHZ_MOUSE_RECEIVER),
         .driver_data = recvr_type_27mhz},

        { /* Logitech MX5000 HID++ / bluetooth receiver keyboard intf. (0xc70e) */
          HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH,
                USB_DEVICE_ID_MX5000_RECEIVER_KBD_DEV),
         .driver_data = recvr_type_bluetooth},
        { /* Logitech MX5000 HID++ / bluetooth receiver mouse intf. (0xc70a) */
          HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH,
                USB_DEVICE_ID_MX5000_RECEIVER_MOUSE_DEV),
         .driver_data = recvr_type_bluetooth},
        { /* Logitech MX5500 HID++ / bluetooth receiver keyboard intf. (0xc71b) */
          HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH,
                USB_DEVICE_ID_MX5500_RECEIVER_KBD_DEV),
         .driver_data = recvr_type_bluetooth},
        { /* Logitech MX5500 HID++ / bluetooth receiver mouse intf. (0xc71c) */
          HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH,
                USB_DEVICE_ID_MX5500_RECEIVER_MOUSE_DEV),
         .driver_data = recvr_type_bluetooth},

        { /* Logitech Dinovo Edge HID++ / bluetooth receiver keyboard intf. (0xc713) */
          HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH,
                USB_DEVICE_ID_DINOVO_EDGE_RECEIVER_KBD_DEV),
         .driver_data = recvr_type_dinovo},
        { /* Logitech Dinovo Edge HID++ / bluetooth receiver mouse intf. (0xc714) */
          HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH,
                USB_DEVICE_ID_DINOVO_EDGE_RECEIVER_MOUSE_DEV),
         .driver_data = recvr_type_dinovo},
        { /* Logitech DiNovo Mini HID++ / bluetooth receiver mouse intf. (0xc71e) */
          HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH,
                USB_DEVICE_ID_DINOVO_MINI_RECEIVER_KBD_DEV),
         .driver_data = recvr_type_dinovo},
        { /* Logitech DiNovo Mini HID++ / bluetooth receiver keyboard intf. (0xc71f) */
          HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH,
                USB_DEVICE_ID_DINOVO_MINI_RECEIVER_MOUSE_DEV),
         .driver_data = recvr_type_dinovo},
        {}
};

MODULE_DEVICE_TABLE(hid, logi_dj_receivers);

static struct hid_driver logi_djreceiver_driver = {
        .name = "logitech-djreceiver",
        .id_table = logi_dj_receivers,
        .probe = logi_dj_probe,
        .remove = logi_dj_remove,
        .raw_event = logi_dj_raw_event,
#ifdef CONFIG_PM
        .reset_resume = logi_dj_reset_resume,
#endif
};

module_hid_driver(logi_djreceiver_driver);

MODULE_DESCRIPTION("HID driver for Logitech receivers");
MODULE_LICENSE("GPL");
MODULE_AUTHOR("Logitech");
MODULE_AUTHOR("Nestor Lopez Casado");
MODULE_AUTHOR("nlopezcasad@logitech.com");




















   43 



   33 




   20 



    9 
   37 









    7 


    2 


    7 












    7 






    1 





    7 
    7 























    8 












   36 





    4 


   34 
   34 
   33 
    7 
   34 






   36 
    8 








    4 




    4 




    3 
    3 
    2 
    1 


    3 
    1 




    4 



    3 



    1 







   49 









   11 










   10 









    3 

    7 




































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * tcp_diag.c        Module for monitoring TCP transport protocols sockets.
 *
 * Authors:        Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
 */

#include <linux/module.h>
#include <linux/net.h>
#include <linux/sock_diag.h>
#include <linux/inet_diag.h>

#include <linux/tcp.h>

#include <net/netlink.h>
#include <net/tcp.h>

static void tcp_diag_get_info(struct sock *sk, struct inet_diag_msg *r,
                              void *_info)
{
        struct tcp_info *info = _info;

        if (inet_sk_state_load(sk) == TCP_LISTEN) {
                r->idiag_rqueue = READ_ONCE(sk->sk_ack_backlog);
                r->idiag_wqueue = READ_ONCE(sk->sk_max_ack_backlog);
        } else if (sk->sk_type == SOCK_STREAM) {
                const struct tcp_sock *tp = tcp_sk(sk);

                r->idiag_rqueue = max_t(int, READ_ONCE(tp->rcv_nxt) -
                                             READ_ONCE(tp->copied_seq), 0);
                r->idiag_wqueue = READ_ONCE(tp->write_seq) - tp->snd_una;
        }
        if (info)
                tcp_get_info(sk, info);
}

#ifdef CONFIG_TCP_MD5SIG
static void tcp_diag_md5sig_fill(struct tcp_diag_md5sig *info,
                                 const struct tcp_md5sig_key *key)
{
        info->tcpm_family = key->family;
        info->tcpm_prefixlen = key->prefixlen;
        info->tcpm_keylen = key->keylen;
        memcpy(info->tcpm_key, key->key, key->keylen);

        if (key->family == AF_INET)
                info->tcpm_addr[0] = key->addr.a4.s_addr;
        #if IS_ENABLED(CONFIG_IPV6)
        else if (key->family == AF_INET6)
                memcpy(&info->tcpm_addr, &key->addr.a6,
                       sizeof(info->tcpm_addr));
        #endif
}

static int tcp_diag_put_md5sig(struct sk_buff *skb,
                               const struct tcp_md5sig_info *md5sig)
{
        const struct tcp_md5sig_key *key;
        struct tcp_diag_md5sig *info;
        struct nlattr *attr;
        int md5sig_count = 0;

        hlist_for_each_entry_rcu(key, &md5sig->head, node)
                md5sig_count++;
        if (md5sig_count == 0)
                return 0;

        attr = nla_reserve(skb, INET_DIAG_MD5SIG,
                           md5sig_count * sizeof(struct tcp_diag_md5sig));
        if (!attr)
                return -EMSGSIZE;

        info = nla_data(attr);
        memset(info, 0, md5sig_count * sizeof(struct tcp_diag_md5sig));
        hlist_for_each_entry_rcu(key, &md5sig->head, node) {
                tcp_diag_md5sig_fill(info++, key);
                if (--md5sig_count == 0)
                        break;
        }

        return 0;
}
#endif

static int tcp_diag_put_ulp(struct sk_buff *skb, struct sock *sk,
                            const struct tcp_ulp_ops *ulp_ops, bool net_admin)
{
        struct nlattr *nest;
        int err;

        nest = nla_nest_start_noflag(skb, INET_DIAG_ULP_INFO);
        if (!nest)
                return -EMSGSIZE;

        err = nla_put_string(skb, INET_ULP_INFO_NAME, ulp_ops->name);
        if (err)
                goto nla_failure;

        if (ulp_ops->get_info)
                err = ulp_ops->get_info(sk, skb, net_admin);
        if (err)
                goto nla_failure;

        nla_nest_end(skb, nest);
        return 0;

nla_failure:
        nla_nest_cancel(skb, nest);
        return err;
}

static int tcp_diag_get_aux(struct sock *sk, bool net_admin,
                            struct sk_buff *skb)
{
        struct inet_connection_sock *icsk = inet_csk(sk);
        const struct tcp_ulp_ops *ulp_ops;
        int err = 0;

#ifdef CONFIG_TCP_MD5SIG
        if (net_admin) {
                struct tcp_md5sig_info *md5sig;

                rcu_read_lock();
                md5sig = rcu_dereference(tcp_sk(sk)->md5sig_info);
                if (md5sig)
                        err = tcp_diag_put_md5sig(skb, md5sig);
                rcu_read_unlock();
                if (err < 0)
                        return err;
        }
#endif

        ulp_ops = icsk->icsk_ulp_ops;
        if (ulp_ops) {
                err = tcp_diag_put_ulp(skb, sk, ulp_ops, net_admin);
                if (err < 0)
                        return err;
        }

        return 0;
}

static size_t tcp_diag_get_aux_size(struct sock *sk, bool net_admin)
{
        struct inet_connection_sock *icsk = inet_csk(sk);
        size_t size = 0;

#ifdef CONFIG_TCP_MD5SIG
        if (net_admin && sk_fullsock(sk)) {
                const struct tcp_md5sig_info *md5sig;
                const struct tcp_md5sig_key *key;
                size_t md5sig_count = 0;

                rcu_read_lock();
                md5sig = rcu_dereference(tcp_sk(sk)->md5sig_info);
                if (md5sig) {
                        hlist_for_each_entry_rcu(key, &md5sig->head, node)
                                md5sig_count++;
                }
                rcu_read_unlock();
                size += nla_total_size(md5sig_count *
                                       sizeof(struct tcp_diag_md5sig));
        }
#endif

        if (sk_fullsock(sk)) {
                const struct tcp_ulp_ops *ulp_ops;

                ulp_ops = icsk->icsk_ulp_ops;
                if (ulp_ops) {
                        size += nla_total_size(0) +
                                nla_total_size(TCP_ULP_NAME_MAX);
                        if (ulp_ops->get_info_size)
                                size += ulp_ops->get_info_size(sk, net_admin);
                }
        }
        return size;
}

static void tcp_diag_dump(struct sk_buff *skb, struct netlink_callback *cb,
                          const struct inet_diag_req_v2 *r)
{
        struct inet_hashinfo *hinfo;

        hinfo = sock_net(cb->skb->sk)->ipv4.tcp_death_row.hashinfo;

        inet_diag_dump_icsk(hinfo, skb, cb, r);
}

static int tcp_diag_dump_one(struct netlink_callback *cb,
                             const struct inet_diag_req_v2 *req)
{
        struct inet_hashinfo *hinfo;

        hinfo = sock_net(cb->skb->sk)->ipv4.tcp_death_row.hashinfo;

        return inet_diag_dump_one_icsk(hinfo, cb, req);
}

#ifdef CONFIG_INET_DIAG_DESTROY
static int tcp_diag_destroy(struct sk_buff *in_skb,
                            const struct inet_diag_req_v2 *req)
{
        struct net *net = sock_net(in_skb->sk);
        struct inet_hashinfo *hinfo;
        struct sock *sk;
        int err;

        hinfo = net->ipv4.tcp_death_row.hashinfo;
        sk = inet_diag_find_one_icsk(net, hinfo, req);

        if (IS_ERR(sk))
                return PTR_ERR(sk);

        err = sock_diag_destroy(sk, ECONNABORTED);

        sock_gen_put(sk);

        return err;
}
#endif

static const struct inet_diag_handler tcp_diag_handler = {
        .owner                        = THIS_MODULE,
        .dump                        = tcp_diag_dump,
        .dump_one                = tcp_diag_dump_one,
        .idiag_get_info                = tcp_diag_get_info,
        .idiag_get_aux                = tcp_diag_get_aux,
        .idiag_get_aux_size        = tcp_diag_get_aux_size,
        .idiag_type                = IPPROTO_TCP,
        .idiag_info_size        = sizeof(struct tcp_info),
#ifdef CONFIG_INET_DIAG_DESTROY
        .destroy                = tcp_diag_destroy,
#endif
};

static int __init tcp_diag_init(void)
{
        return inet_diag_register(&tcp_diag_handler);
}

static void __exit tcp_diag_exit(void)
{
        inet_diag_unregister(&tcp_diag_handler);
}

module_init(tcp_diag_init);
module_exit(tcp_diag_exit);
MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("TCP socket monitoring via SOCK_DIAG");
MODULE_ALIAS_NET_PF_PROTO_TYPE(PF_NETLINK, NETLINK_SOCK_DIAG, 2-6 /* AF_INET - IPPROTO_TCP */);



















    4 








    4 














    4 


























1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 *   Copyright (C) 2020, Microsoft Corporation.
 *
 *   Author(s): Steve French <stfrench@microsoft.com>
 *              Suresh Jayaraman <sjayaraman@suse.de>
 *              Jeff Layton <jlayton@kernel.org>
 */

#include <linux/fs.h>
#include <linux/slab.h>
#include <linux/inet.h>
#include <linux/ctype.h>
#include "cifsglob.h"
#include "cifsproto.h"

/* extract the host portion of the UNC string */
char *extract_hostname(const char *unc)
{
        const char *src;
        char *dst, *delim;
        unsigned int len;

        /* skip double chars at beginning of string */
        /* BB: check validity of these bytes? */
        if (strlen(unc) < 3)
                return ERR_PTR(-EINVAL);
        for (src = unc; *src && *src == '\\'; src++)
                ;
        if (!*src)
                return ERR_PTR(-EINVAL);

        /* delimiter between hostname and sharename is always '\\' now */
        delim = strchr(src, '\\');
        if (!delim)
                return ERR_PTR(-EINVAL);

        len = delim - src;
        dst = kmalloc((len + 1), GFP_KERNEL);
        if (dst == NULL)
                return ERR_PTR(-ENOMEM);

        memcpy(dst, src, len);
        dst[len] = '\0';

        return dst;
}

char *extract_sharename(const char *unc)
{
        const char *src;
        char *delim, *dst;

        /* skip double chars at the beginning */
        src = unc + 2;

        /* share name is always preceded by '\\' now */
        delim = strchr(src, '\\');
        if (!delim)
                return ERR_PTR(-EINVAL);
        delim++;

        /* caller has to free the memory */
        dst = kstrdup(delim, GFP_KERNEL);
        if (!dst)
                return ERR_PTR(-ENOMEM);

        return dst;
}










































































































































































































































































































































































































































































































































































































































































































































































































































































































































    2 


    1 


    1 




    1 





















































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
// SPDX-License-Identifier: GPL-2.0
/*  Copyright(c) 2016-20 Intel Corporation. */

#include <linux/file.h>
#include <linux/freezer.h>
#include <linux/highmem.h>
#include <linux/kthread.h>
#include <linux/miscdevice.h>
#include <linux/node.h>
#include <linux/pagemap.h>
#include <linux/ratelimit.h>
#include <linux/sched/mm.h>
#include <linux/sched/signal.h>
#include <linux/slab.h>
#include <linux/sysfs.h>
#include <linux/vmalloc.h>
#include <asm/msr.h>
#include <asm/sgx.h>
#include "driver.h"
#include "encl.h"
#include "encls.h"

struct sgx_epc_section sgx_epc_sections[SGX_MAX_EPC_SECTIONS];
static int sgx_nr_epc_sections;
static struct task_struct *ksgxd_tsk;
static DECLARE_WAIT_QUEUE_HEAD(ksgxd_waitq);
static DEFINE_XARRAY(sgx_epc_address_space);

/*
 * These variables are part of the state of the reclaimer, and must be accessed
 * with sgx_reclaimer_lock acquired.
 */
static LIST_HEAD(sgx_active_page_list);
static DEFINE_SPINLOCK(sgx_reclaimer_lock);

static atomic_long_t sgx_nr_free_pages = ATOMIC_LONG_INIT(0);

/* Nodes with one or more EPC sections. */
static nodemask_t sgx_numa_mask;

/*
 * Array with one list_head for each possible NUMA node.  Each
 * list contains all the sgx_epc_section's which are on that
 * node.
 */
static struct sgx_numa_node *sgx_numa_nodes;

static LIST_HEAD(sgx_dirty_page_list);

/*
 * Reset post-kexec EPC pages to the uninitialized state. The pages are removed
 * from the input list, and made available for the page allocator. SECS pages
 * prepending their children in the input list are left intact.
 *
 * Return 0 when sanitization was successful or kthread was stopped, and the
 * number of unsanitized pages otherwise.
 */
static unsigned long __sgx_sanitize_pages(struct list_head *dirty_page_list)
{
        unsigned long left_dirty = 0;
        struct sgx_epc_page *page;
        LIST_HEAD(dirty);
        int ret;

        /* dirty_page_list is thread-local, no need for a lock: */
        while (!list_empty(dirty_page_list)) {
                if (kthread_should_stop())
                        return 0;

                page = list_first_entry(dirty_page_list, struct sgx_epc_page, list);

                /*
                 * Checking page->poison without holding the node->lock
                 * is racy, but losing the race (i.e. poison is set just
                 * after the check) just means __eremove() will be uselessly
                 * called for a page that sgx_free_epc_page() will put onto
                 * the node->sgx_poison_page_list later.
                 */
                if (page->poison) {
                        struct sgx_epc_section *section = &sgx_epc_sections[page->section];
                        struct sgx_numa_node *node = section->node;

                        spin_lock(&node->lock);
                        list_move(&page->list, &node->sgx_poison_page_list);
                        spin_unlock(&node->lock);

                        continue;
                }

                ret = __eremove(sgx_get_epc_virt_addr(page));
                if (!ret) {
                        /*
                         * page is now sanitized.  Make it available via the SGX
                         * page allocator:
                         */
                        list_del(&page->list);
                        sgx_free_epc_page(page);
                } else {
                        /* The page is not yet clean - move to the dirty list. */
                        list_move_tail(&page->list, &dirty);
                        left_dirty++;
                }

                cond_resched();
        }

        list_splice(&dirty, dirty_page_list);
        return left_dirty;
}

static bool sgx_reclaimer_age(struct sgx_epc_page *epc_page)
{
        struct sgx_encl_page *page = epc_page->owner;
        struct sgx_encl *encl = page->encl;
        struct sgx_encl_mm *encl_mm;
        bool ret = true;
        int idx;

        idx = srcu_read_lock(&encl->srcu);

        list_for_each_entry_rcu(encl_mm, &encl->mm_list, list) {
                if (!mmget_not_zero(encl_mm->mm))
                        continue;

                mmap_read_lock(encl_mm->mm);
                ret = !sgx_encl_test_and_clear_young(encl_mm->mm, page);
                mmap_read_unlock(encl_mm->mm);

                mmput_async(encl_mm->mm);

                if (!ret)
                        break;
        }

        srcu_read_unlock(&encl->srcu, idx);

        if (!ret)
                return false;

        return true;
}

static void sgx_reclaimer_block(struct sgx_epc_page *epc_page)
{
        struct sgx_encl_page *page = epc_page->owner;
        unsigned long addr = page->desc & PAGE_MASK;
        struct sgx_encl *encl = page->encl;
        int ret;

        sgx_zap_enclave_ptes(encl, addr);

        mutex_lock(&encl->lock);

        ret = __eblock(sgx_get_epc_virt_addr(epc_page));
        if (encls_failed(ret))
                ENCLS_WARN(ret, "EBLOCK");

        mutex_unlock(&encl->lock);
}

static int __sgx_encl_ewb(struct sgx_epc_page *epc_page, void *va_slot,
                          struct sgx_backing *backing)
{
        struct sgx_pageinfo pginfo;
        int ret;

        pginfo.addr = 0;
        pginfo.secs = 0;

        pginfo.contents = (unsigned long)kmap_local_page(backing->contents);
        pginfo.metadata = (unsigned long)kmap_local_page(backing->pcmd) +
                          backing->pcmd_offset;

        ret = __ewb(&pginfo, sgx_get_epc_virt_addr(epc_page), va_slot);
        set_page_dirty(backing->pcmd);
        set_page_dirty(backing->contents);

        kunmap_local((void *)(unsigned long)(pginfo.metadata -
                                              backing->pcmd_offset));
        kunmap_local((void *)(unsigned long)pginfo.contents);

        return ret;
}

void sgx_ipi_cb(void *info)
{
}

/*
 * Swap page to the regular memory transformed to the blocked state by using
 * EBLOCK, which means that it can no longer be referenced (no new TLB entries).
 *
 * The first trial just tries to write the page assuming that some other thread
 * has reset the count for threads inside the enclave by using ETRACK, and
 * previous thread count has been zeroed out. The second trial calls ETRACK
 * before EWB. If that fails we kick all the HW threads out, and then do EWB,
 * which should be guaranteed the succeed.
 */
static void sgx_encl_ewb(struct sgx_epc_page *epc_page,
                         struct sgx_backing *backing)
{
        struct sgx_encl_page *encl_page = epc_page->owner;
        struct sgx_encl *encl = encl_page->encl;
        struct sgx_va_page *va_page;
        unsigned int va_offset;
        void *va_slot;
        int ret;

        encl_page->desc &= ~SGX_ENCL_PAGE_BEING_RECLAIMED;

        va_page = list_first_entry(&encl->va_pages, struct sgx_va_page,
                                   list);
        va_offset = sgx_alloc_va_slot(va_page);
        va_slot = sgx_get_epc_virt_addr(va_page->epc_page) + va_offset;
        if (sgx_va_page_full(va_page))
                list_move_tail(&va_page->list, &encl->va_pages);

        ret = __sgx_encl_ewb(epc_page, va_slot, backing);
        if (ret == SGX_NOT_TRACKED) {
                ret = __etrack(sgx_get_epc_virt_addr(encl->secs.epc_page));
                if (ret) {
                        if (encls_failed(ret))
                                ENCLS_WARN(ret, "ETRACK");
                }

                ret = __sgx_encl_ewb(epc_page, va_slot, backing);
                if (ret == SGX_NOT_TRACKED) {
                        /*
                         * Slow path, send IPIs to kick cpus out of the
                         * enclave.  Note, it's imperative that the cpu
                         * mask is generated *after* ETRACK, else we'll
                         * miss cpus that entered the enclave between
                         * generating the mask and incrementing epoch.
                         */
                        on_each_cpu_mask(sgx_encl_cpumask(encl),
                                         sgx_ipi_cb, NULL, 1);
                        ret = __sgx_encl_ewb(epc_page, va_slot, backing);
                }
        }

        if (ret) {
                if (encls_failed(ret))
                        ENCLS_WARN(ret, "EWB");

                sgx_free_va_slot(va_page, va_offset);
        } else {
                encl_page->desc |= va_offset;
                encl_page->va_page = va_page;
        }
}

static void sgx_reclaimer_write(struct sgx_epc_page *epc_page,
                                struct sgx_backing *backing)
{
        struct sgx_encl_page *encl_page = epc_page->owner;
        struct sgx_encl *encl = encl_page->encl;
        struct sgx_backing secs_backing;
        int ret;

        mutex_lock(&encl->lock);

        sgx_encl_ewb(epc_page, backing);
        encl_page->epc_page = NULL;
        encl->secs_child_cnt--;
        sgx_encl_put_backing(backing);

        if (!encl->secs_child_cnt && test_bit(SGX_ENCL_INITIALIZED, &encl->flags)) {
                ret = sgx_encl_alloc_backing(encl, PFN_DOWN(encl->size),
                                           &secs_backing);
                if (ret)
                        goto out;

                sgx_encl_ewb(encl->secs.epc_page, &secs_backing);

                sgx_encl_free_epc_page(encl->secs.epc_page);
                encl->secs.epc_page = NULL;

                sgx_encl_put_backing(&secs_backing);
        }

out:
        mutex_unlock(&encl->lock);
}

/*
 * Take a fixed number of pages from the head of the active page pool and
 * reclaim them to the enclave's private shmem files. Skip the pages, which have
 * been accessed since the last scan. Move those pages to the tail of active
 * page pool so that the pages get scanned in LRU like fashion.
 *
 * Batch process a chunk of pages (at the moment 16) in order to degrade amount
 * of IPI's and ETRACK's potentially required. sgx_encl_ewb() does degrade a bit
 * among the HW threads with three stage EWB pipeline (EWB, ETRACK + EWB and IPI
 * + EWB) but not sufficiently. Reclaiming one page at a time would also be
 * problematic as it would increase the lock contention too much, which would
 * halt forward progress.
 */
static void sgx_reclaim_pages(void)
{
        struct sgx_epc_page *chunk[SGX_NR_TO_SCAN];
        struct sgx_backing backing[SGX_NR_TO_SCAN];
        struct sgx_encl_page *encl_page;
        struct sgx_epc_page *epc_page;
        pgoff_t page_index;
        int cnt = 0;
        int ret;
        int i;

        spin_lock(&sgx_reclaimer_lock);
        for (i = 0; i < SGX_NR_TO_SCAN; i++) {
                if (list_empty(&sgx_active_page_list))
                        break;

                epc_page = list_first_entry(&sgx_active_page_list,
                                            struct sgx_epc_page, list);
                list_del_init(&epc_page->list);
                encl_page = epc_page->owner;

                if (kref_get_unless_zero(&encl_page->encl->refcount) != 0)
                        chunk[cnt++] = epc_page;
                else
                        /* The owner is freeing the page. No need to add the
                         * page back to the list of reclaimable pages.
                         */
                        epc_page->flags &= ~SGX_EPC_PAGE_RECLAIMER_TRACKED;
        }
        spin_unlock(&sgx_reclaimer_lock);

        for (i = 0; i < cnt; i++) {
                epc_page = chunk[i];
                encl_page = epc_page->owner;

                if (!sgx_reclaimer_age(epc_page))
                        goto skip;

                page_index = PFN_DOWN(encl_page->desc - encl_page->encl->base);

                mutex_lock(&encl_page->encl->lock);
                ret = sgx_encl_alloc_backing(encl_page->encl, page_index, &backing[i]);
                if (ret) {
                        mutex_unlock(&encl_page->encl->lock);
                        goto skip;
                }

                encl_page->desc |= SGX_ENCL_PAGE_BEING_RECLAIMED;
                mutex_unlock(&encl_page->encl->lock);
                continue;

skip:
                spin_lock(&sgx_reclaimer_lock);
                list_add_tail(&epc_page->list, &sgx_active_page_list);
                spin_unlock(&sgx_reclaimer_lock);

                kref_put(&encl_page->encl->refcount, sgx_encl_release);

                chunk[i] = NULL;
        }

        for (i = 0; i < cnt; i++) {
                epc_page = chunk[i];
                if (epc_page)
                        sgx_reclaimer_block(epc_page);
        }

        for (i = 0; i < cnt; i++) {
                epc_page = chunk[i];
                if (!epc_page)
                        continue;

                encl_page = epc_page->owner;
                sgx_reclaimer_write(epc_page, &backing[i]);

                kref_put(&encl_page->encl->refcount, sgx_encl_release);
                epc_page->flags &= ~SGX_EPC_PAGE_RECLAIMER_TRACKED;

                sgx_free_epc_page(epc_page);
        }
}

static bool sgx_should_reclaim(unsigned long watermark)
{
        return atomic_long_read(&sgx_nr_free_pages) < watermark &&
               !list_empty(&sgx_active_page_list);
}

/*
 * sgx_reclaim_direct() should be called (without enclave's mutex held)
 * in locations where SGX memory resources might be low and might be
 * needed in order to make forward progress.
 */
void sgx_reclaim_direct(void)
{
        if (sgx_should_reclaim(SGX_NR_LOW_PAGES))
                sgx_reclaim_pages();
}

static int ksgxd(void *p)
{
        set_freezable();

        /*
         * Sanitize pages in order to recover from kexec(). The 2nd pass is
         * required for SECS pages, whose child pages blocked EREMOVE.
         */
        __sgx_sanitize_pages(&sgx_dirty_page_list);
        WARN_ON(__sgx_sanitize_pages(&sgx_dirty_page_list));

        while (!kthread_should_stop()) {
                if (try_to_freeze())
                        continue;

                wait_event_freezable(ksgxd_waitq,
                                     kthread_should_stop() ||
                                     sgx_should_reclaim(SGX_NR_HIGH_PAGES));

                if (sgx_should_reclaim(SGX_NR_HIGH_PAGES))
                        sgx_reclaim_pages();

                cond_resched();
        }

        return 0;
}

static bool __init sgx_page_reclaimer_init(void)
{
        struct task_struct *tsk;

        tsk = kthread_run(ksgxd, NULL, "ksgxd");
        if (IS_ERR(tsk))
                return false;

        ksgxd_tsk = tsk;

        return true;
}

bool current_is_ksgxd(void)
{
        return current == ksgxd_tsk;
}

static struct sgx_epc_page *__sgx_alloc_epc_page_from_node(int nid)
{
        struct sgx_numa_node *node = &sgx_numa_nodes[nid];
        struct sgx_epc_page *page = NULL;

        spin_lock(&node->lock);

        if (list_empty(&node->free_page_list)) {
                spin_unlock(&node->lock);
                return NULL;
        }

        page = list_first_entry(&node->free_page_list, struct sgx_epc_page, list);
        list_del_init(&page->list);
        page->flags = 0;

        spin_unlock(&node->lock);
        atomic_long_dec(&sgx_nr_free_pages);

        return page;
}

/**
 * __sgx_alloc_epc_page() - Allocate an EPC page
 *
 * Iterate through NUMA nodes and reserve ia free EPC page to the caller. Start
 * from the NUMA node, where the caller is executing.
 *
 * Return:
 * - an EPC page:        A borrowed EPC pages were available.
 * - NULL:                Out of EPC pages.
 */
struct sgx_epc_page *__sgx_alloc_epc_page(void)
{
        struct sgx_epc_page *page;
        int nid_of_current = numa_node_id();
        int nid_start, nid;

        /*
         * Try local node first. If it doesn't have an EPC section,
         * fall back to the non-local NUMA nodes.
         */
        if (node_isset(nid_of_current, sgx_numa_mask))
                nid_start = nid_of_current;
        else
                nid_start = next_node_in(nid_of_current, sgx_numa_mask);

        nid = nid_start;
        do {
                page = __sgx_alloc_epc_page_from_node(nid);
                if (page)
                        return page;

                nid = next_node_in(nid, sgx_numa_mask);
        } while (nid != nid_start);

        return ERR_PTR(-ENOMEM);
}

/**
 * sgx_mark_page_reclaimable() - Mark a page as reclaimable
 * @page:        EPC page
 *
 * Mark a page as reclaimable and add it to the active page list. Pages
 * are automatically removed from the active list when freed.
 */
void sgx_mark_page_reclaimable(struct sgx_epc_page *page)
{
        spin_lock(&sgx_reclaimer_lock);
        page->flags |= SGX_EPC_PAGE_RECLAIMER_TRACKED;
        list_add_tail(&page->list, &sgx_active_page_list);
        spin_unlock(&sgx_reclaimer_lock);
}

/**
 * sgx_unmark_page_reclaimable() - Remove a page from the reclaim list
 * @page:        EPC page
 *
 * Clear the reclaimable flag and remove the page from the active page list.
 *
 * Return:
 *   0 on success,
 *   -EBUSY if the page is in the process of being reclaimed
 */
int sgx_unmark_page_reclaimable(struct sgx_epc_page *page)
{
        spin_lock(&sgx_reclaimer_lock);
        if (page->flags & SGX_EPC_PAGE_RECLAIMER_TRACKED) {
                /* The page is being reclaimed. */
                if (list_empty(&page->list)) {
                        spin_unlock(&sgx_reclaimer_lock);
                        return -EBUSY;
                }

                list_del(&page->list);
                page->flags &= ~SGX_EPC_PAGE_RECLAIMER_TRACKED;
        }
        spin_unlock(&sgx_reclaimer_lock);

        return 0;
}

/**
 * sgx_alloc_epc_page() - Allocate an EPC page
 * @owner:        the owner of the EPC page
 * @reclaim:        reclaim pages if necessary
 *
 * Iterate through EPC sections and borrow a free EPC page to the caller. When a
 * page is no longer needed it must be released with sgx_free_epc_page(). If
 * @reclaim is set to true, directly reclaim pages when we are out of pages. No
 * mm's can be locked when @reclaim is set to true.
 *
 * Finally, wake up ksgxd when the number of pages goes below the watermark
 * before returning back to the caller.
 *
 * Return:
 *   an EPC page,
 *   -errno on error
 */
struct sgx_epc_page *sgx_alloc_epc_page(void *owner, bool reclaim)
{
        struct sgx_epc_page *page;

        for ( ; ; ) {
                page = __sgx_alloc_epc_page();
                if (!IS_ERR(page)) {
                        page->owner = owner;
                        break;
                }

                if (list_empty(&sgx_active_page_list))
                        return ERR_PTR(-ENOMEM);

                if (!reclaim) {
                        page = ERR_PTR(-EBUSY);
                        break;
                }

                if (signal_pending(current)) {
                        page = ERR_PTR(-ERESTARTSYS);
                        break;
                }

                sgx_reclaim_pages();
                cond_resched();
        }

        if (sgx_should_reclaim(SGX_NR_LOW_PAGES))
                wake_up(&ksgxd_waitq);

        return page;
}

/**
 * sgx_free_epc_page() - Free an EPC page
 * @page:        an EPC page
 *
 * Put the EPC page back to the list of free pages. It's the caller's
 * responsibility to make sure that the page is in uninitialized state. In other
 * words, do EREMOVE, EWB or whatever operation is necessary before calling
 * this function.
 */
void sgx_free_epc_page(struct sgx_epc_page *page)
{
        struct sgx_epc_section *section = &sgx_epc_sections[page->section];
        struct sgx_numa_node *node = section->node;

        spin_lock(&node->lock);

        page->owner = NULL;
        if (page->poison)
                list_add(&page->list, &node->sgx_poison_page_list);
        else
                list_add_tail(&page->list, &node->free_page_list);
        page->flags = SGX_EPC_PAGE_IS_FREE;

        spin_unlock(&node->lock);
        atomic_long_inc(&sgx_nr_free_pages);
}

static bool __init sgx_setup_epc_section(u64 phys_addr, u64 size,
                                         unsigned long index,
                                         struct sgx_epc_section *section)
{
        unsigned long nr_pages = size >> PAGE_SHIFT;
        unsigned long i;

        section->virt_addr = memremap(phys_addr, size, MEMREMAP_WB);
        if (!section->virt_addr)
                return false;

        section->pages = vmalloc_array(nr_pages, sizeof(struct sgx_epc_page));
        if (!section->pages) {
                memunmap(section->virt_addr);
                return false;
        }

        section->phys_addr = phys_addr;
        xa_store_range(&sgx_epc_address_space, section->phys_addr,
                       phys_addr + size - 1, section, GFP_KERNEL);

        for (i = 0; i < nr_pages; i++) {
                section->pages[i].section = index;
                section->pages[i].flags = 0;
                section->pages[i].owner = NULL;
                section->pages[i].poison = 0;
                list_add_tail(&section->pages[i].list, &sgx_dirty_page_list);
        }

        return true;
}

bool arch_is_platform_page(u64 paddr)
{
        return !!xa_load(&sgx_epc_address_space, paddr);
}
EXPORT_SYMBOL_GPL(arch_is_platform_page);

static struct sgx_epc_page *sgx_paddr_to_page(u64 paddr)
{
        struct sgx_epc_section *section;

        section = xa_load(&sgx_epc_address_space, paddr);
        if (!section)
                return NULL;

        return &section->pages[PFN_DOWN(paddr - section->phys_addr)];
}

/*
 * Called in process context to handle a hardware reported
 * error in an SGX EPC page.
 * If the MF_ACTION_REQUIRED bit is set in flags, then the
 * context is the task that consumed the poison data. Otherwise
 * this is called from a kernel thread unrelated to the page.
 */
int arch_memory_failure(unsigned long pfn, int flags)
{
        struct sgx_epc_page *page = sgx_paddr_to_page(pfn << PAGE_SHIFT);
        struct sgx_epc_section *section;
        struct sgx_numa_node *node;

        /*
         * mm/memory-failure.c calls this routine for all errors
         * where there isn't a "struct page" for the address. But that
         * includes other address ranges besides SGX.
         */
        if (!page)
                return -ENXIO;

        /*
         * If poison was consumed synchronously. Send a SIGBUS to
         * the task. Hardware has already exited the SGX enclave and
         * will not allow re-entry to an enclave that has a memory
         * error. The signal may help the task understand why the
         * enclave is broken.
         */
        if (flags & MF_ACTION_REQUIRED)
                force_sig(SIGBUS);

        section = &sgx_epc_sections[page->section];
        node = section->node;

        spin_lock(&node->lock);

        /* Already poisoned? Nothing more to do */
        if (page->poison)
                goto out;

        page->poison = 1;

        /*
         * If the page is on a free list, move it to the per-node
         * poison page list.
         */
        if (page->flags & SGX_EPC_PAGE_IS_FREE) {
                list_move(&page->list, &node->sgx_poison_page_list);
                goto out;
        }

        sgx_unmark_page_reclaimable(page);

        /*
         * TBD: Add additional plumbing to enable pre-emptive
         * action for asynchronous poison notification. Until
         * then just hope that the poison:
         * a) is not accessed - sgx_free_epc_page() will deal with it
         *    when the user gives it back
         * b) results in a recoverable machine check rather than
         *    a fatal one
         */
out:
        spin_unlock(&node->lock);
        return 0;
}

/*
 * A section metric is concatenated in a way that @low bits 12-31 define the
 * bits 12-31 of the metric and @high bits 0-19 define the bits 32-51 of the
 * metric.
 */
static inline u64 __init sgx_calc_section_metric(u64 low, u64 high)
{
        return (low & GENMASK_ULL(31, 12)) +
               ((high & GENMASK_ULL(19, 0)) << 32);
}

#ifdef CONFIG_NUMA
static ssize_t sgx_total_bytes_show(struct device *dev, struct device_attribute *attr, char *buf)
{
        return sysfs_emit(buf, "%lu\n", sgx_numa_nodes[dev->id].size);
}
static DEVICE_ATTR_RO(sgx_total_bytes);

static umode_t arch_node_attr_is_visible(struct kobject *kobj,
                struct attribute *attr, int idx)
{
        /* Make all x86/ attributes invisible when SGX is not initialized: */
        if (nodes_empty(sgx_numa_mask))
                return 0;

        return attr->mode;
}

static struct attribute *arch_node_dev_attrs[] = {
        &dev_attr_sgx_total_bytes.attr,
        NULL,
};

const struct attribute_group arch_node_dev_group = {
        .name = "x86",
        .attrs = arch_node_dev_attrs,
        .is_visible = arch_node_attr_is_visible,
};

static void __init arch_update_sysfs_visibility(int nid)
{
        struct node *node = node_devices[nid];
        int ret;

        ret = sysfs_update_group(&node->dev.kobj, &arch_node_dev_group);

        if (ret)
                pr_err("sysfs update failed (%d), files may be invisible", ret);
}
#else /* !CONFIG_NUMA */
static void __init arch_update_sysfs_visibility(int nid) {}
#endif

static bool __init sgx_page_cache_init(void)
{
        u32 eax, ebx, ecx, edx, type;
        u64 pa, size;
        int nid;
        int i;

        sgx_numa_nodes = kmalloc_array(num_possible_nodes(), sizeof(*sgx_numa_nodes), GFP_KERNEL);
        if (!sgx_numa_nodes)
                return false;

        for (i = 0; i < ARRAY_SIZE(sgx_epc_sections); i++) {
                cpuid_count(SGX_CPUID, i + SGX_CPUID_EPC, &eax, &ebx, &ecx, &edx);

                type = eax & SGX_CPUID_EPC_MASK;
                if (type == SGX_CPUID_EPC_INVALID)
                        break;

                if (type != SGX_CPUID_EPC_SECTION) {
                        pr_err_once("Unknown EPC section type: %u\n", type);
                        break;
                }

                pa   = sgx_calc_section_metric(eax, ebx);
                size = sgx_calc_section_metric(ecx, edx);

                pr_info("EPC section 0x%llx-0x%llx\n", pa, pa + size - 1);

                if (!sgx_setup_epc_section(pa, size, i, &sgx_epc_sections[i])) {
                        pr_err("No free memory for an EPC section\n");
                        break;
                }

                nid = numa_map_to_online_node(phys_to_target_node(pa));
                if (nid == NUMA_NO_NODE) {
                        /* The physical address is already printed above. */
                        pr_warn(FW_BUG "Unable to map EPC section to online node. Fallback to the NUMA node 0.\n");
                        nid = 0;
                }

                if (!node_isset(nid, sgx_numa_mask)) {
                        spin_lock_init(&sgx_numa_nodes[nid].lock);
                        INIT_LIST_HEAD(&sgx_numa_nodes[nid].free_page_list);
                        INIT_LIST_HEAD(&sgx_numa_nodes[nid].sgx_poison_page_list);
                        node_set(nid, sgx_numa_mask);
                        sgx_numa_nodes[nid].size = 0;

                        /* Make SGX-specific node sysfs files visible: */
                        arch_update_sysfs_visibility(nid);
                }

                sgx_epc_sections[i].node =  &sgx_numa_nodes[nid];
                sgx_numa_nodes[nid].size += size;

                sgx_nr_epc_sections++;
        }

        if (!sgx_nr_epc_sections) {
                pr_err("There are zero EPC sections.\n");
                return false;
        }

        for_each_online_node(nid) {
                if (!node_isset(nid, sgx_numa_mask) &&
                    node_state(nid, N_MEMORY) && node_state(nid, N_CPU))
                        pr_info("node%d has both CPUs and memory but doesn't have an EPC section\n",
                                nid);
        }

        return true;
}

/*
 * Update the SGX_LEPUBKEYHASH MSRs to the values specified by caller.
 * Bare-metal driver requires to update them to hash of enclave's signer
 * before EINIT. KVM needs to update them to guest's virtual MSR values
 * before doing EINIT from guest.
 */
void sgx_update_lepubkeyhash(u64 *lepubkeyhash)
{
        int i;

        WARN_ON_ONCE(preemptible());

        for (i = 0; i < 4; i++)
                wrmsrq(MSR_IA32_SGXLEPUBKEYHASH0 + i, lepubkeyhash[i]);
}

const struct file_operations sgx_provision_fops = {
        .owner                        = THIS_MODULE,
};

static struct miscdevice sgx_dev_provision = {
        .minor = MISC_DYNAMIC_MINOR,
        .name = "sgx_provision",
        .nodename = "sgx_provision",
        .fops = &sgx_provision_fops,
};

/**
 * sgx_set_attribute() - Update allowed attributes given file descriptor
 * @allowed_attributes:                Pointer to allowed enclave attributes
 * @attribute_fd:                File descriptor for specific attribute
 *
 * Append enclave attribute indicated by file descriptor to allowed
 * attributes. Currently only SGX_ATTR_PROVISIONKEY indicated by
 * /dev/sgx_provision is supported.
 *
 * Return:
 * -0:                SGX_ATTR_PROVISIONKEY is appended to allowed_attributes
 * -EINVAL:        Invalid, or not supported file descriptor
 */
int sgx_set_attribute(unsigned long *allowed_attributes,
                      unsigned int attribute_fd)
{
        CLASS(fd, f)(attribute_fd);

        if (fd_empty(f))
                return -EINVAL;

        if (fd_file(f)->f_op != &sgx_provision_fops)
                return -EINVAL;

        *allowed_attributes |= SGX_ATTR_PROVISIONKEY;
        return 0;
}
EXPORT_SYMBOL_GPL(sgx_set_attribute);

static int __init sgx_init(void)
{
        int ret;
        int i;

        if (!cpu_feature_enabled(X86_FEATURE_SGX))
                return -ENODEV;

        if (!sgx_page_cache_init())
                return -ENOMEM;

        if (!sgx_page_reclaimer_init()) {
                ret = -ENOMEM;
                goto err_page_cache;
        }

        ret = misc_register(&sgx_dev_provision);
        if (ret)
                goto err_kthread;

        /*
         * Always try to initialize the native *and* KVM drivers.
         * The KVM driver is less picky than the native one and
         * can function if the native one is not supported on the
         * current system or fails to initialize.
         *
         * Error out only if both fail to initialize.
         */
        ret = sgx_drv_init();

        if (sgx_vepc_init() && ret)
                goto err_provision;

        return 0;

err_provision:
        misc_deregister(&sgx_dev_provision);

err_kthread:
        kthread_stop(ksgxd_tsk);

err_page_cache:
        for (i = 0; i < sgx_nr_epc_sections; i++) {
                vfree(sgx_epc_sections[i].pages);
                memunmap(sgx_epc_sections[i].virt_addr);
        }

        return ret;
}

device_initcall(sgx_init);





































    1 






    1 





























































































































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
// SPDX-License-Identifier: GPL-2.0-only

#include <linux/net_tstamp.h>
#include <linux/ptp_clock_kernel.h>

#include "netlink.h"
#include "common.h"
#include "bitset.h"
#include "../core/dev.h"
#include "ts.h"

struct tsconfig_req_info {
        struct ethnl_req_info base;
};

struct tsconfig_reply_data {
        struct ethnl_reply_data                base;
        struct hwtstamp_provider_desc        hwprov_desc;
        struct {
                u32 tx_type;
                u32 rx_filter;
                u32 flags;
        } hwtst_config;
};

#define TSCONFIG_REPDATA(__reply_base) \
        container_of(__reply_base, struct tsconfig_reply_data, base)

const struct nla_policy ethnl_tsconfig_get_policy[ETHTOOL_A_TSCONFIG_HEADER + 1] = {
        [ETHTOOL_A_TSCONFIG_HEADER]                =
                NLA_POLICY_NESTED(ethnl_header_policy),
};

static int tsconfig_prepare_data(const struct ethnl_req_info *req_base,
                                 struct ethnl_reply_data *reply_base,
                                 const struct genl_info *info)
{
        struct tsconfig_reply_data *data = TSCONFIG_REPDATA(reply_base);
        struct hwtstamp_provider *hwprov = NULL;
        struct net_device *dev = reply_base->dev;
        struct kernel_hwtstamp_config cfg = {};
        int ret;

        if (!dev->netdev_ops->ndo_hwtstamp_get)
                return -EOPNOTSUPP;

        ret = ethnl_ops_begin(dev);
        if (ret < 0)
                return ret;

        ret = dev_get_hwtstamp_phylib(dev, &cfg);
        if (ret)
                goto out;

        data->hwtst_config.tx_type = BIT(cfg.tx_type);
        data->hwtst_config.rx_filter = BIT(cfg.rx_filter);
        data->hwtst_config.flags = cfg.flags;

        data->hwprov_desc.index = -1;
        hwprov = rtnl_dereference(dev->hwprov);
        if (hwprov) {
                data->hwprov_desc.index = hwprov->desc.index;
                data->hwprov_desc.qualifier = hwprov->desc.qualifier;
        } else {
                struct kernel_ethtool_ts_info ts_info = {};

                ts_info.phc_index = -1;
                ret = __ethtool_get_ts_info(dev, &ts_info);
                if (ret)
                        goto out;

                if (ts_info.phc_index == -1)
                        return -ENODEV;

                data->hwprov_desc.index = ts_info.phc_index;
                data->hwprov_desc.qualifier = ts_info.phc_qualifier;
        }

out:
        ethnl_ops_complete(dev);
        return ret;
}

static int tsconfig_reply_size(const struct ethnl_req_info *req_base,
                               const struct ethnl_reply_data *reply_base)
{
        const struct tsconfig_reply_data *data = TSCONFIG_REPDATA(reply_base);
        bool compact = req_base->flags & ETHTOOL_FLAG_COMPACT_BITSETS;
        int len = 0;
        int ret;

        BUILD_BUG_ON(__HWTSTAMP_TX_CNT > 32);
        BUILD_BUG_ON(__HWTSTAMP_FILTER_CNT > 32);
        BUILD_BUG_ON(__HWTSTAMP_FLAG_CNT > 32);

        if (data->hwtst_config.flags) {
                ret = ethnl_bitset32_size(&data->hwtst_config.flags,
                                          NULL, __HWTSTAMP_FLAG_CNT,
                                          ts_flags_names, compact);
                if (ret < 0)
                        return ret;
                len += ret;        /* _TSCONFIG_HWTSTAMP_FLAGS */
        }

        if (data->hwtst_config.tx_type) {
                ret = ethnl_bitset32_size(&data->hwtst_config.tx_type,
                                          NULL, __HWTSTAMP_TX_CNT,
                                          ts_tx_type_names, compact);
                if (ret < 0)
                        return ret;
                len += ret;        /* _TSCONFIG_TX_TYPES */
        }
        if (data->hwtst_config.rx_filter) {
                ret = ethnl_bitset32_size(&data->hwtst_config.rx_filter,
                                          NULL, __HWTSTAMP_FILTER_CNT,
                                          ts_rx_filter_names, compact);
                if (ret < 0)
                        return ret;
                len += ret;        /* _TSCONFIG_RX_FILTERS */
        }

        if (data->hwprov_desc.index >= 0)
                /* _TSCONFIG_HWTSTAMP_PROVIDER */
                len += nla_total_size(0) +
                       2 * nla_total_size(sizeof(u32));

        return len;
}

static int tsconfig_fill_reply(struct sk_buff *skb,
                               const struct ethnl_req_info *req_base,
                               const struct ethnl_reply_data *reply_base)
{
        const struct tsconfig_reply_data *data = TSCONFIG_REPDATA(reply_base);
        bool compact = req_base->flags & ETHTOOL_FLAG_COMPACT_BITSETS;
        int ret;

        if (data->hwtst_config.flags) {
                ret = ethnl_put_bitset32(skb, ETHTOOL_A_TSCONFIG_HWTSTAMP_FLAGS,
                                         &data->hwtst_config.flags, NULL,
                                         __HWTSTAMP_FLAG_CNT,
                                         ts_flags_names, compact);
                if (ret < 0)
                        return ret;
        }

        if (data->hwtst_config.tx_type) {
                ret = ethnl_put_bitset32(skb, ETHTOOL_A_TSCONFIG_TX_TYPES,
                                         &data->hwtst_config.tx_type, NULL,
                                         __HWTSTAMP_TX_CNT,
                                         ts_tx_type_names, compact);
                if (ret < 0)
                        return ret;
        }

        if (data->hwtst_config.rx_filter) {
                ret = ethnl_put_bitset32(skb, ETHTOOL_A_TSCONFIG_RX_FILTERS,
                                         &data->hwtst_config.rx_filter,
                                         NULL, __HWTSTAMP_FILTER_CNT,
                                         ts_rx_filter_names, compact);
                if (ret < 0)
                        return ret;
        }

        if (data->hwprov_desc.index >= 0) {
                struct nlattr *nest;

                nest = nla_nest_start(skb, ETHTOOL_A_TSCONFIG_HWTSTAMP_PROVIDER);
                if (!nest)
                        return -EMSGSIZE;

                if (nla_put_u32(skb, ETHTOOL_A_TS_HWTSTAMP_PROVIDER_INDEX,
                                data->hwprov_desc.index) ||
                    nla_put_u32(skb,
                                ETHTOOL_A_TS_HWTSTAMP_PROVIDER_QUALIFIER,
                                data->hwprov_desc.qualifier)) {
                        nla_nest_cancel(skb, nest);
                        return -EMSGSIZE;
                }

                nla_nest_end(skb, nest);
        }
        return 0;
}

/* TSCONFIG_SET */
const struct nla_policy ethnl_tsconfig_set_policy[ETHTOOL_A_TSCONFIG_MAX + 1] = {
        [ETHTOOL_A_TSCONFIG_HEADER] = NLA_POLICY_NESTED(ethnl_header_policy),
        [ETHTOOL_A_TSCONFIG_HWTSTAMP_PROVIDER] =
                NLA_POLICY_NESTED(ethnl_ts_hwtst_prov_policy),
        [ETHTOOL_A_TSCONFIG_HWTSTAMP_FLAGS] = { .type = NLA_NESTED },
        [ETHTOOL_A_TSCONFIG_RX_FILTERS] = { .type = NLA_NESTED },
        [ETHTOOL_A_TSCONFIG_TX_TYPES] = { .type = NLA_NESTED },
};

static int tsconfig_send_reply(struct net_device *dev, struct genl_info *info)
{
        struct tsconfig_reply_data *reply_data;
        struct tsconfig_req_info *req_info;
        struct sk_buff *rskb;
        void *reply_payload;
        int reply_len = 0;
        int ret;

        req_info = kzalloc(sizeof(*req_info), GFP_KERNEL);
        if (!req_info)
                return -ENOMEM;
        reply_data = kmalloc(sizeof(*reply_data), GFP_KERNEL);
        if (!reply_data) {
                kfree(req_info);
                return -ENOMEM;
        }

        ASSERT_RTNL();
        reply_data->base.dev = dev;
        ret = tsconfig_prepare_data(&req_info->base, &reply_data->base, info);
        if (ret < 0)
                goto err_cleanup;

        ret = tsconfig_reply_size(&req_info->base, &reply_data->base);
        if (ret < 0)
                goto err_cleanup;

        reply_len = ret + ethnl_reply_header_size();
        rskb = ethnl_reply_init(reply_len, dev, ETHTOOL_MSG_TSCONFIG_SET_REPLY,
                                ETHTOOL_A_TSCONFIG_HEADER, info, &reply_payload);
        if (!rskb)
                goto err_cleanup;

        ret = tsconfig_fill_reply(rskb, &req_info->base, &reply_data->base);
        if (ret < 0)
                goto err_cleanup;

        genlmsg_end(rskb, reply_payload);
        ret = genlmsg_reply(rskb, info);

err_cleanup:
        kfree(reply_data);
        kfree(req_info);
        return ret;
}

static int ethnl_set_tsconfig_validate(struct ethnl_req_info *req_base,
                                       struct genl_info *info)
{
        const struct net_device_ops *ops = req_base->dev->netdev_ops;

        if (!ops->ndo_hwtstamp_set || !ops->ndo_hwtstamp_get)
                return -EOPNOTSUPP;

        return 1;
}

static struct hwtstamp_provider *
tsconfig_set_hwprov_from_desc(struct net_device *dev,
                              struct genl_info *info,
                              struct hwtstamp_provider_desc *hwprov_desc)
{
        struct kernel_ethtool_ts_info ts_info;
        struct hwtstamp_provider *hwprov;
        struct nlattr **tb = info->attrs;
        struct phy_device *phy = NULL;
        enum hwtstamp_source source;
        int ret;

        ret = ethtool_net_get_ts_info_by_phc(dev, &ts_info, hwprov_desc);
        if (!ret) {
                /* Found */
                source = HWTSTAMP_SOURCE_NETDEV;
        } else {
                phy = ethtool_phy_get_ts_info_by_phc(dev, &ts_info, hwprov_desc);
                if (IS_ERR(phy)) {
                        if (PTR_ERR(phy) == -ENODEV)
                                NL_SET_ERR_MSG_ATTR(info->extack,
                                                    tb[ETHTOOL_A_TSCONFIG_HWTSTAMP_PROVIDER],
                                                    "phc not in this net device topology");
                        return ERR_CAST(phy);
                }

                source = HWTSTAMP_SOURCE_PHYLIB;
        }

        hwprov = kzalloc(sizeof(*hwprov), GFP_KERNEL);
        if (!hwprov)
                return ERR_PTR(-ENOMEM);

        hwprov->desc.index = hwprov_desc->index;
        hwprov->desc.qualifier = hwprov_desc->qualifier;
        hwprov->source = source;
        hwprov->phydev = phy;

        return hwprov;
}

static int ethnl_set_tsconfig(struct ethnl_req_info *req_base,
                              struct genl_info *info)
{
        struct kernel_hwtstamp_config hwtst_config = {0};
        bool hwprov_mod = false, config_mod = false;
        struct hwtstamp_provider *hwprov = NULL;
        struct net_device *dev = req_base->dev;
        struct nlattr **tb = info->attrs;
        int ret;

        BUILD_BUG_ON(__HWTSTAMP_TX_CNT >= 32);
        BUILD_BUG_ON(__HWTSTAMP_FILTER_CNT >= 32);
        BUILD_BUG_ON(__HWTSTAMP_FLAG_CNT > 32);

        if (!netif_device_present(dev))
                return -ENODEV;

        if (tb[ETHTOOL_A_TSCONFIG_HWTSTAMP_PROVIDER]) {
                struct hwtstamp_provider_desc __hwprov_desc = {.index = -1};
                struct hwtstamp_provider *__hwprov;

                __hwprov = rtnl_dereference(dev->hwprov);
                if (__hwprov) {
                        __hwprov_desc.index = __hwprov->desc.index;
                        __hwprov_desc.qualifier = __hwprov->desc.qualifier;
                }

                ret = ts_parse_hwtst_provider(tb[ETHTOOL_A_TSCONFIG_HWTSTAMP_PROVIDER],
                                              &__hwprov_desc, info->extack,
                                              &hwprov_mod);
                if (ret < 0)
                        return ret;

                if (hwprov_mod) {
                        hwprov = tsconfig_set_hwprov_from_desc(dev, info,
                                                               &__hwprov_desc);
                        if (IS_ERR(hwprov))
                                return PTR_ERR(hwprov);
                }
        }

        /* Get current hwtstamp config if we are not changing the
         * hwtstamp source. It will be zeroed in the other case.
         */
        if (!hwprov_mod) {
                ret = dev_get_hwtstamp_phylib(dev, &hwtst_config);
                if (ret < 0 && ret != -EOPNOTSUPP)
                        goto err_free_hwprov;
        }

        /* Get the hwtstamp config from netlink */
        if (tb[ETHTOOL_A_TSCONFIG_TX_TYPES]) {
                u32 req_tx_type;

                req_tx_type = BIT(hwtst_config.tx_type);
                ret = ethnl_update_bitset32(&req_tx_type,
                                            __HWTSTAMP_TX_CNT,
                                            tb[ETHTOOL_A_TSCONFIG_TX_TYPES],
                                            ts_tx_type_names, info->extack,
                                            &config_mod);
                if (ret < 0)
                        goto err_free_hwprov;

                /* Select only one tx type at a time */
                if (ffs(req_tx_type) != fls(req_tx_type)) {
                        ret = -EINVAL;
                        goto err_free_hwprov;
                }

                hwtst_config.tx_type = ffs(req_tx_type) - 1;
        }

        if (tb[ETHTOOL_A_TSCONFIG_RX_FILTERS]) {
                u32 req_rx_filter;

                req_rx_filter = BIT(hwtst_config.rx_filter);
                ret = ethnl_update_bitset32(&req_rx_filter,
                                            __HWTSTAMP_FILTER_CNT,
                                            tb[ETHTOOL_A_TSCONFIG_RX_FILTERS],
                                            ts_rx_filter_names, info->extack,
                                            &config_mod);
                if (ret < 0)
                        goto err_free_hwprov;

                /* Select only one rx filter at a time */
                if (ffs(req_rx_filter) != fls(req_rx_filter)) {
                        ret = -EINVAL;
                        goto err_free_hwprov;
                }

                hwtst_config.rx_filter = ffs(req_rx_filter) - 1;
        }

        if (tb[ETHTOOL_A_TSCONFIG_HWTSTAMP_FLAGS]) {
                ret = ethnl_update_bitset32(&hwtst_config.flags,
                                            __HWTSTAMP_FLAG_CNT,
                                            tb[ETHTOOL_A_TSCONFIG_HWTSTAMP_FLAGS],
                                            ts_flags_names, info->extack,
                                            &config_mod);
                if (ret < 0)
                        goto err_free_hwprov;
        }

        ret = net_hwtstamp_validate(&hwtst_config);
        if (ret)
                goto err_free_hwprov;

        if (hwprov_mod) {
                struct kernel_hwtstamp_config zero_config = {0};
                struct hwtstamp_provider *__hwprov;

                /* Disable current time stamping if we try to enable
                 * another one
                 */
                ret = dev_set_hwtstamp_phylib(dev, &zero_config, info->extack);
                if (ret < 0)
                        goto err_free_hwprov;

                /* Change the selected hwtstamp source */
                __hwprov = rcu_replace_pointer_rtnl(dev->hwprov, hwprov);
                if (__hwprov)
                        kfree_rcu(__hwprov, rcu_head);
        }

        if (config_mod) {
                ret = dev_set_hwtstamp_phylib(dev, &hwtst_config,
                                              info->extack);
                if (ret < 0)
                        return ret;
        }

        if (hwprov_mod || config_mod) {
                ret = tsconfig_send_reply(dev, info);
                if (ret && ret != -EOPNOTSUPP) {
                        NL_SET_ERR_MSG(info->extack,
                                       "error while reading the new configuration set");
                        return ret;
                }
        }

        /* tsconfig has no notification */
        return 0;

err_free_hwprov:
        kfree(hwprov);

        return ret;
}

const struct ethnl_request_ops ethnl_tsconfig_request_ops = {
        .request_cmd                = ETHTOOL_MSG_TSCONFIG_GET,
        .reply_cmd                = ETHTOOL_MSG_TSCONFIG_GET_REPLY,
        .hdr_attr                = ETHTOOL_A_TSCONFIG_HEADER,
        .req_info_size                = sizeof(struct tsconfig_req_info),
        .reply_data_size        = sizeof(struct tsconfig_reply_data),

        .prepare_data                = tsconfig_prepare_data,
        .reply_size                = tsconfig_reply_size,
        .fill_reply                = tsconfig_fill_reply,

        .set_validate                = ethnl_set_tsconfig_validate,
        .set                        = ethnl_set_tsconfig,
};














































































































































































































































































   68 







































    4 



































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
/* SPDX-License-Identifier: GPL-2.0 OR MIT */
#ifndef __LINUX_OVERFLOW_H
#define __LINUX_OVERFLOW_H

#include <linux/compiler.h>
#include <linux/limits.h>
#include <linux/const.h>

/*
 * We need to compute the minimum and maximum values representable in a given
 * type. These macros may also be useful elsewhere. It would seem more obvious
 * to do something like:
 *
 * #define type_min(T) (T)(is_signed_type(T) ? (T)1 << (8*sizeof(T)-1) : 0)
 * #define type_max(T) (T)(is_signed_type(T) ? ((T)1 << (8*sizeof(T)-1)) - 1 : ~(T)0)
 *
 * Unfortunately, the middle expressions, strictly speaking, have
 * undefined behaviour, and at least some versions of gcc warn about
 * the type_max expression (but not if -fsanitize=undefined is in
 * effect; in that case, the warning is deferred to runtime...).
 *
 * The slightly excessive casting in type_min is to make sure the
 * macros also produce sensible values for the exotic type _Bool. [The
 * overflow checkers only almost work for _Bool, but that's
 * a-feature-not-a-bug, since people shouldn't be doing arithmetic on
 * _Bools. Besides, the gcc builtins don't allow _Bool* as third
 * argument.]
 *
 * Idea stolen from
 * https://mail-index.netbsd.org/tech-misc/2007/02/05/0000.html -
 * credit to Christian Biere.
 */
#define __type_half_max(type) ((type)1 << (8*sizeof(type) - 1 - is_signed_type(type)))
#define __type_max(T) ((T)((__type_half_max(T) - 1) + __type_half_max(T)))
#define type_max(t)        __type_max(typeof(t))
#define __type_min(T) ((T)((T)-type_max(T)-(T)1))
#define type_min(t)        __type_min(typeof(t))

/*
 * Avoids triggering -Wtype-limits compilation warning,
 * while using unsigned data types to check a < 0.
 */
#define is_non_negative(a) ((a) > 0 || (a) == 0)
#define is_negative(a) (!(is_non_negative(a)))

/*
 * Allows for effectively applying __must_check to a macro so we can have
 * both the type-agnostic benefits of the macros while also being able to
 * enforce that the return value is, in fact, checked.
 */
static inline bool __must_check __must_check_overflow(bool overflow)
{
        return unlikely(overflow);
}

/**
 * check_add_overflow() - Calculate addition with overflow checking
 * @a: first addend
 * @b: second addend
 * @d: pointer to store sum
 *
 * Returns true on wrap-around, false otherwise.
 *
 * *@d holds the results of the attempted addition, regardless of whether
 * wrap-around occurred.
 */
#define check_add_overflow(a, b, d)        \
        __must_check_overflow(__builtin_add_overflow(a, b, d))

/**
 * wrapping_add() - Intentionally perform a wrapping addition
 * @type: type for result of calculation
 * @a: first addend
 * @b: second addend
 *
 * Return the potentially wrapped-around addition without
 * tripping any wrap-around sanitizers that may be enabled.
 */
#define wrapping_add(type, a, b)                                \
        ({                                                        \
                type __val;                                        \
                __builtin_add_overflow(a, b, &__val);                \
                __val;                                                \
        })

/**
 * wrapping_assign_add() - Intentionally perform a wrapping increment assignment
 * @var: variable to be incremented
 * @offset: amount to add
 *
 * Increments @var by @offset with wrap-around. Returns the resulting
 * value of @var. Will not trip any wrap-around sanitizers.
 *
 * Returns the new value of @var.
 */
#define wrapping_assign_add(var, offset)                                \
        ({                                                                \
                typeof(var) *__ptr = &(var);                                \
                *__ptr = wrapping_add(typeof(var), *__ptr, offset);        \
        })

/**
 * check_sub_overflow() - Calculate subtraction with overflow checking
 * @a: minuend; value to subtract from
 * @b: subtrahend; value to subtract from @a
 * @d: pointer to store difference
 *
 * Returns true on wrap-around, false otherwise.
 *
 * *@d holds the results of the attempted subtraction, regardless of whether
 * wrap-around occurred.
 */
#define check_sub_overflow(a, b, d)        \
        __must_check_overflow(__builtin_sub_overflow(a, b, d))

/**
 * wrapping_sub() - Intentionally perform a wrapping subtraction
 * @type: type for result of calculation
 * @a: minuend; value to subtract from
 * @b: subtrahend; value to subtract from @a
 *
 * Return the potentially wrapped-around subtraction without
 * tripping any wrap-around sanitizers that may be enabled.
 */
#define wrapping_sub(type, a, b)                                \
        ({                                                        \
                type __val;                                        \
                __builtin_sub_overflow(a, b, &__val);                \
                __val;                                                \
        })

/**
 * wrapping_assign_sub() - Intentionally perform a wrapping decrement assign
 * @var: variable to be decremented
 * @offset: amount to subtract
 *
 * Decrements @var by @offset with wrap-around. Returns the resulting
 * value of @var. Will not trip any wrap-around sanitizers.
 *
 * Returns the new value of @var.
 */
#define wrapping_assign_sub(var, offset)                                \
        ({                                                                \
                typeof(var) *__ptr = &(var);                                \
                *__ptr = wrapping_sub(typeof(var), *__ptr, offset);        \
        })

/**
 * check_mul_overflow() - Calculate multiplication with overflow checking
 * @a: first factor
 * @b: second factor
 * @d: pointer to store product
 *
 * Returns true on wrap-around, false otherwise.
 *
 * *@d holds the results of the attempted multiplication, regardless of whether
 * wrap-around occurred.
 */
#define check_mul_overflow(a, b, d)        \
        __must_check_overflow(__builtin_mul_overflow(a, b, d))

/**
 * wrapping_mul() - Intentionally perform a wrapping multiplication
 * @type: type for result of calculation
 * @a: first factor
 * @b: second factor
 *
 * Return the potentially wrapped-around multiplication without
 * tripping any wrap-around sanitizers that may be enabled.
 */
#define wrapping_mul(type, a, b)                                \
        ({                                                        \
                type __val;                                        \
                __builtin_mul_overflow(a, b, &__val);                \
                __val;                                                \
        })

/**
 * check_shl_overflow() - Calculate a left-shifted value and check overflow
 * @a: Value to be shifted
 * @s: How many bits left to shift
 * @d: Pointer to where to store the result
 *
 * Computes *@d = (@a << @s)
 *
 * Returns true if '*@d' cannot hold the result or when '@a << @s' doesn't
 * make sense. Example conditions:
 *
 * - '@a << @s' causes bits to be lost when stored in *@d.
 * - '@s' is garbage (e.g. negative) or so large that the result of
 *   '@a << @s' is guaranteed to be 0.
 * - '@a' is negative.
 * - '@a << @s' sets the sign bit, if any, in '*@d'.
 *
 * '*@d' will hold the results of the attempted shift, but is not
 * considered "safe for use" if true is returned.
 */
#define check_shl_overflow(a, s, d) __must_check_overflow(({                \
        typeof(a) _a = a;                                                \
        typeof(s) _s = s;                                                \
        typeof(d) _d = d;                                                \
        unsigned long long _a_full = _a;                                \
        unsigned int _to_shift =                                        \
                is_non_negative(_s) && _s < 8 * sizeof(*d) ? _s : 0;        \
        *_d = (_a_full << _to_shift);                                        \
        (_to_shift != _s || is_negative(*_d) || is_negative(_a) ||        \
        (*_d >> _to_shift) != _a);                                        \
}))

#define __overflows_type_constexpr(x, T) (                        \
        is_unsigned_type(typeof(x)) ?                                \
                (x) > type_max(T) :                                \
        is_unsigned_type(typeof(T)) ?                                \
                (x) < 0 || (x) > type_max(T) :                        \
        (x) < type_min(T) || (x) > type_max(T))

#define __overflows_type(x, T)                ({        \
        typeof(T) v = 0;                        \
        check_add_overflow((x), v, &v);                \
})

/**
 * overflows_type - helper for checking the overflows between value, variables,
 *                    or data type
 *
 * @n: source constant value or variable to be checked
 * @T: destination variable or data type proposed to store @x
 *
 * Compares the @x expression for whether or not it can safely fit in
 * the storage of the type in @T. @x and @T can have different types.
 * If @x is a constant expression, this will also resolve to a constant
 * expression.
 *
 * Returns: true if overflow can occur, false otherwise.
 */
#define overflows_type(n, T)                                        \
        __builtin_choose_expr(__is_constexpr(n),                \
                              __overflows_type_constexpr(n, T),        \
                              __overflows_type(n, T))

/**
 * castable_to_type - like __same_type(), but also allows for casted literals
 *
 * @n: variable or constant value
 * @T: variable or data type
 *
 * Unlike the __same_type() macro, this allows a constant value as the
 * first argument. If this value would not overflow into an assignment
 * of the second argument's type, it returns true. Otherwise, this falls
 * back to __same_type().
 */
#define castable_to_type(n, T)                                                \
        __builtin_choose_expr(__is_constexpr(n),                        \
                              !__overflows_type_constexpr(n, T),        \
                              __same_type(n, T))

/**
 * size_mul() - Calculate size_t multiplication with saturation at SIZE_MAX
 * @factor1: first factor
 * @factor2: second factor
 *
 * Returns: calculate @factor1 * @factor2, both promoted to size_t,
 * with any overflow causing the return value to be SIZE_MAX. The
 * lvalue must be size_t to avoid implicit type conversion.
 */
static inline size_t __must_check size_mul(size_t factor1, size_t factor2)
{
        size_t bytes;

        if (check_mul_overflow(factor1, factor2, &bytes))
                return SIZE_MAX;

        return bytes;
}

/**
 * size_add() - Calculate size_t addition with saturation at SIZE_MAX
 * @addend1: first addend
 * @addend2: second addend
 *
 * Returns: calculate @addend1 + @addend2, both promoted to size_t,
 * with any overflow causing the return value to be SIZE_MAX. The
 * lvalue must be size_t to avoid implicit type conversion.
 */
static inline size_t __must_check size_add(size_t addend1, size_t addend2)
{
        size_t bytes;

        if (check_add_overflow(addend1, addend2, &bytes))
                return SIZE_MAX;

        return bytes;
}

/**
 * size_sub() - Calculate size_t subtraction with saturation at SIZE_MAX
 * @minuend: value to subtract from
 * @subtrahend: value to subtract from @minuend
 *
 * Returns: calculate @minuend - @subtrahend, both promoted to size_t,
 * with any overflow causing the return value to be SIZE_MAX. For
 * composition with the size_add() and size_mul() helpers, neither
 * argument may be SIZE_MAX (or the result with be forced to SIZE_MAX).
 * The lvalue must be size_t to avoid implicit type conversion.
 */
static inline size_t __must_check size_sub(size_t minuend, size_t subtrahend)
{
        size_t bytes;

        if (minuend == SIZE_MAX || subtrahend == SIZE_MAX ||
            check_sub_overflow(minuend, subtrahend, &bytes))
                return SIZE_MAX;

        return bytes;
}

/**
 * array_size() - Calculate size of 2-dimensional array.
 * @a: dimension one
 * @b: dimension two
 *
 * Calculates size of 2-dimensional array: @a * @b.
 *
 * Returns: number of bytes needed to represent the array or SIZE_MAX on
 * overflow.
 */
#define array_size(a, b)        size_mul(a, b)

/**
 * array3_size() - Calculate size of 3-dimensional array.
 * @a: dimension one
 * @b: dimension two
 * @c: dimension three
 *
 * Calculates size of 3-dimensional array: @a * @b * @c.
 *
 * Returns: number of bytes needed to represent the array or SIZE_MAX on
 * overflow.
 */
#define array3_size(a, b, c)        size_mul(size_mul(a, b), c)

/**
 * flex_array_size() - Calculate size of a flexible array member
 *                     within an enclosing structure.
 * @p: Pointer to the structure.
 * @member: Name of the flexible array member.
 * @count: Number of elements in the array.
 *
 * Calculates size of a flexible array of @count number of @member
 * elements, at the end of structure @p.
 *
 * Return: number of bytes needed or SIZE_MAX on overflow.
 */
#define flex_array_size(p, member, count)                                \
        __builtin_choose_expr(__is_constexpr(count),                        \
                (count) * sizeof(*(p)->member) + __must_be_array((p)->member),        \
                size_mul(count, sizeof(*(p)->member) + __must_be_array((p)->member)))

/**
 * struct_size() - Calculate size of structure with trailing flexible array.
 * @p: Pointer to the structure.
 * @member: Name of the array member.
 * @count: Number of elements in the array.
 *
 * Calculates size of memory needed for structure of @p followed by an
 * array of @count number of @member elements.
 *
 * Return: number of bytes needed or SIZE_MAX on overflow.
 */
#define struct_size(p, member, count)                                        \
        __builtin_choose_expr(__is_constexpr(count),                        \
                sizeof(*(p)) + flex_array_size(p, member, count),        \
                size_add(sizeof(*(p)), flex_array_size(p, member, count)))

/**
 * struct_size_t() - Calculate size of structure with trailing flexible array
 * @type: structure type name.
 * @member: Name of the array member.
 * @count: Number of elements in the array.
 *
 * Calculates size of memory needed for structure @type followed by an
 * array of @count number of @member elements. Prefer using struct_size()
 * when possible instead, to keep calculations associated with a specific
 * instance variable of type @type.
 *
 * Return: number of bytes needed or SIZE_MAX on overflow.
 */
#define struct_size_t(type, member, count)                                        \
        struct_size((type *)NULL, member, count)

/**
 * __DEFINE_FLEX() - helper macro for DEFINE_FLEX() family.
 * Enables caller macro to pass arbitrary trailing expressions
 *
 * @type: structure type name, including "struct" keyword.
 * @name: Name for a variable to define.
 * @member: Name of the array member.
 * @count: Number of elements in the array; must be compile-time const.
 * @trailer: Trailing expressions for attributes and/or initializers.
 */
#define __DEFINE_FLEX(type, name, member, count, trailer...)                        \
        _Static_assert(__builtin_constant_p(count),                                \
                       "onstack flex array members require compile-time const count"); \
        union {                                                                        \
                u8 bytes[struct_size_t(type, member, count)];                        \
                type obj;                                                        \
        } name##_u trailer;                                                        \
        type *name = (type *)&name##_u

/**
 * _DEFINE_FLEX() - helper macro for DEFINE_FLEX() family.
 * Enables caller macro to pass (different) initializer.
 *
 * @type: structure type name, including "struct" keyword.
 * @name: Name for a variable to define.
 * @member: Name of the array member.
 * @count: Number of elements in the array; must be compile-time const.
 * @initializer: Initializer expression (e.g., pass `= { }` at minimum).
 */
#define _DEFINE_FLEX(type, name, member, count, initializer...)                        \
        __DEFINE_FLEX(type, name, member, count, = { .obj initializer })

/**
 * DEFINE_RAW_FLEX() - Define an on-stack instance of structure with a trailing
 * flexible array member, when it does not have a __counted_by annotation.
 *
 * @type: structure type name, including "struct" keyword.
 * @name: Name for a variable to define.
 * @member: Name of the array member.
 * @count: Number of elements in the array; must be compile-time const.
 *
 * Define a zeroed, on-stack, instance of @type structure with a trailing
 * flexible array member.
 * Use __struct_size(@name) to get compile-time size of it afterwards.
 * Use __member_size(@name->member) to get compile-time size of @name members.
 * Use STACK_FLEX_ARRAY_SIZE(@name, @member) to get compile-time number of
 * elements in array @member.
 */
#define DEFINE_RAW_FLEX(type, name, member, count)        \
        __DEFINE_FLEX(type, name, member, count, = { })

/**
 * DEFINE_FLEX() - Define an on-stack instance of structure with a trailing
 * flexible array member.
 *
 * @TYPE: structure type name, including "struct" keyword.
 * @NAME: Name for a variable to define.
 * @MEMBER: Name of the array member.
 * @COUNTER: Name of the __counted_by member.
 * @COUNT: Number of elements in the array; must be compile-time const.
 *
 * Define a zeroed, on-stack, instance of @TYPE structure with a trailing
 * flexible array member.
 * Use __struct_size(@NAME) to get compile-time size of it afterwards.
 * Use __member_size(@NAME->member) to get compile-time size of @NAME members.
 * Use STACK_FLEX_ARRAY_SIZE(@name, @member) to get compile-time number of
 * elements in array @member.
 */
#define DEFINE_FLEX(TYPE, NAME, MEMBER, COUNTER, COUNT)        \
        _DEFINE_FLEX(TYPE, NAME, MEMBER, COUNT, = { .COUNTER = COUNT, })

/**
 * STACK_FLEX_ARRAY_SIZE() - helper macro for DEFINE_FLEX() family.
 * Returns the number of elements in @array.
 *
 * @name: Name for a variable defined in DEFINE_RAW_FLEX()/DEFINE_FLEX().
 * @array: Name of the array member.
 */
#define STACK_FLEX_ARRAY_SIZE(name, array)                                                \
        (__member_size((name)->array) / sizeof(*(name)->array) +                        \
                                                __must_be_array((name)->array))

#endif /* __LINUX_OVERFLOW_H */





































   32 
   32 







    1 





    4 



    4 


    4 





























    3 

    1 



    3 





    2 




















    1 



    1 









    2 





    1 





    1 














    9 












    6 



    2 



    6 






    6 










   14 








    1 


   14 

   14 
   10 

   14 




   13 






    1 




















    8 






    1 
















    1 











    1 


    1 









    4 




    6 




    1 

    1 

    4 


    6 




    1 

    1 

    3 































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
// SPDX-License-Identifier: GPL-2.0
/*
 *  fs/signalfd.c
 *
 *  Copyright (C) 2003  Linus Torvalds
 *
 *  Mon Mar 5, 2007: Davide Libenzi <davidel@xmailserver.org>
 *      Changed ->read() to return a siginfo strcture instead of signal number.
 *      Fixed locking in ->poll().
 *      Added sighand-detach notification.
 *      Added fd re-use in sys_signalfd() syscall.
 *      Now using anonymous inode source.
 *      Thanks to Oleg Nesterov for useful code review and suggestions.
 *      More comments and suggestions from Arnd Bergmann.
 *  Sat May 19, 2007: Davi E. M. Arnaut <davi@haxent.com.br>
 *      Retrieve multiple signals with one read() call
 *  Sun Jul 15, 2007: Davide Libenzi <davidel@xmailserver.org>
 *      Attach to the sighand only during read() and poll().
 */

#include <linux/file.h>
#include <linux/poll.h>
#include <linux/init.h>
#include <linux/fs.h>
#include <linux/sched.h>
#include <linux/slab.h>
#include <linux/kernel.h>
#include <linux/signal.h>
#include <linux/list.h>
#include <linux/anon_inodes.h>
#include <linux/signalfd.h>
#include <linux/syscalls.h>
#include <linux/proc_fs.h>
#include <linux/compat.h>

void signalfd_cleanup(struct sighand_struct *sighand)
{
        wake_up_pollfree(&sighand->signalfd_wqh);
}

struct signalfd_ctx {
        sigset_t sigmask;
};

static int signalfd_release(struct inode *inode, struct file *file)
{
        kfree(file->private_data);
        return 0;
}

static __poll_t signalfd_poll(struct file *file, poll_table *wait)
{
        struct signalfd_ctx *ctx = file->private_data;
        __poll_t events = 0;

        poll_wait(file, &current->sighand->signalfd_wqh, wait);

        spin_lock_irq(&current->sighand->siglock);
        if (next_signal(&current->pending, &ctx->sigmask) ||
            next_signal(&current->signal->shared_pending,
                        &ctx->sigmask))
                events |= EPOLLIN;
        spin_unlock_irq(&current->sighand->siglock);

        return events;
}

/*
 * Copied from copy_siginfo_to_user() in kernel/signal.c
 */
static int signalfd_copyinfo(struct iov_iter *to, kernel_siginfo_t const *kinfo)
{
        struct signalfd_siginfo new;

        BUILD_BUG_ON(sizeof(struct signalfd_siginfo) != 128);

        /*
         * Unused members should be zero ...
         */
        memset(&new, 0, sizeof(new));

        /*
         * If you change siginfo_t structure, please be sure
         * this code is fixed accordingly.
         */
        new.ssi_signo = kinfo->si_signo;
        new.ssi_errno = kinfo->si_errno;
        new.ssi_code  = kinfo->si_code;
        switch (siginfo_layout(kinfo->si_signo, kinfo->si_code)) {
        case SIL_KILL:
                new.ssi_pid = kinfo->si_pid;
                new.ssi_uid = kinfo->si_uid;
                break;
        case SIL_TIMER:
                new.ssi_tid = kinfo->si_tid;
                new.ssi_overrun = kinfo->si_overrun;
                new.ssi_ptr = (long) kinfo->si_ptr;
                new.ssi_int = kinfo->si_int;
                break;
        case SIL_POLL:
                new.ssi_band = kinfo->si_band;
                new.ssi_fd   = kinfo->si_fd;
                break;
        case SIL_FAULT_BNDERR:
        case SIL_FAULT_PKUERR:
        case SIL_FAULT_PERF_EVENT:
                /*
                 * Fall through to the SIL_FAULT case.  SIL_FAULT_BNDERR,
                 * SIL_FAULT_PKUERR, and SIL_FAULT_PERF_EVENT are only
                 * generated by faults that deliver them synchronously to
                 * userspace.  In case someone injects one of these signals
                 * and signalfd catches it treat it as SIL_FAULT.
                 */
        case SIL_FAULT:
                new.ssi_addr = (long) kinfo->si_addr;
                break;
        case SIL_FAULT_TRAPNO:
                new.ssi_addr = (long) kinfo->si_addr;
                new.ssi_trapno = kinfo->si_trapno;
                break;
        case SIL_FAULT_MCEERR:
                new.ssi_addr = (long) kinfo->si_addr;
                new.ssi_addr_lsb = (short) kinfo->si_addr_lsb;
                break;
        case SIL_CHLD:
                new.ssi_pid    = kinfo->si_pid;
                new.ssi_uid    = kinfo->si_uid;
                new.ssi_status = kinfo->si_status;
                new.ssi_utime  = kinfo->si_utime;
                new.ssi_stime  = kinfo->si_stime;
                break;
        case SIL_RT:
                /*
                 * This case catches also the signals queued by sigqueue().
                 */
                new.ssi_pid = kinfo->si_pid;
                new.ssi_uid = kinfo->si_uid;
                new.ssi_ptr = (long) kinfo->si_ptr;
                new.ssi_int = kinfo->si_int;
                break;
        case SIL_SYS:
                new.ssi_call_addr = (long) kinfo->si_call_addr;
                new.ssi_syscall   = kinfo->si_syscall;
                new.ssi_arch      = kinfo->si_arch;
                break;
        }

        if (!copy_to_iter_full(&new, sizeof(struct signalfd_siginfo), to))
                return -EFAULT;

        return sizeof(struct signalfd_siginfo);
}

static ssize_t signalfd_dequeue(struct signalfd_ctx *ctx, kernel_siginfo_t *info,
                                int nonblock)
{
        enum pid_type type;
        ssize_t ret;
        DECLARE_WAITQUEUE(wait, current);

        spin_lock_irq(&current->sighand->siglock);
        ret = dequeue_signal(&ctx->sigmask, info, &type);
        switch (ret) {
        case 0:
                if (!nonblock)
                        break;
                ret = -EAGAIN;
                fallthrough;
        default:
                spin_unlock_irq(&current->sighand->siglock);
                return ret;
        }

        add_wait_queue(&current->sighand->signalfd_wqh, &wait);
        for (;;) {
                set_current_state(TASK_INTERRUPTIBLE);
                ret = dequeue_signal(&ctx->sigmask, info, &type);
                if (ret != 0)
                        break;
                if (signal_pending(current)) {
                        ret = -ERESTARTSYS;
                        break;
                }
                spin_unlock_irq(&current->sighand->siglock);
                schedule();
                spin_lock_irq(&current->sighand->siglock);
        }
        spin_unlock_irq(&current->sighand->siglock);

        remove_wait_queue(&current->sighand->signalfd_wqh, &wait);
        __set_current_state(TASK_RUNNING);

        return ret;
}

/*
 * Returns a multiple of the size of a "struct signalfd_siginfo", or a negative
 * error code. The "count" parameter must be at least the size of a
 * "struct signalfd_siginfo".
 */
static ssize_t signalfd_read_iter(struct kiocb *iocb, struct iov_iter *to)
{
        struct file *file = iocb->ki_filp;
        struct signalfd_ctx *ctx = file->private_data;
        size_t count = iov_iter_count(to);
        ssize_t ret, total = 0;
        kernel_siginfo_t info;
        bool nonblock;

        count /= sizeof(struct signalfd_siginfo);
        if (!count)
                return -EINVAL;

        nonblock = file->f_flags & O_NONBLOCK || iocb->ki_flags & IOCB_NOWAIT;
        do {
                ret = signalfd_dequeue(ctx, &info, nonblock);
                if (unlikely(ret <= 0))
                        break;
                ret = signalfd_copyinfo(to, &info);
                if (ret < 0)
                        break;
                total += ret;
                nonblock = 1;
        } while (--count);

        return total ? total: ret;
}

#ifdef CONFIG_PROC_FS
static void signalfd_show_fdinfo(struct seq_file *m, struct file *f)
{
        struct signalfd_ctx *ctx = f->private_data;
        sigset_t sigmask;

        sigmask = ctx->sigmask;
        signotset(&sigmask);
        render_sigset_t(m, "sigmask:\t", &sigmask);
}
#endif

static const struct file_operations signalfd_fops = {
#ifdef CONFIG_PROC_FS
        .show_fdinfo        = signalfd_show_fdinfo,
#endif
        .release        = signalfd_release,
        .poll                = signalfd_poll,
        .read_iter        = signalfd_read_iter,
        .llseek                = noop_llseek,
};

static int do_signalfd4(int ufd, sigset_t *mask, int flags)
{
        struct signalfd_ctx *ctx;

        /* Check the SFD_* constants for consistency.  */
        BUILD_BUG_ON(SFD_CLOEXEC != O_CLOEXEC);
        BUILD_BUG_ON(SFD_NONBLOCK != O_NONBLOCK);

        if (flags & ~(SFD_CLOEXEC | SFD_NONBLOCK))
                return -EINVAL;

        sigdelsetmask(mask, sigmask(SIGKILL) | sigmask(SIGSTOP));
        signotset(mask);

        if (ufd == -1) {
                struct file *file;

                ctx = kmalloc(sizeof(*ctx), GFP_KERNEL);
                if (!ctx)
                        return -ENOMEM;

                ctx->sigmask = *mask;

                ufd = get_unused_fd_flags(flags & O_CLOEXEC);
                if (ufd < 0) {
                        kfree(ctx);
                        return ufd;
                }

                file = anon_inode_getfile_fmode("[signalfd]", &signalfd_fops,
                                        ctx, O_RDWR | (flags & O_NONBLOCK),
                                        FMODE_NOWAIT);
                if (IS_ERR(file)) {
                        put_unused_fd(ufd);
                        kfree(ctx);
                        return PTR_ERR(file);
                }
                fd_install(ufd, file);
        } else {
                CLASS(fd, f)(ufd);
                if (fd_empty(f))
                        return -EBADF;
                ctx = fd_file(f)->private_data;
                if (fd_file(f)->f_op != &signalfd_fops)
                        return -EINVAL;
                spin_lock_irq(&current->sighand->siglock);
                ctx->sigmask = *mask;
                spin_unlock_irq(&current->sighand->siglock);

                wake_up(&current->sighand->signalfd_wqh);
        }

        return ufd;
}

SYSCALL_DEFINE4(signalfd4, int, ufd, sigset_t __user *, user_mask,
                size_t, sizemask, int, flags)
{
        sigset_t mask;

        if (sizemask != sizeof(sigset_t))
                return -EINVAL;
        if (copy_from_user(&mask, user_mask, sizeof(mask)))
                return -EFAULT;
        return do_signalfd4(ufd, &mask, flags);
}

SYSCALL_DEFINE3(signalfd, int, ufd, sigset_t __user *, user_mask,
                size_t, sizemask)
{
        sigset_t mask;

        if (sizemask != sizeof(sigset_t))
                return -EINVAL;
        if (copy_from_user(&mask, user_mask, sizeof(mask)))
                return -EFAULT;
        return do_signalfd4(ufd, &mask, 0);
}

#ifdef CONFIG_COMPAT
static long do_compat_signalfd4(int ufd,
                        const compat_sigset_t __user *user_mask,
                        compat_size_t sigsetsize, int flags)
{
        sigset_t mask;

        if (sigsetsize != sizeof(compat_sigset_t))
                return -EINVAL;
        if (get_compat_sigset(&mask, user_mask))
                return -EFAULT;
        return do_signalfd4(ufd, &mask, flags);
}

COMPAT_SYSCALL_DEFINE4(signalfd4, int, ufd,
                     const compat_sigset_t __user *, user_mask,
                     compat_size_t, sigsetsize,
                     int, flags)
{
        return do_compat_signalfd4(ufd, user_mask, sigsetsize, flags);
}

COMPAT_SYSCALL_DEFINE3(signalfd, int, ufd,
                     const compat_sigset_t __user *, user_mask,
                     compat_size_t, sigsetsize)
{
        return do_compat_signalfd4(ufd, user_mask, sigsetsize, 0);
}
#endif




























































   26 











   26 

   26 


   26 










































































































































































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
// SPDX-License-Identifier: GPL-2.0-only
/*
 * HT handling
 *
 * Copyright 2003, Jouni Malinen <jkmaline@cc.hut.fi>
 * Copyright 2002-2005, Instant802 Networks, Inc.
 * Copyright 2005-2006, Devicescape Software, Inc.
 * Copyright 2006-2007        Jiri Benc <jbenc@suse.cz>
 * Copyright 2007, Michael Wu <flamingice@sourmilk.net>
 * Copyright 2007-2010, Intel Corporation
 * Copyright(c) 2015-2017 Intel Deutschland GmbH
 * Copyright (C) 2018-2024 Intel Corporation
 */

/**
 * DOC: RX A-MPDU aggregation
 *
 * Aggregation on the RX side requires only implementing the
 * @ampdu_action callback that is invoked to start/stop any
 * block-ack sessions for RX aggregation.
 *
 * When RX aggregation is started by the peer, the driver is
 * notified via @ampdu_action function, with the
 * %IEEE80211_AMPDU_RX_START action, and may reject the request
 * in which case a negative response is sent to the peer, if it
 * accepts it a positive response is sent.
 *
 * While the session is active, the device/driver are required
 * to de-aggregate frames and pass them up one by one to mac80211,
 * which will handle the reorder buffer.
 *
 * When the aggregation session is stopped again by the peer or
 * ourselves, the driver's @ampdu_action function will be called
 * with the action %IEEE80211_AMPDU_RX_STOP. In this case, the
 * call must not fail.
 */

#include <linux/ieee80211.h>
#include <linux/slab.h>
#include <linux/export.h>
#include <net/mac80211.h>
#include "ieee80211_i.h"
#include "driver-ops.h"

static void ieee80211_free_tid_rx(struct rcu_head *h)
{
        struct tid_ampdu_rx *tid_rx =
                container_of(h, struct tid_ampdu_rx, rcu_head);
        int i;

        for (i = 0; i < tid_rx->buf_size; i++)
                __skb_queue_purge(&tid_rx->reorder_buf[i]);
        kfree(tid_rx->reorder_buf);
        kfree(tid_rx->reorder_time);
        kfree(tid_rx);
}

void __ieee80211_stop_rx_ba_session(struct sta_info *sta, u16 tid,
                                    u16 initiator, u16 reason, bool tx)
{
        struct ieee80211_local *local = sta->local;
        struct tid_ampdu_rx *tid_rx;
        struct ieee80211_ampdu_params params = {
                .sta = &sta->sta,
                .action = IEEE80211_AMPDU_RX_STOP,
                .tid = tid,
                .amsdu = false,
                .timeout = 0,
                .ssn = 0,
        };

        lockdep_assert_wiphy(sta->local->hw.wiphy);

        tid_rx = rcu_dereference_protected(sta->ampdu_mlme.tid_rx[tid],
                                        lockdep_is_held(&sta->local->hw.wiphy->mtx));

        if (!test_bit(tid, sta->ampdu_mlme.agg_session_valid))
                return;

        RCU_INIT_POINTER(sta->ampdu_mlme.tid_rx[tid], NULL);
        __clear_bit(tid, sta->ampdu_mlme.agg_session_valid);

        ht_dbg(sta->sdata,
               "Rx BA session stop requested for %pM tid %u %s reason: %d\n",
               sta->sta.addr, tid,
               initiator == WLAN_BACK_RECIPIENT ? "recipient" : "initiator",
               (int)reason);

        if (drv_ampdu_action(local, sta->sdata, &params))
                sdata_info(sta->sdata,
                           "HW problem - can not stop rx aggregation for %pM tid %d\n",
                           sta->sta.addr, tid);

        /* check if this is a self generated aggregation halt */
        if (initiator == WLAN_BACK_RECIPIENT && tx)
                ieee80211_send_delba(sta->sdata, sta->sta.addr,
                                     tid, WLAN_BACK_RECIPIENT, reason);

        /*
         * return here in case tid_rx is not assigned - which will happen if
         * IEEE80211_HW_SUPPORTS_REORDERING_BUFFER is set.
         */
        if (!tid_rx)
                return;

        timer_delete_sync(&tid_rx->session_timer);

        /* make sure ieee80211_sta_reorder_release() doesn't re-arm the timer */
        spin_lock_bh(&tid_rx->reorder_lock);
        tid_rx->removed = true;
        spin_unlock_bh(&tid_rx->reorder_lock);
        timer_delete_sync(&tid_rx->reorder_timer);

        call_rcu(&tid_rx->rcu_head, ieee80211_free_tid_rx);
}

void ieee80211_stop_rx_ba_session(struct ieee80211_vif *vif, u16 ba_rx_bitmap,
                                  const u8 *addr)
{
        struct ieee80211_sub_if_data *sdata = vif_to_sdata(vif);
        struct sta_info *sta;
        int i;

        rcu_read_lock();
        sta = sta_info_get_bss(sdata, addr);
        if (!sta) {
                rcu_read_unlock();
                return;
        }

        for (i = 0; i < IEEE80211_NUM_TIDS; i++)
                if (ba_rx_bitmap & BIT(i))
                        set_bit(i, sta->ampdu_mlme.tid_rx_stop_requested);

        wiphy_work_queue(sta->local->hw.wiphy, &sta->ampdu_mlme.work);
        rcu_read_unlock();
}
EXPORT_SYMBOL(ieee80211_stop_rx_ba_session);

/*
 * After accepting the AddBA Request we activated a timer,
 * resetting it after each frame that arrives from the originator.
 */
static void sta_rx_agg_session_timer_expired(struct timer_list *t)
{
        struct tid_ampdu_rx *tid_rx = timer_container_of(tid_rx, t,
                                                         session_timer);
        struct sta_info *sta = tid_rx->sta;
        u8 tid = tid_rx->tid;
        unsigned long timeout;

        timeout = tid_rx->last_rx + TU_TO_JIFFIES(tid_rx->timeout);
        if (time_is_after_jiffies(timeout)) {
                mod_timer(&tid_rx->session_timer, timeout);
                return;
        }

        ht_dbg(sta->sdata, "RX session timer expired on %pM tid %d\n",
               sta->sta.addr, tid);

        set_bit(tid, sta->ampdu_mlme.tid_rx_timer_expired);
        wiphy_work_queue(sta->local->hw.wiphy, &sta->ampdu_mlme.work);
}

static void sta_rx_agg_reorder_timer_expired(struct timer_list *t)
{
        struct tid_ampdu_rx *tid_rx = timer_container_of(tid_rx, t,
                                                         reorder_timer);

        rcu_read_lock();
        ieee80211_release_reorder_timeout(tid_rx->sta, tid_rx->tid);
        rcu_read_unlock();
}

void ieee80211_add_addbaext(struct sk_buff *skb,
                            const u8 req_addba_ext_data,
                            u16 buf_size)
{
        struct ieee80211_addba_ext_ie *addba_ext;
        u8 *pos;

        pos = skb_put_zero(skb, 2 + sizeof(struct ieee80211_addba_ext_ie));
        *pos++ = WLAN_EID_ADDBA_EXT;
        *pos++ = sizeof(struct ieee80211_addba_ext_ie);
        addba_ext = (struct ieee80211_addba_ext_ie *)pos;

        addba_ext->data = IEEE80211_ADDBA_EXT_NO_FRAG;
        if (req_addba_ext_data)
                addba_ext->data &= req_addba_ext_data;

        addba_ext->data |=
                u8_encode_bits(buf_size >> IEEE80211_ADDBA_EXT_BUF_SIZE_SHIFT,
                               IEEE80211_ADDBA_EXT_BUF_SIZE_MASK);
}

u8 ieee80211_retrieve_addba_ext_data(struct sta_info *sta,
                                     const void *elem_data, ssize_t elem_len,
                                     u16 *buf_size)
{
        struct ieee802_11_elems *elems;
        u8 buf_size_1k, data = 0;

        if (!sta->sta.deflink.he_cap.has_he)
                return 0;

        if (elem_len <= 0)
                return 0;

        elems = ieee802_11_parse_elems(elem_data, elem_len, true, NULL);

        if (!elems || elems->parse_error || !elems->addba_ext_ie)
                goto free;

        data = elems->addba_ext_ie->data;

        if (buf_size &&
            (sta->sta.valid_links || sta->sta.deflink.eht_cap.has_eht)) {
                buf_size_1k = u8_get_bits(elems->addba_ext_ie->data,
                                          IEEE80211_ADDBA_EXT_BUF_SIZE_MASK);
                *buf_size |= (u16)buf_size_1k <<
                             IEEE80211_ADDBA_EXT_BUF_SIZE_SHIFT;
        }

free:
        kfree(elems);

        return data;
}

static void ieee80211_send_addba_resp(struct sta_info *sta, u8 *da, u16 tid,
                                      u8 dialog_token, u16 status, u16 policy,
                                      u16 buf_size, u16 timeout,
                                      const u8 req_addba_ext_data)
{
        struct ieee80211_sub_if_data *sdata = sta->sdata;
        struct ieee80211_local *local = sdata->local;
        struct sk_buff *skb;
        struct ieee80211_mgmt *mgmt;
        bool amsdu = ieee80211_hw_check(&local->hw, SUPPORTS_AMSDU_IN_AMPDU);
        u16 capab;

        skb = dev_alloc_skb(sizeof(*mgmt) +
                    2 + sizeof(struct ieee80211_addba_ext_ie) +
                    local->hw.extra_tx_headroom);
        if (!skb)
                return;

        skb_reserve(skb, local->hw.extra_tx_headroom);
        mgmt = ieee80211_mgmt_ba(skb, da, sdata);

        skb_put(skb, 1 + sizeof(mgmt->u.action.u.addba_resp));
        mgmt->u.action.category = WLAN_CATEGORY_BACK;
        mgmt->u.action.u.addba_resp.action_code = WLAN_ACTION_ADDBA_RESP;
        mgmt->u.action.u.addba_resp.dialog_token = dialog_token;

        capab = u16_encode_bits(amsdu, IEEE80211_ADDBA_PARAM_AMSDU_MASK);
        capab |= u16_encode_bits(policy, IEEE80211_ADDBA_PARAM_POLICY_MASK);
        capab |= u16_encode_bits(tid, IEEE80211_ADDBA_PARAM_TID_MASK);
        capab |= u16_encode_bits(buf_size, IEEE80211_ADDBA_PARAM_BUF_SIZE_MASK);

        mgmt->u.action.u.addba_resp.capab = cpu_to_le16(capab);
        mgmt->u.action.u.addba_resp.timeout = cpu_to_le16(timeout);
        mgmt->u.action.u.addba_resp.status = cpu_to_le16(status);

        if (sta->sta.valid_links || sta->sta.deflink.he_cap.has_he)
                ieee80211_add_addbaext(skb, req_addba_ext_data, buf_size);

        ieee80211_tx_skb(sdata, skb);
}

void __ieee80211_start_rx_ba_session(struct sta_info *sta,
                                     u8 dialog_token, u16 timeout,
                                     u16 start_seq_num, u16 ba_policy, u16 tid,
                                     u16 buf_size, bool tx, bool auto_seq,
                                     const u8 addba_ext_data)
{
        struct ieee80211_local *local = sta->sdata->local;
        struct tid_ampdu_rx *tid_agg_rx;
        struct ieee80211_ampdu_params params = {
                .sta = &sta->sta,
                .action = IEEE80211_AMPDU_RX_START,
                .tid = tid,
                .amsdu = false,
                .timeout = timeout,
                .ssn = start_seq_num,
        };
        int i, ret = -EOPNOTSUPP;
        u16 status = WLAN_STATUS_REQUEST_DECLINED;
        u16 max_buf_size;

        lockdep_assert_wiphy(sta->local->hw.wiphy);

        if (tid >= IEEE80211_FIRST_TSPEC_TSID) {
                ht_dbg(sta->sdata,
                       "STA %pM requests BA session on unsupported tid %d\n",
                       sta->sta.addr, tid);
                goto end;
        }

        if (!sta->sta.valid_links &&
            !sta->sta.deflink.ht_cap.ht_supported &&
            !sta->sta.deflink.he_cap.has_he &&
            !sta->sta.deflink.s1g_cap.s1g) {
                ht_dbg(sta->sdata,
                       "STA %pM erroneously requests BA session on tid %d w/o HT\n",
                       sta->sta.addr, tid);
                /* send a response anyway, it's an error case if we get here */
                goto end;
        }

        if (test_sta_flag(sta, WLAN_STA_BLOCK_BA)) {
                ht_dbg(sta->sdata,
                       "Suspend in progress - Denying ADDBA request (%pM tid %d)\n",
                       sta->sta.addr, tid);
                goto end;
        }

        if (sta->sta.valid_links || sta->sta.deflink.eht_cap.has_eht)
                max_buf_size = IEEE80211_MAX_AMPDU_BUF_EHT;
        else if (sta->sta.deflink.he_cap.has_he)
                max_buf_size = IEEE80211_MAX_AMPDU_BUF_HE;
        else
                max_buf_size = IEEE80211_MAX_AMPDU_BUF_HT;

        /* sanity check for incoming parameters:
         * check if configuration can support the BA policy
         * and if buffer size does not exceeds max value */
        /* XXX: check own ht delayed BA capability?? */
        if (((ba_policy != 1) &&
             (sta->sta.valid_links ||
              !(sta->sta.deflink.ht_cap.cap & IEEE80211_HT_CAP_DELAY_BA) ||
              !(sta->sta.deflink.s1g_cap.cap[3] & S1G_CAP3_HT_DELAYED_BA))) ||
            (buf_size > max_buf_size)) {
                status = WLAN_STATUS_INVALID_QOS_PARAM;
                ht_dbg_ratelimited(sta->sdata,
                                   "AddBA Req with bad params from %pM on tid %u. policy %d, buffer size %d\n",
                                   sta->sta.addr, tid, ba_policy, buf_size);
                goto end;
        }
        /* determine default buffer size */
        if (buf_size == 0)
                buf_size = max_buf_size;

        /* make sure the size doesn't exceed the maximum supported by the hw */
        if (buf_size > sta->sta.max_rx_aggregation_subframes)
                buf_size = sta->sta.max_rx_aggregation_subframes;
        params.buf_size = buf_size;

        ht_dbg(sta->sdata, "AddBA Req buf_size=%d for %pM\n",
               buf_size, sta->sta.addr);

        if (test_bit(tid, sta->ampdu_mlme.agg_session_valid)) {
                if (sta->ampdu_mlme.tid_rx_token[tid] == dialog_token) {
                        struct tid_ampdu_rx *tid_rx;

                        ht_dbg_ratelimited(sta->sdata,
                                           "updated AddBA Req from %pM on tid %u\n",
                                           sta->sta.addr, tid);
                        /* We have no API to update the timeout value in the
                         * driver so reject the timeout update if the timeout
                         * changed. If it did not change, i.e., no real update,
                         * just reply with success.
                         */
                        rcu_read_lock();
                        tid_rx = rcu_dereference(sta->ampdu_mlme.tid_rx[tid]);
                        if (tid_rx && tid_rx->timeout == timeout)
                                status = WLAN_STATUS_SUCCESS;
                        else
                                status = WLAN_STATUS_REQUEST_DECLINED;
                        rcu_read_unlock();
                        goto end;
                }

                ht_dbg_ratelimited(sta->sdata,
                                   "unexpected AddBA Req from %pM on tid %u\n",
                                   sta->sta.addr, tid);

                /* delete existing Rx BA session on the same tid */
                __ieee80211_stop_rx_ba_session(sta, tid, WLAN_BACK_RECIPIENT,
                                               WLAN_STATUS_UNSPECIFIED_QOS,
                                               false);
        }

        if (ieee80211_hw_check(&local->hw, SUPPORTS_REORDERING_BUFFER)) {
                ret = drv_ampdu_action(local, sta->sdata, &params);
                ht_dbg(sta->sdata,
                       "Rx A-MPDU request on %pM tid %d result %d\n",
                       sta->sta.addr, tid, ret);
                if (!ret)
                        status = WLAN_STATUS_SUCCESS;
                goto end;
        }

        /* prepare A-MPDU MLME for Rx aggregation */
        tid_agg_rx = kzalloc(sizeof(*tid_agg_rx), GFP_KERNEL);
        if (!tid_agg_rx)
                goto end;

        spin_lock_init(&tid_agg_rx->reorder_lock);

        /* rx timer */
        timer_setup(&tid_agg_rx->session_timer,
                    sta_rx_agg_session_timer_expired, TIMER_DEFERRABLE);

        /* rx reorder timer */
        timer_setup(&tid_agg_rx->reorder_timer,
                    sta_rx_agg_reorder_timer_expired, 0);

        /* prepare reordering buffer */
        tid_agg_rx->reorder_buf =
                kcalloc(buf_size, sizeof(struct sk_buff_head), GFP_KERNEL);
        tid_agg_rx->reorder_time =
                kcalloc(buf_size, sizeof(unsigned long), GFP_KERNEL);
        if (!tid_agg_rx->reorder_buf || !tid_agg_rx->reorder_time) {
                kfree(tid_agg_rx->reorder_buf);
                kfree(tid_agg_rx->reorder_time);
                kfree(tid_agg_rx);
                goto end;
        }

        for (i = 0; i < buf_size; i++)
                __skb_queue_head_init(&tid_agg_rx->reorder_buf[i]);

        ret = drv_ampdu_action(local, sta->sdata, &params);
        ht_dbg(sta->sdata, "Rx A-MPDU request on %pM tid %d result %d\n",
               sta->sta.addr, tid, ret);
        if (ret) {
                kfree(tid_agg_rx->reorder_buf);
                kfree(tid_agg_rx->reorder_time);
                kfree(tid_agg_rx);
                goto end;
        }

        /* update data */
        tid_agg_rx->ssn = start_seq_num;
        tid_agg_rx->head_seq_num = start_seq_num;
        tid_agg_rx->buf_size = buf_size;
        tid_agg_rx->timeout = timeout;
        tid_agg_rx->stored_mpdu_num = 0;
        tid_agg_rx->auto_seq = auto_seq;
        tid_agg_rx->started = false;
        tid_agg_rx->reorder_buf_filtered = 0;
        tid_agg_rx->tid = tid;
        tid_agg_rx->sta = sta;
        status = WLAN_STATUS_SUCCESS;

        /* activate it for RX */
        rcu_assign_pointer(sta->ampdu_mlme.tid_rx[tid], tid_agg_rx);

        if (timeout) {
                mod_timer(&tid_agg_rx->session_timer, TU_TO_EXP_TIME(timeout));
                tid_agg_rx->last_rx = jiffies;
        }

end:
        if (status == WLAN_STATUS_SUCCESS) {
                __set_bit(tid, sta->ampdu_mlme.agg_session_valid);
                __clear_bit(tid, sta->ampdu_mlme.unexpected_agg);
                sta->ampdu_mlme.tid_rx_token[tid] = dialog_token;
        }

        if (tx)
                ieee80211_send_addba_resp(sta, sta->sta.addr, tid,
                                          dialog_token, status, 1, buf_size,
                                          timeout, addba_ext_data);
}

void ieee80211_process_addba_request(struct ieee80211_local *local,
                                     struct sta_info *sta,
                                     struct ieee80211_mgmt *mgmt,
                                     size_t len)
{
        u16 capab, tid, timeout, ba_policy, buf_size, start_seq_num;
        u8 dialog_token, addba_ext_data;

        /* extract session parameters from addba request frame */
        dialog_token = mgmt->u.action.u.addba_req.dialog_token;
        timeout = le16_to_cpu(mgmt->u.action.u.addba_req.timeout);
        start_seq_num =
                le16_to_cpu(mgmt->u.action.u.addba_req.start_seq_num) >> 4;

        capab = le16_to_cpu(mgmt->u.action.u.addba_req.capab);
        ba_policy = (capab & IEEE80211_ADDBA_PARAM_POLICY_MASK) >> 1;
        tid = (capab & IEEE80211_ADDBA_PARAM_TID_MASK) >> 2;
        buf_size = (capab & IEEE80211_ADDBA_PARAM_BUF_SIZE_MASK) >> 6;

        addba_ext_data =
                ieee80211_retrieve_addba_ext_data(sta,
                                                  mgmt->u.action.u.addba_req.variable,
                                                  len -
                                                  offsetof(typeof(*mgmt),
                                                           u.action.u.addba_req.variable),
                                                  &buf_size);

        __ieee80211_start_rx_ba_session(sta, dialog_token, timeout,
                                        start_seq_num, ba_policy, tid,
                                        buf_size, true, false, addba_ext_data);
}

void ieee80211_manage_rx_ba_offl(struct ieee80211_vif *vif,
                                 const u8 *addr, unsigned int tid)
{
        struct ieee80211_sub_if_data *sdata = vif_to_sdata(vif);
        struct sta_info *sta;

        rcu_read_lock();
        sta = sta_info_get_bss(sdata, addr);
        if (!sta)
                goto unlock;

        set_bit(tid, sta->ampdu_mlme.tid_rx_manage_offl);
        wiphy_work_queue(sta->local->hw.wiphy, &sta->ampdu_mlme.work);
 unlock:
        rcu_read_unlock();
}
EXPORT_SYMBOL(ieee80211_manage_rx_ba_offl);

void ieee80211_rx_ba_timer_expired(struct ieee80211_vif *vif,
                                   const u8 *addr, unsigned int tid)
{
        struct ieee80211_sub_if_data *sdata = vif_to_sdata(vif);
        struct sta_info *sta;

        rcu_read_lock();
        sta = sta_info_get_bss(sdata, addr);
        if (!sta)
                goto unlock;

        set_bit(tid, sta->ampdu_mlme.tid_rx_timer_expired);
        wiphy_work_queue(sta->local->hw.wiphy, &sta->ampdu_mlme.work);

 unlock:
        rcu_read_unlock();
}
EXPORT_SYMBOL(ieee80211_rx_ba_timer_expired);































   72 












    4 
 12918 


















 13027 
 2503 



















    2 
    8 












































 1013 


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
/* SPDX-License-Identifier: GPL-2.0-only */
/*
 * kref.h - library routines for handling generic reference counted objects
 *
 * Copyright (C) 2004 Greg Kroah-Hartman <greg@kroah.com>
 * Copyright (C) 2004 IBM Corp.
 *
 * based on kobject.h which was:
 * Copyright (C) 2002-2003 Patrick Mochel <mochel@osdl.org>
 * Copyright (C) 2002-2003 Open Source Development Labs
 */

#ifndef _KREF_H_
#define _KREF_H_

#include <linux/spinlock.h>
#include <linux/refcount.h>

struct kref {
        refcount_t refcount;
};

#define KREF_INIT(n)        { .refcount = REFCOUNT_INIT(n), }

/**
 * kref_init - initialize object.
 * @kref: object in question.
 */
static inline void kref_init(struct kref *kref)
{
        refcount_set(&kref->refcount, 1);
}

static inline unsigned int kref_read(const struct kref *kref)
{
        return refcount_read(&kref->refcount);
}

/**
 * kref_get - increment refcount for object.
 * @kref: object.
 */
static inline void kref_get(struct kref *kref)
{
        refcount_inc(&kref->refcount);
}

/**
 * kref_put - Decrement refcount for object
 * @kref: Object
 * @release: Pointer to the function that will clean up the object when the
 *             last reference to the object is released.
 *
 * Decrement the refcount, and if 0, call @release.  The caller may not
 * pass NULL or kfree() as the release function.
 *
 * Return: 1 if this call removed the object, otherwise return 0.  Beware,
 * if this function returns 0, another caller may have removed the object
 * by the time this function returns.  The return value is only certain
 * if you want to see if the object is definitely released.
 */
static inline int kref_put(struct kref *kref, void (*release)(struct kref *kref))
{
        if (refcount_dec_and_test(&kref->refcount)) {
                release(kref);
                return 1;
        }
        return 0;
}

/**
 * kref_put_mutex - Decrement refcount for object
 * @kref: Object
 * @release: Pointer to the function that will clean up the object when the
 *             last reference to the object is released.
 * @mutex: Mutex which protects the release function.
 *
 * This variant of kref_lock() calls the @release function with the @mutex
 * held.  The @release function will release the mutex.
 */
static inline int kref_put_mutex(struct kref *kref,
                                 void (*release)(struct kref *kref),
                                 struct mutex *mutex)
{
        if (refcount_dec_and_mutex_lock(&kref->refcount, mutex)) {
                release(kref);
                return 1;
        }
        return 0;
}

/**
 * kref_put_lock - Decrement refcount for object
 * @kref: Object
 * @release: Pointer to the function that will clean up the object when the
 *             last reference to the object is released.
 * @lock: Spinlock which protects the release function.
 *
 * This variant of kref_lock() calls the @release function with the @lock
 * held.  The @release function will release the lock.
 */
static inline int kref_put_lock(struct kref *kref,
                                void (*release)(struct kref *kref),
                                spinlock_t *lock)
{
        if (refcount_dec_and_lock(&kref->refcount, lock)) {
                release(kref);
                return 1;
        }
        return 0;
}

/**
 * kref_get_unless_zero - Increment refcount for object unless it is zero.
 * @kref: object.
 *
 * This function is intended to simplify locking around refcounting for
 * objects that can be looked up from a lookup structure, and which are
 * removed from that lookup structure in the object destructor.
 * Operations on such objects require at least a read lock around
 * lookup + kref_get, and a write lock around kref_put + remove from lookup
 * structure. Furthermore, RCU implementations become extremely tricky.
 * With a lookup followed by a kref_get_unless_zero *with return value check*
 * locking in the kref_put path can be deferred to the actual removal from
 * the lookup structure and RCU lookups become trivial.
 *
 * Return: non-zero if the increment succeeded. Otherwise return 0.
 */
static inline int __must_check kref_get_unless_zero(struct kref *kref)
{
        return refcount_inc_not_zero(&kref->refcount);
}
#endif /* _KREF_H_ */




























































   17 



    2 




    1 



    1 


   11 
    2 

    1 
    8 

    2 














    3 





    7 













    8 










    2 








    2 
















    2 






    2 



    2 
































   11 




    4 


    7 





    1 







    5 







    1 


































    5 







    1 







    1 







    1 




















   19 
   12 


    1 



    5 

    1 
























    1 




    1 



















































































    1 
    1 



















































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright (c) 2008-2009 Patrick McHardy <kaber@trash.net>
 *
 * Development of this code funded by Astaro AG (http://www.astaro.com/)
 */

#include <linux/kernel.h>
#include <linux/init.h>
#include <linux/module.h>
#include <linux/spinlock.h>
#include <linux/netlink.h>
#include <linux/netfilter.h>
#include <linux/netfilter/nf_tables.h>
#include <net/netfilter/nf_tables.h>

struct nft_limit {
        spinlock_t        lock;
        u64                last;
        u64                tokens;
};

struct nft_limit_priv {
        struct nft_limit *limit;
        u64                tokens_max;
        u64                rate;
        u64                nsecs;
        u32                burst;
        bool                invert;
};

static inline bool nft_limit_eval(struct nft_limit_priv *priv, u64 cost)
{
        u64 now, tokens;
        s64 delta;

        spin_lock_bh(&priv->limit->lock);
        now = ktime_get_ns();
        tokens = priv->limit->tokens + now - priv->limit->last;
        if (tokens > priv->tokens_max)
                tokens = priv->tokens_max;

        priv->limit->last = now;
        delta = tokens - cost;
        if (delta >= 0) {
                priv->limit->tokens = delta;
                spin_unlock_bh(&priv->limit->lock);
                return priv->invert;
        }
        priv->limit->tokens = tokens;
        spin_unlock_bh(&priv->limit->lock);
        return !priv->invert;
}

/* Use same default as in iptables. */
#define NFT_LIMIT_PKT_BURST_DEFAULT        5

static int nft_limit_init(struct nft_limit_priv *priv,
                          const struct nlattr * const tb[], bool pkts)
{
        u64 unit, tokens, rate_with_burst;
        bool invert = false;

        if (tb[NFTA_LIMIT_RATE] == NULL ||
            tb[NFTA_LIMIT_UNIT] == NULL)
                return -EINVAL;

        priv->rate = be64_to_cpu(nla_get_be64(tb[NFTA_LIMIT_RATE]));
        if (priv->rate == 0)
                return -EINVAL;

        unit = be64_to_cpu(nla_get_be64(tb[NFTA_LIMIT_UNIT]));
        if (check_mul_overflow(unit, NSEC_PER_SEC, &priv->nsecs))
                return -EOVERFLOW;

        if (tb[NFTA_LIMIT_BURST])
                priv->burst = ntohl(nla_get_be32(tb[NFTA_LIMIT_BURST]));

        if (pkts && priv->burst == 0)
                priv->burst = NFT_LIMIT_PKT_BURST_DEFAULT;

        if (check_add_overflow(priv->rate, priv->burst, &rate_with_burst))
                return -EOVERFLOW;

        if (pkts) {
                u64 tmp = div64_u64(priv->nsecs, priv->rate);

                if (check_mul_overflow(tmp, priv->burst, &tokens))
                        return -EOVERFLOW;
        } else {
                u64 tmp;

                /* The token bucket size limits the number of tokens can be
                 * accumulated. tokens_max specifies the bucket size.
                 * tokens_max = unit * (rate + burst) / rate.
                 */
                if (check_mul_overflow(priv->nsecs, rate_with_burst, &tmp))
                        return -EOVERFLOW;

                tokens = div64_u64(tmp, priv->rate);
        }

        if (tb[NFTA_LIMIT_FLAGS]) {
                u32 flags = ntohl(nla_get_be32(tb[NFTA_LIMIT_FLAGS]));

                if (flags & ~NFT_LIMIT_F_INV)
                        return -EOPNOTSUPP;

                if (flags & NFT_LIMIT_F_INV)
                        invert = true;
        }

        priv->limit = kmalloc(sizeof(*priv->limit), GFP_KERNEL_ACCOUNT);
        if (!priv->limit)
                return -ENOMEM;

        priv->limit->tokens = tokens;
        priv->tokens_max = priv->limit->tokens;
        priv->invert = invert;
        priv->limit->last = ktime_get_ns();
        spin_lock_init(&priv->limit->lock);

        return 0;
}

static int nft_limit_dump(struct sk_buff *skb, const struct nft_limit_priv *priv,
                          enum nft_limit_type type)
{
        u32 flags = priv->invert ? NFT_LIMIT_F_INV : 0;
        u64 secs = div_u64(priv->nsecs, NSEC_PER_SEC);

        if (nla_put_be64(skb, NFTA_LIMIT_RATE, cpu_to_be64(priv->rate),
                         NFTA_LIMIT_PAD) ||
            nla_put_be64(skb, NFTA_LIMIT_UNIT, cpu_to_be64(secs),
                         NFTA_LIMIT_PAD) ||
            nla_put_be32(skb, NFTA_LIMIT_BURST, htonl(priv->burst)) ||
            nla_put_be32(skb, NFTA_LIMIT_TYPE, htonl(type)) ||
            nla_put_be32(skb, NFTA_LIMIT_FLAGS, htonl(flags)))
                goto nla_put_failure;
        return 0;

nla_put_failure:
        return -1;
}

static void nft_limit_destroy(const struct nft_ctx *ctx,
                              const struct nft_limit_priv *priv)
{
        kfree(priv->limit);
}

static int nft_limit_clone(struct nft_limit_priv *priv_dst,
                           const struct nft_limit_priv *priv_src, gfp_t gfp)
{
        priv_dst->tokens_max = priv_src->tokens_max;
        priv_dst->rate = priv_src->rate;
        priv_dst->nsecs = priv_src->nsecs;
        priv_dst->burst = priv_src->burst;
        priv_dst->invert = priv_src->invert;

        priv_dst->limit = kmalloc(sizeof(*priv_dst->limit), gfp);
        if (!priv_dst->limit)
                return -ENOMEM;

        spin_lock_init(&priv_dst->limit->lock);
        priv_dst->limit->tokens = priv_src->tokens_max;
        priv_dst->limit->last = ktime_get_ns();

        return 0;
}

struct nft_limit_priv_pkts {
        struct nft_limit_priv        limit;
        u64                        cost;
};

static void nft_limit_pkts_eval(const struct nft_expr *expr,
                                struct nft_regs *regs,
                                const struct nft_pktinfo *pkt)
{
        struct nft_limit_priv_pkts *priv = nft_expr_priv(expr);

        if (nft_limit_eval(&priv->limit, priv->cost))
                regs->verdict.code = NFT_BREAK;
}

static const struct nla_policy nft_limit_policy[NFTA_LIMIT_MAX + 1] = {
        [NFTA_LIMIT_RATE]        = { .type = NLA_U64 },
        [NFTA_LIMIT_UNIT]        = { .type = NLA_U64 },
        [NFTA_LIMIT_BURST]        = { .type = NLA_U32 },
        [NFTA_LIMIT_TYPE]        = { .type = NLA_U32 },
        [NFTA_LIMIT_FLAGS]        = { .type = NLA_U32 },
};

static int nft_limit_pkts_init(const struct nft_ctx *ctx,
                               const struct nft_expr *expr,
                               const struct nlattr * const tb[])
{
        struct nft_limit_priv_pkts *priv = nft_expr_priv(expr);
        int err;

        err = nft_limit_init(&priv->limit, tb, true);
        if (err < 0)
                return err;

        priv->cost = div64_u64(priv->limit.nsecs, priv->limit.rate);
        return 0;
}

static int nft_limit_pkts_dump(struct sk_buff *skb,
                               const struct nft_expr *expr, bool reset)
{
        const struct nft_limit_priv_pkts *priv = nft_expr_priv(expr);

        return nft_limit_dump(skb, &priv->limit, NFT_LIMIT_PKTS);
}

static void nft_limit_pkts_destroy(const struct nft_ctx *ctx,
                                   const struct nft_expr *expr)
{
        const struct nft_limit_priv_pkts *priv = nft_expr_priv(expr);

        nft_limit_destroy(ctx, &priv->limit);
}

static int nft_limit_pkts_clone(struct nft_expr *dst, const struct nft_expr *src,
                                gfp_t gfp)
{
        struct nft_limit_priv_pkts *priv_dst = nft_expr_priv(dst);
        struct nft_limit_priv_pkts *priv_src = nft_expr_priv(src);

        priv_dst->cost = priv_src->cost;

        return nft_limit_clone(&priv_dst->limit, &priv_src->limit, gfp);
}

static struct nft_expr_type nft_limit_type;
static const struct nft_expr_ops nft_limit_pkts_ops = {
        .type                = &nft_limit_type,
        .size                = NFT_EXPR_SIZE(sizeof(struct nft_limit_priv_pkts)),
        .eval                = nft_limit_pkts_eval,
        .init                = nft_limit_pkts_init,
        .destroy        = nft_limit_pkts_destroy,
        .clone                = nft_limit_pkts_clone,
        .dump                = nft_limit_pkts_dump,
        .reduce                = NFT_REDUCE_READONLY,
};

static void nft_limit_bytes_eval(const struct nft_expr *expr,
                                 struct nft_regs *regs,
                                 const struct nft_pktinfo *pkt)
{
        struct nft_limit_priv *priv = nft_expr_priv(expr);
        u64 cost = div64_u64(priv->nsecs * pkt->skb->len, priv->rate);

        if (nft_limit_eval(priv, cost))
                regs->verdict.code = NFT_BREAK;
}

static int nft_limit_bytes_init(const struct nft_ctx *ctx,
                                const struct nft_expr *expr,
                                const struct nlattr * const tb[])
{
        struct nft_limit_priv *priv = nft_expr_priv(expr);

        return nft_limit_init(priv, tb, false);
}

static int nft_limit_bytes_dump(struct sk_buff *skb,
                                const struct nft_expr *expr, bool reset)
{
        const struct nft_limit_priv *priv = nft_expr_priv(expr);

        return nft_limit_dump(skb, priv, NFT_LIMIT_PKT_BYTES);
}

static void nft_limit_bytes_destroy(const struct nft_ctx *ctx,
                                    const struct nft_expr *expr)
{
        const struct nft_limit_priv *priv = nft_expr_priv(expr);

        nft_limit_destroy(ctx, priv);
}

static int nft_limit_bytes_clone(struct nft_expr *dst, const struct nft_expr *src,
                                 gfp_t gfp)
{
        struct nft_limit_priv *priv_dst = nft_expr_priv(dst);
        struct nft_limit_priv *priv_src = nft_expr_priv(src);

        return nft_limit_clone(priv_dst, priv_src, gfp);
}

static const struct nft_expr_ops nft_limit_bytes_ops = {
        .type                = &nft_limit_type,
        .size                = NFT_EXPR_SIZE(sizeof(struct nft_limit_priv)),
        .eval                = nft_limit_bytes_eval,
        .init                = nft_limit_bytes_init,
        .dump                = nft_limit_bytes_dump,
        .clone                = nft_limit_bytes_clone,
        .destroy        = nft_limit_bytes_destroy,
        .reduce                = NFT_REDUCE_READONLY,
};

static const struct nft_expr_ops *
nft_limit_select_ops(const struct nft_ctx *ctx,
                     const struct nlattr * const tb[])
{
        if (tb[NFTA_LIMIT_TYPE] == NULL)
                return &nft_limit_pkts_ops;

        switch (ntohl(nla_get_be32(tb[NFTA_LIMIT_TYPE]))) {
        case NFT_LIMIT_PKTS:
                return &nft_limit_pkts_ops;
        case NFT_LIMIT_PKT_BYTES:
                return &nft_limit_bytes_ops;
        }
        return ERR_PTR(-EOPNOTSUPP);
}

static struct nft_expr_type nft_limit_type __read_mostly = {
        .name                = "limit",
        .select_ops        = nft_limit_select_ops,
        .policy                = nft_limit_policy,
        .maxattr        = NFTA_LIMIT_MAX,
        .flags                = NFT_EXPR_STATEFUL,
        .owner                = THIS_MODULE,
};

static void nft_limit_obj_pkts_eval(struct nft_object *obj,
                                    struct nft_regs *regs,
                                    const struct nft_pktinfo *pkt)
{
        struct nft_limit_priv_pkts *priv = nft_obj_data(obj);

        if (nft_limit_eval(&priv->limit, priv->cost))
                regs->verdict.code = NFT_BREAK;
}

static int nft_limit_obj_pkts_init(const struct nft_ctx *ctx,
                                   const struct nlattr * const tb[],
                                   struct nft_object *obj)
{
        struct nft_limit_priv_pkts *priv = nft_obj_data(obj);
        int err;

        err = nft_limit_init(&priv->limit, tb, true);
        if (err < 0)
                return err;

        priv->cost = div64_u64(priv->limit.nsecs, priv->limit.rate);
        return 0;
}

static int nft_limit_obj_pkts_dump(struct sk_buff *skb,
                                   struct nft_object *obj,
                                   bool reset)
{
        const struct nft_limit_priv_pkts *priv = nft_obj_data(obj);

        return nft_limit_dump(skb, &priv->limit, NFT_LIMIT_PKTS);
}

static void nft_limit_obj_pkts_destroy(const struct nft_ctx *ctx,
                                       struct nft_object *obj)
{
        struct nft_limit_priv_pkts *priv = nft_obj_data(obj);

        nft_limit_destroy(ctx, &priv->limit);
}

static struct nft_object_type nft_limit_obj_type;
static const struct nft_object_ops nft_limit_obj_pkts_ops = {
        .type                = &nft_limit_obj_type,
        .size                = NFT_EXPR_SIZE(sizeof(struct nft_limit_priv_pkts)),
        .init                = nft_limit_obj_pkts_init,
        .destroy        = nft_limit_obj_pkts_destroy,
        .eval                = nft_limit_obj_pkts_eval,
        .dump                = nft_limit_obj_pkts_dump,
};

static void nft_limit_obj_bytes_eval(struct nft_object *obj,
                                     struct nft_regs *regs,
                                     const struct nft_pktinfo *pkt)
{
        struct nft_limit_priv *priv = nft_obj_data(obj);
        u64 cost = div64_u64(priv->nsecs * pkt->skb->len, priv->rate);

        if (nft_limit_eval(priv, cost))
                regs->verdict.code = NFT_BREAK;
}

static int nft_limit_obj_bytes_init(const struct nft_ctx *ctx,
                                    const struct nlattr * const tb[],
                                    struct nft_object *obj)
{
        struct nft_limit_priv *priv = nft_obj_data(obj);

        return nft_limit_init(priv, tb, false);
}

static int nft_limit_obj_bytes_dump(struct sk_buff *skb,
                                    struct nft_object *obj,
                                    bool reset)
{
        const struct nft_limit_priv *priv = nft_obj_data(obj);

        return nft_limit_dump(skb, priv, NFT_LIMIT_PKT_BYTES);
}

static void nft_limit_obj_bytes_destroy(const struct nft_ctx *ctx,
                                        struct nft_object *obj)
{
        struct nft_limit_priv *priv = nft_obj_data(obj);

        nft_limit_destroy(ctx, priv);
}

static struct nft_object_type nft_limit_obj_type;
static const struct nft_object_ops nft_limit_obj_bytes_ops = {
        .type                = &nft_limit_obj_type,
        .size                = sizeof(struct nft_limit_priv),
        .init                = nft_limit_obj_bytes_init,
        .destroy        = nft_limit_obj_bytes_destroy,
        .eval                = nft_limit_obj_bytes_eval,
        .dump                = nft_limit_obj_bytes_dump,
};

static const struct nft_object_ops *
nft_limit_obj_select_ops(const struct nft_ctx *ctx,
                         const struct nlattr * const tb[])
{
        if (!tb[NFTA_LIMIT_TYPE])
                return &nft_limit_obj_pkts_ops;

        switch (ntohl(nla_get_be32(tb[NFTA_LIMIT_TYPE]))) {
        case NFT_LIMIT_PKTS:
                return &nft_limit_obj_pkts_ops;
        case NFT_LIMIT_PKT_BYTES:
                return &nft_limit_obj_bytes_ops;
        }
        return ERR_PTR(-EOPNOTSUPP);
}

static struct nft_object_type nft_limit_obj_type __read_mostly = {
        .select_ops        = nft_limit_obj_select_ops,
        .type                = NFT_OBJECT_LIMIT,
        .maxattr        = NFTA_LIMIT_MAX,
        .policy                = nft_limit_policy,
        .owner                = THIS_MODULE,
};

static int __init nft_limit_module_init(void)
{
        int err;

        err = nft_register_obj(&nft_limit_obj_type);
        if (err < 0)
                return err;

        err = nft_register_expr(&nft_limit_type);
        if (err < 0)
                goto err1;

        return 0;
err1:
        nft_unregister_obj(&nft_limit_obj_type);
        return err;
}

static void __exit nft_limit_module_exit(void)
{
        nft_unregister_expr(&nft_limit_type);
        nft_unregister_obj(&nft_limit_obj_type);
}

module_init(nft_limit_module_init);
module_exit(nft_limit_module_exit);

MODULE_LICENSE("GPL");
MODULE_AUTHOR("Patrick McHardy <kaber@trash.net>");
MODULE_ALIAS_NFT_EXPR("limit");
MODULE_ALIAS_NFT_OBJ(NFT_OBJECT_LIMIT);
MODULE_DESCRIPTION("nftables limit expression support");































   78 






   80 




    1 











   69 

   15 







   69 

   15 







    2 
    1 
    2 



    2 
    2 

    2 













    2 








    2 




    2 




































    3 





    1 










    2 













   20 


    7 

   60 
    1 
    2 












   43 
   50 







    2 


    7 
    7 

    5 

    5 


    4 

    4 




    5 






    1 




    1 






























    4 





    1 

    2 



    1 












    2 
    1 




























































    1 



    6 

    7 


    7 


















1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
// SPDX-License-Identifier: GPL-2.0
/*
 * High-level sync()-related operations
 */

#include <linux/blkdev.h>
#include <linux/kernel.h>
#include <linux/file.h>
#include <linux/fs.h>
#include <linux/slab.h>
#include <linux/export.h>
#include <linux/namei.h>
#include <linux/sched.h>
#include <linux/writeback.h>
#include <linux/syscalls.h>
#include <linux/linkage.h>
#include <linux/pagemap.h>
#include <linux/quotaops.h>
#include <linux/backing-dev.h>
#include "internal.h"

#define VALID_FLAGS (SYNC_FILE_RANGE_WAIT_BEFORE|SYNC_FILE_RANGE_WRITE| \
                        SYNC_FILE_RANGE_WAIT_AFTER)

/*
 * Write out and wait upon all dirty data associated with this
 * superblock.  Filesystem data as well as the underlying block
 * device.  Takes the superblock lock.
 */
int sync_filesystem(struct super_block *sb)
{
        int ret = 0;

        /*
         * We need to be protected against the filesystem going from
         * r/o to r/w or vice versa.
         */
        WARN_ON(!rwsem_is_locked(&sb->s_umount));

        /*
         * No point in syncing out anything if the filesystem is read-only.
         */
        if (sb_rdonly(sb))
                return 0;

        /*
         * Do the filesystem syncing work.  For simple filesystems
         * writeback_inodes_sb(sb) just dirties buffers with inodes so we have
         * to submit I/O for these buffers via sync_blockdev().  This also
         * speeds up the wait == 1 case since in that case write_inode()
         * methods call sync_dirty_buffer() and thus effectively write one block
         * at a time.
         */
        writeback_inodes_sb(sb, WB_REASON_SYNC);
        if (sb->s_op->sync_fs) {
                ret = sb->s_op->sync_fs(sb, 0);
                if (ret)
                        return ret;
        }
        ret = sync_blockdev_nowait(sb->s_bdev);
        if (ret)
                return ret;

        sync_inodes_sb(sb);
        if (sb->s_op->sync_fs) {
                ret = sb->s_op->sync_fs(sb, 1);
                if (ret)
                        return ret;
        }
        return sync_blockdev(sb->s_bdev);
}
EXPORT_SYMBOL(sync_filesystem);

static void sync_inodes_one_sb(struct super_block *sb, void *arg)
{
        if (!sb_rdonly(sb))
                sync_inodes_sb(sb);
}

static void sync_fs_one_sb(struct super_block *sb, void *arg)
{
        if (!sb_rdonly(sb) && !(sb->s_iflags & SB_I_SKIP_SYNC) &&
            sb->s_op->sync_fs)
                sb->s_op->sync_fs(sb, *(int *)arg);
}

/*
 * Sync everything. We start by waking flusher threads so that most of
 * writeback runs on all devices in parallel. Then we sync all inodes reliably
 * which effectively also waits for all flusher threads to finish doing
 * writeback. At this point all data is on disk so metadata should be stable
 * and we tell filesystems to sync their metadata via ->sync_fs() calls.
 * Finally, we writeout all block devices because some filesystems (e.g. ext2)
 * just write metadata (such as inodes or bitmaps) to block device page cache
 * and do not sync it on their own in ->sync_fs().
 */
void ksys_sync(void)
{
        int nowait = 0, wait = 1;

        wakeup_flusher_threads(WB_REASON_SYNC);
        iterate_supers(sync_inodes_one_sb, NULL);
        iterate_supers(sync_fs_one_sb, &nowait);
        iterate_supers(sync_fs_one_sb, &wait);
        sync_bdevs(false);
        sync_bdevs(true);
        if (unlikely(laptop_mode))
                laptop_sync_completion();
}

SYSCALL_DEFINE0(sync)
{
        ksys_sync();
        return 0;
}

static void do_sync_work(struct work_struct *work)
{
        int nowait = 0;

        /*
         * Sync twice to reduce the possibility we skipped some inodes / pages
         * because they were temporarily locked
         */
        iterate_supers(sync_inodes_one_sb, &nowait);
        iterate_supers(sync_fs_one_sb, &nowait);
        sync_bdevs(false);
        iterate_supers(sync_inodes_one_sb, &nowait);
        iterate_supers(sync_fs_one_sb, &nowait);
        sync_bdevs(false);
        printk("Emergency Sync complete\n");
        kfree(work);
}

void emergency_sync(void)
{
        struct work_struct *work;

        work = kmalloc(sizeof(*work), GFP_ATOMIC);
        if (work) {
                INIT_WORK(work, do_sync_work);
                schedule_work(work);
        }
}

/*
 * sync a single super
 */
SYSCALL_DEFINE1(syncfs, int, fd)
{
        CLASS(fd, f)(fd);
        struct super_block *sb;
        int ret, ret2;

        if (fd_empty(f))
                return -EBADF;
        sb = fd_file(f)->f_path.dentry->d_sb;

        down_read(&sb->s_umount);
        ret = sync_filesystem(sb);
        up_read(&sb->s_umount);

        ret2 = errseq_check_and_advance(&sb->s_wb_err, &fd_file(f)->f_sb_err);

        return ret ? ret : ret2;
}

/**
 * vfs_fsync_range - helper to sync a range of data & metadata to disk
 * @file:                file to sync
 * @start:                offset in bytes of the beginning of data range to sync
 * @end:                offset in bytes of the end of data range (inclusive)
 * @datasync:                perform only datasync
 *
 * Write back data in range @start..@end and metadata for @file to disk.  If
 * @datasync is set only metadata needed to access modified file data is
 * written.
 */
int vfs_fsync_range(struct file *file, loff_t start, loff_t end, int datasync)
{
        struct inode *inode = file->f_mapping->host;

        if (!file->f_op->fsync)
                return -EINVAL;
        if (!datasync && (inode->i_state & I_DIRTY_TIME))
                mark_inode_dirty_sync(inode);
        return file->f_op->fsync(file, start, end, datasync);
}
EXPORT_SYMBOL(vfs_fsync_range);

/**
 * vfs_fsync - perform a fsync or fdatasync on a file
 * @file:                file to sync
 * @datasync:                only perform a fdatasync operation
 *
 * Write back data and metadata for @file to disk.  If @datasync is
 * set only metadata needed to access modified file data is written.
 */
int vfs_fsync(struct file *file, int datasync)
{
        return vfs_fsync_range(file, 0, LLONG_MAX, datasync);
}
EXPORT_SYMBOL(vfs_fsync);

static int do_fsync(unsigned int fd, int datasync)
{
        CLASS(fd, f)(fd);

        if (fd_empty(f))
                return -EBADF;

        return vfs_fsync(fd_file(f), datasync);
}

SYSCALL_DEFINE1(fsync, unsigned int, fd)
{
        return do_fsync(fd, 0);
}

SYSCALL_DEFINE1(fdatasync, unsigned int, fd)
{
        return do_fsync(fd, 1);
}

int sync_file_range(struct file *file, loff_t offset, loff_t nbytes,
                    unsigned int flags)
{
        int ret;
        struct address_space *mapping;
        loff_t endbyte;                        /* inclusive */
        umode_t i_mode;

        ret = -EINVAL;
        if (flags & ~VALID_FLAGS)
                goto out;

        endbyte = offset + nbytes;

        if ((s64)offset < 0)
                goto out;
        if ((s64)endbyte < 0)
                goto out;
        if (endbyte < offset)
                goto out;

        if (sizeof(pgoff_t) == 4) {
                if (offset >= (0x100000000ULL << PAGE_SHIFT)) {
                        /*
                         * The range starts outside a 32 bit machine's
                         * pagecache addressing capabilities.  Let it "succeed"
                         */
                        ret = 0;
                        goto out;
                }
                if (endbyte >= (0x100000000ULL << PAGE_SHIFT)) {
                        /*
                         * Out to EOF
                         */
                        nbytes = 0;
                }
        }

        if (nbytes == 0)
                endbyte = LLONG_MAX;
        else
                endbyte--;                /* inclusive */

        i_mode = file_inode(file)->i_mode;
        ret = -ESPIPE;
        if (!S_ISREG(i_mode) && !S_ISBLK(i_mode) && !S_ISDIR(i_mode) &&
                        !S_ISLNK(i_mode))
                goto out;

        mapping = file->f_mapping;
        ret = 0;
        if (flags & SYNC_FILE_RANGE_WAIT_BEFORE) {
                ret = file_fdatawait_range(file, offset, endbyte);
                if (ret < 0)
                        goto out;
        }

        if (flags & SYNC_FILE_RANGE_WRITE) {
                int sync_mode = WB_SYNC_NONE;

                if ((flags & SYNC_FILE_RANGE_WRITE_AND_WAIT) ==
                             SYNC_FILE_RANGE_WRITE_AND_WAIT)
                        sync_mode = WB_SYNC_ALL;

                ret = __filemap_fdatawrite_range(mapping, offset, endbyte,
                                                 sync_mode);
                if (ret < 0)
                        goto out;
        }

        if (flags & SYNC_FILE_RANGE_WAIT_AFTER)
                ret = file_fdatawait_range(file, offset, endbyte);

out:
        return ret;
}

/*
 * ksys_sync_file_range() permits finely controlled syncing over a segment of
 * a file in the range offset .. (offset+nbytes-1) inclusive.  If nbytes is
 * zero then ksys_sync_file_range() will operate from offset out to EOF.
 *
 * The flag bits are:
 *
 * SYNC_FILE_RANGE_WAIT_BEFORE: wait upon writeout of all pages in the range
 * before performing the write.
 *
 * SYNC_FILE_RANGE_WRITE: initiate writeout of all those dirty pages in the
 * range which are not presently under writeback. Note that this may block for
 * significant periods due to exhaustion of disk request structures.
 *
 * SYNC_FILE_RANGE_WAIT_AFTER: wait upon writeout of all pages in the range
 * after performing the write.
 *
 * Useful combinations of the flag bits are:
 *
 * SYNC_FILE_RANGE_WAIT_BEFORE|SYNC_FILE_RANGE_WRITE: ensures that all pages
 * in the range which were dirty on entry to ksys_sync_file_range() are placed
 * under writeout.  This is a start-write-for-data-integrity operation.
 *
 * SYNC_FILE_RANGE_WRITE: start writeout of all dirty pages in the range which
 * are not presently under writeout.  This is an asynchronous flush-to-disk
 * operation.  Not suitable for data integrity operations.
 *
 * SYNC_FILE_RANGE_WAIT_BEFORE (or SYNC_FILE_RANGE_WAIT_AFTER): wait for
 * completion of writeout of all pages in the range.  This will be used after an
 * earlier SYNC_FILE_RANGE_WAIT_BEFORE|SYNC_FILE_RANGE_WRITE operation to wait
 * for that operation to complete and to return the result.
 *
 * SYNC_FILE_RANGE_WAIT_BEFORE|SYNC_FILE_RANGE_WRITE|SYNC_FILE_RANGE_WAIT_AFTER
 * (a.k.a. SYNC_FILE_RANGE_WRITE_AND_WAIT):
 * a traditional sync() operation.  This is a write-for-data-integrity operation
 * which will ensure that all pages in the range which were dirty on entry to
 * ksys_sync_file_range() are written to disk.  It should be noted that disk
 * caches are not flushed by this call, so there are no guarantees here that the
 * data will be available on disk after a crash.
 *
 *
 * SYNC_FILE_RANGE_WAIT_BEFORE and SYNC_FILE_RANGE_WAIT_AFTER will detect any
 * I/O errors or ENOSPC conditions and will return those to the caller, after
 * clearing the EIO and ENOSPC flags in the address_space.
 *
 * It should be noted that none of these operations write out the file's
 * metadata.  So unless the application is strictly performing overwrites of
 * already-instantiated disk blocks, there are no guarantees here that the data
 * will be available after a crash.
 */
int ksys_sync_file_range(int fd, loff_t offset, loff_t nbytes,
                         unsigned int flags)
{
        CLASS(fd, f)(fd);

        if (fd_empty(f))
                return -EBADF;

        return sync_file_range(fd_file(f), offset, nbytes, flags);
}

SYSCALL_DEFINE4(sync_file_range, int, fd, loff_t, offset, loff_t, nbytes,
                                unsigned int, flags)
{
        return ksys_sync_file_range(fd, offset, nbytes, flags);
}

#if defined(CONFIG_COMPAT) && defined(__ARCH_WANT_COMPAT_SYNC_FILE_RANGE)
COMPAT_SYSCALL_DEFINE6(sync_file_range, int, fd, compat_arg_u64_dual(offset),
                       compat_arg_u64_dual(nbytes), unsigned int, flags)
{
        return ksys_sync_file_range(fd, compat_arg_u64_glue(offset),
                                    compat_arg_u64_glue(nbytes), flags);
}
#endif

/* It would be nice if people remember that not all the world's an i386
   when they introduce new system calls */
SYSCALL_DEFINE4(sync_file_range2, int, fd, unsigned int, flags,
                                 loff_t, offset, loff_t, nbytes)
{
        return ksys_sync_file_range(fd, offset, nbytes, flags);
}









































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































    5 










    5 







    5 





    5 















































    5 






























































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965
2966
2967
2968
2969
2970
2971
2972
2973
2974
2975
2976
2977
2978
2979
2980
2981
2982
2983
2984
2985
2986
2987
2988
2989
2990
2991
2992
2993
2994
2995
2996
2997
2998
2999
3000
3001
3002
3003
3004
3005
3006
3007
3008
3009
3010
3011
3012
3013
3014
3015
3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029
3030
3031
3032
3033
3034
3035
3036
3037
3038
3039
3040
3041
3042
3043
3044
3045
3046
3047
3048
3049
3050
3051
3052
3053
3054
3055
3056
3057
3058
3059
3060
3061
3062
3063
3064
3065
3066
3067
3068
3069
3070
3071
3072
3073
3074
3075
3076
3077
3078
3079
3080
3081
3082
3083
3084
3085
3086
3087
3088
3089
3090
3091
3092
3093
3094
3095
3096
3097
3098
3099
3100
3101
3102
3103
3104
3105
3106
3107
3108
3109
3110
3111
3112
3113
3114
3115
3116
3117
3118
3119
3120
3121
3122
3123
3124
3125
3126
3127
3128
3129
3130
3131
3132
3133
3134
3135
3136
3137
3138
3139
3140
3141
3142
3143
3144
3145
3146
3147
3148
3149
3150
3151
3152
3153
3154
3155
3156
3157
3158
3159
3160
3161
3162
3163
3164
3165
3166
3167
3168
3169
3170
3171
3172
3173
3174
3175
3176
3177
3178
3179
3180
3181
3182
3183
3184
3185
3186
3187
3188
3189
3190
3191
3192
3193
3194
3195
3196
3197
3198
3199
3200
3201
3202
3203
3204
3205
3206
3207
3208
3209
3210
3211
3212
3213
3214
3215
3216
3217
3218
3219
3220
3221
3222
3223
3224
3225
3226
3227
3228
3229
3230
3231
3232
3233
3234
3235
3236
3237
3238
3239
3240
3241
3242
3243
3244
3245
3246
3247
3248
3249
3250
3251
3252
3253
3254
3255
3256
3257
3258
3259
3260
3261
3262
3263
3264
3265
3266
3267
3268
3269
3270
3271
3272
3273
3274
3275
3276
3277
3278
3279
3280
3281
3282
3283
3284
3285
3286
3287
3288
3289
3290
3291
3292
3293
3294
3295
3296
3297
3298
3299
3300
3301
3302
3303
3304
3305
3306
3307
3308
3309
3310
3311
3312
3313
3314
3315
3316
3317
3318
3319
3320
3321
3322
3323
3324
3325
3326
3327
3328
3329
3330
3331
3332
3333
3334
3335
3336
3337
3338
3339
3340
3341
3342
3343
3344
3345
3346
3347
3348
3349
3350
3351
3352
3353
3354
3355
3356
3357
3358
3359
3360
3361
3362
3363
3364
3365
3366
3367
3368
3369
3370
3371
3372
3373
3374
3375
3376
3377
3378
3379
3380
3381
3382
3383
3384
3385
3386
3387
3388
3389
3390
3391
3392
3393
3394
3395
3396
3397
3398
3399
3400
3401
3402
3403
3404
3405
3406
3407
3408
3409
3410
3411
3412
3413
3414
3415
3416
3417
3418
3419
3420
3421
3422
3423
3424
3425
3426
3427
3428
3429
3430
3431
3432
3433
3434
3435
3436
3437
3438
3439
3440
3441
3442
3443
3444
3445
3446
3447
3448
3449
3450
3451
3452
3453
3454
3455
3456
3457
3458
3459
3460
3461
3462
3463
3464
3465
3466
3467
3468
3469
3470
3471
3472
3473
3474
3475
3476
3477
3478
3479
3480
3481
3482
3483
3484
3485
3486
3487
3488
3489
3490
3491
3492
3493
3494
3495
3496
3497
3498
3499
3500
3501
3502
3503
3504
3505
3506
3507
3508
3509
3510
3511
3512
3513
3514
3515
3516
3517
3518
3519
3520
3521
3522
3523
3524
3525
3526
3527
3528
3529
3530
3531
3532
3533
3534
3535
3536
3537
3538
3539
3540
3541
3542
3543
3544
3545
3546
3547
3548
3549
3550
3551
3552
3553
3554
3555
3556
3557
3558
3559
3560
3561
3562
3563
3564
3565
3566
3567
3568
3569
3570
3571
3572
3573
3574
3575
3576
3577
3578
3579
3580
3581
3582
3583
3584
3585
3586
3587
3588
3589
3590
3591
3592
3593
3594
3595
3596
3597
3598
3599
3600
3601
3602
3603
3604
3605
3606
3607
3608
3609
3610
3611
3612
3613
3614
3615
3616
3617
3618
3619
3620
3621
3622
3623
3624
3625
3626
3627
3628
3629
3630
3631
3632
3633
3634
3635
3636
3637
3638
3639
3640
3641
3642
3643
3644
3645
3646
3647
3648
3649
3650
3651
3652
3653
3654
3655
3656
3657
3658
3659
3660
3661
3662
3663
3664
3665
3666
3667
3668
3669
3670
3671
3672
3673
3674
3675
3676
3677
3678
3679
3680
3681
3682
3683
3684
3685
3686
3687
3688
3689
3690
3691
3692
3693
3694
3695
3696
3697
3698
3699
3700
3701
3702
3703
3704
3705
3706
3707
3708
3709
3710
3711
3712
3713
3714
3715
3716
3717
3718
3719
3720
3721
3722
3723
3724
3725
3726
3727
3728
3729
3730
3731
3732
3733
3734
3735
3736
3737
3738
3739
3740
3741
3742
3743
3744
3745
3746
3747
3748
3749
3750
3751
3752
3753
3754
3755
3756
3757
3758
3759
3760
3761
3762
3763
3764
3765
3766
3767
3768
3769
3770
3771
3772
3773
3774
3775
3776
3777
3778
3779
3780
3781
3782
3783
3784
3785
3786
3787
3788
3789
3790
3791
3792
3793
3794
3795
3796
3797
3798
3799
3800
3801
3802
3803
3804
3805
3806
3807
3808
3809
3810
3811
3812
3813
3814
3815
3816
3817
3818
3819
3820
3821
3822
3823
3824
3825
3826
3827
3828
3829
3830
3831
3832
3833
3834
3835
3836
3837
3838
3839
3840
3841
3842
3843
3844
3845
3846
3847
3848
3849
3850
3851
3852
3853
3854
3855
3856
3857
3858
3859
3860
3861
3862
3863
3864
3865
3866
3867
3868
3869
3870
3871
3872
3873
3874
3875
3876
3877
3878
3879
3880
3881
3882
3883
3884
3885
3886
3887
3888
3889
3890
3891
3892
3893
3894
3895
3896
3897
3898
3899
3900
3901
3902
3903
3904
3905
3906
3907
3908
3909
3910
3911
3912
3913
3914
3915
3916
3917
3918
3919
3920
3921
3922
3923
3924
3925
3926
3927
3928
3929
3930
3931
3932
3933
3934
3935
3936
3937
3938
3939
3940
3941
3942
3943
3944
3945
3946
3947
3948
3949
3950
3951
3952
3953
3954
3955
3956
3957
3958
3959
3960
3961
3962
3963
3964
3965
3966
3967
3968
3969
3970
3971
3972
3973
3974
3975
3976
3977
3978
3979
3980
3981
3982
3983
3984
3985
3986
3987
3988
3989
3990
3991
3992
3993
3994
3995
3996
3997
3998
3999
4000
4001
4002
4003
4004
4005
4006
4007
4008
4009
4010
4011
4012
4013
4014
4015
4016
4017
4018
4019
4020
4021
4022
4023
4024
4025
4026
4027
4028
4029
4030
4031
4032
4033
4034
4035
4036
4037
4038
4039
4040
4041
4042
4043
4044
4045
4046
4047
4048
4049
4050
4051
4052
4053
4054
4055
4056
4057
4058
4059
4060
4061
4062
4063
4064
4065
4066
4067
4068
4069
4070
4071
4072
4073
4074
4075
4076
4077
4078
4079
4080
4081
4082
4083
4084
4085
4086
4087
4088
4089
4090
4091
4092
4093
4094
4095
4096
4097
4098
4099
4100
4101
4102
4103
4104
4105
4106
4107
4108
4109
4110
4111
4112
4113
4114
4115
4116
4117
4118
4119
4120
4121
4122
4123
4124
4125
4126
4127
4128
4129
4130
4131
4132
4133
4134
4135
4136
4137
4138
4139
4140
4141
4142
4143
4144
4145
4146
4147
4148
4149
4150
4151
4152
4153
4154
4155
4156
4157
4158
4159
4160
4161
4162
4163
4164
4165
4166
4167
4168
4169
4170
4171
4172
4173
4174
4175
4176
4177
4178
4179
4180
4181
4182
4183
4184
4185
4186
4187
4188
4189
4190
4191
4192
4193
4194
4195
4196
4197
4198
4199
4200
4201
4202
4203
4204
4205
4206
4207
4208
4209
4210
4211
4212
4213
4214
4215
4216
4217
4218
4219
4220
4221
4222
4223
4224
4225
4226
4227
4228
4229
4230
4231
4232
4233
4234
4235
4236
4237
4238
4239
4240
4241
4242
4243
4244
4245
4246
4247
4248
4249
4250
4251
4252
4253
4254
4255
4256
4257
4258
4259
4260
4261
4262
4263
4264
4265
4266
4267
4268
4269
4270
4271
4272
4273
4274
4275
4276
4277
4278
4279
4280
4281
4282
4283
4284
4285
4286
4287
4288
4289
4290
4291
4292
4293
4294
4295
4296
4297
4298
4299
4300
4301
4302
4303
4304
4305
4306
4307
4308
4309
4310
4311
4312
4313
4314
4315
4316
4317
4318
4319
4320
4321
4322
4323
4324
4325
4326
4327
4328
4329
4330
4331
4332
4333
4334
4335
4336
4337
4338
4339
4340
4341
4342
4343
4344
4345
4346
4347
4348
4349
4350
4351
4352
4353
4354
4355
4356
4357
4358
4359
4360
4361
4362
4363
4364
4365
4366
4367
4368
4369
4370
4371
4372
4373
4374
4375
4376
4377
4378
4379
4380
4381
4382
4383
4384
4385
4386
4387
4388
4389
4390
4391
4392
4393
4394
4395
4396
4397
4398
4399
4400
4401
4402
4403
4404
4405
4406
4407
4408
4409
4410
4411
4412
4413
4414
4415
4416
4417
4418
4419
4420
4421
4422
4423
4424
4425
4426
4427
4428
4429
4430
4431
4432
4433
4434
4435
4436
4437
4438
4439
4440
4441
4442
4443
4444
4445
4446
4447
4448
4449
4450
4451
4452
4453
4454
4455
4456
4457
4458
4459
4460
4461
4462
4463
4464
4465
4466
4467
4468
4469
4470
4471
4472
4473
4474
4475
4476
4477
4478
4479
4480
4481
4482
4483
4484
4485
4486
4487
4488
4489
4490
4491
4492
4493
4494
4495
4496
4497
4498
4499
4500
4501
4502
4503
4504
4505
4506
4507
4508
4509
4510
4511
4512
4513
4514
4515
4516
4517
4518
4519
4520
4521
4522
4523
4524
4525
4526
4527
4528
4529
4530
4531
4532
4533
4534
4535
4536
4537
4538
4539
4540
4541
4542
4543
4544
4545
4546
4547
4548
4549
4550
4551
4552
4553
4554
4555
4556
4557
4558
4559
4560
4561
4562
4563
4564
4565
4566
4567
4568
4569
4570
4571
4572
4573
4574
4575
4576
4577
4578
4579
4580
4581
4582
4583
4584
4585
4586
4587
4588
4589
4590
4591
4592
4593
4594
4595
4596
4597
4598
4599
4600
4601
4602
4603
4604
4605
4606
4607
4608
4609
4610
4611
4612
4613
4614
4615
4616
4617
4618
4619
4620
4621
4622
4623
4624
4625
4626
4627
4628
4629
4630
4631
4632
4633
4634
4635
4636
4637
4638
4639
4640
4641
4642
4643
4644
4645
4646
4647
4648
4649
4650
4651
4652
4653
4654
4655
4656
4657
4658
4659
4660
4661
4662
4663
4664
4665
4666
4667
4668
4669
4670
4671
4672
4673
4674
4675
4676
4677
4678
4679
4680
4681
4682
4683
4684
4685
4686
4687
4688
4689
4690
4691
4692
4693
4694
4695
4696
4697
4698
4699
4700
4701
4702
4703
4704
4705
4706
4707
4708
4709
4710
4711
4712
4713
4714
4715
4716
4717
4718
4719
4720
4721
4722
4723
4724
4725
4726
4727
4728
4729
4730
4731
4732
4733
4734
4735
4736
4737
4738
4739
4740
4741
4742
4743
4744
4745
4746
4747
4748
4749
4750
4751
4752
4753
4754
4755
4756
4757
4758
4759
4760
4761
4762
4763
4764
4765
4766
4767
4768
4769
4770
4771
4772
4773
4774
4775
4776
4777
4778
4779
4780
4781
4782
4783
4784
4785
4786
4787
4788
4789
4790
4791
4792
4793
4794
4795
4796
4797
4798
4799
4800
4801
4802
4803
4804
4805
4806
4807
4808
4809
4810
4811
4812
4813
4814
4815
4816
4817
4818
4819
4820
4821
4822
4823
4824
4825
4826
4827
4828
4829
4830
4831
4832
4833
4834
4835
4836
4837
4838
4839
4840
4841
4842
4843
4844
4845
4846
4847
4848
4849
4850
4851
4852
4853
4854
4855
4856
4857
4858
4859
4860
4861
4862
4863
4864
4865
4866
4867
4868
4869
4870
4871
4872
4873
4874
4875
4876
4877
4878
4879
4880
4881
4882
4883
4884
4885
4886
4887
4888
4889
4890
4891
4892
4893
4894
4895
4896
4897
4898
4899
4900
4901
4902
4903
4904
4905
4906
4907
4908
4909
4910
4911
4912
4913
4914
4915
4916
4917
4918
4919
4920
4921
4922
4923
4924
4925
4926
4927
4928
4929
4930
4931
4932
4933
4934
4935
4936
4937
4938
4939
4940
4941
4942
4943
4944
4945
4946
4947
4948
4949
4950
4951
4952
4953
4954
4955
4956
4957
4958
4959
4960
4961
4962
4963
4964
4965
4966
4967
4968
4969
4970
4971
4972
4973
4974
4975
4976
4977
4978
4979
4980
4981
4982
4983
4984
4985
4986
4987
4988
4989
4990
4991
4992
4993
4994
4995
4996
4997
4998
4999
5000
5001
5002
5003
5004
5005
5006
5007
5008
5009
5010
5011
5012
5013
5014
5015
5016
5017
5018
5019
5020
5021
5022
5023
5024
5025
5026
5027
5028
5029
5030
5031
5032
5033
5034
5035
5036
5037
5038
5039
5040
5041
5042
5043
5044
5045
5046
5047
5048
5049
5050
5051
5052
5053
5054
5055
5056
5057
5058
5059
5060
5061
5062
5063
5064
5065
5066
5067
5068
5069
5070
5071
5072
5073
5074
5075
5076
5077
5078
5079
5080
5081
5082
5083
5084
5085
5086
5087
5088
5089
5090
5091
5092
5093
5094
5095
5096
5097
5098
5099
5100
5101
5102
5103
5104
5105
5106
5107
5108
5109
5110
5111
5112
5113
5114
5115
5116
5117
5118
5119
5120
5121
5122
5123
5124
5125
5126
5127
5128
5129
5130
5131
5132
5133
5134
5135
5136
5137
5138
5139
5140
5141
5142
5143
5144
5145
5146
5147
5148
5149
5150
5151
5152
5153
5154
5155
5156
5157
5158
5159
5160
5161
5162
5163
5164
5165
5166
5167
5168
5169
5170
5171
5172
5173
5174
5175
5176
5177
5178
5179
5180
5181
5182
5183
5184
5185
5186
5187
5188
5189
5190
5191
5192
5193
5194
5195
5196
5197
5198
5199
5200
5201
5202
5203
5204
5205
5206
5207
5208
5209
5210
5211
5212
5213
5214
5215
5216
5217
5218
5219
5220
5221
5222
5223
5224
5225
5226
5227
5228
5229
5230
5231
5232
5233
5234
5235
5236
5237
5238
5239
5240
5241
5242
5243
5244
5245
5246
5247
5248
5249
5250
5251
5252
5253
5254
5255
5256
5257
5258
5259
5260
5261
5262
5263
5264
5265
5266
5267
5268
5269
5270
5271
5272
5273
5274
5275
5276
5277
5278
5279
5280
5281
5282
5283
5284
5285
5286
5287
5288
5289
5290
5291
5292
5293
5294
5295
5296
5297
5298
5299
5300
5301
5302
5303
5304
5305
5306
5307
5308
5309
5310
5311
5312
5313
5314
5315
5316
5317
5318
5319
5320
5321
5322
5323
5324
5325
5326
5327
5328
5329
5330
5331
5332
5333
5334
5335
5336
5337
5338
5339
5340
5341
5342
5343
5344
5345
5346
5347
5348
5349
5350
5351
5352
5353
5354
5355
5356
5357
5358
5359
5360
5361
5362
5363
5364
5365
5366
5367
5368
5369
5370
5371
5372
5373
5374
5375
5376
5377
5378
5379
5380
5381
5382
5383
5384
5385
5386
5387
5388
5389
5390
5391
5392
5393
5394
5395
5396
5397
5398
5399
5400
5401
5402
5403
5404
5405
5406
5407
5408
5409
5410
5411
5412
5413
5414
5415
5416
5417
5418
5419
5420
5421
5422
5423
5424
5425
5426
5427
5428
5429
5430
5431
5432
5433
5434
5435
5436
5437
5438
5439
5440
5441
5442
5443
5444
5445
5446
5447
5448
5449
5450
5451
5452
5453
5454
5455
5456
5457
5458
5459
5460
5461
5462
5463
5464
5465
5466
5467
5468
5469
5470
5471
5472
5473
5474
5475
5476
5477
5478
5479
5480
5481
5482
5483
5484
5485
5486
5487
5488
5489
5490
5491
5492
5493
5494
5495
5496
5497
5498
5499
5500
5501
5502
5503
5504
5505
5506
5507
5508
5509
5510
5511
5512
5513
5514
5515
5516
5517
5518
5519
5520
5521
5522
5523
5524
5525
5526
5527
5528
5529
5530
5531
5532
5533
5534
5535
5536
5537
5538
5539
5540
5541
5542
5543
5544
5545
5546
5547
5548
5549
5550
5551
5552
5553
5554
5555
5556
5557
5558
5559
5560
5561
5562
5563
5564
5565
5566
5567
5568
5569
5570
5571
5572
5573
5574
5575
5576
5577
5578
5579
5580
5581
5582
5583
5584
5585
5586
5587
5588
5589
5590
5591
5592
5593
5594
5595
5596
5597
5598
5599
5600
5601
5602
5603
5604
5605
5606
5607
5608
5609
5610
5611
5612
5613
5614
5615
5616
5617
5618
5619
5620
5621
5622
5623
5624
5625
5626
5627
5628
5629
5630
5631
5632
5633
5634
5635
5636
5637
5638
5639
5640
5641
5642
5643
5644
5645
5646
5647
5648
5649
5650
5651
5652
5653
5654
5655
5656
5657
5658
5659
5660
5661
5662
5663
5664
5665
5666
5667
5668
5669
5670
5671
5672
5673
5674
5675
5676
5677
5678
5679
5680
5681
5682
5683
5684
5685
5686
5687
5688
5689
5690
5691
5692
5693
5694
5695
5696
5697
5698
5699
5700
5701
5702
5703
5704
5705
5706
5707
5708
5709
5710
5711
5712
5713
5714
5715
5716
5717
5718
5719
5720
5721
5722
5723
5724
5725
5726
5727
5728
5729
5730
5731
5732
5733
5734
5735
5736
5737
5738
5739
5740
5741
5742
5743
5744
5745
5746
5747
5748
5749
5750
5751
5752
5753
5754
5755
5756
5757
5758
5759
5760
5761
5762
5763
5764
5765
5766
5767
5768
5769
5770
5771
5772
5773
5774
5775
5776
5777
5778
5779
5780
5781
5782
5783
5784
5785
5786
5787
5788
5789
5790
5791
5792
5793
5794
5795
5796
5797
5798
5799
5800
5801
5802
5803
5804
5805
5806
5807
5808
5809
5810
5811
5812
5813
5814
5815
5816
5817
5818
5819
5820
5821
5822
5823
5824
5825
5826
5827
5828
5829
5830
5831
5832
5833
5834
5835
5836
5837
5838
5839
5840
5841
5842
5843
5844
5845
5846
5847
5848
5849
5850
5851
5852
5853
5854
5855
5856
5857
5858
5859
5860
5861
5862
5863
5864
5865
5866
5867
5868
5869
5870
5871
5872
5873
5874
5875
5876
5877
5878
5879
5880
5881
5882
5883
5884
5885
5886
5887
5888
5889
5890
5891
5892
5893
5894
5895
5896
5897
5898
5899
5900
5901
5902
5903
5904
5905
5906
5907
5908
5909
5910
5911
5912
5913
5914
5915
5916
5917
5918
5919
5920
5921
5922
5923
5924
5925
5926
5927
5928
5929
5930
5931
5932
5933
5934
5935
5936
5937
5938
5939
5940
5941
5942
5943
5944
5945
5946
5947
5948
5949
5950
5951
5952
5953
5954
5955
5956
5957
5958
5959
5960
5961
5962
5963
5964
5965
5966
5967
5968
5969
5970
5971
5972
5973
5974
5975
5976
5977
5978
5979
5980
5981
5982
5983
5984
5985
5986
5987
5988
5989
5990
5991
5992
5993
5994
5995
5996
5997
5998
5999
6000
6001
6002
6003
6004
6005
6006
6007
6008
6009
6010
6011
6012
6013
6014
6015
6016
6017
6018
6019
6020
6021
6022
6023
6024
6025
6026
6027
6028
6029
6030
6031
6032
6033
6034
6035
6036
6037
6038
6039
6040
6041
6042
6043
6044
6045
6046
6047
6048
6049
6050
6051
6052
6053
6054
6055
6056
6057
6058
6059
6060
6061
6062
6063
6064
6065
6066
6067
6068
6069
6070
6071
6072
6073
6074
6075
6076
6077
6078
6079
6080
6081
6082
6083
6084
6085
6086
6087
6088
6089
6090
6091
6092
6093
6094
6095
6096
6097
6098
6099
6100
6101
6102
6103
6104
6105
6106
6107
6108
6109
6110
6111
6112
6113
6114
6115
6116
6117
6118
6119
6120
6121
6122
6123
6124
6125
6126
6127
6128
6129
6130
6131
6132
6133
6134
6135
6136
6137
6138
6139
6140
6141
6142
6143
6144
6145
6146
6147
6148
6149
6150
6151
6152
6153
6154
6155
6156
6157
6158
6159
6160
6161
6162
6163
6164
6165
6166
6167
6168
6169
6170
6171
6172
6173
6174
6175
6176
6177
6178
6179
6180
6181
6182
6183
6184
6185
6186
6187
6188
6189
6190
6191
6192
6193
6194
6195
6196
6197
6198
6199
6200
6201
6202
6203
6204
6205
6206
6207
6208
6209
6210
6211
6212
6213
6214
6215
6216
6217
6218
6219
6220
6221
6222
6223
6224
6225
6226
6227
6228
6229
6230
6231
6232
6233
6234
6235
6236
6237
6238
6239
6240
6241
6242
6243
6244
6245
6246
6247
6248
6249
6250
6251
6252
6253
6254
6255
6256
6257
6258
6259
6260
6261
6262
6263
6264
6265
6266
6267
6268
6269
6270
6271
6272
6273
6274
6275
6276
6277
6278
6279
6280
6281
6282
6283
6284
6285
6286
6287
6288
6289
6290
6291
6292
6293
6294
6295
6296
6297
6298
6299
6300
6301
6302
6303
6304
6305
6306
6307
6308
6309
6310
6311
6312
6313
6314
6315
6316
6317
6318
6319
6320
6321
6322
6323
6324
6325
6326
6327
6328
6329
6330
6331
6332
6333
6334
6335
6336
6337
6338
6339
6340
6341
6342
6343
6344
6345
6346
6347
6348
6349
6350
6351
6352
6353
6354
6355
6356
6357
6358
6359
6360
6361
6362
6363
6364
6365
6366
6367
6368
6369
6370
6371
6372
6373
6374
6375
6376
6377
6378
6379
6380
6381
6382
6383
6384
6385
6386
6387
6388
6389
6390
6391
6392
6393
6394
6395
6396
6397
6398
6399
6400
6401
6402
6403
6404
6405
6406
6407
6408
6409
6410
6411
6412
6413
6414
6415
6416
6417
6418
6419
6420
6421
6422
6423
6424
6425
6426
6427
6428
6429
6430
6431
6432
6433
6434
6435
6436
6437
6438
6439
6440
6441
6442
6443
6444
6445
6446
6447
6448
6449
6450
6451
6452
6453
6454
6455
6456
6457
6458
6459
6460
6461
6462
6463
6464
6465
6466
6467
6468
6469
6470
6471
6472
6473
6474
6475
6476
6477
6478
6479
6480
6481
6482
6483
6484
6485
6486
6487
6488
6489
6490
6491
6492
6493
6494
6495
6496
6497
6498
6499
6500
6501
6502
6503
6504
6505
6506
6507
6508
6509
6510
6511
6512
6513
6514
6515
6516
6517
6518
6519
6520
6521
6522
6523
6524
6525
6526
6527
6528
6529
6530
6531
6532
6533
6534
6535
6536
6537
6538
6539
6540
6541
6542
6543
6544
6545
6546
6547
6548
6549
6550
6551
6552
6553
6554
6555
6556
6557
6558
6559
6560
6561
6562
6563
6564
6565
6566
6567
6568
6569
6570
6571
6572
6573
6574
6575
6576
6577
6578
6579
6580
6581
6582
6583
6584
6585
6586
6587
6588
6589
6590
6591
6592
6593
6594
6595
6596
6597
6598
6599
6600
6601
6602
6603
6604
6605
6606
6607
6608
6609
6610
6611
6612
6613
6614
6615
6616
6617
6618
6619
6620
6621
6622
6623
6624
6625
6626
6627
6628
6629
6630
6631
6632
6633
6634
6635
6636
6637
6638
6639
6640
6641
6642
6643
6644
6645
6646
6647
6648
6649
6650
6651
6652
6653
6654
6655
6656
6657
6658
6659
6660
6661
6662
6663
6664
6665
6666
6667
6668
6669
6670
6671
6672
6673
6674
6675
6676
6677
6678
6679
6680
6681
6682
6683
6684
6685
6686
6687
6688
6689
6690
6691
6692
6693
6694
6695
6696
6697
6698
6699
6700
6701
6702
6703
6704
6705
6706
6707
6708
6709
6710
6711
6712
6713
6714
6715
6716
6717
6718
6719
6720
6721
6722
6723
6724
6725
6726
6727
6728
6729
6730
6731
6732
6733
6734
6735
6736
6737
6738
6739
6740
6741
6742
6743
6744
6745
6746
6747
6748
6749
6750
6751
6752
6753
6754
6755
6756
6757
6758
6759
6760
6761
6762
6763
6764
6765
6766
6767
6768
6769
6770
6771
6772
6773
6774
6775
6776
6777
6778
6779
6780
6781
6782
6783
6784
6785
6786
6787
6788
6789
6790
6791
6792
6793
6794
6795
6796
6797
6798
6799
6800
6801
6802
6803
6804
6805
6806
6807
6808
6809
6810
6811
6812
6813
6814
6815
6816
6817
6818
6819
6820
6821
6822
6823
6824
6825
6826
6827
6828
6829
6830
6831
6832
6833
6834
6835
6836
6837
6838
6839
6840
6841
6842
6843
6844
6845
6846
6847
6848
6849
6850
6851
6852
6853
6854
6855
6856
6857
6858
6859
6860
6861
6862
6863
6864
6865
6866
6867
6868
6869
6870
6871
6872
6873
6874
6875
6876
6877
6878
6879
6880
6881
6882
6883
6884
6885
6886
6887
6888
6889
6890
6891
6892
6893
6894
6895
6896
6897
6898
6899
6900
6901
6902
6903
6904
6905
6906
6907
6908
6909
6910
6911
6912
6913
6914
6915
6916
6917
6918
6919
6920
6921
6922
6923
6924
6925
6926
6927
6928
6929
6930
6931
6932
6933
6934
6935
6936
6937
6938
6939
6940
6941
6942
6943
6944
6945
6946
6947
6948
6949
6950
6951
6952
6953
6954
6955
6956
6957
6958
6959
6960
6961
6962
6963
6964
6965
6966
6967
6968
6969
6970
6971
6972
6973
6974
6975
6976
6977
6978
6979
6980
6981
6982
6983
6984
6985
6986
6987
6988
6989
6990
6991
6992
6993
6994
6995
6996
6997
6998
6999
7000
7001
7002
7003
7004
7005
7006
7007
7008
7009
7010
7011
7012
7013
7014
7015
7016
7017
7018
7019
7020
7021
7022
7023
7024
7025
7026
7027
7028
7029
7030
7031
7032
7033
7034
7035
7036
7037
7038
7039
7040
7041
7042
7043
7044
7045
7046
7047
7048
7049
7050
7051
7052
7053
7054
7055
7056
// SPDX-License-Identifier: GPL-2.0
/*
 * Copyright (c) 2003-2006, Cluster File Systems, Inc, info@clusterfs.com
 * Written by Alex Tomas <alex@clusterfs.com>
 */


/*
 * mballoc.c contains the multiblocks allocation routines
 */

#include "ext4_jbd2.h"
#include "mballoc.h"
#include <linux/log2.h>
#include <linux/module.h>
#include <linux/slab.h>
#include <linux/nospec.h>
#include <linux/backing-dev.h>
#include <linux/freezer.h>
#include <trace/events/ext4.h>
#include <kunit/static_stub.h>

/*
 * MUSTDO:
 *   - test ext4_ext_search_left() and ext4_ext_search_right()
 *   - search for metadata in few groups
 *
 * TODO v4:
 *   - normalization should take into account whether file is still open
 *   - discard preallocations if no free space left (policy?)
 *   - don't normalize tails
 *   - quota
 *   - reservation for superuser
 *
 * TODO v3:
 *   - bitmap read-ahead (proposed by Oleg Drokin aka green)
 *   - track min/max extents in each group for better group selection
 *   - mb_mark_used() may allocate chunk right after splitting buddy
 *   - tree of groups sorted by number of free blocks
 *   - error handling
 */

/*
 * The allocation request involve request for multiple number of blocks
 * near to the goal(block) value specified.
 *
 * During initialization phase of the allocator we decide to use the
 * group preallocation or inode preallocation depending on the size of
 * the file. The size of the file could be the resulting file size we
 * would have after allocation, or the current file size, which ever
 * is larger. If the size is less than sbi->s_mb_stream_request we
 * select to use the group preallocation. The default value of
 * s_mb_stream_request is 16 blocks. This can also be tuned via
 * /sys/fs/ext4/<partition>/mb_stream_req. The value is represented in
 * terms of number of blocks.
 *
 * The main motivation for having small file use group preallocation is to
 * ensure that we have small files closer together on the disk.
 *
 * First stage the allocator looks at the inode prealloc list,
 * ext4_inode_info->i_prealloc_list, which contains list of prealloc
 * spaces for this particular inode. The inode prealloc space is
 * represented as:
 *
 * pa_lstart -> the logical start block for this prealloc space
 * pa_pstart -> the physical start block for this prealloc space
 * pa_len    -> length for this prealloc space (in clusters)
 * pa_free   ->  free space available in this prealloc space (in clusters)
 *
 * The inode preallocation space is used looking at the _logical_ start
 * block. If only the logical file block falls within the range of prealloc
 * space we will consume the particular prealloc space. This makes sure that
 * we have contiguous physical blocks representing the file blocks
 *
 * The important thing to be noted in case of inode prealloc space is that
 * we don't modify the values associated to inode prealloc space except
 * pa_free.
 *
 * If we are not able to find blocks in the inode prealloc space and if we
 * have the group allocation flag set then we look at the locality group
 * prealloc space. These are per CPU prealloc list represented as
 *
 * ext4_sb_info.s_locality_groups[smp_processor_id()]
 *
 * The reason for having a per cpu locality group is to reduce the contention
 * between CPUs. It is possible to get scheduled at this point.
 *
 * The locality group prealloc space is used looking at whether we have
 * enough free space (pa_free) within the prealloc space.
 *
 * If we can't allocate blocks via inode prealloc or/and locality group
 * prealloc then we look at the buddy cache. The buddy cache is represented
 * by ext4_sb_info.s_buddy_cache (struct inode) whose file offset gets
 * mapped to the buddy and bitmap information regarding different
 * groups. The buddy information is attached to buddy cache inode so that
 * we can access them through the page cache. The information regarding
 * each group is loaded via ext4_mb_load_buddy.  The information involve
 * block bitmap and buddy information. The information are stored in the
 * inode as:
 *
 *  {                        page                        }
 *  [ group 0 bitmap][ group 0 buddy] [group 1][ group 1]...
 *
 *
 * one block each for bitmap and buddy information.  So for each group we
 * take up 2 blocks. A page can contain blocks_per_page (PAGE_SIZE /
 * blocksize) blocks.  So it can have information regarding groups_per_page
 * which is blocks_per_page/2
 *
 * The buddy cache inode is not stored on disk. The inode is thrown
 * away when the filesystem is unmounted.
 *
 * We look for count number of blocks in the buddy cache. If we were able
 * to locate that many free blocks we return with additional information
 * regarding rest of the contiguous physical block available
 *
 * Before allocating blocks via buddy cache we normalize the request
 * blocks. This ensure we ask for more blocks that we needed. The extra
 * blocks that we get after allocation is added to the respective prealloc
 * list. In case of inode preallocation we follow a list of heuristics
 * based on file size. This can be found in ext4_mb_normalize_request. If
 * we are doing a group prealloc we try to normalize the request to
 * sbi->s_mb_group_prealloc.  The default value of s_mb_group_prealloc is
 * dependent on the cluster size; for non-bigalloc file systems, it is
 * 512 blocks. This can be tuned via
 * /sys/fs/ext4/<partition>/mb_group_prealloc. The value is represented in
 * terms of number of blocks. If we have mounted the file system with -O
 * stripe=<value> option the group prealloc request is normalized to the
 * smallest multiple of the stripe value (sbi->s_stripe) which is
 * greater than the default mb_group_prealloc.
 *
 * If "mb_optimize_scan" mount option is set, we maintain in memory group info
 * structures in two data structures:
 *
 * 1) Array of largest free order lists (sbi->s_mb_largest_free_orders)
 *
 *    Locking: sbi->s_mb_largest_free_orders_locks(array of rw locks)
 *
 *    This is an array of lists where the index in the array represents the
 *    largest free order in the buddy bitmap of the participating group infos of
 *    that list. So, there are exactly MB_NUM_ORDERS(sb) (which means total
 *    number of buddy bitmap orders possible) number of lists. Group-infos are
 *    placed in appropriate lists.
 *
 * 2) Average fragment size lists (sbi->s_mb_avg_fragment_size)
 *
 *    Locking: sbi->s_mb_avg_fragment_size_locks(array of rw locks)
 *
 *    This is an array of lists where in the i-th list there are groups with
 *    average fragment size >= 2^i and < 2^(i+1). The average fragment size
 *    is computed as ext4_group_info->bb_free / ext4_group_info->bb_fragments.
 *    Note that we don't bother with a special list for completely empty groups
 *    so we only have MB_NUM_ORDERS(sb) lists.
 *
 * When "mb_optimize_scan" mount option is set, mballoc consults the above data
 * structures to decide the order in which groups are to be traversed for
 * fulfilling an allocation request.
 *
 * At CR_POWER2_ALIGNED , we look for groups which have the largest_free_order
 * >= the order of the request. We directly look at the largest free order list
 * in the data structure (1) above where largest_free_order = order of the
 * request. If that list is empty, we look at remaining list in the increasing
 * order of largest_free_order. This allows us to perform CR_POWER2_ALIGNED
 * lookup in O(1) time.
 *
 * At CR_GOAL_LEN_FAST, we only consider groups where
 * average fragment size > request size. So, we lookup a group which has average
 * fragment size just above or equal to request size using our average fragment
 * size group lists (data structure 2) in O(1) time.
 *
 * At CR_BEST_AVAIL_LEN, we aim to optimize allocations which can't be satisfied
 * in CR_GOAL_LEN_FAST. The fact that we couldn't find a group in
 * CR_GOAL_LEN_FAST suggests that there is no BG that has avg
 * fragment size > goal length. So before falling to the slower
 * CR_GOAL_LEN_SLOW, in CR_BEST_AVAIL_LEN we proactively trim goal length and
 * then use the same fragment lists as CR_GOAL_LEN_FAST to find a BG with a big
 * enough average fragment size. This increases the chances of finding a
 * suitable block group in O(1) time and results in faster allocation at the
 * cost of reduced size of allocation.
 *
 * If "mb_optimize_scan" mount option is not set, mballoc traverses groups in
 * linear order which requires O(N) search time for each CR_POWER2_ALIGNED and
 * CR_GOAL_LEN_FAST phase.
 *
 * The regular allocator (using the buddy cache) supports a few tunables.
 *
 * /sys/fs/ext4/<partition>/mb_min_to_scan
 * /sys/fs/ext4/<partition>/mb_max_to_scan
 * /sys/fs/ext4/<partition>/mb_order2_req
 * /sys/fs/ext4/<partition>/mb_max_linear_groups
 *
 * The regular allocator uses buddy scan only if the request len is power of
 * 2 blocks and the order of allocation is >= sbi->s_mb_order2_reqs. The
 * value of s_mb_order2_reqs can be tuned via
 * /sys/fs/ext4/<partition>/mb_order2_req.  If the request len is equal to
 * stripe size (sbi->s_stripe), we try to search for contiguous block in
 * stripe size. This should result in better allocation on RAID setups. If
 * not, we search in the specific group using bitmap for best extents. The
 * tunable min_to_scan and max_to_scan control the behaviour here.
 * min_to_scan indicate how long the mballoc __must__ look for a best
 * extent and max_to_scan indicates how long the mballoc __can__ look for a
 * best extent in the found extents. Searching for the blocks starts with
 * the group specified as the goal value in allocation context via
 * ac_g_ex. Each group is first checked based on the criteria whether it
 * can be used for allocation. ext4_mb_good_group explains how the groups are
 * checked.
 *
 * When "mb_optimize_scan" is turned on, as mentioned above, the groups may not
 * get traversed linearly. That may result in subsequent allocations being not
 * close to each other. And so, the underlying device may get filled up in a
 * non-linear fashion. While that may not matter on non-rotational devices, for
 * rotational devices that may result in higher seek times. "mb_max_linear_groups"
 * tells mballoc how many groups mballoc should search linearly before
 * performing consulting above data structures for more efficient lookups. For
 * non rotational devices, this value defaults to 0 and for rotational devices
 * this is set to MB_DEFAULT_LINEAR_LIMIT.
 *
 * Both the prealloc space are getting populated as above. So for the first
 * request we will hit the buddy cache which will result in this prealloc
 * space getting filled. The prealloc space is then later used for the
 * subsequent request.
 */

/*
 * mballoc operates on the following data:
 *  - on-disk bitmap
 *  - in-core buddy (actually includes buddy and bitmap)
 *  - preallocation descriptors (PAs)
 *
 * there are two types of preallocations:
 *  - inode
 *    assiged to specific inode and can be used for this inode only.
 *    it describes part of inode's space preallocated to specific
 *    physical blocks. any block from that preallocated can be used
 *    independent. the descriptor just tracks number of blocks left
 *    unused. so, before taking some block from descriptor, one must
 *    make sure corresponded logical block isn't allocated yet. this
 *    also means that freeing any block within descriptor's range
 *    must discard all preallocated blocks.
 *  - locality group
 *    assigned to specific locality group which does not translate to
 *    permanent set of inodes: inode can join and leave group. space
 *    from this type of preallocation can be used for any inode. thus
 *    it's consumed from the beginning to the end.
 *
 * relation between them can be expressed as:
 *    in-core buddy = on-disk bitmap + preallocation descriptors
 *
 * this mean blocks mballoc considers used are:
 *  - allocated blocks (persistent)
 *  - preallocated blocks (non-persistent)
 *
 * consistency in mballoc world means that at any time a block is either
 * free or used in ALL structures. notice: "any time" should not be read
 * literally -- time is discrete and delimited by locks.
 *
 *  to keep it simple, we don't use block numbers, instead we count number of
 *  blocks: how many blocks marked used/free in on-disk bitmap, buddy and PA.
 *
 * all operations can be expressed as:
 *  - init buddy:                        buddy = on-disk + PAs
 *  - new PA:                                buddy += N; PA = N
 *  - use inode PA:                        on-disk += N; PA -= N
 *  - discard inode PA                        buddy -= on-disk - PA; PA = 0
 *  - use locality group PA                on-disk += N; PA -= N
 *  - discard locality group PA                buddy -= PA; PA = 0
 *  note: 'buddy -= on-disk - PA' is used to show that on-disk bitmap
 *        is used in real operation because we can't know actual used
 *        bits from PA, only from on-disk bitmap
 *
 * if we follow this strict logic, then all operations above should be atomic.
 * given some of them can block, we'd have to use something like semaphores
 * killing performance on high-end SMP hardware. let's try to relax it using
 * the following knowledge:
 *  1) if buddy is referenced, it's already initialized
 *  2) while block is used in buddy and the buddy is referenced,
 *     nobody can re-allocate that block
 *  3) we work on bitmaps and '+' actually means 'set bits'. if on-disk has
 *     bit set and PA claims same block, it's OK. IOW, one can set bit in
 *     on-disk bitmap if buddy has same bit set or/and PA covers corresponded
 *     block
 *
 * so, now we're building a concurrency table:
 *  - init buddy vs.
 *    - new PA
 *      blocks for PA are allocated in the buddy, buddy must be referenced
 *      until PA is linked to allocation group to avoid concurrent buddy init
 *    - use inode PA
 *      we need to make sure that either on-disk bitmap or PA has uptodate data
 *      given (3) we care that PA-=N operation doesn't interfere with init
 *    - discard inode PA
 *      the simplest way would be to have buddy initialized by the discard
 *    - use locality group PA
 *      again PA-=N must be serialized with init
 *    - discard locality group PA
 *      the simplest way would be to have buddy initialized by the discard
 *  - new PA vs.
 *    - use inode PA
 *      i_data_sem serializes them
 *    - discard inode PA
 *      discard process must wait until PA isn't used by another process
 *    - use locality group PA
 *      some mutex should serialize them
 *    - discard locality group PA
 *      discard process must wait until PA isn't used by another process
 *  - use inode PA
 *    - use inode PA
 *      i_data_sem or another mutex should serializes them
 *    - discard inode PA
 *      discard process must wait until PA isn't used by another process
 *    - use locality group PA
 *      nothing wrong here -- they're different PAs covering different blocks
 *    - discard locality group PA
 *      discard process must wait until PA isn't used by another process
 *
 * now we're ready to make few consequences:
 *  - PA is referenced and while it is no discard is possible
 *  - PA is referenced until block isn't marked in on-disk bitmap
 *  - PA changes only after on-disk bitmap
 *  - discard must not compete with init. either init is done before
 *    any discard or they're serialized somehow
 *  - buddy init as sum of on-disk bitmap and PAs is done atomically
 *
 * a special case when we've used PA to emptiness. no need to modify buddy
 * in this case, but we should care about concurrent init
 *
 */

 /*
 * Logic in few words:
 *
 *  - allocation:
 *    load group
 *    find blocks
 *    mark bits in on-disk bitmap
 *    release group
 *
 *  - use preallocation:
 *    find proper PA (per-inode or group)
 *    load group
 *    mark bits in on-disk bitmap
 *    release group
 *    release PA
 *
 *  - free:
 *    load group
 *    mark bits in on-disk bitmap
 *    release group
 *
 *  - discard preallocations in group:
 *    mark PAs deleted
 *    move them onto local list
 *    load on-disk bitmap
 *    load group
 *    remove PA from object (inode or locality group)
 *    mark free blocks in-core
 *
 *  - discard inode's preallocations:
 */

/*
 * Locking rules
 *
 * Locks:
 *  - bitlock on a group        (group)
 *  - object (inode/locality)        (object)
 *  - per-pa lock                (pa)
 *  - cr_power2_aligned lists lock        (cr_power2_aligned)
 *  - cr_goal_len_fast lists lock        (cr_goal_len_fast)
 *
 * Paths:
 *  - new pa
 *    object
 *    group
 *
 *  - find and use pa:
 *    pa
 *
 *  - release consumed pa:
 *    pa
 *    group
 *    object
 *
 *  - generate in-core bitmap:
 *    group
 *        pa
 *
 *  - discard all for given object (inode, locality group):
 *    object
 *        pa
 *    group
 *
 *  - discard all for given group:
 *    group
 *        pa
 *    group
 *        object
 *
 *  - allocation path (ext4_mb_regular_allocator)
 *    group
 *    cr_power2_aligned/cr_goal_len_fast
 */
static struct kmem_cache *ext4_pspace_cachep;
static struct kmem_cache *ext4_ac_cachep;
static struct kmem_cache *ext4_free_data_cachep;

/* We create slab caches for groupinfo data structures based on the
 * superblock block size.  There will be one per mounted filesystem for
 * each unique s_blocksize_bits */
#define NR_GRPINFO_CACHES 8
static struct kmem_cache *ext4_groupinfo_caches[NR_GRPINFO_CACHES];

static const char * const ext4_groupinfo_slab_names[NR_GRPINFO_CACHES] = {
        "ext4_groupinfo_1k", "ext4_groupinfo_2k", "ext4_groupinfo_4k",
        "ext4_groupinfo_8k", "ext4_groupinfo_16k", "ext4_groupinfo_32k",
        "ext4_groupinfo_64k", "ext4_groupinfo_128k"
};

static void ext4_mb_generate_from_pa(struct super_block *sb, void *bitmap,
                                        ext4_group_t group);
static void ext4_mb_new_preallocation(struct ext4_allocation_context *ac);

static bool ext4_mb_good_group(struct ext4_allocation_context *ac,
                               ext4_group_t group, enum criteria cr);

static int ext4_try_to_trim_range(struct super_block *sb,
                struct ext4_buddy *e4b, ext4_grpblk_t start,
                ext4_grpblk_t max, ext4_grpblk_t minblocks);

/*
 * The algorithm using this percpu seq counter goes below:
 * 1. We sample the percpu discard_pa_seq counter before trying for block
 *    allocation in ext4_mb_new_blocks().
 * 2. We increment this percpu discard_pa_seq counter when we either allocate
 *    or free these blocks i.e. while marking those blocks as used/free in
 *    mb_mark_used()/mb_free_blocks().
 * 3. We also increment this percpu seq counter when we successfully identify
 *    that the bb_prealloc_list is not empty and hence proceed for discarding
 *    of those PAs inside ext4_mb_discard_group_preallocations().
 *
 * Now to make sure that the regular fast path of block allocation is not
 * affected, as a small optimization we only sample the percpu seq counter
 * on that cpu. Only when the block allocation fails and when freed blocks
 * found were 0, that is when we sample percpu seq counter for all cpus using
 * below function ext4_get_discard_pa_seq_sum(). This happens after making
 * sure that all the PAs on grp->bb_prealloc_list got freed or if it's empty.
 */
static DEFINE_PER_CPU(u64, discard_pa_seq);
static inline u64 ext4_get_discard_pa_seq_sum(void)
{
        int __cpu;
        u64 __seq = 0;

        for_each_possible_cpu(__cpu)
                __seq += per_cpu(discard_pa_seq, __cpu);
        return __seq;
}

static inline void *mb_correct_addr_and_bit(int *bit, void *addr)
{
#if BITS_PER_LONG == 64
        *bit += ((unsigned long) addr & 7UL) << 3;
        addr = (void *) ((unsigned long) addr & ~7UL);
#elif BITS_PER_LONG == 32
        *bit += ((unsigned long) addr & 3UL) << 3;
        addr = (void *) ((unsigned long) addr & ~3UL);
#else
#error "how many bits you are?!"
#endif
        return addr;
}

static inline int mb_test_bit(int bit, void *addr)
{
        /*
         * ext4_test_bit on architecture like powerpc
         * needs unsigned long aligned address
         */
        addr = mb_correct_addr_and_bit(&bit, addr);
        return ext4_test_bit(bit, addr);
}

static inline void mb_set_bit(int bit, void *addr)
{
        addr = mb_correct_addr_and_bit(&bit, addr);
        ext4_set_bit(bit, addr);
}

static inline void mb_clear_bit(int bit, void *addr)
{
        addr = mb_correct_addr_and_bit(&bit, addr);
        ext4_clear_bit(bit, addr);
}

static inline int mb_test_and_clear_bit(int bit, void *addr)
{
        addr = mb_correct_addr_and_bit(&bit, addr);
        return ext4_test_and_clear_bit(bit, addr);
}

static inline int mb_find_next_zero_bit(void *addr, int max, int start)
{
        int fix = 0, ret, tmpmax;
        addr = mb_correct_addr_and_bit(&fix, addr);
        tmpmax = max + fix;
        start += fix;

        ret = ext4_find_next_zero_bit(addr, tmpmax, start) - fix;
        if (ret > max)
                return max;
        return ret;
}

static inline int mb_find_next_bit(void *addr, int max, int start)
{
        int fix = 0, ret, tmpmax;
        addr = mb_correct_addr_and_bit(&fix, addr);
        tmpmax = max + fix;
        start += fix;

        ret = ext4_find_next_bit(addr, tmpmax, start) - fix;
        if (ret > max)
                return max;
        return ret;
}

static void *mb_find_buddy(struct ext4_buddy *e4b, int order, int *max)
{
        char *bb;

        BUG_ON(e4b->bd_bitmap == e4b->bd_buddy);
        BUG_ON(max == NULL);

        if (order > e4b->bd_blkbits + 1) {
                *max = 0;
                return NULL;
        }

        /* at order 0 we see each particular block */
        if (order == 0) {
                *max = 1 << (e4b->bd_blkbits + 3);
                return e4b->bd_bitmap;
        }

        bb = e4b->bd_buddy + EXT4_SB(e4b->bd_sb)->s_mb_offsets[order];
        *max = EXT4_SB(e4b->bd_sb)->s_mb_maxs[order];

        return bb;
}

#ifdef DOUBLE_CHECK
static void mb_free_blocks_double(struct inode *inode, struct ext4_buddy *e4b,
                           int first, int count)
{
        int i;
        struct super_block *sb = e4b->bd_sb;

        if (unlikely(e4b->bd_info->bb_bitmap == NULL))
                return;
        assert_spin_locked(ext4_group_lock_ptr(sb, e4b->bd_group));
        for (i = 0; i < count; i++) {
                if (!mb_test_bit(first + i, e4b->bd_info->bb_bitmap)) {
                        ext4_fsblk_t blocknr;

                        blocknr = ext4_group_first_block_no(sb, e4b->bd_group);
                        blocknr += EXT4_C2B(EXT4_SB(sb), first + i);
                        ext4_mark_group_bitmap_corrupted(sb, e4b->bd_group,
                                        EXT4_GROUP_INFO_BBITMAP_CORRUPT);
                        ext4_grp_locked_error(sb, e4b->bd_group,
                                              inode ? inode->i_ino : 0,
                                              blocknr,
                                              "freeing block already freed "
                                              "(bit %u)",
                                              first + i);
                }
                mb_clear_bit(first + i, e4b->bd_info->bb_bitmap);
        }
}

static void mb_mark_used_double(struct ext4_buddy *e4b, int first, int count)
{
        int i;

        if (unlikely(e4b->bd_info->bb_bitmap == NULL))
                return;
        assert_spin_locked(ext4_group_lock_ptr(e4b->bd_sb, e4b->bd_group));
        for (i = 0; i < count; i++) {
                BUG_ON(mb_test_bit(first + i, e4b->bd_info->bb_bitmap));
                mb_set_bit(first + i, e4b->bd_info->bb_bitmap);
        }
}

static void mb_cmp_bitmaps(struct ext4_buddy *e4b, void *bitmap)
{
        if (unlikely(e4b->bd_info->bb_bitmap == NULL))
                return;
        if (memcmp(e4b->bd_info->bb_bitmap, bitmap, e4b->bd_sb->s_blocksize)) {
                unsigned char *b1, *b2;
                int i;
                b1 = (unsigned char *) e4b->bd_info->bb_bitmap;
                b2 = (unsigned char *) bitmap;
                for (i = 0; i < e4b->bd_sb->s_blocksize; i++) {
                        if (b1[i] != b2[i]) {
                                ext4_msg(e4b->bd_sb, KERN_ERR,
                                         "corruption in group %u "
                                         "at byte %u(%u): %x in copy != %x "
                                         "on disk/prealloc",
                                         e4b->bd_group, i, i * 8, b1[i], b2[i]);
                                BUG();
                        }
                }
        }
}

static void mb_group_bb_bitmap_alloc(struct super_block *sb,
                        struct ext4_group_info *grp, ext4_group_t group)
{
        struct buffer_head *bh;

        grp->bb_bitmap = kmalloc(sb->s_blocksize, GFP_NOFS);
        if (!grp->bb_bitmap)
                return;

        bh = ext4_read_block_bitmap(sb, group);
        if (IS_ERR_OR_NULL(bh)) {
                kfree(grp->bb_bitmap);
                grp->bb_bitmap = NULL;
                return;
        }

        memcpy(grp->bb_bitmap, bh->b_data, sb->s_blocksize);
        put_bh(bh);
}

static void mb_group_bb_bitmap_free(struct ext4_group_info *grp)
{
        kfree(grp->bb_bitmap);
}

#else
static inline void mb_free_blocks_double(struct inode *inode,
                                struct ext4_buddy *e4b, int first, int count)
{
        return;
}
static inline void mb_mark_used_double(struct ext4_buddy *e4b,
                                                int first, int count)
{
        return;
}
static inline void mb_cmp_bitmaps(struct ext4_buddy *e4b, void *bitmap)
{
        return;
}

static inline void mb_group_bb_bitmap_alloc(struct super_block *sb,
                        struct ext4_group_info *grp, ext4_group_t group)
{
        return;
}

static inline void mb_group_bb_bitmap_free(struct ext4_group_info *grp)
{
        return;
}
#endif

#ifdef AGGRESSIVE_CHECK

#define MB_CHECK_ASSERT(assert)                                                \
do {                                                                        \
        if (!(assert)) {                                                \
                printk(KERN_EMERG                                        \
                        "Assertion failure in %s() at %s:%d: \"%s\"\n",        \
                        function, file, line, # assert);                \
                BUG();                                                        \
        }                                                                \
} while (0)

static void __mb_check_buddy(struct ext4_buddy *e4b, char *file,
                                const char *function, int line)
{
        struct super_block *sb = e4b->bd_sb;
        int order = e4b->bd_blkbits + 1;
        int max;
        int max2;
        int i;
        int j;
        int k;
        int count;
        struct ext4_group_info *grp;
        int fragments = 0;
        int fstart;
        struct list_head *cur;
        void *buddy;
        void *buddy2;

        if (e4b->bd_info->bb_check_counter++ % 10)
                return;

        while (order > 1) {
                buddy = mb_find_buddy(e4b, order, &max);
                MB_CHECK_ASSERT(buddy);
                buddy2 = mb_find_buddy(e4b, order - 1, &max2);
                MB_CHECK_ASSERT(buddy2);
                MB_CHECK_ASSERT(buddy != buddy2);
                MB_CHECK_ASSERT(max * 2 == max2);

                count = 0;
                for (i = 0; i < max; i++) {

                        if (mb_test_bit(i, buddy)) {
                                /* only single bit in buddy2 may be 0 */
                                if (!mb_test_bit(i << 1, buddy2)) {
                                        MB_CHECK_ASSERT(
                                                mb_test_bit((i<<1)+1, buddy2));
                                }
                                continue;
                        }

                        /* both bits in buddy2 must be 1 */
                        MB_CHECK_ASSERT(mb_test_bit(i << 1, buddy2));
                        MB_CHECK_ASSERT(mb_test_bit((i << 1) + 1, buddy2));

                        for (j = 0; j < (1 << order); j++) {
                                k = (i * (1 << order)) + j;
                                MB_CHECK_ASSERT(
                                        !mb_test_bit(k, e4b->bd_bitmap));
                        }
                        count++;
                }
                MB_CHECK_ASSERT(e4b->bd_info->bb_counters[order] == count);
                order--;
        }

        fstart = -1;
        buddy = mb_find_buddy(e4b, 0, &max);
        for (i = 0; i < max; i++) {
                if (!mb_test_bit(i, buddy)) {
                        MB_CHECK_ASSERT(i >= e4b->bd_info->bb_first_free);
                        if (fstart == -1) {
                                fragments++;
                                fstart = i;
                        }
                        continue;
                }
                fstart = -1;
                /* check used bits only */
                for (j = 0; j < e4b->bd_blkbits + 1; j++) {
                        buddy2 = mb_find_buddy(e4b, j, &max2);
                        k = i >> j;
                        MB_CHECK_ASSERT(k < max2);
                        MB_CHECK_ASSERT(mb_test_bit(k, buddy2));
                }
        }
        MB_CHECK_ASSERT(!EXT4_MB_GRP_NEED_INIT(e4b->bd_info));
        MB_CHECK_ASSERT(e4b->bd_info->bb_fragments == fragments);

        grp = ext4_get_group_info(sb, e4b->bd_group);
        if (!grp)
                return;
        list_for_each(cur, &grp->bb_prealloc_list) {
                ext4_group_t groupnr;
                struct ext4_prealloc_space *pa;
                pa = list_entry(cur, struct ext4_prealloc_space, pa_group_list);
                ext4_get_group_no_and_offset(sb, pa->pa_pstart, &groupnr, &k);
                MB_CHECK_ASSERT(groupnr == e4b->bd_group);
                for (i = 0; i < pa->pa_len; i++)
                        MB_CHECK_ASSERT(mb_test_bit(k + i, buddy));
        }
}
#undef MB_CHECK_ASSERT
#define mb_check_buddy(e4b) __mb_check_buddy(e4b,        \
                                        __FILE__, __func__, __LINE__)
#else
#define mb_check_buddy(e4b)
#endif

/*
 * Divide blocks started from @first with length @len into
 * smaller chunks with power of 2 blocks.
 * Clear the bits in bitmap which the blocks of the chunk(s) covered,
 * then increase bb_counters[] for corresponded chunk size.
 */
static void ext4_mb_mark_free_simple(struct super_block *sb,
                                void *buddy, ext4_grpblk_t first, ext4_grpblk_t len,
                                        struct ext4_group_info *grp)
{
        struct ext4_sb_info *sbi = EXT4_SB(sb);
        ext4_grpblk_t min;
        ext4_grpblk_t max;
        ext4_grpblk_t chunk;
        unsigned int border;

        BUG_ON(len > EXT4_CLUSTERS_PER_GROUP(sb));

        border = 2 << sb->s_blocksize_bits;

        while (len > 0) {
                /* find how many blocks can be covered since this position */
                max = ffs(first | border) - 1;

                /* find how many blocks of power 2 we need to mark */
                min = fls(len) - 1;

                if (max < min)
                        min = max;
                chunk = 1 << min;

                /* mark multiblock chunks only */
                grp->bb_counters[min]++;
                if (min > 0)
                        mb_clear_bit(first >> min,
                                     buddy + sbi->s_mb_offsets[min]);

                len -= chunk;
                first += chunk;
        }
}

static int mb_avg_fragment_size_order(struct super_block *sb, ext4_grpblk_t len)
{
        int order;

        /*
         * We don't bother with a special lists groups with only 1 block free
         * extents and for completely empty groups.
         */
        order = fls(len) - 2;
        if (order < 0)
                return 0;
        if (order == MB_NUM_ORDERS(sb))
                order--;
        if (WARN_ON_ONCE(order > MB_NUM_ORDERS(sb)))
                order = MB_NUM_ORDERS(sb) - 1;
        return order;
}

/* Move group to appropriate avg_fragment_size list */
static void
mb_update_avg_fragment_size(struct super_block *sb, struct ext4_group_info *grp)
{
        struct ext4_sb_info *sbi = EXT4_SB(sb);
        int new_order;

        if (!test_opt2(sb, MB_OPTIMIZE_SCAN) || grp->bb_fragments == 0)
                return;

        new_order = mb_avg_fragment_size_order(sb,
                                        grp->bb_free / grp->bb_fragments);
        if (new_order == grp->bb_avg_fragment_size_order)
                return;

        if (grp->bb_avg_fragment_size_order != -1) {
                write_lock(&sbi->s_mb_avg_fragment_size_locks[
                                        grp->bb_avg_fragment_size_order]);
                list_del(&grp->bb_avg_fragment_size_node);
                write_unlock(&sbi->s_mb_avg_fragment_size_locks[
                                        grp->bb_avg_fragment_size_order]);
        }
        grp->bb_avg_fragment_size_order = new_order;
        write_lock(&sbi->s_mb_avg_fragment_size_locks[
                                        grp->bb_avg_fragment_size_order]);
        list_add_tail(&grp->bb_avg_fragment_size_node,
                &sbi->s_mb_avg_fragment_size[grp->bb_avg_fragment_size_order]);
        write_unlock(&sbi->s_mb_avg_fragment_size_locks[
                                        grp->bb_avg_fragment_size_order]);
}

/*
 * Choose next group by traversing largest_free_order lists. Updates *new_cr if
 * cr level needs an update.
 */
static void ext4_mb_choose_next_group_p2_aligned(struct ext4_allocation_context *ac,
                        enum criteria *new_cr, ext4_group_t *group)
{
        struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);
        struct ext4_group_info *iter;
        int i;

        if (ac->ac_status == AC_STATUS_FOUND)
                return;

        if (unlikely(sbi->s_mb_stats && ac->ac_flags & EXT4_MB_CR_POWER2_ALIGNED_OPTIMIZED))
                atomic_inc(&sbi->s_bal_p2_aligned_bad_suggestions);

        for (i = ac->ac_2order; i < MB_NUM_ORDERS(ac->ac_sb); i++) {
                if (list_empty(&sbi->s_mb_largest_free_orders[i]))
                        continue;
                read_lock(&sbi->s_mb_largest_free_orders_locks[i]);
                if (list_empty(&sbi->s_mb_largest_free_orders[i])) {
                        read_unlock(&sbi->s_mb_largest_free_orders_locks[i]);
                        continue;
                }
                list_for_each_entry(iter, &sbi->s_mb_largest_free_orders[i],
                                    bb_largest_free_order_node) {
                        if (sbi->s_mb_stats)
                                atomic64_inc(&sbi->s_bal_cX_groups_considered[CR_POWER2_ALIGNED]);
                        if (likely(ext4_mb_good_group(ac, iter->bb_group, CR_POWER2_ALIGNED))) {
                                *group = iter->bb_group;
                                ac->ac_flags |= EXT4_MB_CR_POWER2_ALIGNED_OPTIMIZED;
                                read_unlock(&sbi->s_mb_largest_free_orders_locks[i]);
                                return;
                        }
                }
                read_unlock(&sbi->s_mb_largest_free_orders_locks[i]);
        }

        /* Increment cr and search again if no group is found */
        *new_cr = CR_GOAL_LEN_FAST;
}

/*
 * Find a suitable group of given order from the average fragments list.
 */
static struct ext4_group_info *
ext4_mb_find_good_group_avg_frag_lists(struct ext4_allocation_context *ac, int order)
{
        struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);
        struct list_head *frag_list = &sbi->s_mb_avg_fragment_size[order];
        rwlock_t *frag_list_lock = &sbi->s_mb_avg_fragment_size_locks[order];
        struct ext4_group_info *grp = NULL, *iter;
        enum criteria cr = ac->ac_criteria;

        if (list_empty(frag_list))
                return NULL;
        read_lock(frag_list_lock);
        if (list_empty(frag_list)) {
                read_unlock(frag_list_lock);
                return NULL;
        }
        list_for_each_entry(iter, frag_list, bb_avg_fragment_size_node) {
                if (sbi->s_mb_stats)
                        atomic64_inc(&sbi->s_bal_cX_groups_considered[cr]);
                if (likely(ext4_mb_good_group(ac, iter->bb_group, cr))) {
                        grp = iter;
                        break;
                }
        }
        read_unlock(frag_list_lock);
        return grp;
}

/*
 * Choose next group by traversing average fragment size list of suitable
 * order. Updates *new_cr if cr level needs an update.
 */
static void ext4_mb_choose_next_group_goal_fast(struct ext4_allocation_context *ac,
                enum criteria *new_cr, ext4_group_t *group)
{
        struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);
        struct ext4_group_info *grp = NULL;
        int i;

        if (unlikely(ac->ac_flags & EXT4_MB_CR_GOAL_LEN_FAST_OPTIMIZED)) {
                if (sbi->s_mb_stats)
                        atomic_inc(&sbi->s_bal_goal_fast_bad_suggestions);
        }

        for (i = mb_avg_fragment_size_order(ac->ac_sb, ac->ac_g_ex.fe_len);
             i < MB_NUM_ORDERS(ac->ac_sb); i++) {
                grp = ext4_mb_find_good_group_avg_frag_lists(ac, i);
                if (grp) {
                        *group = grp->bb_group;
                        ac->ac_flags |= EXT4_MB_CR_GOAL_LEN_FAST_OPTIMIZED;
                        return;
                }
        }

        /*
         * CR_BEST_AVAIL_LEN works based on the concept that we have
         * a larger normalized goal len request which can be trimmed to
         * a smaller goal len such that it can still satisfy original
         * request len. However, allocation request for non-regular
         * files never gets normalized.
         * See function ext4_mb_normalize_request() (EXT4_MB_HINT_DATA).
         */
        if (ac->ac_flags & EXT4_MB_HINT_DATA)
                *new_cr = CR_BEST_AVAIL_LEN;
        else
                *new_cr = CR_GOAL_LEN_SLOW;
}

/*
 * We couldn't find a group in CR_GOAL_LEN_FAST so try to find the highest free fragment
 * order we have and proactively trim the goal request length to that order to
 * find a suitable group faster.
 *
 * This optimizes allocation speed at the cost of slightly reduced
 * preallocations. However, we make sure that we don't trim the request too
 * much and fall to CR_GOAL_LEN_SLOW in that case.
 */
static void ext4_mb_choose_next_group_best_avail(struct ext4_allocation_context *ac,
                enum criteria *new_cr, ext4_group_t *group)
{
        struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);
        struct ext4_group_info *grp = NULL;
        int i, order, min_order;
        unsigned long num_stripe_clusters = 0;

        if (unlikely(ac->ac_flags & EXT4_MB_CR_BEST_AVAIL_LEN_OPTIMIZED)) {
                if (sbi->s_mb_stats)
                        atomic_inc(&sbi->s_bal_best_avail_bad_suggestions);
        }

        /*
         * mb_avg_fragment_size_order() returns order in a way that makes
         * retrieving back the length using (1 << order) inaccurate. Hence, use
         * fls() instead since we need to know the actual length while modifying
         * goal length.
         */
        order = fls(ac->ac_g_ex.fe_len) - 1;
        if (WARN_ON_ONCE(order - 1 > MB_NUM_ORDERS(ac->ac_sb)))
                order = MB_NUM_ORDERS(ac->ac_sb);
        min_order = order - sbi->s_mb_best_avail_max_trim_order;
        if (min_order < 0)
                min_order = 0;

        if (sbi->s_stripe > 0) {
                /*
                 * We are assuming that stripe size is always a multiple of
                 * cluster ratio otherwise __ext4_fill_super exists early.
                 */
                num_stripe_clusters = EXT4_NUM_B2C(sbi, sbi->s_stripe);
                if (1 << min_order < num_stripe_clusters)
                        /*
                         * We consider 1 order less because later we round
                         * up the goal len to num_stripe_clusters
                         */
                        min_order = fls(num_stripe_clusters) - 1;
        }

        if (1 << min_order < ac->ac_o_ex.fe_len)
                min_order = fls(ac->ac_o_ex.fe_len);

        for (i = order; i >= min_order; i--) {
                int frag_order;
                /*
                 * Scale down goal len to make sure we find something
                 * in the free fragments list. Basically, reduce
                 * preallocations.
                 */
                ac->ac_g_ex.fe_len = 1 << i;

                if (num_stripe_clusters > 0) {
                        /*
                         * Try to round up the adjusted goal length to
                         * stripe size (in cluster units) multiple for
                         * efficiency.
                         */
                        ac->ac_g_ex.fe_len = roundup(ac->ac_g_ex.fe_len,
                                                     num_stripe_clusters);
                }

                frag_order = mb_avg_fragment_size_order(ac->ac_sb,
                                                        ac->ac_g_ex.fe_len);

                grp = ext4_mb_find_good_group_avg_frag_lists(ac, frag_order);
                if (grp) {
                        *group = grp->bb_group;
                        ac->ac_flags |= EXT4_MB_CR_BEST_AVAIL_LEN_OPTIMIZED;
                        return;
                }
        }

        /* Reset goal length to original goal length before falling into CR_GOAL_LEN_SLOW */
        ac->ac_g_ex.fe_len = ac->ac_orig_goal_len;
        *new_cr = CR_GOAL_LEN_SLOW;
}

static inline int should_optimize_scan(struct ext4_allocation_context *ac)
{
        if (unlikely(!test_opt2(ac->ac_sb, MB_OPTIMIZE_SCAN)))
                return 0;
        if (ac->ac_criteria >= CR_GOAL_LEN_SLOW)
                return 0;
        if (!ext4_test_inode_flag(ac->ac_inode, EXT4_INODE_EXTENTS))
                return 0;
        return 1;
}

/*
 * Return next linear group for allocation.
 */
static ext4_group_t
next_linear_group(ext4_group_t group, ext4_group_t ngroups)
{
        /*
         * Artificially restricted ngroups for non-extent
         * files makes group > ngroups possible on first loop.
         */
        return group + 1 >= ngroups ? 0 : group + 1;
}

/*
 * ext4_mb_choose_next_group: choose next group for allocation.
 *
 * @ac        Allocation Context
 * @new_cr    This is an output parameter. If the there is no good group
 *            available at current CR level, this field is updated to indicate
 *            the new cr level that should be used.
 * @group     This is an input / output parameter. As an input it indicates the
 *            next group that the allocator intends to use for allocation. As
 *            output, this field indicates the next group that should be used as
 *            determined by the optimization functions.
 * @ngroups   Total number of groups
 */
static void ext4_mb_choose_next_group(struct ext4_allocation_context *ac,
                enum criteria *new_cr, ext4_group_t *group, ext4_group_t ngroups)
{
        *new_cr = ac->ac_criteria;

        if (!should_optimize_scan(ac)) {
                *group = next_linear_group(*group, ngroups);
                return;
        }

        /*
         * Optimized scanning can return non adjacent groups which can cause
         * seek overhead for rotational disks. So try few linear groups before
         * trying optimized scan.
         */
        if (ac->ac_groups_linear_remaining) {
                *group = next_linear_group(*group, ngroups);
                ac->ac_groups_linear_remaining--;
                return;
        }

        if (*new_cr == CR_POWER2_ALIGNED) {
                ext4_mb_choose_next_group_p2_aligned(ac, new_cr, group);
        } else if (*new_cr == CR_GOAL_LEN_FAST) {
                ext4_mb_choose_next_group_goal_fast(ac, new_cr, group);
        } else if (*new_cr == CR_BEST_AVAIL_LEN) {
                ext4_mb_choose_next_group_best_avail(ac, new_cr, group);
        } else {
                /*
                 * TODO: For CR_GOAL_LEN_SLOW, we can arrange groups in an
                 * rb tree sorted by bb_free. But until that happens, we should
                 * never come here.
                 */
                WARN_ON(1);
        }
}

/*
 * Cache the order of the largest free extent we have available in this block
 * group.
 */
static void
mb_set_largest_free_order(struct super_block *sb, struct ext4_group_info *grp)
{
        struct ext4_sb_info *sbi = EXT4_SB(sb);
        int i;

        for (i = MB_NUM_ORDERS(sb) - 1; i >= 0; i--)
                if (grp->bb_counters[i] > 0)
                        break;
        /* No need to move between order lists? */
        if (!test_opt2(sb, MB_OPTIMIZE_SCAN) ||
            i == grp->bb_largest_free_order) {
                grp->bb_largest_free_order = i;
                return;
        }

        if (grp->bb_largest_free_order >= 0) {
                write_lock(&sbi->s_mb_largest_free_orders_locks[
                                              grp->bb_largest_free_order]);
                list_del_init(&grp->bb_largest_free_order_node);
                write_unlock(&sbi->s_mb_largest_free_orders_locks[
                                              grp->bb_largest_free_order]);
        }
        grp->bb_largest_free_order = i;
        if (grp->bb_largest_free_order >= 0 && grp->bb_free) {
                write_lock(&sbi->s_mb_largest_free_orders_locks[
                                              grp->bb_largest_free_order]);
                list_add_tail(&grp->bb_largest_free_order_node,
                      &sbi->s_mb_largest_free_orders[grp->bb_largest_free_order]);
                write_unlock(&sbi->s_mb_largest_free_orders_locks[
                                              grp->bb_largest_free_order]);
        }
}

static noinline_for_stack
void ext4_mb_generate_buddy(struct super_block *sb,
                            void *buddy, void *bitmap, ext4_group_t group,
                            struct ext4_group_info *grp)
{
        struct ext4_sb_info *sbi = EXT4_SB(sb);
        ext4_grpblk_t max = EXT4_CLUSTERS_PER_GROUP(sb);
        ext4_grpblk_t i = 0;
        ext4_grpblk_t first;
        ext4_grpblk_t len;
        unsigned free = 0;
        unsigned fragments = 0;
        unsigned long long period = get_cycles();

        /* initialize buddy from bitmap which is aggregation
         * of on-disk bitmap and preallocations */
        i = mb_find_next_zero_bit(bitmap, max, 0);
        grp->bb_first_free = i;
        while (i < max) {
                fragments++;
                first = i;
                i = mb_find_next_bit(bitmap, max, i);
                len = i - first;
                free += len;
                if (len > 1)
                        ext4_mb_mark_free_simple(sb, buddy, first, len, grp);
                else
                        grp->bb_counters[0]++;
                if (i < max)
                        i = mb_find_next_zero_bit(bitmap, max, i);
        }
        grp->bb_fragments = fragments;

        if (free != grp->bb_free) {
                ext4_grp_locked_error(sb, group, 0, 0,
                                      "block bitmap and bg descriptor "
                                      "inconsistent: %u vs %u free clusters",
                                      free, grp->bb_free);
                /*
                 * If we intend to continue, we consider group descriptor
                 * corrupt and update bb_free using bitmap value
                 */
                grp->bb_free = free;
                ext4_mark_group_bitmap_corrupted(sb, group,
                                        EXT4_GROUP_INFO_BBITMAP_CORRUPT);
        }
        mb_set_largest_free_order(sb, grp);
        mb_update_avg_fragment_size(sb, grp);

        clear_bit(EXT4_GROUP_INFO_NEED_INIT_BIT, &(grp->bb_state));

        period = get_cycles() - period;
        atomic_inc(&sbi->s_mb_buddies_generated);
        atomic64_add(period, &sbi->s_mb_generation_time);
}

static void mb_regenerate_buddy(struct ext4_buddy *e4b)
{
        int count;
        int order = 1;
        void *buddy;

        while ((buddy = mb_find_buddy(e4b, order++, &count)))
                mb_set_bits(buddy, 0, count);

        e4b->bd_info->bb_fragments = 0;
        memset(e4b->bd_info->bb_counters, 0,
                sizeof(*e4b->bd_info->bb_counters) *
                (e4b->bd_sb->s_blocksize_bits + 2));

        ext4_mb_generate_buddy(e4b->bd_sb, e4b->bd_buddy,
                e4b->bd_bitmap, e4b->bd_group, e4b->bd_info);
}

/* The buddy information is attached the buddy cache inode
 * for convenience. The information regarding each group
 * is loaded via ext4_mb_load_buddy. The information involve
 * block bitmap and buddy information. The information are
 * stored in the inode as
 *
 * {                        page                        }
 * [ group 0 bitmap][ group 0 buddy] [group 1][ group 1]...
 *
 *
 * one block each for bitmap and buddy information.
 * So for each group we take up 2 blocks. A page can
 * contain blocks_per_page (PAGE_SIZE / blocksize)  blocks.
 * So it can have information regarding groups_per_page which
 * is blocks_per_page/2
 *
 * Locking note:  This routine takes the block group lock of all groups
 * for this page; do not hold this lock when calling this routine!
 */

static int ext4_mb_init_cache(struct folio *folio, char *incore, gfp_t gfp)
{
        ext4_group_t ngroups;
        unsigned int blocksize;
        int blocks_per_page;
        int groups_per_page;
        int err = 0;
        int i;
        ext4_group_t first_group, group;
        int first_block;
        struct super_block *sb;
        struct buffer_head *bhs;
        struct buffer_head **bh = NULL;
        struct inode *inode;
        char *data;
        char *bitmap;
        struct ext4_group_info *grinfo;

        inode = folio->mapping->host;
        sb = inode->i_sb;
        ngroups = ext4_get_groups_count(sb);
        blocksize = i_blocksize(inode);
        blocks_per_page = PAGE_SIZE / blocksize;

        mb_debug(sb, "init folio %lu\n", folio->index);

        groups_per_page = blocks_per_page >> 1;
        if (groups_per_page == 0)
                groups_per_page = 1;

        /* allocate buffer_heads to read bitmaps */
        if (groups_per_page > 1) {
                i = sizeof(struct buffer_head *) * groups_per_page;
                bh = kzalloc(i, gfp);
                if (bh == NULL)
                        return -ENOMEM;
        } else
                bh = &bhs;

        first_group = folio->index * blocks_per_page / 2;

        /* read all groups the folio covers into the cache */
        for (i = 0, group = first_group; i < groups_per_page; i++, group++) {
                if (group >= ngroups)
                        break;

                grinfo = ext4_get_group_info(sb, group);
                if (!grinfo)
                        continue;
                /*
                 * If page is uptodate then we came here after online resize
                 * which added some new uninitialized group info structs, so
                 * we must skip all initialized uptodate buddies on the folio,
                 * which may be currently in use by an allocating task.
                 */
                if (folio_test_uptodate(folio) &&
                                !EXT4_MB_GRP_NEED_INIT(grinfo)) {
                        bh[i] = NULL;
                        continue;
                }
                bh[i] = ext4_read_block_bitmap_nowait(sb, group, false);
                if (IS_ERR(bh[i])) {
                        err = PTR_ERR(bh[i]);
                        bh[i] = NULL;
                        goto out;
                }
                mb_debug(sb, "read bitmap for group %u\n", group);
        }

        /* wait for I/O completion */
        for (i = 0, group = first_group; i < groups_per_page; i++, group++) {
                int err2;

                if (!bh[i])
                        continue;
                err2 = ext4_wait_block_bitmap(sb, group, bh[i]);
                if (!err)
                        err = err2;
        }

        first_block = folio->index * blocks_per_page;
        for (i = 0; i < blocks_per_page; i++) {
                group = (first_block + i) >> 1;
                if (group >= ngroups)
                        break;

                if (!bh[group - first_group])
                        /* skip initialized uptodate buddy */
                        continue;

                if (!buffer_verified(bh[group - first_group]))
                        /* Skip faulty bitmaps */
                        continue;
                err = 0;

                /*
                 * data carry information regarding this
                 * particular group in the format specified
                 * above
                 *
                 */
                data = folio_address(folio) + (i * blocksize);
                bitmap = bh[group - first_group]->b_data;

                /*
                 * We place the buddy block and bitmap block
                 * close together
                 */
                grinfo = ext4_get_group_info(sb, group);
                if (!grinfo) {
                        err = -EFSCORRUPTED;
                        goto out;
                }
                if ((first_block + i) & 1) {
                        /* this is block of buddy */
                        BUG_ON(incore == NULL);
                        mb_debug(sb, "put buddy for group %u in folio %lu/%x\n",
                                group, folio->index, i * blocksize);
                        trace_ext4_mb_buddy_bitmap_load(sb, group);
                        grinfo->bb_fragments = 0;
                        memset(grinfo->bb_counters, 0,
                               sizeof(*grinfo->bb_counters) *
                               (MB_NUM_ORDERS(sb)));
                        /*
                         * incore got set to the group block bitmap below
                         */
                        ext4_lock_group(sb, group);
                        /* init the buddy */
                        memset(data, 0xff, blocksize);
                        ext4_mb_generate_buddy(sb, data, incore, group, grinfo);
                        ext4_unlock_group(sb, group);
                        incore = NULL;
                } else {
                        /* this is block of bitmap */
                        BUG_ON(incore != NULL);
                        mb_debug(sb, "put bitmap for group %u in folio %lu/%x\n",
                                group, folio->index, i * blocksize);
                        trace_ext4_mb_bitmap_load(sb, group);

                        /* see comments in ext4_mb_put_pa() */
                        ext4_lock_group(sb, group);
                        memcpy(data, bitmap, blocksize);

                        /* mark all preallocated blks used in in-core bitmap */
                        ext4_mb_generate_from_pa(sb, data, group);
                        WARN_ON_ONCE(!RB_EMPTY_ROOT(&grinfo->bb_free_root));
                        ext4_unlock_group(sb, group);

                        /* set incore so that the buddy information can be
                         * generated using this
                         */
                        incore = data;
                }
        }
        folio_mark_uptodate(folio);

out:
        if (bh) {
                for (i = 0; i < groups_per_page; i++)
                        brelse(bh[i]);
                if (bh != &bhs)
                        kfree(bh);
        }
        return err;
}

/*
 * Lock the buddy and bitmap pages. This make sure other parallel init_group
 * on the same buddy page doesn't happen whild holding the buddy page lock.
 * Return locked buddy and bitmap pages on e4b struct. If buddy and bitmap
 * are on the same page e4b->bd_buddy_folio is NULL and return value is 0.
 */
static int ext4_mb_get_buddy_page_lock(struct super_block *sb,
                ext4_group_t group, struct ext4_buddy *e4b, gfp_t gfp)
{
        struct inode *inode = EXT4_SB(sb)->s_buddy_cache;
        int block, pnum, poff;
        int blocks_per_page;
        struct folio *folio;

        e4b->bd_buddy_folio = NULL;
        e4b->bd_bitmap_folio = NULL;

        blocks_per_page = PAGE_SIZE / sb->s_blocksize;
        /*
         * the buddy cache inode stores the block bitmap
         * and buddy information in consecutive blocks.
         * So for each group we need two blocks.
         */
        block = group * 2;
        pnum = block / blocks_per_page;
        poff = block % blocks_per_page;
        folio = __filemap_get_folio(inode->i_mapping, pnum,
                        FGP_LOCK | FGP_ACCESSED | FGP_CREAT, gfp);
        if (IS_ERR(folio))
                return PTR_ERR(folio);
        BUG_ON(folio->mapping != inode->i_mapping);
        e4b->bd_bitmap_folio = folio;
        e4b->bd_bitmap = folio_address(folio) + (poff * sb->s_blocksize);

        if (blocks_per_page >= 2) {
                /* buddy and bitmap are on the same page */
                return 0;
        }

        /* blocks_per_page == 1, hence we need another page for the buddy */
        folio = __filemap_get_folio(inode->i_mapping, block + 1,
                        FGP_LOCK | FGP_ACCESSED | FGP_CREAT, gfp);
        if (IS_ERR(folio))
                return PTR_ERR(folio);
        BUG_ON(folio->mapping != inode->i_mapping);
        e4b->bd_buddy_folio = folio;
        return 0;
}

static void ext4_mb_put_buddy_page_lock(struct ext4_buddy *e4b)
{
        if (e4b->bd_bitmap_folio) {
                folio_unlock(e4b->bd_bitmap_folio);
                folio_put(e4b->bd_bitmap_folio);
        }
        if (e4b->bd_buddy_folio) {
                folio_unlock(e4b->bd_buddy_folio);
                folio_put(e4b->bd_buddy_folio);
        }
}

/*
 * Locking note:  This routine calls ext4_mb_init_cache(), which takes the
 * block group lock of all groups for this page; do not hold the BG lock when
 * calling this routine!
 */
static noinline_for_stack
int ext4_mb_init_group(struct super_block *sb, ext4_group_t group, gfp_t gfp)
{

        struct ext4_group_info *this_grp;
        struct ext4_buddy e4b;
        struct folio *folio;
        int ret = 0;

        might_sleep();
        mb_debug(sb, "init group %u\n", group);
        this_grp = ext4_get_group_info(sb, group);
        if (!this_grp)
                return -EFSCORRUPTED;

        /*
         * This ensures that we don't reinit the buddy cache
         * page which map to the group from which we are already
         * allocating. If we are looking at the buddy cache we would
         * have taken a reference using ext4_mb_load_buddy and that
         * would have pinned buddy page to page cache.
         * The call to ext4_mb_get_buddy_page_lock will mark the
         * page accessed.
         */
        ret = ext4_mb_get_buddy_page_lock(sb, group, &e4b, gfp);
        if (ret || !EXT4_MB_GRP_NEED_INIT(this_grp)) {
                /*
                 * somebody initialized the group
                 * return without doing anything
                 */
                goto err;
        }

        folio = e4b.bd_bitmap_folio;
        ret = ext4_mb_init_cache(folio, NULL, gfp);
        if (ret)
                goto err;
        if (!folio_test_uptodate(folio)) {
                ret = -EIO;
                goto err;
        }

        if (e4b.bd_buddy_folio == NULL) {
                /*
                 * If both the bitmap and buddy are in
                 * the same page we don't need to force
                 * init the buddy
                 */
                ret = 0;
                goto err;
        }
        /* init buddy cache */
        folio = e4b.bd_buddy_folio;
        ret = ext4_mb_init_cache(folio, e4b.bd_bitmap, gfp);
        if (ret)
                goto err;
        if (!folio_test_uptodate(folio)) {
                ret = -EIO;
                goto err;
        }
err:
        ext4_mb_put_buddy_page_lock(&e4b);
        return ret;
}

/*
 * Locking note:  This routine calls ext4_mb_init_cache(), which takes the
 * block group lock of all groups for this page; do not hold the BG lock when
 * calling this routine!
 */
static noinline_for_stack int
ext4_mb_load_buddy_gfp(struct super_block *sb, ext4_group_t group,
                       struct ext4_buddy *e4b, gfp_t gfp)
{
        int blocks_per_page;
        int block;
        int pnum;
        int poff;
        struct folio *folio;
        int ret;
        struct ext4_group_info *grp;
        struct ext4_sb_info *sbi = EXT4_SB(sb);
        struct inode *inode = sbi->s_buddy_cache;

        might_sleep();
        mb_debug(sb, "load group %u\n", group);

        blocks_per_page = PAGE_SIZE / sb->s_blocksize;
        grp = ext4_get_group_info(sb, group);
        if (!grp)
                return -EFSCORRUPTED;

        e4b->bd_blkbits = sb->s_blocksize_bits;
        e4b->bd_info = grp;
        e4b->bd_sb = sb;
        e4b->bd_group = group;
        e4b->bd_buddy_folio = NULL;
        e4b->bd_bitmap_folio = NULL;

        if (unlikely(EXT4_MB_GRP_NEED_INIT(grp))) {
                /*
                 * we need full data about the group
                 * to make a good selection
                 */
                ret = ext4_mb_init_group(sb, group, gfp);
                if (ret)
                        return ret;
        }

        /*
         * the buddy cache inode stores the block bitmap
         * and buddy information in consecutive blocks.
         * So for each group we need two blocks.
         */
        block = group * 2;
        pnum = block / blocks_per_page;
        poff = block % blocks_per_page;

        /* Avoid locking the folio in the fast path ... */
        folio = __filemap_get_folio(inode->i_mapping, pnum, FGP_ACCESSED, 0);
        if (IS_ERR(folio) || !folio_test_uptodate(folio)) {
                if (!IS_ERR(folio))
                        /*
                         * drop the folio reference and try
                         * to get the folio with lock. If we
                         * are not uptodate that implies
                         * somebody just created the folio but
                         * is yet to initialize it. So
                         * wait for it to initialize.
                         */
                        folio_put(folio);
                folio = __filemap_get_folio(inode->i_mapping, pnum,
                                FGP_LOCK | FGP_ACCESSED | FGP_CREAT, gfp);
                if (!IS_ERR(folio)) {
                        if (WARN_RATELIMIT(folio->mapping != inode->i_mapping,
        "ext4: bitmap's mapping != inode->i_mapping\n")) {
                                /* should never happen */
                                folio_unlock(folio);
                                ret = -EINVAL;
                                goto err;
                        }
                        if (!folio_test_uptodate(folio)) {
                                ret = ext4_mb_init_cache(folio, NULL, gfp);
                                if (ret) {
                                        folio_unlock(folio);
                                        goto err;
                                }
                                mb_cmp_bitmaps(e4b, folio_address(folio) +
                                               (poff * sb->s_blocksize));
                        }
                        folio_unlock(folio);
                }
        }
        if (IS_ERR(folio)) {
                ret = PTR_ERR(folio);
                goto err;
        }
        if (!folio_test_uptodate(folio)) {
                ret = -EIO;
                goto err;
        }

        /* Folios marked accessed already */
        e4b->bd_bitmap_folio = folio;
        e4b->bd_bitmap = folio_address(folio) + (poff * sb->s_blocksize);

        block++;
        pnum = block / blocks_per_page;
        poff = block % blocks_per_page;

        folio = __filemap_get_folio(inode->i_mapping, pnum, FGP_ACCESSED, 0);
        if (IS_ERR(folio) || !folio_test_uptodate(folio)) {
                if (!IS_ERR(folio))
                        folio_put(folio);
                folio = __filemap_get_folio(inode->i_mapping, pnum,
                                FGP_LOCK | FGP_ACCESSED | FGP_CREAT, gfp);
                if (!IS_ERR(folio)) {
                        if (WARN_RATELIMIT(folio->mapping != inode->i_mapping,
        "ext4: buddy bitmap's mapping != inode->i_mapping\n")) {
                                /* should never happen */
                                folio_unlock(folio);
                                ret = -EINVAL;
                                goto err;
                        }
                        if (!folio_test_uptodate(folio)) {
                                ret = ext4_mb_init_cache(folio, e4b->bd_bitmap,
                                                         gfp);
                                if (ret) {
                                        folio_unlock(folio);
                                        goto err;
                                }
                        }
                        folio_unlock(folio);
                }
        }
        if (IS_ERR(folio)) {
                ret = PTR_ERR(folio);
                goto err;
        }
        if (!folio_test_uptodate(folio)) {
                ret = -EIO;
                goto err;
        }

        /* Folios marked accessed already */
        e4b->bd_buddy_folio = folio;
        e4b->bd_buddy = folio_address(folio) + (poff * sb->s_blocksize);

        return 0;

err:
        if (!IS_ERR_OR_NULL(folio))
                folio_put(folio);
        if (e4b->bd_bitmap_folio)
                folio_put(e4b->bd_bitmap_folio);

        e4b->bd_buddy = NULL;
        e4b->bd_bitmap = NULL;
        return ret;
}

static int ext4_mb_load_buddy(struct super_block *sb, ext4_group_t group,
                              struct ext4_buddy *e4b)
{
        return ext4_mb_load_buddy_gfp(sb, group, e4b, GFP_NOFS);
}

static void ext4_mb_unload_buddy(struct ext4_buddy *e4b)
{
        if (e4b->bd_bitmap_folio)
                folio_put(e4b->bd_bitmap_folio);
        if (e4b->bd_buddy_folio)
                folio_put(e4b->bd_buddy_folio);
}


static int mb_find_order_for_block(struct ext4_buddy *e4b, int block)
{
        int order = 1, max;
        void *bb;

        BUG_ON(e4b->bd_bitmap == e4b->bd_buddy);
        BUG_ON(block >= (1 << (e4b->bd_blkbits + 3)));

        while (order <= e4b->bd_blkbits + 1) {
                bb = mb_find_buddy(e4b, order, &max);
                if (!mb_test_bit(block >> order, bb)) {
                        /* this block is part of buddy of order 'order' */
                        return order;
                }
                order++;
        }
        return 0;
}

static void mb_clear_bits(void *bm, int cur, int len)
{
        __u32 *addr;

        len = cur + len;
        while (cur < len) {
                if ((cur & 31) == 0 && (len - cur) >= 32) {
                        /* fast path: clear whole word at once */
                        addr = bm + (cur >> 3);
                        *addr = 0;
                        cur += 32;
                        continue;
                }
                mb_clear_bit(cur, bm);
                cur++;
        }
}

/* clear bits in given range
 * will return first found zero bit if any, -1 otherwise
 */
static int mb_test_and_clear_bits(void *bm, int cur, int len)
{
        __u32 *addr;
        int zero_bit = -1;

        len = cur + len;
        while (cur < len) {
                if ((cur & 31) == 0 && (len - cur) >= 32) {
                        /* fast path: clear whole word at once */
                        addr = bm + (cur >> 3);
                        if (*addr != (__u32)(-1) && zero_bit == -1)
                                zero_bit = cur + mb_find_next_zero_bit(addr, 32, 0);
                        *addr = 0;
                        cur += 32;
                        continue;
                }
                if (!mb_test_and_clear_bit(cur, bm) && zero_bit == -1)
                        zero_bit = cur;
                cur++;
        }

        return zero_bit;
}

void mb_set_bits(void *bm, int cur, int len)
{
        __u32 *addr;

        len = cur + len;
        while (cur < len) {
                if ((cur & 31) == 0 && (len - cur) >= 32) {
                        /* fast path: set whole word at once */
                        addr = bm + (cur >> 3);
                        *addr = 0xffffffff;
                        cur += 32;
                        continue;
                }
                mb_set_bit(cur, bm);
                cur++;
        }
}

static inline int mb_buddy_adjust_border(int* bit, void* bitmap, int side)
{
        if (mb_test_bit(*bit + side, bitmap)) {
                mb_clear_bit(*bit, bitmap);
                (*bit) -= side;
                return 1;
        }
        else {
                (*bit) += side;
                mb_set_bit(*bit, bitmap);
                return -1;
        }
}

static void mb_buddy_mark_free(struct ext4_buddy *e4b, int first, int last)
{
        int max;
        int order = 1;
        void *buddy = mb_find_buddy(e4b, order, &max);

        while (buddy) {
                void *buddy2;

                /* Bits in range [first; last] are known to be set since
                 * corresponding blocks were allocated. Bits in range
                 * (first; last) will stay set because they form buddies on
                 * upper layer. We just deal with borders if they don't
                 * align with upper layer and then go up.
                 * Releasing entire group is all about clearing
                 * single bit of highest order buddy.
                 */

                /* Example:
                 * ---------------------------------
                 * |   1   |   1   |   1   |   1   |
                 * ---------------------------------
                 * | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 |
                 * ---------------------------------
                 *   0   1   2   3   4   5   6   7
                 *      \_____________________/
                 *
                 * Neither [1] nor [6] is aligned to above layer.
                 * Left neighbour [0] is free, so mark it busy,
                 * decrease bb_counters and extend range to
                 * [0; 6]
                 * Right neighbour [7] is busy. It can't be coaleasced with [6], so
                 * mark [6] free, increase bb_counters and shrink range to
                 * [0; 5].
                 * Then shift range to [0; 2], go up and do the same.
                 */


                if (first & 1)
                        e4b->bd_info->bb_counters[order] += mb_buddy_adjust_border(&first, buddy, -1);
                if (!(last & 1))
                        e4b->bd_info->bb_counters[order] += mb_buddy_adjust_border(&last, buddy, 1);
                if (first > last)
                        break;
                order++;

                buddy2 = mb_find_buddy(e4b, order, &max);
                if (!buddy2) {
                        mb_clear_bits(buddy, first, last - first + 1);
                        e4b->bd_info->bb_counters[order - 1] += last - first + 1;
                        break;
                }
                first >>= 1;
                last >>= 1;
                buddy = buddy2;
        }
}

static void mb_free_blocks(struct inode *inode, struct ext4_buddy *e4b,
                           int first, int count)
{
        int left_is_free = 0;
        int right_is_free = 0;
        int block;
        int last = first + count - 1;
        struct super_block *sb = e4b->bd_sb;

        if (WARN_ON(count == 0))
                return;
        BUG_ON(last >= (sb->s_blocksize << 3));
        assert_spin_locked(ext4_group_lock_ptr(sb, e4b->bd_group));
        /* Don't bother if the block group is corrupt. */
        if (unlikely(EXT4_MB_GRP_BBITMAP_CORRUPT(e4b->bd_info)))
                return;

        mb_check_buddy(e4b);
        mb_free_blocks_double(inode, e4b, first, count);

        /* access memory sequentially: check left neighbour,
         * clear range and then check right neighbour
         */
        if (first != 0)
                left_is_free = !mb_test_bit(first - 1, e4b->bd_bitmap);
        block = mb_test_and_clear_bits(e4b->bd_bitmap, first, count);
        if (last + 1 < EXT4_SB(sb)->s_mb_maxs[0])
                right_is_free = !mb_test_bit(last + 1, e4b->bd_bitmap);

        if (unlikely(block != -1)) {
                struct ext4_sb_info *sbi = EXT4_SB(sb);
                ext4_fsblk_t blocknr;

                /*
                 * Fastcommit replay can free already freed blocks which
                 * corrupts allocation info. Regenerate it.
                 */
                if (sbi->s_mount_state & EXT4_FC_REPLAY) {
                        mb_regenerate_buddy(e4b);
                        goto check;
                }

                blocknr = ext4_group_first_block_no(sb, e4b->bd_group);
                blocknr += EXT4_C2B(sbi, block);
                ext4_mark_group_bitmap_corrupted(sb, e4b->bd_group,
                                EXT4_GROUP_INFO_BBITMAP_CORRUPT);
                ext4_grp_locked_error(sb, e4b->bd_group,
                                      inode ? inode->i_ino : 0, blocknr,
                                      "freeing already freed block (bit %u); block bitmap corrupt.",
                                      block);
                return;
        }

        this_cpu_inc(discard_pa_seq);
        e4b->bd_info->bb_free += count;
        if (first < e4b->bd_info->bb_first_free)
                e4b->bd_info->bb_first_free = first;

        /* let's maintain fragments counter */
        if (left_is_free && right_is_free)
                e4b->bd_info->bb_fragments--;
        else if (!left_is_free && !right_is_free)
                e4b->bd_info->bb_fragments++;

        /* buddy[0] == bd_bitmap is a special case, so handle
         * it right away and let mb_buddy_mark_free stay free of
         * zero order checks.
         * Check if neighbours are to be coaleasced,
         * adjust bitmap bb_counters and borders appropriately.
         */
        if (first & 1) {
                first += !left_is_free;
                e4b->bd_info->bb_counters[0] += left_is_free ? -1 : 1;
        }
        if (!(last & 1)) {
                last -= !right_is_free;
                e4b->bd_info->bb_counters[0] += right_is_free ? -1 : 1;
        }

        if (first <= last)
                mb_buddy_mark_free(e4b, first >> 1, last >> 1);

        mb_set_largest_free_order(sb, e4b->bd_info);
        mb_update_avg_fragment_size(sb, e4b->bd_info);
check:
        mb_check_buddy(e4b);
}

static int mb_find_extent(struct ext4_buddy *e4b, int block,
                                int needed, struct ext4_free_extent *ex)
{
        int max, order, next;
        void *buddy;

        assert_spin_locked(ext4_group_lock_ptr(e4b->bd_sb, e4b->bd_group));
        BUG_ON(ex == NULL);

        buddy = mb_find_buddy(e4b, 0, &max);
        BUG_ON(buddy == NULL);
        BUG_ON(block >= max);
        if (mb_test_bit(block, buddy)) {
                ex->fe_len = 0;
                ex->fe_start = 0;
                ex->fe_group = 0;
                return 0;
        }

        /* find actual order */
        order = mb_find_order_for_block(e4b, block);

        ex->fe_len = (1 << order) - (block & ((1 << order) - 1));
        ex->fe_start = block;
        ex->fe_group = e4b->bd_group;

        block = block >> order;

        while (needed > ex->fe_len &&
               mb_find_buddy(e4b, order, &max)) {

                if (block + 1 >= max)
                        break;

                next = (block + 1) * (1 << order);
                if (mb_test_bit(next, e4b->bd_bitmap))
                        break;

                order = mb_find_order_for_block(e4b, next);

                block = next >> order;
                ex->fe_len += 1 << order;
        }

        if (ex->fe_start + ex->fe_len > EXT4_CLUSTERS_PER_GROUP(e4b->bd_sb)) {
                /* Should never happen! (but apparently sometimes does?!?) */
                WARN_ON(1);
                ext4_grp_locked_error(e4b->bd_sb, e4b->bd_group, 0, 0,
                        "corruption or bug in mb_find_extent "
                        "block=%d, order=%d needed=%d ex=%u/%d/%d@%u",
                        block, order, needed, ex->fe_group, ex->fe_start,
                        ex->fe_len, ex->fe_logical);
                ex->fe_len = 0;
                ex->fe_start = 0;
                ex->fe_group = 0;
        }
        return ex->fe_len;
}

static int mb_mark_used(struct ext4_buddy *e4b, struct ext4_free_extent *ex)
{
        int ord;
        int mlen = 0;
        int max = 0;
        int start = ex->fe_start;
        int len = ex->fe_len;
        unsigned ret = 0;
        int len0 = len;
        void *buddy;
        int ord_start, ord_end;

        BUG_ON(start + len > (e4b->bd_sb->s_blocksize << 3));
        BUG_ON(e4b->bd_group != ex->fe_group);
        assert_spin_locked(ext4_group_lock_ptr(e4b->bd_sb, e4b->bd_group));
        mb_check_buddy(e4b);
        mb_mark_used_double(e4b, start, len);

        this_cpu_inc(discard_pa_seq);
        e4b->bd_info->bb_free -= len;
        if (e4b->bd_info->bb_first_free == start)
                e4b->bd_info->bb_first_free += len;

        /* let's maintain fragments counter */
        if (start != 0)
                mlen = !mb_test_bit(start - 1, e4b->bd_bitmap);
        if (start + len < EXT4_SB(e4b->bd_sb)->s_mb_maxs[0])
                max = !mb_test_bit(start + len, e4b->bd_bitmap);
        if (mlen && max)
                e4b->bd_info->bb_fragments++;
        else if (!mlen && !max)
                e4b->bd_info->bb_fragments--;

        /* let's maintain buddy itself */
        while (len) {
                ord = mb_find_order_for_block(e4b, start);

                if (((start >> ord) << ord) == start && len >= (1 << ord)) {
                        /* the whole chunk may be allocated at once! */
                        mlen = 1 << ord;
                        buddy = mb_find_buddy(e4b, ord, &max);
                        BUG_ON((start >> ord) >= max);
                        mb_set_bit(start >> ord, buddy);
                        e4b->bd_info->bb_counters[ord]--;
                        start += mlen;
                        len -= mlen;
                        BUG_ON(len < 0);
                        continue;
                }

                /* store for history */
                if (ret == 0)
                        ret = len | (ord << 16);

                BUG_ON(ord <= 0);
                buddy = mb_find_buddy(e4b, ord, &max);
                mb_set_bit(start >> ord, buddy);
                e4b->bd_info->bb_counters[ord]--;

                ord_start = (start >> ord) << ord;
                ord_end = ord_start + (1 << ord);
                /* first chunk */
                if (start > ord_start)
                        ext4_mb_mark_free_simple(e4b->bd_sb, e4b->bd_buddy,
                                                 ord_start, start - ord_start,
                                                 e4b->bd_info);

                /* last chunk */
                if (start + len < ord_end) {
                        ext4_mb_mark_free_simple(e4b->bd_sb, e4b->bd_buddy,
                                                 start + len,
                                                 ord_end - (start + len),
                                                 e4b->bd_info);
                        break;
                }
                len = start + len - ord_end;
                start = ord_end;
        }
        mb_set_largest_free_order(e4b->bd_sb, e4b->bd_info);

        mb_update_avg_fragment_size(e4b->bd_sb, e4b->bd_info);
        mb_set_bits(e4b->bd_bitmap, ex->fe_start, len0);
        mb_check_buddy(e4b);

        return ret;
}

/*
 * Must be called under group lock!
 */
static void ext4_mb_use_best_found(struct ext4_allocation_context *ac,
                                        struct ext4_buddy *e4b)
{
        struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);
        int ret;

        BUG_ON(ac->ac_b_ex.fe_group != e4b->bd_group);
        BUG_ON(ac->ac_status == AC_STATUS_FOUND);

        ac->ac_b_ex.fe_len = min(ac->ac_b_ex.fe_len, ac->ac_g_ex.fe_len);
        ac->ac_b_ex.fe_logical = ac->ac_g_ex.fe_logical;
        ret = mb_mark_used(e4b, &ac->ac_b_ex);

        /* preallocation can change ac_b_ex, thus we store actually
         * allocated blocks for history */
        ac->ac_f_ex = ac->ac_b_ex;

        ac->ac_status = AC_STATUS_FOUND;
        ac->ac_tail = ret & 0xffff;
        ac->ac_buddy = ret >> 16;

        /*
         * take the page reference. We want the page to be pinned
         * so that we don't get a ext4_mb_init_cache_call for this
         * group until we update the bitmap. That would mean we
         * double allocate blocks. The reference is dropped
         * in ext4_mb_release_context
         */
        ac->ac_bitmap_folio = e4b->bd_bitmap_folio;
        folio_get(ac->ac_bitmap_folio);
        ac->ac_buddy_folio = e4b->bd_buddy_folio;
        folio_get(ac->ac_buddy_folio);
        /* store last allocated for subsequent stream allocation */
        if (ac->ac_flags & EXT4_MB_STREAM_ALLOC) {
                spin_lock(&sbi->s_md_lock);
                sbi->s_mb_last_group = ac->ac_f_ex.fe_group;
                sbi->s_mb_last_start = ac->ac_f_ex.fe_start;
                spin_unlock(&sbi->s_md_lock);
        }
        /*
         * As we've just preallocated more space than
         * user requested originally, we store allocated
         * space in a special descriptor.
         */
        if (ac->ac_o_ex.fe_len < ac->ac_b_ex.fe_len)
                ext4_mb_new_preallocation(ac);

}

static void ext4_mb_check_limits(struct ext4_allocation_context *ac,
                                        struct ext4_buddy *e4b,
                                        int finish_group)
{
        struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);
        struct ext4_free_extent *bex = &ac->ac_b_ex;
        struct ext4_free_extent *gex = &ac->ac_g_ex;

        if (ac->ac_status == AC_STATUS_FOUND)
                return;
        /*
         * We don't want to scan for a whole year
         */
        if (ac->ac_found > sbi->s_mb_max_to_scan &&
                        !(ac->ac_flags & EXT4_MB_HINT_FIRST)) {
                ac->ac_status = AC_STATUS_BREAK;
                return;
        }

        /*
         * Haven't found good chunk so far, let's continue
         */
        if (bex->fe_len < gex->fe_len)
                return;

        if (finish_group || ac->ac_found > sbi->s_mb_min_to_scan)
                ext4_mb_use_best_found(ac, e4b);
}

/*
 * The routine checks whether found extent is good enough. If it is,
 * then the extent gets marked used and flag is set to the context
 * to stop scanning. Otherwise, the extent is compared with the
 * previous found extent and if new one is better, then it's stored
 * in the context. Later, the best found extent will be used, if
 * mballoc can't find good enough extent.
 *
 * The algorithm used is roughly as follows:
 *
 * * If free extent found is exactly as big as goal, then
 *   stop the scan and use it immediately
 *
 * * If free extent found is smaller than goal, then keep retrying
 *   upto a max of sbi->s_mb_max_to_scan times (default 200). After
 *   that stop scanning and use whatever we have.
 *
 * * If free extent found is bigger than goal, then keep retrying
 *   upto a max of sbi->s_mb_min_to_scan times (default 10) before
 *   stopping the scan and using the extent.
 *
 *
 * FIXME: real allocation policy is to be designed yet!
 */
static void ext4_mb_measure_extent(struct ext4_allocation_context *ac,
                                        struct ext4_free_extent *ex,
                                        struct ext4_buddy *e4b)
{
        struct ext4_free_extent *bex = &ac->ac_b_ex;
        struct ext4_free_extent *gex = &ac->ac_g_ex;

        BUG_ON(ex->fe_len <= 0);
        BUG_ON(ex->fe_len > EXT4_CLUSTERS_PER_GROUP(ac->ac_sb));
        BUG_ON(ex->fe_start >= EXT4_CLUSTERS_PER_GROUP(ac->ac_sb));
        BUG_ON(ac->ac_status != AC_STATUS_CONTINUE);

        ac->ac_found++;
        ac->ac_cX_found[ac->ac_criteria]++;

        /*
         * The special case - take what you catch first
         */
        if (unlikely(ac->ac_flags & EXT4_MB_HINT_FIRST)) {
                *bex = *ex;
                ext4_mb_use_best_found(ac, e4b);
                return;
        }

        /*
         * Let's check whether the chuck is good enough
         */
        if (ex->fe_len == gex->fe_len) {
                *bex = *ex;
                ext4_mb_use_best_found(ac, e4b);
                return;
        }

        /*
         * If this is first found extent, just store it in the context
         */
        if (bex->fe_len == 0) {
                *bex = *ex;
                return;
        }

        /*
         * If new found extent is better, store it in the context
         */
        if (bex->fe_len < gex->fe_len) {
                /* if the request isn't satisfied, any found extent
                 * larger than previous best one is better */
                if (ex->fe_len > bex->fe_len)
                        *bex = *ex;
        } else if (ex->fe_len > gex->fe_len) {
                /* if the request is satisfied, then we try to find
                 * an extent that still satisfy the request, but is
                 * smaller than previous one */
                if (ex->fe_len < bex->fe_len)
                        *bex = *ex;
        }

        ext4_mb_check_limits(ac, e4b, 0);
}

static noinline_for_stack
void ext4_mb_try_best_found(struct ext4_allocation_context *ac,
                                        struct ext4_buddy *e4b)
{
        struct ext4_free_extent ex = ac->ac_b_ex;
        ext4_group_t group = ex.fe_group;
        int max;
        int err;

        BUG_ON(ex.fe_len <= 0);
        err = ext4_mb_load_buddy(ac->ac_sb, group, e4b);
        if (err)
                return;

        ext4_lock_group(ac->ac_sb, group);
        if (unlikely(EXT4_MB_GRP_BBITMAP_CORRUPT(e4b->bd_info)))
                goto out;

        max = mb_find_extent(e4b, ex.fe_start, ex.fe_len, &ex);

        if (max > 0) {
                ac->ac_b_ex = ex;
                ext4_mb_use_best_found(ac, e4b);
        }

out:
        ext4_unlock_group(ac->ac_sb, group);
        ext4_mb_unload_buddy(e4b);
}

static noinline_for_stack
int ext4_mb_find_by_goal(struct ext4_allocation_context *ac,
                                struct ext4_buddy *e4b)
{
        ext4_group_t group = ac->ac_g_ex.fe_group;
        int max;
        int err;
        struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);
        struct ext4_group_info *grp = ext4_get_group_info(ac->ac_sb, group);
        struct ext4_free_extent ex;

        if (!grp)
                return -EFSCORRUPTED;
        if (!(ac->ac_flags & (EXT4_MB_HINT_TRY_GOAL | EXT4_MB_HINT_GOAL_ONLY)))
                return 0;
        if (grp->bb_free == 0)
                return 0;

        err = ext4_mb_load_buddy(ac->ac_sb, group, e4b);
        if (err)
                return err;

        ext4_lock_group(ac->ac_sb, group);
        if (unlikely(EXT4_MB_GRP_BBITMAP_CORRUPT(e4b->bd_info)))
                goto out;

        max = mb_find_extent(e4b, ac->ac_g_ex.fe_start,
                             ac->ac_g_ex.fe_len, &ex);
        ex.fe_logical = 0xDEADFA11; /* debug value */

        if (max >= ac->ac_g_ex.fe_len &&
            ac->ac_g_ex.fe_len == EXT4_NUM_B2C(sbi, sbi->s_stripe)) {
                ext4_fsblk_t start;

                start = ext4_grp_offs_to_block(ac->ac_sb, &ex);
                /* use do_div to get remainder (would be 64-bit modulo) */
                if (do_div(start, sbi->s_stripe) == 0) {
                        ac->ac_found++;
                        ac->ac_b_ex = ex;
                        ext4_mb_use_best_found(ac, e4b);
                }
        } else if (max >= ac->ac_g_ex.fe_len) {
                BUG_ON(ex.fe_len <= 0);
                BUG_ON(ex.fe_group != ac->ac_g_ex.fe_group);
                BUG_ON(ex.fe_start != ac->ac_g_ex.fe_start);
                ac->ac_found++;
                ac->ac_b_ex = ex;
                ext4_mb_use_best_found(ac, e4b);
        } else if (max > 0 && (ac->ac_flags & EXT4_MB_HINT_MERGE)) {
                /* Sometimes, caller may want to merge even small
                 * number of blocks to an existing extent */
                BUG_ON(ex.fe_len <= 0);
                BUG_ON(ex.fe_group != ac->ac_g_ex.fe_group);
                BUG_ON(ex.fe_start != ac->ac_g_ex.fe_start);
                ac->ac_found++;
                ac->ac_b_ex = ex;
                ext4_mb_use_best_found(ac, e4b);
        }
out:
        ext4_unlock_group(ac->ac_sb, group);
        ext4_mb_unload_buddy(e4b);

        return 0;
}

/*
 * The routine scans buddy structures (not bitmap!) from given order
 * to max order and tries to find big enough chunk to satisfy the req
 */
static noinline_for_stack
void ext4_mb_simple_scan_group(struct ext4_allocation_context *ac,
                                        struct ext4_buddy *e4b)
{
        struct super_block *sb = ac->ac_sb;
        struct ext4_group_info *grp = e4b->bd_info;
        void *buddy;
        int i;
        int k;
        int max;

        BUG_ON(ac->ac_2order <= 0);
        for (i = ac->ac_2order; i < MB_NUM_ORDERS(sb); i++) {
                if (grp->bb_counters[i] == 0)
                        continue;

                buddy = mb_find_buddy(e4b, i, &max);
                if (WARN_RATELIMIT(buddy == NULL,
                         "ext4: mb_simple_scan_group: mb_find_buddy failed, (%d)\n", i))
                        continue;

                k = mb_find_next_zero_bit(buddy, max, 0);
                if (k >= max) {
                        ext4_mark_group_bitmap_corrupted(ac->ac_sb,
                                        e4b->bd_group,
                                        EXT4_GROUP_INFO_BBITMAP_CORRUPT);
                        ext4_grp_locked_error(ac->ac_sb, e4b->bd_group, 0, 0,
                                "%d free clusters of order %d. But found 0",
                                grp->bb_counters[i], i);
                        break;
                }
                ac->ac_found++;
                ac->ac_cX_found[ac->ac_criteria]++;

                ac->ac_b_ex.fe_len = 1 << i;
                ac->ac_b_ex.fe_start = k << i;
                ac->ac_b_ex.fe_group = e4b->bd_group;

                ext4_mb_use_best_found(ac, e4b);

                BUG_ON(ac->ac_f_ex.fe_len != ac->ac_g_ex.fe_len);

                if (EXT4_SB(sb)->s_mb_stats)
                        atomic_inc(&EXT4_SB(sb)->s_bal_2orders);

                break;
        }
}

/*
 * The routine scans the group and measures all found extents.
 * In order to optimize scanning, caller must pass number of
 * free blocks in the group, so the routine can know upper limit.
 */
static noinline_for_stack
void ext4_mb_complex_scan_group(struct ext4_allocation_context *ac,
                                        struct ext4_buddy *e4b)
{
        struct super_block *sb = ac->ac_sb;
        void *bitmap = e4b->bd_bitmap;
        struct ext4_free_extent ex;
        int i, j, freelen;
        int free;

        free = e4b->bd_info->bb_free;
        if (WARN_ON(free <= 0))
                return;

        i = e4b->bd_info->bb_first_free;

        while (free && ac->ac_status == AC_STATUS_CONTINUE) {
                i = mb_find_next_zero_bit(bitmap,
                                                EXT4_CLUSTERS_PER_GROUP(sb), i);
                if (i >= EXT4_CLUSTERS_PER_GROUP(sb)) {
                        /*
                         * IF we have corrupt bitmap, we won't find any
                         * free blocks even though group info says we
                         * have free blocks
                         */
                        ext4_mark_group_bitmap_corrupted(sb, e4b->bd_group,
                                        EXT4_GROUP_INFO_BBITMAP_CORRUPT);
                        ext4_grp_locked_error(sb, e4b->bd_group, 0, 0,
                                        "%d free clusters as per "
                                        "group info. But bitmap says 0",
                                        free);
                        break;
                }

                if (!ext4_mb_cr_expensive(ac->ac_criteria)) {
                        /*
                         * In CR_GOAL_LEN_FAST and CR_BEST_AVAIL_LEN, we are
                         * sure that this group will have a large enough
                         * continuous free extent, so skip over the smaller free
                         * extents
                         */
                        j = mb_find_next_bit(bitmap,
                                                EXT4_CLUSTERS_PER_GROUP(sb), i);
                        freelen = j - i;

                        if (freelen < ac->ac_g_ex.fe_len) {
                                i = j;
                                free -= freelen;
                                continue;
                        }
                }

                mb_find_extent(e4b, i, ac->ac_g_ex.fe_len, &ex);
                if (WARN_ON(ex.fe_len <= 0))
                        break;
                if (free < ex.fe_len) {
                        ext4_mark_group_bitmap_corrupted(sb, e4b->bd_group,
                                        EXT4_GROUP_INFO_BBITMAP_CORRUPT);
                        ext4_grp_locked_error(sb, e4b->bd_group, 0, 0,
                                        "%d free clusters as per "
                                        "group info. But got %d blocks",
                                        free, ex.fe_len);
                        /*
                         * The number of free blocks differs. This mostly
                         * indicate that the bitmap is corrupt. So exit
                         * without claiming the space.
                         */
                        break;
                }
                ex.fe_logical = 0xDEADC0DE; /* debug value */
                ext4_mb_measure_extent(ac, &ex, e4b);

                i += ex.fe_len;
                free -= ex.fe_len;
        }

        ext4_mb_check_limits(ac, e4b, 1);
}

/*
 * This is a special case for storages like raid5
 * we try to find stripe-aligned chunks for stripe-size-multiple requests
 */
static noinline_for_stack
void ext4_mb_scan_aligned(struct ext4_allocation_context *ac,
                                 struct ext4_buddy *e4b)
{
        struct super_block *sb = ac->ac_sb;
        struct ext4_sb_info *sbi = EXT4_SB(sb);
        void *bitmap = e4b->bd_bitmap;
        struct ext4_free_extent ex;
        ext4_fsblk_t first_group_block;
        ext4_fsblk_t a;
        ext4_grpblk_t i, stripe;
        int max;

        BUG_ON(sbi->s_stripe == 0);

        /* find first stripe-aligned block in group */
        first_group_block = ext4_group_first_block_no(sb, e4b->bd_group);

        a = first_group_block + sbi->s_stripe - 1;
        do_div(a, sbi->s_stripe);
        i = (a * sbi->s_stripe) - first_group_block;

        stripe = EXT4_NUM_B2C(sbi, sbi->s_stripe);
        i = EXT4_B2C(sbi, i);
        while (i < EXT4_CLUSTERS_PER_GROUP(sb)) {
                if (!mb_test_bit(i, bitmap)) {
                        max = mb_find_extent(e4b, i, stripe, &ex);
                        if (max >= stripe) {
                                ac->ac_found++;
                                ac->ac_cX_found[ac->ac_criteria]++;
                                ex.fe_logical = 0xDEADF00D; /* debug value */
                                ac->ac_b_ex = ex;
                                ext4_mb_use_best_found(ac, e4b);
                                break;
                        }
                }
                i += stripe;
        }
}

/*
 * This is also called BEFORE we load the buddy bitmap.
 * Returns either 1 or 0 indicating that the group is either suitable
 * for the allocation or not.
 */
static bool ext4_mb_good_group(struct ext4_allocation_context *ac,
                                ext4_group_t group, enum criteria cr)
{
        ext4_grpblk_t free, fragments;
        int flex_size = ext4_flex_bg_size(EXT4_SB(ac->ac_sb));
        struct ext4_group_info *grp = ext4_get_group_info(ac->ac_sb, group);

        BUG_ON(cr < CR_POWER2_ALIGNED || cr >= EXT4_MB_NUM_CRS);

        if (unlikely(!grp || EXT4_MB_GRP_BBITMAP_CORRUPT(grp)))
                return false;

        free = grp->bb_free;
        if (free == 0)
                return false;

        fragments = grp->bb_fragments;
        if (fragments == 0)
                return false;

        switch (cr) {
        case CR_POWER2_ALIGNED:
                BUG_ON(ac->ac_2order == 0);

                /* Avoid using the first bg of a flexgroup for data files */
                if ((ac->ac_flags & EXT4_MB_HINT_DATA) &&
                    (flex_size >= EXT4_FLEX_SIZE_DIR_ALLOC_SCHEME) &&
                    ((group % flex_size) == 0))
                        return false;

                if (free < ac->ac_g_ex.fe_len)
                        return false;

                if (ac->ac_2order >= MB_NUM_ORDERS(ac->ac_sb))
                        return true;

                if (grp->bb_largest_free_order < ac->ac_2order)
                        return false;

                return true;
        case CR_GOAL_LEN_FAST:
        case CR_BEST_AVAIL_LEN:
                if ((free / fragments) >= ac->ac_g_ex.fe_len)
                        return true;
                break;
        case CR_GOAL_LEN_SLOW:
                if (free >= ac->ac_g_ex.fe_len)
                        return true;
                break;
        case CR_ANY_FREE:
                return true;
        default:
                BUG();
        }

        return false;
}

/*
 * This could return negative error code if something goes wrong
 * during ext4_mb_init_group(). This should not be called with
 * ext4_lock_group() held.
 *
 * Note: because we are conditionally operating with the group lock in
 * the EXT4_MB_STRICT_CHECK case, we need to fake out sparse in this
 * function using __acquire and __release.  This means we need to be
 * super careful before messing with the error path handling via "goto
 * out"!
 */
static int ext4_mb_good_group_nolock(struct ext4_allocation_context *ac,
                                     ext4_group_t group, enum criteria cr)
{
        struct ext4_group_info *grp = ext4_get_group_info(ac->ac_sb, group);
        struct super_block *sb = ac->ac_sb;
        struct ext4_sb_info *sbi = EXT4_SB(sb);
        bool should_lock = ac->ac_flags & EXT4_MB_STRICT_CHECK;
        ext4_grpblk_t free;
        int ret = 0;

        if (!grp)
                return -EFSCORRUPTED;
        if (sbi->s_mb_stats)
                atomic64_inc(&sbi->s_bal_cX_groups_considered[ac->ac_criteria]);
        if (should_lock) {
                ext4_lock_group(sb, group);
                __release(ext4_group_lock_ptr(sb, group));
        }
        free = grp->bb_free;
        if (free == 0)
                goto out;
        /*
         * In all criterias except CR_ANY_FREE we try to avoid groups that
         * can't possibly satisfy the full goal request due to insufficient
         * free blocks.
         */
        if (cr < CR_ANY_FREE && free < ac->ac_g_ex.fe_len)
                goto out;
        if (unlikely(EXT4_MB_GRP_BBITMAP_CORRUPT(grp)))
                goto out;
        if (should_lock) {
                __acquire(ext4_group_lock_ptr(sb, group));
                ext4_unlock_group(sb, group);
        }

        /* We only do this if the grp has never been initialized */
        if (unlikely(EXT4_MB_GRP_NEED_INIT(grp))) {
                struct ext4_group_desc *gdp =
                        ext4_get_group_desc(sb, group, NULL);
                int ret;

                /*
                 * CR_POWER2_ALIGNED/CR_GOAL_LEN_FAST is a very optimistic
                 * search to find large good chunks almost for free. If buddy
                 * data is not ready, then this optimization makes no sense. But
                 * we never skip the first block group in a flex_bg, since this
                 * gets used for metadata block allocation, and we want to make
                 * sure we locate metadata blocks in the first block group in
                 * the flex_bg if possible.
                 */
                if (!ext4_mb_cr_expensive(cr) &&
                    (!sbi->s_log_groups_per_flex ||
                     ((group & ((1 << sbi->s_log_groups_per_flex) - 1)) != 0)) &&
                    !(ext4_has_group_desc_csum(sb) &&
                      (gdp->bg_flags & cpu_to_le16(EXT4_BG_BLOCK_UNINIT))))
                        return 0;
                ret = ext4_mb_init_group(sb, group, GFP_NOFS);
                if (ret)
                        return ret;
        }

        if (should_lock) {
                ext4_lock_group(sb, group);
                __release(ext4_group_lock_ptr(sb, group));
        }
        ret = ext4_mb_good_group(ac, group, cr);
out:
        if (should_lock) {
                __acquire(ext4_group_lock_ptr(sb, group));
                ext4_unlock_group(sb, group);
        }
        return ret;
}

/*
 * Start prefetching @nr block bitmaps starting at @group.
 * Return the next group which needs to be prefetched.
 */
ext4_group_t ext4_mb_prefetch(struct super_block *sb, ext4_group_t group,
                              unsigned int nr, int *cnt)
{
        ext4_group_t ngroups = ext4_get_groups_count(sb);
        struct buffer_head *bh;
        struct blk_plug plug;

        blk_start_plug(&plug);
        while (nr-- > 0) {
                struct ext4_group_desc *gdp = ext4_get_group_desc(sb, group,
                                                                  NULL);
                struct ext4_group_info *grp = ext4_get_group_info(sb, group);

                /*
                 * Prefetch block groups with free blocks; but don't
                 * bother if it is marked uninitialized on disk, since
                 * it won't require I/O to read.  Also only try to
                 * prefetch once, so we avoid getblk() call, which can
                 * be expensive.
                 */
                if (gdp && grp && !EXT4_MB_GRP_TEST_AND_SET_READ(grp) &&
                    EXT4_MB_GRP_NEED_INIT(grp) &&
                    ext4_free_group_clusters(sb, gdp) > 0 ) {
                        bh = ext4_read_block_bitmap_nowait(sb, group, true);
                        if (bh && !IS_ERR(bh)) {
                                if (!buffer_uptodate(bh) && cnt)
                                        (*cnt)++;
                                brelse(bh);
                        }
                }
                if (++group >= ngroups)
                        group = 0;
        }
        blk_finish_plug(&plug);
        return group;
}

/*
 * Prefetching reads the block bitmap into the buffer cache; but we
 * need to make sure that the buddy bitmap in the page cache has been
 * initialized.  Note that ext4_mb_init_group() will block if the I/O
 * is not yet completed, or indeed if it was not initiated by
 * ext4_mb_prefetch did not start the I/O.
 *
 * TODO: We should actually kick off the buddy bitmap setup in a work
 * queue when the buffer I/O is completed, so that we don't block
 * waiting for the block allocation bitmap read to finish when
 * ext4_mb_prefetch_fini is called from ext4_mb_regular_allocator().
 */
void ext4_mb_prefetch_fini(struct super_block *sb, ext4_group_t group,
                           unsigned int nr)
{
        struct ext4_group_desc *gdp;
        struct ext4_group_info *grp;

        while (nr-- > 0) {
                if (!group)
                        group = ext4_get_groups_count(sb);
                group--;
                gdp = ext4_get_group_desc(sb, group, NULL);
                grp = ext4_get_group_info(sb, group);

                if (grp && gdp && EXT4_MB_GRP_NEED_INIT(grp) &&
                    ext4_free_group_clusters(sb, gdp) > 0) {
                        if (ext4_mb_init_group(sb, group, GFP_NOFS))
                                break;
                }
        }
}

static noinline_for_stack int
ext4_mb_regular_allocator(struct ext4_allocation_context *ac)
{
        ext4_group_t prefetch_grp = 0, ngroups, group, i;
        enum criteria new_cr, cr = CR_GOAL_LEN_FAST;
        int err = 0, first_err = 0;
        unsigned int nr = 0, prefetch_ios = 0;
        struct ext4_sb_info *sbi;
        struct super_block *sb;
        struct ext4_buddy e4b;
        int lost;

        sb = ac->ac_sb;
        sbi = EXT4_SB(sb);
        ngroups = ext4_get_groups_count(sb);
        /* non-extent files are limited to low blocks/groups */
        if (!(ext4_test_inode_flag(ac->ac_inode, EXT4_INODE_EXTENTS)))
                ngroups = sbi->s_blockfile_groups;

        BUG_ON(ac->ac_status == AC_STATUS_FOUND);

        /* first, try the goal */
        err = ext4_mb_find_by_goal(ac, &e4b);
        if (err || ac->ac_status == AC_STATUS_FOUND)
                goto out;

        if (unlikely(ac->ac_flags & EXT4_MB_HINT_GOAL_ONLY))
                goto out;

        /*
         * ac->ac_2order is set only if the fe_len is a power of 2
         * if ac->ac_2order is set we also set criteria to CR_POWER2_ALIGNED
         * so that we try exact allocation using buddy.
         */
        i = fls(ac->ac_g_ex.fe_len);
        ac->ac_2order = 0;
        /*
         * We search using buddy data only if the order of the request
         * is greater than equal to the sbi_s_mb_order2_reqs
         * You can tune it via /sys/fs/ext4/<partition>/mb_order2_req
         * We also support searching for power-of-two requests only for
         * requests upto maximum buddy size we have constructed.
         */
        if (i >= sbi->s_mb_order2_reqs && i <= MB_NUM_ORDERS(sb)) {
                if (is_power_of_2(ac->ac_g_ex.fe_len))
                        ac->ac_2order = array_index_nospec(i - 1,
                                                           MB_NUM_ORDERS(sb));
        }

        /* if stream allocation is enabled, use global goal */
        if (ac->ac_flags & EXT4_MB_STREAM_ALLOC) {
                /* TBD: may be hot point */
                spin_lock(&sbi->s_md_lock);
                ac->ac_g_ex.fe_group = sbi->s_mb_last_group;
                ac->ac_g_ex.fe_start = sbi->s_mb_last_start;
                spin_unlock(&sbi->s_md_lock);
        }

        /*
         * Let's just scan groups to find more-less suitable blocks We
         * start with CR_GOAL_LEN_FAST, unless it is power of 2
         * aligned, in which case let's do that faster approach first.
         */
        if (ac->ac_2order)
                cr = CR_POWER2_ALIGNED;
repeat:
        for (; cr < EXT4_MB_NUM_CRS && ac->ac_status == AC_STATUS_CONTINUE; cr++) {
                ac->ac_criteria = cr;
                /*
                 * searching for the right group start
                 * from the goal value specified
                 */
                group = ac->ac_g_ex.fe_group;
                ac->ac_groups_linear_remaining = sbi->s_mb_max_linear_groups;
                prefetch_grp = group;
                nr = 0;

                for (i = 0, new_cr = cr; i < ngroups; i++,
                     ext4_mb_choose_next_group(ac, &new_cr, &group, ngroups)) {
                        int ret = 0;

                        cond_resched();
                        if (new_cr != cr) {
                                cr = new_cr;
                                goto repeat;
                        }

                        /*
                         * Batch reads of the block allocation bitmaps
                         * to get multiple READs in flight; limit
                         * prefetching at inexpensive CR, otherwise mballoc
                         * can spend a lot of time loading imperfect groups
                         */
                        if ((prefetch_grp == group) &&
                            (ext4_mb_cr_expensive(cr) ||
                             prefetch_ios < sbi->s_mb_prefetch_limit)) {
                                nr = sbi->s_mb_prefetch;
                                if (ext4_has_feature_flex_bg(sb)) {
                                        nr = 1 << sbi->s_log_groups_per_flex;
                                        nr -= group & (nr - 1);
                                        nr = min(nr, sbi->s_mb_prefetch);
                                }
                                prefetch_grp = ext4_mb_prefetch(sb, group,
                                                        nr, &prefetch_ios);
                        }

                        /* This now checks without needing the buddy page */
                        ret = ext4_mb_good_group_nolock(ac, group, cr);
                        if (ret <= 0) {
                                if (!first_err)
                                        first_err = ret;
                                continue;
                        }

                        err = ext4_mb_load_buddy(sb, group, &e4b);
                        if (err)
                                goto out;

                        ext4_lock_group(sb, group);

                        /*
                         * We need to check again after locking the
                         * block group
                         */
                        ret = ext4_mb_good_group(ac, group, cr);
                        if (ret == 0) {
                                ext4_unlock_group(sb, group);
                                ext4_mb_unload_buddy(&e4b);
                                continue;
                        }

                        ac->ac_groups_scanned++;
                        if (cr == CR_POWER2_ALIGNED)
                                ext4_mb_simple_scan_group(ac, &e4b);
                        else {
                                bool is_stripe_aligned =
                                        (sbi->s_stripe >=
                                         sbi->s_cluster_ratio) &&
                                        !(ac->ac_g_ex.fe_len %
                                          EXT4_NUM_B2C(sbi, sbi->s_stripe));

                                if ((cr == CR_GOAL_LEN_FAST ||
                                     cr == CR_BEST_AVAIL_LEN) &&
                                    is_stripe_aligned)
                                        ext4_mb_scan_aligned(ac, &e4b);

                                if (ac->ac_status == AC_STATUS_CONTINUE)
                                        ext4_mb_complex_scan_group(ac, &e4b);
                        }

                        ext4_unlock_group(sb, group);
                        ext4_mb_unload_buddy(&e4b);

                        if (ac->ac_status != AC_STATUS_CONTINUE)
                                break;
                }
                /* Processed all groups and haven't found blocks */
                if (sbi->s_mb_stats && i == ngroups)
                        atomic64_inc(&sbi->s_bal_cX_failed[cr]);

                if (i == ngroups && ac->ac_criteria == CR_BEST_AVAIL_LEN)
                        /* Reset goal length to original goal length before
                         * falling into CR_GOAL_LEN_SLOW */
                        ac->ac_g_ex.fe_len = ac->ac_orig_goal_len;
        }

        if (ac->ac_b_ex.fe_len > 0 && ac->ac_status != AC_STATUS_FOUND &&
            !(ac->ac_flags & EXT4_MB_HINT_FIRST)) {
                /*
                 * We've been searching too long. Let's try to allocate
                 * the best chunk we've found so far
                 */
                ext4_mb_try_best_found(ac, &e4b);
                if (ac->ac_status != AC_STATUS_FOUND) {
                        /*
                         * Someone more lucky has already allocated it.
                         * The only thing we can do is just take first
                         * found block(s)
                         */
                        lost = atomic_inc_return(&sbi->s_mb_lost_chunks);
                        mb_debug(sb, "lost chunk, group: %u, start: %d, len: %d, lost: %d\n",
                                 ac->ac_b_ex.fe_group, ac->ac_b_ex.fe_start,
                                 ac->ac_b_ex.fe_len, lost);

                        ac->ac_b_ex.fe_group = 0;
                        ac->ac_b_ex.fe_start = 0;
                        ac->ac_b_ex.fe_len = 0;
                        ac->ac_status = AC_STATUS_CONTINUE;
                        ac->ac_flags |= EXT4_MB_HINT_FIRST;
                        cr = CR_ANY_FREE;
                        goto repeat;
                }
        }

        if (sbi->s_mb_stats && ac->ac_status == AC_STATUS_FOUND)
                atomic64_inc(&sbi->s_bal_cX_hits[ac->ac_criteria]);
out:
        if (!err && ac->ac_status != AC_STATUS_FOUND && first_err)
                err = first_err;

        mb_debug(sb, "Best len %d, origin len %d, ac_status %u, ac_flags 0x%x, cr %d ret %d\n",
                 ac->ac_b_ex.fe_len, ac->ac_o_ex.fe_len, ac->ac_status,
                 ac->ac_flags, cr, err);

        if (nr)
                ext4_mb_prefetch_fini(sb, prefetch_grp, nr);

        return err;
}

static void *ext4_mb_seq_groups_start(struct seq_file *seq, loff_t *pos)
{
        struct super_block *sb = pde_data(file_inode(seq->file));
        ext4_group_t group;

        if (*pos < 0 || *pos >= ext4_get_groups_count(sb))
                return NULL;
        group = *pos + 1;
        return (void *) ((unsigned long) group);
}

static void *ext4_mb_seq_groups_next(struct seq_file *seq, void *v, loff_t *pos)
{
        struct super_block *sb = pde_data(file_inode(seq->file));
        ext4_group_t group;

        ++*pos;
        if (*pos < 0 || *pos >= ext4_get_groups_count(sb))
                return NULL;
        group = *pos + 1;
        return (void *) ((unsigned long) group);
}

static int ext4_mb_seq_groups_show(struct seq_file *seq, void *v)
{
        struct super_block *sb = pde_data(file_inode(seq->file));
        ext4_group_t group = (ext4_group_t) ((unsigned long) v);
        int i, err;
        char nbuf[16];
        struct ext4_buddy e4b;
        struct ext4_group_info *grinfo;
        unsigned char blocksize_bits = min_t(unsigned char,
                                             sb->s_blocksize_bits,
                                             EXT4_MAX_BLOCK_LOG_SIZE);
        DEFINE_RAW_FLEX(struct ext4_group_info, sg, bb_counters,
                        EXT4_MAX_BLOCK_LOG_SIZE + 2);

        group--;
        if (group == 0)
                seq_puts(seq, "#group: free  frags first ["
                              " 2^0   2^1   2^2   2^3   2^4   2^5   2^6  "
                              " 2^7   2^8   2^9   2^10  2^11  2^12  2^13  ]\n");

        i = (blocksize_bits + 2) * sizeof(sg->bb_counters[0]) +
                sizeof(struct ext4_group_info);

        grinfo = ext4_get_group_info(sb, group);
        if (!grinfo)
                return 0;
        /* Load the group info in memory only if not already loaded. */
        if (unlikely(EXT4_MB_GRP_NEED_INIT(grinfo))) {
                err = ext4_mb_load_buddy(sb, group, &e4b);
                if (err) {
                        seq_printf(seq, "#%-5u: %s\n", group, ext4_decode_error(NULL, err, nbuf));
                        return 0;
                }
                ext4_mb_unload_buddy(&e4b);
        }

        /*
         * We care only about free space counters in the group info and
         * these are safe to access even after the buddy has been unloaded
         */
        memcpy(sg, grinfo, i);
        seq_printf(seq, "#%-5u: %-5u %-5u %-5u [", group, sg->bb_free,
                        sg->bb_fragments, sg->bb_first_free);
        for (i = 0; i <= 13; i++)
                seq_printf(seq, " %-5u", i <= blocksize_bits + 1 ?
                                sg->bb_counters[i] : 0);
        seq_puts(seq, " ]");
        if (EXT4_MB_GRP_BBITMAP_CORRUPT(sg))
                seq_puts(seq, " Block bitmap corrupted!");
        seq_putc(seq, '\n');
        return 0;
}

static void ext4_mb_seq_groups_stop(struct seq_file *seq, void *v)
{
}

const struct seq_operations ext4_mb_seq_groups_ops = {
        .start  = ext4_mb_seq_groups_start,
        .next   = ext4_mb_seq_groups_next,
        .stop   = ext4_mb_seq_groups_stop,
        .show   = ext4_mb_seq_groups_show,
};

int ext4_seq_mb_stats_show(struct seq_file *seq, void *offset)
{
        struct super_block *sb = seq->private;
        struct ext4_sb_info *sbi = EXT4_SB(sb);

        seq_puts(seq, "mballoc:\n");
        if (!sbi->s_mb_stats) {
                seq_puts(seq, "\tmb stats collection turned off.\n");
                seq_puts(
                        seq,
                        "\tTo enable, please write \"1\" to sysfs file mb_stats.\n");
                return 0;
        }
        seq_printf(seq, "\treqs: %u\n", atomic_read(&sbi->s_bal_reqs));
        seq_printf(seq, "\tsuccess: %u\n", atomic_read(&sbi->s_bal_success));

        seq_printf(seq, "\tgroups_scanned: %u\n",
                   atomic_read(&sbi->s_bal_groups_scanned));

        /* CR_POWER2_ALIGNED stats */
        seq_puts(seq, "\tcr_p2_aligned_stats:\n");
        seq_printf(seq, "\t\thits: %llu\n",
                   atomic64_read(&sbi->s_bal_cX_hits[CR_POWER2_ALIGNED]));
        seq_printf(
                seq, "\t\tgroups_considered: %llu\n",
                atomic64_read(
                        &sbi->s_bal_cX_groups_considered[CR_POWER2_ALIGNED]));
        seq_printf(seq, "\t\textents_scanned: %u\n",
                   atomic_read(&sbi->s_bal_cX_ex_scanned[CR_POWER2_ALIGNED]));
        seq_printf(seq, "\t\tuseless_loops: %llu\n",
                   atomic64_read(&sbi->s_bal_cX_failed[CR_POWER2_ALIGNED]));
        seq_printf(seq, "\t\tbad_suggestions: %u\n",
                   atomic_read(&sbi->s_bal_p2_aligned_bad_suggestions));

        /* CR_GOAL_LEN_FAST stats */
        seq_puts(seq, "\tcr_goal_fast_stats:\n");
        seq_printf(seq, "\t\thits: %llu\n",
                   atomic64_read(&sbi->s_bal_cX_hits[CR_GOAL_LEN_FAST]));
        seq_printf(seq, "\t\tgroups_considered: %llu\n",
                   atomic64_read(
                           &sbi->s_bal_cX_groups_considered[CR_GOAL_LEN_FAST]));
        seq_printf(seq, "\t\textents_scanned: %u\n",
                   atomic_read(&sbi->s_bal_cX_ex_scanned[CR_GOAL_LEN_FAST]));
        seq_printf(seq, "\t\tuseless_loops: %llu\n",
                   atomic64_read(&sbi->s_bal_cX_failed[CR_GOAL_LEN_FAST]));
        seq_printf(seq, "\t\tbad_suggestions: %u\n",
                   atomic_read(&sbi->s_bal_goal_fast_bad_suggestions));

        /* CR_BEST_AVAIL_LEN stats */
        seq_puts(seq, "\tcr_best_avail_stats:\n");
        seq_printf(seq, "\t\thits: %llu\n",
                   atomic64_read(&sbi->s_bal_cX_hits[CR_BEST_AVAIL_LEN]));
        seq_printf(
                seq, "\t\tgroups_considered: %llu\n",
                atomic64_read(
                        &sbi->s_bal_cX_groups_considered[CR_BEST_AVAIL_LEN]));
        seq_printf(seq, "\t\textents_scanned: %u\n",
                   atomic_read(&sbi->s_bal_cX_ex_scanned[CR_BEST_AVAIL_LEN]));
        seq_printf(seq, "\t\tuseless_loops: %llu\n",
                   atomic64_read(&sbi->s_bal_cX_failed[CR_BEST_AVAIL_LEN]));
        seq_printf(seq, "\t\tbad_suggestions: %u\n",
                   atomic_read(&sbi->s_bal_best_avail_bad_suggestions));

        /* CR_GOAL_LEN_SLOW stats */
        seq_puts(seq, "\tcr_goal_slow_stats:\n");
        seq_printf(seq, "\t\thits: %llu\n",
                   atomic64_read(&sbi->s_bal_cX_hits[CR_GOAL_LEN_SLOW]));
        seq_printf(seq, "\t\tgroups_considered: %llu\n",
                   atomic64_read(
                           &sbi->s_bal_cX_groups_considered[CR_GOAL_LEN_SLOW]));
        seq_printf(seq, "\t\textents_scanned: %u\n",
                   atomic_read(&sbi->s_bal_cX_ex_scanned[CR_GOAL_LEN_SLOW]));
        seq_printf(seq, "\t\tuseless_loops: %llu\n",
                   atomic64_read(&sbi->s_bal_cX_failed[CR_GOAL_LEN_SLOW]));

        /* CR_ANY_FREE stats */
        seq_puts(seq, "\tcr_any_free_stats:\n");
        seq_printf(seq, "\t\thits: %llu\n",
                   atomic64_read(&sbi->s_bal_cX_hits[CR_ANY_FREE]));
        seq_printf(
                seq, "\t\tgroups_considered: %llu\n",
                atomic64_read(&sbi->s_bal_cX_groups_considered[CR_ANY_FREE]));
        seq_printf(seq, "\t\textents_scanned: %u\n",
                   atomic_read(&sbi->s_bal_cX_ex_scanned[CR_ANY_FREE]));
        seq_printf(seq, "\t\tuseless_loops: %llu\n",
                   atomic64_read(&sbi->s_bal_cX_failed[CR_ANY_FREE]));

        /* Aggregates */
        seq_printf(seq, "\textents_scanned: %u\n",
                   atomic_read(&sbi->s_bal_ex_scanned));
        seq_printf(seq, "\t\tgoal_hits: %u\n", atomic_read(&sbi->s_bal_goals));
        seq_printf(seq, "\t\tlen_goal_hits: %u\n",
                   atomic_read(&sbi->s_bal_len_goals));
        seq_printf(seq, "\t\t2^n_hits: %u\n", atomic_read(&sbi->s_bal_2orders));
        seq_printf(seq, "\t\tbreaks: %u\n", atomic_read(&sbi->s_bal_breaks));
        seq_printf(seq, "\t\tlost: %u\n", atomic_read(&sbi->s_mb_lost_chunks));
        seq_printf(seq, "\tbuddies_generated: %u/%u\n",
                   atomic_read(&sbi->s_mb_buddies_generated),
                   ext4_get_groups_count(sb));
        seq_printf(seq, "\tbuddies_time_used: %llu\n",
                   atomic64_read(&sbi->s_mb_generation_time));
        seq_printf(seq, "\tpreallocated: %u\n",
                   atomic_read(&sbi->s_mb_preallocated));
        seq_printf(seq, "\tdiscarded: %u\n", atomic_read(&sbi->s_mb_discarded));
        return 0;
}

static void *ext4_mb_seq_structs_summary_start(struct seq_file *seq, loff_t *pos)
{
        struct super_block *sb = pde_data(file_inode(seq->file));
        unsigned long position;

        if (*pos < 0 || *pos >= 2*MB_NUM_ORDERS(sb))
                return NULL;
        position = *pos + 1;
        return (void *) ((unsigned long) position);
}

static void *ext4_mb_seq_structs_summary_next(struct seq_file *seq, void *v, loff_t *pos)
{
        struct super_block *sb = pde_data(file_inode(seq->file));
        unsigned long position;

        ++*pos;
        if (*pos < 0 || *pos >= 2*MB_NUM_ORDERS(sb))
                return NULL;
        position = *pos + 1;
        return (void *) ((unsigned long) position);
}

static int ext4_mb_seq_structs_summary_show(struct seq_file *seq, void *v)
{
        struct super_block *sb = pde_data(file_inode(seq->file));
        struct ext4_sb_info *sbi = EXT4_SB(sb);
        unsigned long position = ((unsigned long) v);
        struct ext4_group_info *grp;
        unsigned int count;

        position--;
        if (position >= MB_NUM_ORDERS(sb)) {
                position -= MB_NUM_ORDERS(sb);
                if (position == 0)
                        seq_puts(seq, "avg_fragment_size_lists:\n");

                count = 0;
                read_lock(&sbi->s_mb_avg_fragment_size_locks[position]);
                list_for_each_entry(grp, &sbi->s_mb_avg_fragment_size[position],
                                    bb_avg_fragment_size_node)
                        count++;
                read_unlock(&sbi->s_mb_avg_fragment_size_locks[position]);
                seq_printf(seq, "\tlist_order_%u_groups: %u\n",
                                        (unsigned int)position, count);
                return 0;
        }

        if (position == 0) {
                seq_printf(seq, "optimize_scan: %d\n",
                           test_opt2(sb, MB_OPTIMIZE_SCAN) ? 1 : 0);
                seq_puts(seq, "max_free_order_lists:\n");
        }
        count = 0;
        read_lock(&sbi->s_mb_largest_free_orders_locks[position]);
        list_for_each_entry(grp, &sbi->s_mb_largest_free_orders[position],
                            bb_largest_free_order_node)
                count++;
        read_unlock(&sbi->s_mb_largest_free_orders_locks[position]);
        seq_printf(seq, "\tlist_order_%u_groups: %u\n",
                   (unsigned int)position, count);

        return 0;
}

static void ext4_mb_seq_structs_summary_stop(struct seq_file *seq, void *v)
{
}

const struct seq_operations ext4_mb_seq_structs_summary_ops = {
        .start  = ext4_mb_seq_structs_summary_start,
        .next   = ext4_mb_seq_structs_summary_next,
        .stop   = ext4_mb_seq_structs_summary_stop,
        .show   = ext4_mb_seq_structs_summary_show,
};

static struct kmem_cache *get_groupinfo_cache(int blocksize_bits)
{
        int cache_index = blocksize_bits - EXT4_MIN_BLOCK_LOG_SIZE;
        struct kmem_cache *cachep = ext4_groupinfo_caches[cache_index];

        BUG_ON(!cachep);
        return cachep;
}

/*
 * Allocate the top-level s_group_info array for the specified number
 * of groups
 */
int ext4_mb_alloc_groupinfo(struct super_block *sb, ext4_group_t ngroups)
{
        struct ext4_sb_info *sbi = EXT4_SB(sb);
        unsigned size;
        struct ext4_group_info ***old_groupinfo, ***new_groupinfo;

        size = (ngroups + EXT4_DESC_PER_BLOCK(sb) - 1) >>
                EXT4_DESC_PER_BLOCK_BITS(sb);
        if (size <= sbi->s_group_info_size)
                return 0;

        size = roundup_pow_of_two(sizeof(*sbi->s_group_info) * size);
        new_groupinfo = kvzalloc(size, GFP_KERNEL);
        if (!new_groupinfo) {
                ext4_msg(sb, KERN_ERR, "can't allocate buddy meta group");
                return -ENOMEM;
        }
        rcu_read_lock();
        old_groupinfo = rcu_dereference(sbi->s_group_info);
        if (old_groupinfo)
                memcpy(new_groupinfo, old_groupinfo,
                       sbi->s_group_info_size * sizeof(*sbi->s_group_info));
        rcu_read_unlock();
        rcu_assign_pointer(sbi->s_group_info, new_groupinfo);
        sbi->s_group_info_size = size / sizeof(*sbi->s_group_info);
        if (old_groupinfo)
                ext4_kvfree_array_rcu(old_groupinfo);
        ext4_debug("allocated s_groupinfo array for %d meta_bg's\n",
                   sbi->s_group_info_size);
        return 0;
}

/* Create and initialize ext4_group_info data for the given group. */
int ext4_mb_add_groupinfo(struct super_block *sb, ext4_group_t group,
                          struct ext4_group_desc *desc)
{
        int i;
        int metalen = 0;
        int idx = group >> EXT4_DESC_PER_BLOCK_BITS(sb);
        struct ext4_sb_info *sbi = EXT4_SB(sb);
        struct ext4_group_info **meta_group_info;
        struct kmem_cache *cachep = get_groupinfo_cache(sb->s_blocksize_bits);

        /*
         * First check if this group is the first of a reserved block.
         * If it's true, we have to allocate a new table of pointers
         * to ext4_group_info structures
         */
        if (group % EXT4_DESC_PER_BLOCK(sb) == 0) {
                metalen = sizeof(*meta_group_info) <<
                        EXT4_DESC_PER_BLOCK_BITS(sb);
                meta_group_info = kmalloc(metalen, GFP_NOFS);
                if (meta_group_info == NULL) {
                        ext4_msg(sb, KERN_ERR, "can't allocate mem "
                                 "for a buddy group");
                        return -ENOMEM;
                }
                rcu_read_lock();
                rcu_dereference(sbi->s_group_info)[idx] = meta_group_info;
                rcu_read_unlock();
        }

        meta_group_info = sbi_array_rcu_deref(sbi, s_group_info, idx);
        i = group & (EXT4_DESC_PER_BLOCK(sb) - 1);

        meta_group_info[i] = kmem_cache_zalloc(cachep, GFP_NOFS);
        if (meta_group_info[i] == NULL) {
                ext4_msg(sb, KERN_ERR, "can't allocate buddy mem");
                goto exit_group_info;
        }
        set_bit(EXT4_GROUP_INFO_NEED_INIT_BIT,
                &(meta_group_info[i]->bb_state));

        /*
         * initialize bb_free to be able to skip
         * empty groups without initialization
         */
        if (ext4_has_group_desc_csum(sb) &&
            (desc->bg_flags & cpu_to_le16(EXT4_BG_BLOCK_UNINIT))) {
                meta_group_info[i]->bb_free =
                        ext4_free_clusters_after_init(sb, group, desc);
        } else {
                meta_group_info[i]->bb_free =
                        ext4_free_group_clusters(sb, desc);
        }

        INIT_LIST_HEAD(&meta_group_info[i]->bb_prealloc_list);
        init_rwsem(&meta_group_info[i]->alloc_sem);
        meta_group_info[i]->bb_free_root = RB_ROOT;
        INIT_LIST_HEAD(&meta_group_info[i]->bb_largest_free_order_node);
        INIT_LIST_HEAD(&meta_group_info[i]->bb_avg_fragment_size_node);
        meta_group_info[i]->bb_largest_free_order = -1;  /* uninit */
        meta_group_info[i]->bb_avg_fragment_size_order = -1;  /* uninit */
        meta_group_info[i]->bb_group = group;

        mb_group_bb_bitmap_alloc(sb, meta_group_info[i], group);
        return 0;

exit_group_info:
        /* If a meta_group_info table has been allocated, release it now */
        if (group % EXT4_DESC_PER_BLOCK(sb) == 0) {
                struct ext4_group_info ***group_info;

                rcu_read_lock();
                group_info = rcu_dereference(sbi->s_group_info);
                kfree(group_info[idx]);
                group_info[idx] = NULL;
                rcu_read_unlock();
        }
        return -ENOMEM;
} /* ext4_mb_add_groupinfo */

static int ext4_mb_init_backend(struct super_block *sb)
{
        ext4_group_t ngroups = ext4_get_groups_count(sb);
        ext4_group_t i;
        struct ext4_sb_info *sbi = EXT4_SB(sb);
        int err;
        struct ext4_group_desc *desc;
        struct ext4_group_info ***group_info;
        struct kmem_cache *cachep;

        err = ext4_mb_alloc_groupinfo(sb, ngroups);
        if (err)
                return err;

        sbi->s_buddy_cache = new_inode(sb);
        if (sbi->s_buddy_cache == NULL) {
                ext4_msg(sb, KERN_ERR, "can't get new inode");
                goto err_freesgi;
        }
        /* To avoid potentially colliding with an valid on-disk inode number,
         * use EXT4_BAD_INO for the buddy cache inode number.  This inode is
         * not in the inode hash, so it should never be found by iget(), but
         * this will avoid confusion if it ever shows up during debugging. */
        sbi->s_buddy_cache->i_ino = EXT4_BAD_INO;
        EXT4_I(sbi->s_buddy_cache)->i_disksize = 0;
        for (i = 0; i < ngroups; i++) {
                cond_resched();
                desc = ext4_get_group_desc(sb, i, NULL);
                if (desc == NULL) {
                        ext4_msg(sb, KERN_ERR, "can't read descriptor %u", i);
                        goto err_freebuddy;
                }
                if (ext4_mb_add_groupinfo(sb, i, desc) != 0)
                        goto err_freebuddy;
        }

        if (ext4_has_feature_flex_bg(sb)) {
                /* a single flex group is supposed to be read by a single IO.
                 * 2 ^ s_log_groups_per_flex != UINT_MAX as s_mb_prefetch is
                 * unsigned integer, so the maximum shift is 32.
                 */
                if (sbi->s_es->s_log_groups_per_flex >= 32) {
                        ext4_msg(sb, KERN_ERR, "too many log groups per flexible block group");
                        goto err_freebuddy;
                }
                sbi->s_mb_prefetch = min_t(uint, 1 << sbi->s_es->s_log_groups_per_flex,
                        BLK_MAX_SEGMENT_SIZE >> (sb->s_blocksize_bits - 9));
                sbi->s_mb_prefetch *= 8; /* 8 prefetch IOs in flight at most */
        } else {
                sbi->s_mb_prefetch = 32;
        }
        if (sbi->s_mb_prefetch > ext4_get_groups_count(sb))
                sbi->s_mb_prefetch = ext4_get_groups_count(sb);
        /*
         * now many real IOs to prefetch within a single allocation at
         * CR_POWER2_ALIGNED. Given CR_POWER2_ALIGNED is an CPU-related
         * optimization we shouldn't try to load too many groups, at some point
         * we should start to use what we've got in memory.
         * with an average random access time 5ms, it'd take a second to get
         * 200 groups (* N with flex_bg), so let's make this limit 4
         */
        sbi->s_mb_prefetch_limit = sbi->s_mb_prefetch * 4;
        if (sbi->s_mb_prefetch_limit > ext4_get_groups_count(sb))
                sbi->s_mb_prefetch_limit = ext4_get_groups_count(sb);

        return 0;

err_freebuddy:
        cachep = get_groupinfo_cache(sb->s_blocksize_bits);
        while (i-- > 0) {
                struct ext4_group_info *grp = ext4_get_group_info(sb, i);

                if (grp)
                        kmem_cache_free(cachep, grp);
        }
        i = sbi->s_group_info_size;
        rcu_read_lock();
        group_info = rcu_dereference(sbi->s_group_info);
        while (i-- > 0)
                kfree(group_info[i]);
        rcu_read_unlock();
        iput(sbi->s_buddy_cache);
err_freesgi:
        rcu_read_lock();
        kvfree(rcu_dereference(sbi->s_group_info));
        rcu_read_unlock();
        return -ENOMEM;
}

static void ext4_groupinfo_destroy_slabs(void)
{
        int i;

        for (i = 0; i < NR_GRPINFO_CACHES; i++) {
                kmem_cache_destroy(ext4_groupinfo_caches[i]);
                ext4_groupinfo_caches[i] = NULL;
        }
}

static int ext4_groupinfo_create_slab(size_t size)
{
        static DEFINE_MUTEX(ext4_grpinfo_slab_create_mutex);
        int slab_size;
        int blocksize_bits = order_base_2(size);
        int cache_index = blocksize_bits - EXT4_MIN_BLOCK_LOG_SIZE;
        struct kmem_cache *cachep;

        if (cache_index >= NR_GRPINFO_CACHES)
                return -EINVAL;

        if (unlikely(cache_index < 0))
                cache_index = 0;

        mutex_lock(&ext4_grpinfo_slab_create_mutex);
        if (ext4_groupinfo_caches[cache_index]) {
                mutex_unlock(&ext4_grpinfo_slab_create_mutex);
                return 0;        /* Already created */
        }

        slab_size = offsetof(struct ext4_group_info,
                                bb_counters[blocksize_bits + 2]);

        cachep = kmem_cache_create(ext4_groupinfo_slab_names[cache_index],
                                        slab_size, 0, SLAB_RECLAIM_ACCOUNT,
                                        NULL);

        ext4_groupinfo_caches[cache_index] = cachep;

        mutex_unlock(&ext4_grpinfo_slab_create_mutex);
        if (!cachep) {
                printk(KERN_EMERG
                       "EXT4-fs: no memory for groupinfo slab cache\n");
                return -ENOMEM;
        }

        return 0;
}

static void ext4_discard_work(struct work_struct *work)
{
        struct ext4_sb_info *sbi = container_of(work,
                        struct ext4_sb_info, s_discard_work);
        struct super_block *sb = sbi->s_sb;
        struct ext4_free_data *fd, *nfd;
        struct ext4_buddy e4b;
        LIST_HEAD(discard_list);
        ext4_group_t grp, load_grp;
        int err = 0;

        spin_lock(&sbi->s_md_lock);
        list_splice_init(&sbi->s_discard_list, &discard_list);
        spin_unlock(&sbi->s_md_lock);

        load_grp = UINT_MAX;
        list_for_each_entry_safe(fd, nfd, &discard_list, efd_list) {
                /*
                 * If filesystem is umounting or no memory or suffering
                 * from no space, give up the discard
                 */
                if ((sb->s_flags & SB_ACTIVE) && !err &&
                    !atomic_read(&sbi->s_retry_alloc_pending)) {
                        grp = fd->efd_group;
                        if (grp != load_grp) {
                                if (load_grp != UINT_MAX)
                                        ext4_mb_unload_buddy(&e4b);

                                err = ext4_mb_load_buddy(sb, grp, &e4b);
                                if (err) {
                                        kmem_cache_free(ext4_free_data_cachep, fd);
                                        load_grp = UINT_MAX;
                                        continue;
                                } else {
                                        load_grp = grp;
                                }
                        }

                        ext4_lock_group(sb, grp);
                        ext4_try_to_trim_range(sb, &e4b, fd->efd_start_cluster,
                                                fd->efd_start_cluster + fd->efd_count - 1, 1);
                        ext4_unlock_group(sb, grp);
                }
                kmem_cache_free(ext4_free_data_cachep, fd);
        }

        if (load_grp != UINT_MAX)
                ext4_mb_unload_buddy(&e4b);
}

int ext4_mb_init(struct super_block *sb)
{
        struct ext4_sb_info *sbi = EXT4_SB(sb);
        unsigned i, j;
        unsigned offset, offset_incr;
        unsigned max;
        int ret;

        i = MB_NUM_ORDERS(sb) * sizeof(*sbi->s_mb_offsets);

        sbi->s_mb_offsets = kmalloc(i, GFP_KERNEL);
        if (sbi->s_mb_offsets == NULL) {
                ret = -ENOMEM;
                goto out;
        }

        i = MB_NUM_ORDERS(sb) * sizeof(*sbi->s_mb_maxs);
        sbi->s_mb_maxs = kmalloc(i, GFP_KERNEL);
        if (sbi->s_mb_maxs == NULL) {
                ret = -ENOMEM;
                goto out;
        }

        ret = ext4_groupinfo_create_slab(sb->s_blocksize);
        if (ret < 0)
                goto out;

        /* order 0 is regular bitmap */
        sbi->s_mb_maxs[0] = sb->s_blocksize << 3;
        sbi->s_mb_offsets[0] = 0;

        i = 1;
        offset = 0;
        offset_incr = 1 << (sb->s_blocksize_bits - 1);
        max = sb->s_blocksize << 2;
        do {
                sbi->s_mb_offsets[i] = offset;
                sbi->s_mb_maxs[i] = max;
                offset += offset_incr;
                offset_incr = offset_incr >> 1;
                max = max >> 1;
                i++;
        } while (i < MB_NUM_ORDERS(sb));

        sbi->s_mb_avg_fragment_size =
                kmalloc_array(MB_NUM_ORDERS(sb), sizeof(struct list_head),
                        GFP_KERNEL);
        if (!sbi->s_mb_avg_fragment_size) {
                ret = -ENOMEM;
                goto out;
        }
        sbi->s_mb_avg_fragment_size_locks =
                kmalloc_array(MB_NUM_ORDERS(sb), sizeof(rwlock_t),
                        GFP_KERNEL);
        if (!sbi->s_mb_avg_fragment_size_locks) {
                ret = -ENOMEM;
                goto out;
        }
        for (i = 0; i < MB_NUM_ORDERS(sb); i++) {
                INIT_LIST_HEAD(&sbi->s_mb_avg_fragment_size[i]);
                rwlock_init(&sbi->s_mb_avg_fragment_size_locks[i]);
        }
        sbi->s_mb_largest_free_orders =
                kmalloc_array(MB_NUM_ORDERS(sb), sizeof(struct list_head),
                        GFP_KERNEL);
        if (!sbi->s_mb_largest_free_orders) {
                ret = -ENOMEM;
                goto out;
        }
        sbi->s_mb_largest_free_orders_locks =
                kmalloc_array(MB_NUM_ORDERS(sb), sizeof(rwlock_t),
                        GFP_KERNEL);
        if (!sbi->s_mb_largest_free_orders_locks) {
                ret = -ENOMEM;
                goto out;
        }
        for (i = 0; i < MB_NUM_ORDERS(sb); i++) {
                INIT_LIST_HEAD(&sbi->s_mb_largest_free_orders[i]);
                rwlock_init(&sbi->s_mb_largest_free_orders_locks[i]);
        }

        spin_lock_init(&sbi->s_md_lock);
        sbi->s_mb_free_pending = 0;
        INIT_LIST_HEAD(&sbi->s_freed_data_list[0]);
        INIT_LIST_HEAD(&sbi->s_freed_data_list[1]);
        INIT_LIST_HEAD(&sbi->s_discard_list);
        INIT_WORK(&sbi->s_discard_work, ext4_discard_work);
        atomic_set(&sbi->s_retry_alloc_pending, 0);

        sbi->s_mb_max_to_scan = MB_DEFAULT_MAX_TO_SCAN;
        sbi->s_mb_min_to_scan = MB_DEFAULT_MIN_TO_SCAN;
        sbi->s_mb_stats = MB_DEFAULT_STATS;
        sbi->s_mb_stream_request = MB_DEFAULT_STREAM_THRESHOLD;
        sbi->s_mb_order2_reqs = MB_DEFAULT_ORDER2_REQS;
        sbi->s_mb_best_avail_max_trim_order = MB_DEFAULT_BEST_AVAIL_TRIM_ORDER;

        /*
         * The default group preallocation is 512, which for 4k block
         * sizes translates to 2 megabytes.  However for bigalloc file
         * systems, this is probably too big (i.e, if the cluster size
         * is 1 megabyte, then group preallocation size becomes half a
         * gigabyte!).  As a default, we will keep a two megabyte
         * group pralloc size for cluster sizes up to 64k, and after
         * that, we will force a minimum group preallocation size of
         * 32 clusters.  This translates to 8 megs when the cluster
         * size is 256k, and 32 megs when the cluster size is 1 meg,
         * which seems reasonable as a default.
         */
        sbi->s_mb_group_prealloc = max(MB_DEFAULT_GROUP_PREALLOC >>
                                       sbi->s_cluster_bits, 32);
        /*
         * If there is a s_stripe > 1, then we set the s_mb_group_prealloc
         * to the lowest multiple of s_stripe which is bigger than
         * the s_mb_group_prealloc as determined above. We want
         * the preallocation size to be an exact multiple of the
         * RAID stripe size so that preallocations don't fragment
         * the stripes.
         */
        if (sbi->s_stripe > 1) {
                sbi->s_mb_group_prealloc = roundup(
                        sbi->s_mb_group_prealloc, EXT4_NUM_B2C(sbi, sbi->s_stripe));
        }

        sbi->s_locality_groups = alloc_percpu(struct ext4_locality_group);
        if (sbi->s_locality_groups == NULL) {
                ret = -ENOMEM;
                goto out;
        }
        for_each_possible_cpu(i) {
                struct ext4_locality_group *lg;
                lg = per_cpu_ptr(sbi->s_locality_groups, i);
                mutex_init(&lg->lg_mutex);
                for (j = 0; j < PREALLOC_TB_SIZE; j++)
                        INIT_LIST_HEAD(&lg->lg_prealloc_list[j]);
                spin_lock_init(&lg->lg_prealloc_lock);
        }

        if (bdev_nonrot(sb->s_bdev))
                sbi->s_mb_max_linear_groups = 0;
        else
                sbi->s_mb_max_linear_groups = MB_DEFAULT_LINEAR_LIMIT;
        /* init file for buddy data */
        ret = ext4_mb_init_backend(sb);
        if (ret != 0)
                goto out_free_locality_groups;

        return 0;

out_free_locality_groups:
        free_percpu(sbi->s_locality_groups);
        sbi->s_locality_groups = NULL;
out:
        kfree(sbi->s_mb_avg_fragment_size);
        kfree(sbi->s_mb_avg_fragment_size_locks);
        kfree(sbi->s_mb_largest_free_orders);
        kfree(sbi->s_mb_largest_free_orders_locks);
        kfree(sbi->s_mb_offsets);
        sbi->s_mb_offsets = NULL;
        kfree(sbi->s_mb_maxs);
        sbi->s_mb_maxs = NULL;
        return ret;
}

/* need to called with the ext4 group lock held */
static int ext4_mb_cleanup_pa(struct ext4_group_info *grp)
{
        struct ext4_prealloc_space *pa;
        struct list_head *cur, *tmp;
        int count = 0;

        list_for_each_safe(cur, tmp, &grp->bb_prealloc_list) {
                pa = list_entry(cur, struct ext4_prealloc_space, pa_group_list);
                list_del(&pa->pa_group_list);
                count++;
                kmem_cache_free(ext4_pspace_cachep, pa);
        }
        return count;
}

void ext4_mb_release(struct super_block *sb)
{
        ext4_group_t ngroups = ext4_get_groups_count(sb);
        ext4_group_t i;
        int num_meta_group_infos;
        struct ext4_group_info *grinfo, ***group_info;
        struct ext4_sb_info *sbi = EXT4_SB(sb);
        struct kmem_cache *cachep = get_groupinfo_cache(sb->s_blocksize_bits);
        int count;

        if (test_opt(sb, DISCARD)) {
                /*
                 * wait the discard work to drain all of ext4_free_data
                 */
                flush_work(&sbi->s_discard_work);
                WARN_ON_ONCE(!list_empty(&sbi->s_discard_list));
        }

        if (sbi->s_group_info) {
                for (i = 0; i < ngroups; i++) {
                        cond_resched();
                        grinfo = ext4_get_group_info(sb, i);
                        if (!grinfo)
                                continue;
                        mb_group_bb_bitmap_free(grinfo);
                        ext4_lock_group(sb, i);
                        count = ext4_mb_cleanup_pa(grinfo);
                        if (count)
                                mb_debug(sb, "mballoc: %d PAs left\n",
                                         count);
                        ext4_unlock_group(sb, i);
                        kmem_cache_free(cachep, grinfo);
                }
                num_meta_group_infos = (ngroups +
                                EXT4_DESC_PER_BLOCK(sb) - 1) >>
                        EXT4_DESC_PER_BLOCK_BITS(sb);
                rcu_read_lock();
                group_info = rcu_dereference(sbi->s_group_info);
                for (i = 0; i < num_meta_group_infos; i++)
                        kfree(group_info[i]);
                kvfree(group_info);
                rcu_read_unlock();
        }
        kfree(sbi->s_mb_avg_fragment_size);
        kfree(sbi->s_mb_avg_fragment_size_locks);
        kfree(sbi->s_mb_largest_free_orders);
        kfree(sbi->s_mb_largest_free_orders_locks);
        kfree(sbi->s_mb_offsets);
        kfree(sbi->s_mb_maxs);
        iput(sbi->s_buddy_cache);
        if (sbi->s_mb_stats) {
                ext4_msg(sb, KERN_INFO,
                       "mballoc: %u blocks %u reqs (%u success)",
                                atomic_read(&sbi->s_bal_allocated),
                                atomic_read(&sbi->s_bal_reqs),
                                atomic_read(&sbi->s_bal_success));
                ext4_msg(sb, KERN_INFO,
                      "mballoc: %u extents scanned, %u groups scanned, %u goal hits, "
                                "%u 2^N hits, %u breaks, %u lost",
                                atomic_read(&sbi->s_bal_ex_scanned),
                                atomic_read(&sbi->s_bal_groups_scanned),
                                atomic_read(&sbi->s_bal_goals),
                                atomic_read(&sbi->s_bal_2orders),
                                atomic_read(&sbi->s_bal_breaks),
                                atomic_read(&sbi->s_mb_lost_chunks));
                ext4_msg(sb, KERN_INFO,
                       "mballoc: %u generated and it took %llu",
                                atomic_read(&sbi->s_mb_buddies_generated),
                                atomic64_read(&sbi->s_mb_generation_time));
                ext4_msg(sb, KERN_INFO,
                       "mballoc: %u preallocated, %u discarded",
                                atomic_read(&sbi->s_mb_preallocated),
                                atomic_read(&sbi->s_mb_discarded));
        }

        free_percpu(sbi->s_locality_groups);
}

static inline int ext4_issue_discard(struct super_block *sb,
                ext4_group_t block_group, ext4_grpblk_t cluster, int count)
{
        ext4_fsblk_t discard_block;

        discard_block = (EXT4_C2B(EXT4_SB(sb), cluster) +
                         ext4_group_first_block_no(sb, block_group));
        count = EXT4_C2B(EXT4_SB(sb), count);
        trace_ext4_discard_blocks(sb,
                        (unsigned long long) discard_block, count);

        return sb_issue_discard(sb, discard_block, count, GFP_NOFS, 0);
}

static void ext4_free_data_in_buddy(struct super_block *sb,
                                    struct ext4_free_data *entry)
{
        struct ext4_buddy e4b;
        struct ext4_group_info *db;
        int err, count = 0;

        mb_debug(sb, "gonna free %u blocks in group %u (0x%p):",
                 entry->efd_count, entry->efd_group, entry);

        err = ext4_mb_load_buddy(sb, entry->efd_group, &e4b);
        /* we expect to find existing buddy because it's pinned */
        BUG_ON(err != 0);

        spin_lock(&EXT4_SB(sb)->s_md_lock);
        EXT4_SB(sb)->s_mb_free_pending -= entry->efd_count;
        spin_unlock(&EXT4_SB(sb)->s_md_lock);

        db = e4b.bd_info;
        /* there are blocks to put in buddy to make them really free */
        count += entry->efd_count;
        ext4_lock_group(sb, entry->efd_group);
        /* Take it out of per group rb tree */
        rb_erase(&entry->efd_node, &(db->bb_free_root));
        mb_free_blocks(NULL, &e4b, entry->efd_start_cluster, entry->efd_count);

        /*
         * Clear the trimmed flag for the group so that the next
         * ext4_trim_fs can trim it.
         */
        EXT4_MB_GRP_CLEAR_TRIMMED(db);

        if (!db->bb_free_root.rb_node) {
                /* No more items in the per group rb tree
                 * balance refcounts from ext4_mb_free_metadata()
                 */
                folio_put(e4b.bd_buddy_folio);
                folio_put(e4b.bd_bitmap_folio);
        }
        ext4_unlock_group(sb, entry->efd_group);
        ext4_mb_unload_buddy(&e4b);

        mb_debug(sb, "freed %d blocks in 1 structures\n", count);
}

/*
 * This function is called by the jbd2 layer once the commit has finished,
 * so we know we can free the blocks that were released with that commit.
 */
void ext4_process_freed_data(struct super_block *sb, tid_t commit_tid)
{
        struct ext4_sb_info *sbi = EXT4_SB(sb);
        struct ext4_free_data *entry, *tmp;
        LIST_HEAD(freed_data_list);
        struct list_head *s_freed_head = &sbi->s_freed_data_list[commit_tid & 1];
        bool wake;

        list_replace_init(s_freed_head, &freed_data_list);

        list_for_each_entry(entry, &freed_data_list, efd_list)
                ext4_free_data_in_buddy(sb, entry);

        if (test_opt(sb, DISCARD)) {
                spin_lock(&sbi->s_md_lock);
                wake = list_empty(&sbi->s_discard_list);
                list_splice_tail(&freed_data_list, &sbi->s_discard_list);
                spin_unlock(&sbi->s_md_lock);
                if (wake)
                        queue_work(system_unbound_wq, &sbi->s_discard_work);
        } else {
                list_for_each_entry_safe(entry, tmp, &freed_data_list, efd_list)
                        kmem_cache_free(ext4_free_data_cachep, entry);
        }
}

int __init ext4_init_mballoc(void)
{
        ext4_pspace_cachep = KMEM_CACHE(ext4_prealloc_space,
                                        SLAB_RECLAIM_ACCOUNT);
        if (ext4_pspace_cachep == NULL)
                goto out;

        ext4_ac_cachep = KMEM_CACHE(ext4_allocation_context,
                                    SLAB_RECLAIM_ACCOUNT);
        if (ext4_ac_cachep == NULL)
                goto out_pa_free;

        ext4_free_data_cachep = KMEM_CACHE(ext4_free_data,
                                           SLAB_RECLAIM_ACCOUNT);
        if (ext4_free_data_cachep == NULL)
                goto out_ac_free;

        return 0;

out_ac_free:
        kmem_cache_destroy(ext4_ac_cachep);
out_pa_free:
        kmem_cache_destroy(ext4_pspace_cachep);
out:
        return -ENOMEM;
}

void ext4_exit_mballoc(void)
{
        /*
         * Wait for completion of call_rcu()'s on ext4_pspace_cachep
         * before destroying the slab cache.
         */
        rcu_barrier();
        kmem_cache_destroy(ext4_pspace_cachep);
        kmem_cache_destroy(ext4_ac_cachep);
        kmem_cache_destroy(ext4_free_data_cachep);
        ext4_groupinfo_destroy_slabs();
}

#define EXT4_MB_BITMAP_MARKED_CHECK 0x0001
#define EXT4_MB_SYNC_UPDATE 0x0002
static int
ext4_mb_mark_context(handle_t *handle, struct super_block *sb, bool state,
                     ext4_group_t group, ext4_grpblk_t blkoff,
                     ext4_grpblk_t len, int flags, ext4_grpblk_t *ret_changed)
{
        struct ext4_sb_info *sbi = EXT4_SB(sb);
        struct buffer_head *bitmap_bh = NULL;
        struct ext4_group_desc *gdp;
        struct buffer_head *gdp_bh;
        int err;
        unsigned int i, already, changed = len;

        KUNIT_STATIC_STUB_REDIRECT(ext4_mb_mark_context,
                                   handle, sb, state, group, blkoff, len,
                                   flags, ret_changed);

        if (ret_changed)
                *ret_changed = 0;
        bitmap_bh = ext4_read_block_bitmap(sb, group);
        if (IS_ERR(bitmap_bh))
                return PTR_ERR(bitmap_bh);

        if (handle) {
                BUFFER_TRACE(bitmap_bh, "getting write access");
                err = ext4_journal_get_write_access(handle, sb, bitmap_bh,
                                                    EXT4_JTR_NONE);
                if (err)
                        goto out_err;
        }

        err = -EIO;
        gdp = ext4_get_group_desc(sb, group, &gdp_bh);
        if (!gdp)
                goto out_err;

        if (handle) {
                BUFFER_TRACE(gdp_bh, "get_write_access");
                err = ext4_journal_get_write_access(handle, sb, gdp_bh,
                                                    EXT4_JTR_NONE);
                if (err)
                        goto out_err;
        }

        ext4_lock_group(sb, group);
        if (ext4_has_group_desc_csum(sb) &&
            (gdp->bg_flags & cpu_to_le16(EXT4_BG_BLOCK_UNINIT))) {
                gdp->bg_flags &= cpu_to_le16(~EXT4_BG_BLOCK_UNINIT);
                ext4_free_group_clusters_set(sb, gdp,
                        ext4_free_clusters_after_init(sb, group, gdp));
        }

        if (flags & EXT4_MB_BITMAP_MARKED_CHECK) {
                already = 0;
                for (i = 0; i < len; i++)
                        if (mb_test_bit(blkoff + i, bitmap_bh->b_data) ==
                                        state)
                                already++;
                changed = len - already;
        }

        if (state) {
                mb_set_bits(bitmap_bh->b_data, blkoff, len);
                ext4_free_group_clusters_set(sb, gdp,
                        ext4_free_group_clusters(sb, gdp) - changed);
        } else {
                mb_clear_bits(bitmap_bh->b_data, blkoff, len);
                ext4_free_group_clusters_set(sb, gdp,
                        ext4_free_group_clusters(sb, gdp) + changed);
        }

        ext4_block_bitmap_csum_set(sb, gdp, bitmap_bh);
        ext4_group_desc_csum_set(sb, group, gdp);
        ext4_unlock_group(sb, group);
        if (ret_changed)
                *ret_changed = changed;

        if (sbi->s_log_groups_per_flex) {
                ext4_group_t flex_group = ext4_flex_group(sbi, group);
                struct flex_groups *fg = sbi_array_rcu_deref(sbi,
                                           s_flex_groups, flex_group);

                if (state)
                        atomic64_sub(changed, &fg->free_clusters);
                else
                        atomic64_add(changed, &fg->free_clusters);
        }

        err = ext4_handle_dirty_metadata(handle, NULL, bitmap_bh);
        if (err)
                goto out_err;
        err = ext4_handle_dirty_metadata(handle, NULL, gdp_bh);
        if (err)
                goto out_err;

        if (flags & EXT4_MB_SYNC_UPDATE) {
                sync_dirty_buffer(bitmap_bh);
                sync_dirty_buffer(gdp_bh);
        }

out_err:
        brelse(bitmap_bh);
        return err;
}

/*
 * Check quota and mark chosen space (ac->ac_b_ex) non-free in bitmaps
 * Returns 0 if success or error code
 */
static noinline_for_stack int
ext4_mb_mark_diskspace_used(struct ext4_allocation_context *ac,
                                handle_t *handle, unsigned int reserv_clstrs)
{
        struct ext4_group_desc *gdp;
        struct ext4_sb_info *sbi;
        struct super_block *sb;
        ext4_fsblk_t block;
        int err, len;
        int flags = 0;
        ext4_grpblk_t changed;

        BUG_ON(ac->ac_status != AC_STATUS_FOUND);
        BUG_ON(ac->ac_b_ex.fe_len <= 0);

        sb = ac->ac_sb;
        sbi = EXT4_SB(sb);

        gdp = ext4_get_group_desc(sb, ac->ac_b_ex.fe_group, NULL);
        if (!gdp)
                return -EIO;
        ext4_debug("using block group %u(%d)\n", ac->ac_b_ex.fe_group,
                        ext4_free_group_clusters(sb, gdp));

        block = ext4_grp_offs_to_block(sb, &ac->ac_b_ex);
        len = EXT4_C2B(sbi, ac->ac_b_ex.fe_len);
        if (!ext4_inode_block_valid(ac->ac_inode, block, len)) {
                ext4_error(sb, "Allocating blocks %llu-%llu which overlap "
                           "fs metadata", block, block+len);
                /* File system mounted not to panic on error
                 * Fix the bitmap and return EFSCORRUPTED
                 * We leak some of the blocks here.
                 */
                err = ext4_mb_mark_context(handle, sb, true,
                                           ac->ac_b_ex.fe_group,
                                           ac->ac_b_ex.fe_start,
                                           ac->ac_b_ex.fe_len,
                                           0, NULL);
                if (!err)
                        err = -EFSCORRUPTED;
                return err;
        }

#ifdef AGGRESSIVE_CHECK
        flags |= EXT4_MB_BITMAP_MARKED_CHECK;
#endif
        err = ext4_mb_mark_context(handle, sb, true, ac->ac_b_ex.fe_group,
                                   ac->ac_b_ex.fe_start, ac->ac_b_ex.fe_len,
                                   flags, &changed);

        if (err && changed == 0)
                return err;

#ifdef AGGRESSIVE_CHECK
        BUG_ON(changed != ac->ac_b_ex.fe_len);
#endif
        percpu_counter_sub(&sbi->s_freeclusters_counter, ac->ac_b_ex.fe_len);
        /*
         * Now reduce the dirty block count also. Should not go negative
         */
        if (!(ac->ac_flags & EXT4_MB_DELALLOC_RESERVED))
                /* release all the reserved blocks if non delalloc */
                percpu_counter_sub(&sbi->s_dirtyclusters_counter,
                                   reserv_clstrs);

        return err;
}

/*
 * Idempotent helper for Ext4 fast commit replay path to set the state of
 * blocks in bitmaps and update counters.
 */
void ext4_mb_mark_bb(struct super_block *sb, ext4_fsblk_t block,
                     int len, bool state)
{
        struct ext4_sb_info *sbi = EXT4_SB(sb);
        ext4_group_t group;
        ext4_grpblk_t blkoff;
        int err = 0;
        unsigned int clen, thisgrp_len;

        while (len > 0) {
                ext4_get_group_no_and_offset(sb, block, &group, &blkoff);

                /*
                 * Check to see if we are freeing blocks across a group
                 * boundary.
                 * In case of flex_bg, this can happen that (block, len) may
                 * span across more than one group. In that case we need to
                 * get the corresponding group metadata to work with.
                 * For this we have goto again loop.
                 */
                thisgrp_len = min_t(unsigned int, (unsigned int)len,
                        EXT4_BLOCKS_PER_GROUP(sb) - EXT4_C2B(sbi, blkoff));
                clen = EXT4_NUM_B2C(sbi, thisgrp_len);

                if (!ext4_sb_block_valid(sb, NULL, block, thisgrp_len)) {
                        ext4_error(sb, "Marking blocks in system zone - "
                                   "Block = %llu, len = %u",
                                   block, thisgrp_len);
                        break;
                }

                err = ext4_mb_mark_context(NULL, sb, state,
                                           group, blkoff, clen,
                                           EXT4_MB_BITMAP_MARKED_CHECK |
                                           EXT4_MB_SYNC_UPDATE,
                                           NULL);
                if (err)
                        break;

                block += thisgrp_len;
                len -= thisgrp_len;
                BUG_ON(len < 0);
        }
}

/*
 * here we normalize request for locality group
 * Group request are normalized to s_mb_group_prealloc, which goes to
 * s_strip if we set the same via mount option.
 * s_mb_group_prealloc can be configured via
 * /sys/fs/ext4/<partition>/mb_group_prealloc
 *
 * XXX: should we try to preallocate more than the group has now?
 */
static void ext4_mb_normalize_group_request(struct ext4_allocation_context *ac)
{
        struct super_block *sb = ac->ac_sb;
        struct ext4_locality_group *lg = ac->ac_lg;

        BUG_ON(lg == NULL);
        ac->ac_g_ex.fe_len = EXT4_SB(sb)->s_mb_group_prealloc;
        mb_debug(sb, "goal %u blocks for locality group\n", ac->ac_g_ex.fe_len);
}

/*
 * This function returns the next element to look at during inode
 * PA rbtree walk. We assume that we have held the inode PA rbtree lock
 * (ei->i_prealloc_lock)
 *
 * new_start        The start of the range we want to compare
 * cur_start        The existing start that we are comparing against
 * node        The node of the rb_tree
 */
static inline struct rb_node*
ext4_mb_pa_rb_next_iter(ext4_lblk_t new_start, ext4_lblk_t cur_start, struct rb_node *node)
{
        if (new_start < cur_start)
                return node->rb_left;
        else
                return node->rb_right;
}

static inline void
ext4_mb_pa_assert_overlap(struct ext4_allocation_context *ac,
                          ext4_lblk_t start, loff_t end)
{
        struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);
        struct ext4_inode_info *ei = EXT4_I(ac->ac_inode);
        struct ext4_prealloc_space *tmp_pa;
        ext4_lblk_t tmp_pa_start;
        loff_t tmp_pa_end;
        struct rb_node *iter;

        read_lock(&ei->i_prealloc_lock);
        for (iter = ei->i_prealloc_node.rb_node; iter;
             iter = ext4_mb_pa_rb_next_iter(start, tmp_pa_start, iter)) {
                tmp_pa = rb_entry(iter, struct ext4_prealloc_space,
                                  pa_node.inode_node);
                tmp_pa_start = tmp_pa->pa_lstart;
                tmp_pa_end = pa_logical_end(sbi, tmp_pa);

                spin_lock(&tmp_pa->pa_lock);
                if (tmp_pa->pa_deleted == 0)
                        BUG_ON(!(start >= tmp_pa_end || end <= tmp_pa_start));
                spin_unlock(&tmp_pa->pa_lock);
        }
        read_unlock(&ei->i_prealloc_lock);
}

/*
 * Given an allocation context "ac" and a range "start", "end", check
 * and adjust boundaries if the range overlaps with any of the existing
 * preallocatoins stored in the corresponding inode of the allocation context.
 *
 * Parameters:
 *        ac                        allocation context
 *        start                        start of the new range
 *        end                        end of the new range
 */
static inline void
ext4_mb_pa_adjust_overlap(struct ext4_allocation_context *ac,
                          ext4_lblk_t *start, loff_t *end)
{
        struct ext4_inode_info *ei = EXT4_I(ac->ac_inode);
        struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);
        struct ext4_prealloc_space *tmp_pa = NULL, *left_pa = NULL, *right_pa = NULL;
        struct rb_node *iter;
        ext4_lblk_t new_start, tmp_pa_start, right_pa_start = -1;
        loff_t new_end, tmp_pa_end, left_pa_end = -1;

        new_start = *start;
        new_end = *end;

        /*
         * Adjust the normalized range so that it doesn't overlap with any
         * existing preallocated blocks(PAs). Make sure to hold the rbtree lock
         * so it doesn't change underneath us.
         */
        read_lock(&ei->i_prealloc_lock);

        /* Step 1: find any one immediate neighboring PA of the normalized range */
        for (iter = ei->i_prealloc_node.rb_node; iter;
             iter = ext4_mb_pa_rb_next_iter(ac->ac_o_ex.fe_logical,
                                            tmp_pa_start, iter)) {
                tmp_pa = rb_entry(iter, struct ext4_prealloc_space,
                                  pa_node.inode_node);
                tmp_pa_start = tmp_pa->pa_lstart;
                tmp_pa_end = pa_logical_end(sbi, tmp_pa);

                /* PA must not overlap original request */
                spin_lock(&tmp_pa->pa_lock);
                if (tmp_pa->pa_deleted == 0)
                        BUG_ON(!(ac->ac_o_ex.fe_logical >= tmp_pa_end ||
                                 ac->ac_o_ex.fe_logical < tmp_pa_start));
                spin_unlock(&tmp_pa->pa_lock);
        }

        /*
         * Step 2: check if the found PA is left or right neighbor and
         * get the other neighbor
         */
        if (tmp_pa) {
                if (tmp_pa->pa_lstart < ac->ac_o_ex.fe_logical) {
                        struct rb_node *tmp;

                        left_pa = tmp_pa;
                        tmp = rb_next(&left_pa->pa_node.inode_node);
                        if (tmp) {
                                right_pa = rb_entry(tmp,
                                                    struct ext4_prealloc_space,
                                                    pa_node.inode_node);
                        }
                } else {
                        struct rb_node *tmp;

                        right_pa = tmp_pa;
                        tmp = rb_prev(&right_pa->pa_node.inode_node);
                        if (tmp) {
                                left_pa = rb_entry(tmp,
                                                   struct ext4_prealloc_space,
                                                   pa_node.inode_node);
                        }
                }
        }

        /* Step 3: get the non deleted neighbors */
        if (left_pa) {
                for (iter = &left_pa->pa_node.inode_node;;
                     iter = rb_prev(iter)) {
                        if (!iter) {
                                left_pa = NULL;
                                break;
                        }

                        tmp_pa = rb_entry(iter, struct ext4_prealloc_space,
                                          pa_node.inode_node);
                        left_pa = tmp_pa;
                        spin_lock(&tmp_pa->pa_lock);
                        if (tmp_pa->pa_deleted == 0) {
                                spin_unlock(&tmp_pa->pa_lock);
                                break;
                        }
                        spin_unlock(&tmp_pa->pa_lock);
                }
        }

        if (right_pa) {
                for (iter = &right_pa->pa_node.inode_node;;
                     iter = rb_next(iter)) {
                        if (!iter) {
                                right_pa = NULL;
                                break;
                        }

                        tmp_pa = rb_entry(iter, struct ext4_prealloc_space,
                                          pa_node.inode_node);
                        right_pa = tmp_pa;
                        spin_lock(&tmp_pa->pa_lock);
                        if (tmp_pa->pa_deleted == 0) {
                                spin_unlock(&tmp_pa->pa_lock);
                                break;
                        }
                        spin_unlock(&tmp_pa->pa_lock);
                }
        }

        if (left_pa) {
                left_pa_end = pa_logical_end(sbi, left_pa);
                BUG_ON(left_pa_end > ac->ac_o_ex.fe_logical);
        }

        if (right_pa) {
                right_pa_start = right_pa->pa_lstart;
                BUG_ON(right_pa_start <= ac->ac_o_ex.fe_logical);
        }

        /* Step 4: trim our normalized range to not overlap with the neighbors */
        if (left_pa) {
                if (left_pa_end > new_start)
                        new_start = left_pa_end;
        }

        if (right_pa) {
                if (right_pa_start < new_end)
                        new_end = right_pa_start;
        }
        read_unlock(&ei->i_prealloc_lock);

        /* XXX: extra loop to check we really don't overlap preallocations */
        ext4_mb_pa_assert_overlap(ac, new_start, new_end);

        *start = new_start;
        *end = new_end;
}

/*
 * Normalization means making request better in terms of
 * size and alignment
 */
static noinline_for_stack void
ext4_mb_normalize_request(struct ext4_allocation_context *ac,
                                struct ext4_allocation_request *ar)
{
        struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);
        struct ext4_super_block *es = sbi->s_es;
        int bsbits, max;
        loff_t size, start_off, end;
        loff_t orig_size __maybe_unused;
        ext4_lblk_t start;

        /* do normalize only data requests, metadata requests
           do not need preallocation */
        if (!(ac->ac_flags & EXT4_MB_HINT_DATA))
                return;

        /* sometime caller may want exact blocks */
        if (unlikely(ac->ac_flags & EXT4_MB_HINT_GOAL_ONLY))
                return;

        /* caller may indicate that preallocation isn't
         * required (it's a tail, for example) */
        if (ac->ac_flags & EXT4_MB_HINT_NOPREALLOC)
                return;

        if (ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC) {
                ext4_mb_normalize_group_request(ac);
                return ;
        }

        bsbits = ac->ac_sb->s_blocksize_bits;

        /* first, let's learn actual file size
         * given current request is allocated */
        size = extent_logical_end(sbi, &ac->ac_o_ex);
        size = size << bsbits;
        if (size < i_size_read(ac->ac_inode))
                size = i_size_read(ac->ac_inode);
        orig_size = size;

        /* max size of free chunks */
        max = 2 << bsbits;

#define NRL_CHECK_SIZE(req, size, max, chunk_size)        \
                (req <= (size) || max <= (chunk_size))

        /* first, try to predict filesize */
        /* XXX: should this table be tunable? */
        start_off = 0;
        if (size <= 16 * 1024) {
                size = 16 * 1024;
        } else if (size <= 32 * 1024) {
                size = 32 * 1024;
        } else if (size <= 64 * 1024) {
                size = 64 * 1024;
        } else if (size <= 128 * 1024) {
                size = 128 * 1024;
        } else if (size <= 256 * 1024) {
                size = 256 * 1024;
        } else if (size <= 512 * 1024) {
                size = 512 * 1024;
        } else if (size <= 1024 * 1024) {
                size = 1024 * 1024;
        } else if (NRL_CHECK_SIZE(size, 4 * 1024 * 1024, max, 2 * 1024)) {
                start_off = ((loff_t)ac->ac_o_ex.fe_logical >>
                                                (21 - bsbits)) << 21;
                size = 2 * 1024 * 1024;
        } else if (NRL_CHECK_SIZE(size, 8 * 1024 * 1024, max, 4 * 1024)) {
                start_off = ((loff_t)ac->ac_o_ex.fe_logical >>
                                                        (22 - bsbits)) << 22;
                size = 4 * 1024 * 1024;
        } else if (NRL_CHECK_SIZE(EXT4_C2B(sbi, ac->ac_o_ex.fe_len),
                                        (8<<20)>>bsbits, max, 8 * 1024)) {
                start_off = ((loff_t)ac->ac_o_ex.fe_logical >>
                                                        (23 - bsbits)) << 23;
                size = 8 * 1024 * 1024;
        } else {
                start_off = (loff_t) ac->ac_o_ex.fe_logical << bsbits;
                size          = (loff_t) EXT4_C2B(sbi,
                                              ac->ac_o_ex.fe_len) << bsbits;
        }
        size = size >> bsbits;
        start = start_off >> bsbits;

        /*
         * For tiny groups (smaller than 8MB) the chosen allocation
         * alignment may be larger than group size. Make sure the
         * alignment does not move allocation to a different group which
         * makes mballoc fail assertions later.
         */
        start = max(start, rounddown(ac->ac_o_ex.fe_logical,
                        (ext4_lblk_t)EXT4_BLOCKS_PER_GROUP(ac->ac_sb)));

        /* avoid unnecessary preallocation that may trigger assertions */
        if (start + size > EXT_MAX_BLOCKS)
                size = EXT_MAX_BLOCKS - start;

        /* don't cover already allocated blocks in selected range */
        if (ar->pleft && start <= ar->lleft) {
                size -= ar->lleft + 1 - start;
                start = ar->lleft + 1;
        }
        if (ar->pright && start + size - 1 >= ar->lright)
                size -= start + size - ar->lright;

        /*
         * Trim allocation request for filesystems with artificially small
         * groups.
         */
        if (size > EXT4_BLOCKS_PER_GROUP(ac->ac_sb))
                size = EXT4_BLOCKS_PER_GROUP(ac->ac_sb);

        end = start + size;

        ext4_mb_pa_adjust_overlap(ac, &start, &end);

        size = end - start;

        /*
         * In this function "start" and "size" are normalized for better
         * alignment and length such that we could preallocate more blocks.
         * This normalization is done such that original request of
         * ac->ac_o_ex.fe_logical & fe_len should always lie within "start" and
         * "size" boundaries.
         * (Note fe_len can be relaxed since FS block allocation API does not
         * provide gurantee on number of contiguous blocks allocation since that
         * depends upon free space left, etc).
         * In case of inode pa, later we use the allocated blocks
         * [pa_pstart + fe_logical - pa_lstart, fe_len/size] from the preallocated
         * range of goal/best blocks [start, size] to put it at the
         * ac_o_ex.fe_logical extent of this inode.
         * (See ext4_mb_use_inode_pa() for more details)
         */
        if (start + size <= ac->ac_o_ex.fe_logical ||
                        start > ac->ac_o_ex.fe_logical) {
                ext4_msg(ac->ac_sb, KERN_ERR,
                         "start %lu, size %lu, fe_logical %lu",
                         (unsigned long) start, (unsigned long) size,
                         (unsigned long) ac->ac_o_ex.fe_logical);
                BUG();
        }
        BUG_ON(size <= 0 || size > EXT4_BLOCKS_PER_GROUP(ac->ac_sb));

        /* now prepare goal request */

        /* XXX: is it better to align blocks WRT to logical
         * placement or satisfy big request as is */
        ac->ac_g_ex.fe_logical = start;
        ac->ac_g_ex.fe_len = EXT4_NUM_B2C(sbi, size);
        ac->ac_orig_goal_len = ac->ac_g_ex.fe_len;

        /* define goal start in order to merge */
        if (ar->pright && (ar->lright == (start + size)) &&
            ar->pright >= size &&
            ar->pright - size >= le32_to_cpu(es->s_first_data_block)) {
                /* merge to the right */
                ext4_get_group_no_and_offset(ac->ac_sb, ar->pright - size,
                                                &ac->ac_g_ex.fe_group,
                                                &ac->ac_g_ex.fe_start);
                ac->ac_flags |= EXT4_MB_HINT_TRY_GOAL;
        }
        if (ar->pleft && (ar->lleft + 1 == start) &&
            ar->pleft + 1 < ext4_blocks_count(es)) {
                /* merge to the left */
                ext4_get_group_no_and_offset(ac->ac_sb, ar->pleft + 1,
                                                &ac->ac_g_ex.fe_group,
                                                &ac->ac_g_ex.fe_start);
                ac->ac_flags |= EXT4_MB_HINT_TRY_GOAL;
        }

        mb_debug(ac->ac_sb, "goal: %lld(was %lld) blocks at %u\n", size,
                 orig_size, start);
}

static void ext4_mb_collect_stats(struct ext4_allocation_context *ac)
{
        struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);

        if (sbi->s_mb_stats && ac->ac_g_ex.fe_len >= 1) {
                atomic_inc(&sbi->s_bal_reqs);
                atomic_add(ac->ac_b_ex.fe_len, &sbi->s_bal_allocated);
                if (ac->ac_b_ex.fe_len >= ac->ac_o_ex.fe_len)
                        atomic_inc(&sbi->s_bal_success);

                atomic_add(ac->ac_found, &sbi->s_bal_ex_scanned);
                for (int i=0; i<EXT4_MB_NUM_CRS; i++) {
                        atomic_add(ac->ac_cX_found[i], &sbi->s_bal_cX_ex_scanned[i]);
                }

                atomic_add(ac->ac_groups_scanned, &sbi->s_bal_groups_scanned);
                if (ac->ac_g_ex.fe_start == ac->ac_b_ex.fe_start &&
                                ac->ac_g_ex.fe_group == ac->ac_b_ex.fe_group)
                        atomic_inc(&sbi->s_bal_goals);
                /* did we allocate as much as normalizer originally wanted? */
                if (ac->ac_f_ex.fe_len == ac->ac_orig_goal_len)
                        atomic_inc(&sbi->s_bal_len_goals);

                if (ac->ac_found > sbi->s_mb_max_to_scan)
                        atomic_inc(&sbi->s_bal_breaks);
        }

        if (ac->ac_op == EXT4_MB_HISTORY_ALLOC)
                trace_ext4_mballoc_alloc(ac);
        else
                trace_ext4_mballoc_prealloc(ac);
}

/*
 * Called on failure; free up any blocks from the inode PA for this
 * context.  We don't need this for MB_GROUP_PA because we only change
 * pa_free in ext4_mb_release_context(), but on failure, we've already
 * zeroed out ac->ac_b_ex.fe_len, so group_pa->pa_free is not changed.
 */
static void ext4_discard_allocated_blocks(struct ext4_allocation_context *ac)
{
        struct ext4_prealloc_space *pa = ac->ac_pa;
        struct ext4_buddy e4b;
        int err;

        if (pa == NULL) {
                if (ac->ac_f_ex.fe_len == 0)
                        return;
                err = ext4_mb_load_buddy(ac->ac_sb, ac->ac_f_ex.fe_group, &e4b);
                if (WARN_RATELIMIT(err,
                                   "ext4: mb_load_buddy failed (%d)", err))
                        /*
                         * This should never happen since we pin the
                         * pages in the ext4_allocation_context so
                         * ext4_mb_load_buddy() should never fail.
                         */
                        return;
                ext4_lock_group(ac->ac_sb, ac->ac_f_ex.fe_group);
                mb_free_blocks(ac->ac_inode, &e4b, ac->ac_f_ex.fe_start,
                               ac->ac_f_ex.fe_len);
                ext4_unlock_group(ac->ac_sb, ac->ac_f_ex.fe_group);
                ext4_mb_unload_buddy(&e4b);
                return;
        }
        if (pa->pa_type == MB_INODE_PA) {
                spin_lock(&pa->pa_lock);
                pa->pa_free += ac->ac_b_ex.fe_len;
                spin_unlock(&pa->pa_lock);
        }
}

/*
 * use blocks preallocated to inode
 */
static void ext4_mb_use_inode_pa(struct ext4_allocation_context *ac,
                                struct ext4_prealloc_space *pa)
{
        struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);
        ext4_fsblk_t start;
        ext4_fsblk_t end;
        int len;

        /* found preallocated blocks, use them */
        start = pa->pa_pstart + (ac->ac_o_ex.fe_logical - pa->pa_lstart);
        end = min(pa->pa_pstart + EXT4_C2B(sbi, pa->pa_len),
                  start + EXT4_C2B(sbi, ac->ac_o_ex.fe_len));
        len = EXT4_NUM_B2C(sbi, end - start);
        ext4_get_group_no_and_offset(ac->ac_sb, start, &ac->ac_b_ex.fe_group,
                                        &ac->ac_b_ex.fe_start);
        ac->ac_b_ex.fe_len = len;
        ac->ac_status = AC_STATUS_FOUND;
        ac->ac_pa = pa;

        BUG_ON(start < pa->pa_pstart);
        BUG_ON(end > pa->pa_pstart + EXT4_C2B(sbi, pa->pa_len));
        BUG_ON(pa->pa_free < len);
        BUG_ON(ac->ac_b_ex.fe_len <= 0);
        pa->pa_free -= len;

        mb_debug(ac->ac_sb, "use %llu/%d from inode pa %p\n", start, len, pa);
}

/*
 * use blocks preallocated to locality group
 */
static void ext4_mb_use_group_pa(struct ext4_allocation_context *ac,
                                struct ext4_prealloc_space *pa)
{
        unsigned int len = ac->ac_o_ex.fe_len;

        ext4_get_group_no_and_offset(ac->ac_sb, pa->pa_pstart,
                                        &ac->ac_b_ex.fe_group,
                                        &ac->ac_b_ex.fe_start);
        ac->ac_b_ex.fe_len = len;
        ac->ac_status = AC_STATUS_FOUND;
        ac->ac_pa = pa;

        /* we don't correct pa_pstart or pa_len here to avoid
         * possible race when the group is being loaded concurrently
         * instead we correct pa later, after blocks are marked
         * in on-disk bitmap -- see ext4_mb_release_context()
         * Other CPUs are prevented from allocating from this pa by lg_mutex
         */
        mb_debug(ac->ac_sb, "use %u/%u from group pa %p\n",
                 pa->pa_lstart, len, pa);
}

/*
 * Return the prealloc space that have minimal distance
 * from the goal block. @cpa is the prealloc
 * space that is having currently known minimal distance
 * from the goal block.
 */
static struct ext4_prealloc_space *
ext4_mb_check_group_pa(ext4_fsblk_t goal_block,
                        struct ext4_prealloc_space *pa,
                        struct ext4_prealloc_space *cpa)
{
        ext4_fsblk_t cur_distance, new_distance;

        if (cpa == NULL) {
                atomic_inc(&pa->pa_count);
                return pa;
        }
        cur_distance = abs(goal_block - cpa->pa_pstart);
        new_distance = abs(goal_block - pa->pa_pstart);

        if (cur_distance <= new_distance)
                return cpa;

        /* drop the previous reference */
        atomic_dec(&cpa->pa_count);
        atomic_inc(&pa->pa_count);
        return pa;
}

/*
 * check if found pa meets EXT4_MB_HINT_GOAL_ONLY
 */
static bool
ext4_mb_pa_goal_check(struct ext4_allocation_context *ac,
                      struct ext4_prealloc_space *pa)
{
        struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);
        ext4_fsblk_t start;

        if (likely(!(ac->ac_flags & EXT4_MB_HINT_GOAL_ONLY)))
                return true;

        /*
         * If EXT4_MB_HINT_GOAL_ONLY is set, ac_g_ex will not be adjusted
         * in ext4_mb_normalize_request and will keep same with ac_o_ex
         * from ext4_mb_initialize_context. Choose ac_g_ex here to keep
         * consistent with ext4_mb_find_by_goal.
         */
        start = pa->pa_pstart +
                (ac->ac_g_ex.fe_logical - pa->pa_lstart);
        if (ext4_grp_offs_to_block(ac->ac_sb, &ac->ac_g_ex) != start)
                return false;

        if (ac->ac_g_ex.fe_len > pa->pa_len -
            EXT4_B2C(sbi, ac->ac_g_ex.fe_logical - pa->pa_lstart))
                return false;

        return true;
}

/*
 * search goal blocks in preallocated space
 */
static noinline_for_stack bool
ext4_mb_use_preallocated(struct ext4_allocation_context *ac)
{
        struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);
        int order, i;
        struct ext4_inode_info *ei = EXT4_I(ac->ac_inode);
        struct ext4_locality_group *lg;
        struct ext4_prealloc_space *tmp_pa = NULL, *cpa = NULL;
        struct rb_node *iter;
        ext4_fsblk_t goal_block;

        /* only data can be preallocated */
        if (!(ac->ac_flags & EXT4_MB_HINT_DATA))
                return false;

        /*
         * first, try per-file preallocation by searching the inode pa rbtree.
         *
         * Here, we can't do a direct traversal of the tree because
         * ext4_mb_discard_group_preallocation() can paralelly mark the pa
         * deleted and that can cause direct traversal to skip some entries.
         */
        read_lock(&ei->i_prealloc_lock);

        if (RB_EMPTY_ROOT(&ei->i_prealloc_node)) {
                goto try_group_pa;
        }

        /*
         * Step 1: Find a pa with logical start immediately adjacent to the
         * original logical start. This could be on the left or right.
         *
         * (tmp_pa->pa_lstart never changes so we can skip locking for it).
         */
        for (iter = ei->i_prealloc_node.rb_node; iter;
             iter = ext4_mb_pa_rb_next_iter(ac->ac_o_ex.fe_logical,
                                            tmp_pa->pa_lstart, iter)) {
                tmp_pa = rb_entry(iter, struct ext4_prealloc_space,
                                  pa_node.inode_node);
        }

        /*
         * Step 2: The adjacent pa might be to the right of logical start, find
         * the left adjacent pa. After this step we'd have a valid tmp_pa whose
         * logical start is towards the left of original request's logical start
         */
        if (tmp_pa->pa_lstart > ac->ac_o_ex.fe_logical) {
                struct rb_node *tmp;
                tmp = rb_prev(&tmp_pa->pa_node.inode_node);

                if (tmp) {
                        tmp_pa = rb_entry(tmp, struct ext4_prealloc_space,
                                            pa_node.inode_node);
                } else {
                        /*
                         * If there is no adjacent pa to the left then finding
                         * an overlapping pa is not possible hence stop searching
                         * inode pa tree
                         */
                        goto try_group_pa;
                }
        }

        BUG_ON(!(tmp_pa && tmp_pa->pa_lstart <= ac->ac_o_ex.fe_logical));

        /*
         * Step 3: If the left adjacent pa is deleted, keep moving left to find
         * the first non deleted adjacent pa. After this step we should have a
         * valid tmp_pa which is guaranteed to be non deleted.
         */
        for (iter = &tmp_pa->pa_node.inode_node;; iter = rb_prev(iter)) {
                if (!iter) {
                        /*
                         * no non deleted left adjacent pa, so stop searching
                         * inode pa tree
                         */
                        goto try_group_pa;
                }
                tmp_pa = rb_entry(iter, struct ext4_prealloc_space,
                                  pa_node.inode_node);
                spin_lock(&tmp_pa->pa_lock);
                if (tmp_pa->pa_deleted == 0) {
                        /*
                         * We will keep holding the pa_lock from
                         * this point on because we don't want group discard
                         * to delete this pa underneath us. Since group
                         * discard is anyways an ENOSPC operation it
                         * should be okay for it to wait a few more cycles.
                         */
                        break;
                } else {
                        spin_unlock(&tmp_pa->pa_lock);
                }
        }

        BUG_ON(!(tmp_pa && tmp_pa->pa_lstart <= ac->ac_o_ex.fe_logical));
        BUG_ON(tmp_pa->pa_deleted == 1);

        /*
         * Step 4: We now have the non deleted left adjacent pa. Only this
         * pa can possibly satisfy the request hence check if it overlaps
         * original logical start and stop searching if it doesn't.
         */
        if (ac->ac_o_ex.fe_logical >= pa_logical_end(sbi, tmp_pa)) {
                spin_unlock(&tmp_pa->pa_lock);
                goto try_group_pa;
        }

        /* non-extent files can't have physical blocks past 2^32 */
        if (!(ext4_test_inode_flag(ac->ac_inode, EXT4_INODE_EXTENTS)) &&
            (tmp_pa->pa_pstart + EXT4_C2B(sbi, tmp_pa->pa_len) >
             EXT4_MAX_BLOCK_FILE_PHYS)) {
                /*
                 * Since PAs don't overlap, we won't find any other PA to
                 * satisfy this.
                 */
                spin_unlock(&tmp_pa->pa_lock);
                goto try_group_pa;
        }

        if (tmp_pa->pa_free && likely(ext4_mb_pa_goal_check(ac, tmp_pa))) {
                atomic_inc(&tmp_pa->pa_count);
                ext4_mb_use_inode_pa(ac, tmp_pa);
                spin_unlock(&tmp_pa->pa_lock);
                read_unlock(&ei->i_prealloc_lock);
                return true;
        } else {
                /*
                 * We found a valid overlapping pa but couldn't use it because
                 * it had no free blocks. This should ideally never happen
                 * because:
                 *
                 * 1. When a new inode pa is added to rbtree it must have
                 *    pa_free > 0 since otherwise we won't actually need
                 *    preallocation.
                 *
                 * 2. An inode pa that is in the rbtree can only have it's
                 *    pa_free become zero when another thread calls:
                 *      ext4_mb_new_blocks
                 *       ext4_mb_use_preallocated
                 *        ext4_mb_use_inode_pa
                 *
                 * 3. Further, after the above calls make pa_free == 0, we will
                 *    immediately remove it from the rbtree in:
                 *      ext4_mb_new_blocks
                 *       ext4_mb_release_context
                 *        ext4_mb_put_pa
                 *
                 * 4. Since the pa_free becoming 0 and pa_free getting removed
                 * from tree both happen in ext4_mb_new_blocks, which is always
                 * called with i_data_sem held for data allocations, we can be
                 * sure that another process will never see a pa in rbtree with
                 * pa_free == 0.
                 */
                WARN_ON_ONCE(tmp_pa->pa_free == 0);
        }
        spin_unlock(&tmp_pa->pa_lock);
try_group_pa:
        read_unlock(&ei->i_prealloc_lock);

        /* can we use group allocation? */
        if (!(ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC))
                return false;

        /* inode may have no locality group for some reason */
        lg = ac->ac_lg;
        if (lg == NULL)
                return false;
        order  = fls(ac->ac_o_ex.fe_len) - 1;
        if (order > PREALLOC_TB_SIZE - 1)
                /* The max size of hash table is PREALLOC_TB_SIZE */
                order = PREALLOC_TB_SIZE - 1;

        goal_block = ext4_grp_offs_to_block(ac->ac_sb, &ac->ac_g_ex);
        /*
         * search for the prealloc space that is having