Total coverage: 65634 (5%)of 1584490 






































    5 


















































































    5 














   11 














  976 







  977 





    9 























    5 







    2 

























































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
// SPDX-License-Identifier: GPL-2.0-only
#include <linux/etherdevice.h>
#include <linux/if_macvlan.h>
#include <linux/if_tap.h>
#include <linux/if_vlan.h>
#include <linux/interrupt.h>
#include <linux/nsproxy.h>
#include <linux/compat.h>
#include <linux/if_tun.h>
#include <linux/module.h>
#include <linux/skbuff.h>
#include <linux/cache.h>
#include <linux/sched/signal.h>
#include <linux/types.h>
#include <linux/slab.h>
#include <linux/wait.h>
#include <linux/cdev.h>
#include <linux/idr.h>
#include <linux/fs.h>
#include <linux/uio.h>

#include <net/net_namespace.h>
#include <net/rtnetlink.h>
#include <net/sock.h>
#include <linux/virtio_net.h>
#include <linux/skb_array.h>

struct macvtap_dev {
        struct macvlan_dev vlan;
        struct tap_dev    tap;
};

/*
 * Variables for dealing with macvtaps device numbers.
 */
static dev_t macvtap_major;

static const void *macvtap_net_namespace(struct device *d)
{
        struct net_device *dev = to_net_dev(d->parent);
        return dev_net(dev);
}

static struct class macvtap_class = {
        .name = "macvtap",
        .owner = THIS_MODULE,
        .ns_type = &net_ns_type_operations,
        .namespace = macvtap_net_namespace,
};
static struct cdev macvtap_cdev;

#define TUN_OFFLOADS (NETIF_F_HW_CSUM | NETIF_F_TSO_ECN | NETIF_F_TSO | \
                      NETIF_F_TSO6)

static void macvtap_count_tx_dropped(struct tap_dev *tap)
{
        struct macvtap_dev *vlantap = container_of(tap, struct macvtap_dev, tap);
        struct macvlan_dev *vlan = &vlantap->vlan;

        this_cpu_inc(vlan->pcpu_stats->tx_dropped);
}

static void macvtap_count_rx_dropped(struct tap_dev *tap)
{
        struct macvtap_dev *vlantap = container_of(tap, struct macvtap_dev, tap);
        struct macvlan_dev *vlan = &vlantap->vlan;

        macvlan_count_rx(vlan, 0, 0, 0);
}

static void macvtap_update_features(struct tap_dev *tap,
                                    netdev_features_t features)
{
        struct macvtap_dev *vlantap = container_of(tap, struct macvtap_dev, tap);
        struct macvlan_dev *vlan = &vlantap->vlan;

        vlan->set_features = features;
        netdev_update_features(vlan->dev);
}

static int macvtap_newlink(struct net *src_net, struct net_device *dev,
                           struct nlattr *tb[], struct nlattr *data[],
                           struct netlink_ext_ack *extack)
{
        struct macvtap_dev *vlantap = netdev_priv(dev);
        int err;

        INIT_LIST_HEAD(&vlantap->tap.queue_list);

        /* Since macvlan supports all offloads by default, make
         * tap support all offloads also.
         */
        vlantap->tap.tap_features = TUN_OFFLOADS;

        /* Register callbacks for rx/tx drops accounting and updating
         * net_device features
         */
        vlantap->tap.count_tx_dropped = macvtap_count_tx_dropped;
        vlantap->tap.count_rx_dropped = macvtap_count_rx_dropped;
        vlantap->tap.update_features  = macvtap_update_features;

        err = netdev_rx_handler_register(dev, tap_handle_frame, &vlantap->tap);
        if (err)
                return err;

        /* Don't put anything that may fail after macvlan_common_newlink
         * because we can't undo what it does.
         */
        err = macvlan_common_newlink(src_net, dev, tb, data, extack);
        if (err) {
                netdev_rx_handler_unregister(dev);
                return err;
        }

        vlantap->tap.dev = vlantap->vlan.dev;

        return 0;
}

static void macvtap_dellink(struct net_device *dev,
                            struct list_head *head)
{
        struct macvtap_dev *vlantap = netdev_priv(dev);

        netdev_rx_handler_unregister(dev);
        tap_del_queues(&vlantap->tap);
        macvlan_dellink(dev, head);
}

static void macvtap_setup(struct net_device *dev)
{
        macvlan_common_setup(dev);
        dev->tx_queue_len = TUN_READQ_SIZE;
}

static struct net *macvtap_link_net(const struct net_device *dev)
{
        return dev_net(macvlan_dev_real_dev(dev));
}

static struct rtnl_link_ops macvtap_link_ops __read_mostly = {
        .kind                = "macvtap",
        .setup                = macvtap_setup,
        .newlink        = macvtap_newlink,
        .dellink        = macvtap_dellink,
        .get_link_net        = macvtap_link_net,
        .priv_size      = sizeof(struct macvtap_dev),
};

static int macvtap_device_event(struct notifier_block *unused,
                                unsigned long event, void *ptr)
{
        struct net_device *dev = netdev_notifier_info_to_dev(ptr);
        struct macvtap_dev *vlantap;
        struct device *classdev;
        dev_t devt;
        int err;
        char tap_name[IFNAMSIZ];

        if (dev->rtnl_link_ops != &macvtap_link_ops)
                return NOTIFY_DONE;

        snprintf(tap_name, IFNAMSIZ, "tap%d", dev->ifindex);
        vlantap = netdev_priv(dev);

        switch (event) {
        case NETDEV_REGISTER:
                /* Create the device node here after the network device has
                 * been registered but before register_netdevice has
                 * finished running.
                 */
                err = tap_get_minor(macvtap_major, &vlantap->tap);
                if (err)
                        return notifier_from_errno(err);

                devt = MKDEV(MAJOR(macvtap_major), vlantap->tap.minor);
                classdev = device_create(&macvtap_class, &dev->dev, devt,
                                         dev, tap_name);
                if (IS_ERR(classdev)) {
                        tap_free_minor(macvtap_major, &vlantap->tap);
                        return notifier_from_errno(PTR_ERR(classdev));
                }
                err = sysfs_create_link(&dev->dev.kobj, &classdev->kobj,
                                        tap_name);
                if (err)
                        return notifier_from_errno(err);
                break;
        case NETDEV_UNREGISTER:
                /* vlan->minor == 0 if NETDEV_REGISTER above failed */
                if (vlantap->tap.minor == 0)
                        break;
                sysfs_remove_link(&dev->dev.kobj, tap_name);
                devt = MKDEV(MAJOR(macvtap_major), vlantap->tap.minor);
                device_destroy(&macvtap_class, devt);
                tap_free_minor(macvtap_major, &vlantap->tap);
                break;
        case NETDEV_CHANGE_TX_QUEUE_LEN:
                if (tap_queue_resize(&vlantap->tap))
                        return NOTIFY_BAD;
                break;
        }

        return NOTIFY_DONE;
}

static struct notifier_block macvtap_notifier_block __read_mostly = {
        .notifier_call        = macvtap_device_event,
};

static int macvtap_init(void)
{
        int err;

        err = tap_create_cdev(&macvtap_cdev, &macvtap_major, "macvtap",
                              THIS_MODULE);
        if (err)
                goto out1;

        err = class_register(&macvtap_class);
        if (err)
                goto out2;

        err = register_netdevice_notifier(&macvtap_notifier_block);
        if (err)
                goto out3;

        err = macvlan_link_register(&macvtap_link_ops);
        if (err)
                goto out4;

        return 0;

out4:
        unregister_netdevice_notifier(&macvtap_notifier_block);
out3:
        class_unregister(&macvtap_class);
out2:
        tap_destroy_cdev(macvtap_major, &macvtap_cdev);
out1:
        return err;
}
module_init(macvtap_init);

static void macvtap_exit(void)
{
        rtnl_link_unregister(&macvtap_link_ops);
        unregister_netdevice_notifier(&macvtap_notifier_block);
        class_unregister(&macvtap_class);
        tap_destroy_cdev(macvtap_major, &macvtap_cdev);
}
module_exit(macvtap_exit);

MODULE_ALIAS_RTNL_LINK("macvtap");
MODULE_AUTHOR("Arnd Bergmann <arnd@arndb.de>");
MODULE_LICENSE("GPL");














































































   49 


   49 

























   49 
   49 








   49 









1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
// SPDX-License-Identifier: GPL-2.0
/*
 * procfs-based user access to knfsd statistics
 *
 * /proc/net/rpc/nfsd
 *
 * Format:
 *        rc <hits> <misses> <nocache>
 *                        Statistsics for the reply cache
 *        fh <stale> <deprecated filehandle cache stats>
 *                        statistics for filehandle lookup
 *        io <bytes-read> <bytes-written>
 *                        statistics for IO throughput
 *        th <threads> <deprecated thread usage histogram stats>
 *                        number of threads
 *        ra <deprecated ra-cache stats>
 *
 *        plus generic RPC stats (see net/sunrpc/stats.c)
 *
 * Copyright (C) 1995, 1996, 1997 Olaf Kirch <okir@monad.swb.de>
 */

#include <linux/seq_file.h>
#include <linux/module.h>
#include <linux/sunrpc/stats.h>
#include <net/net_namespace.h>

#include "nfsd.h"

static int nfsd_show(struct seq_file *seq, void *v)
{
        struct net *net = PDE_DATA(file_inode(seq->file));
        struct nfsd_net *nn = net_generic(net, nfsd_net_id);
        int i;

        seq_printf(seq, "rc %lld %lld %lld\nfh %lld 0 0 0 0\nio %lld %lld\n",
                   percpu_counter_sum_positive(&nn->counter[NFSD_STATS_RC_HITS]),
                   percpu_counter_sum_positive(&nn->counter[NFSD_STATS_RC_MISSES]),
                   percpu_counter_sum_positive(&nn->counter[NFSD_STATS_RC_NOCACHE]),
                   percpu_counter_sum_positive(&nn->counter[NFSD_STATS_FH_STALE]),
                   percpu_counter_sum_positive(&nn->counter[NFSD_STATS_IO_READ]),
                   percpu_counter_sum_positive(&nn->counter[NFSD_STATS_IO_WRITE]));

        /* thread usage: */
        seq_printf(seq, "th %u 0", atomic_read(&nfsd_th_cnt));

        /* deprecated thread usage histogram stats */
        for (i = 0; i < 10; i++)
                seq_puts(seq, " 0.000");

        /* deprecated ra-cache stats */
        seq_puts(seq, "\nra 0 0 0 0 0 0 0 0 0 0 0 0\n");

        /* show my rpc info */
        svc_seq_show(seq, &nn->nfsd_svcstats);

#ifdef CONFIG_NFSD_V4
        /* Show count for individual nfsv4 operations */
        /* Writing operation numbers 0 1 2 also for maintaining uniformity */
        seq_printf(seq,"proc4ops %u", LAST_NFS4_OP + 1);
        for (i = 0; i <= LAST_NFS4_OP; i++) {
                seq_printf(seq, " %lld",
                           percpu_counter_sum_positive(&nn->counter[NFSD_STATS_NFS4_OP(i)]));
        }

        seq_putc(seq, '\n');
#endif

        return 0;
}

DEFINE_PROC_SHOW_ATTRIBUTE(nfsd);

int nfsd_percpu_counters_init(struct percpu_counter *counters, int num)
{
        int i, err = 0;

        for (i = 0; !err && i < num; i++)
                err = percpu_counter_init(&counters[i], 0, GFP_KERNEL);

        if (!err)
                return 0;

        for (; i > 0; i--)
                percpu_counter_destroy(&counters[i-1]);

        return err;
}

void nfsd_percpu_counters_reset(struct percpu_counter counters[], int num)
{
        int i;

        for (i = 0; i < num; i++)
                percpu_counter_set(&counters[i], 0);
}

void nfsd_percpu_counters_destroy(struct percpu_counter counters[], int num)
{
        int i;

        for (i = 0; i < num; i++)
                percpu_counter_destroy(&counters[i]);
}

int nfsd_stat_counters_init(struct nfsd_net *nn)
{
        return nfsd_percpu_counters_init(nn->counter, NFSD_STATS_COUNTERS_NUM);
}

void nfsd_stat_counters_destroy(struct nfsd_net *nn)
{
        nfsd_percpu_counters_destroy(nn->counter, NFSD_STATS_COUNTERS_NUM);
}

void nfsd_proc_stat_init(struct net *net)
{
        struct nfsd_net *nn = net_generic(net, nfsd_net_id);

        svc_proc_register(net, &nn->nfsd_svcstats, &nfsd_proc_ops);
}

void nfsd_proc_stat_shutdown(struct net *net)
{
        svc_proc_unregister(net, "nfsd");
}













































































































    4 















1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
// SPDX-License-Identifier: GPL-2.0
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/device.h>
#include <linux/netdevice.h>

#include <net/bonding.h>
#include <net/bond_alb.h>

#if defined(CONFIG_DEBUG_FS) && !defined(CONFIG_NET_NS)

#include <linux/debugfs.h>
#include <linux/seq_file.h>

static struct dentry *bonding_debug_root;

/* Show RLB hash table */
static int bond_debug_rlb_hash_show(struct seq_file *m, void *v)
{
        struct bonding *bond = m->private;
        struct alb_bond_info *bond_info = &(BOND_ALB_INFO(bond));
        struct rlb_client_info *client_info;
        u32 hash_index;

        if (BOND_MODE(bond) != BOND_MODE_ALB)
                return 0;

        seq_printf(m, "SourceIP        DestinationIP   "
                        "Destination MAC   DEV\n");

        spin_lock_bh(&bond->mode_lock);

        hash_index = bond_info->rx_hashtbl_used_head;
        for (; hash_index != RLB_NULL_INDEX;
             hash_index = client_info->used_next) {
                client_info = &(bond_info->rx_hashtbl[hash_index]);
                seq_printf(m, "%-15pI4 %-15pI4 %-17pM %s\n",
                        &client_info->ip_src,
                        &client_info->ip_dst,
                        &client_info->mac_dst,
                        client_info->slave->dev->name);
        }

        spin_unlock_bh(&bond->mode_lock);

        return 0;
}
DEFINE_SHOW_ATTRIBUTE(bond_debug_rlb_hash);

void bond_debug_register(struct bonding *bond)
{
        if (!bonding_debug_root)
                return;

        bond->debug_dir =
                debugfs_create_dir(bond->dev->name, bonding_debug_root);

        debugfs_create_file("rlb_hash_table", 0400, bond->debug_dir,
                                bond, &bond_debug_rlb_hash_fops);
}

void bond_debug_unregister(struct bonding *bond)
{
        if (!bonding_debug_root)
                return;

        debugfs_remove_recursive(bond->debug_dir);
}

void bond_debug_reregister(struct bonding *bond)
{
        struct dentry *d;

        if (!bonding_debug_root)
                return;

        d = debugfs_rename(bonding_debug_root, bond->debug_dir,
                           bonding_debug_root, bond->dev->name);
        if (!IS_ERR(d)) {
                bond->debug_dir = d;
        } else {
                netdev_warn(bond->dev, "failed to reregister, so just unregister old one\n");
                bond_debug_unregister(bond);
        }
}

void bond_create_debugfs(void)
{
        bonding_debug_root = debugfs_create_dir("bonding", NULL);

        if (!bonding_debug_root)
                pr_warn("Warning: Cannot create bonding directory in debugfs\n");
}

void bond_destroy_debugfs(void)
{
        debugfs_remove_recursive(bonding_debug_root);
        bonding_debug_root = NULL;
}


#else /* !CONFIG_DEBUG_FS */

void bond_debug_register(struct bonding *bond)
{
}

void bond_debug_unregister(struct bonding *bond)
{
}

void bond_debug_reregister(struct bonding *bond)
{
}

void bond_create_debugfs(void)
{
}

void bond_destroy_debugfs(void)
{
}

#endif /* CONFIG_DEBUG_FS */































   18 









   18 






















































    7 





































    7 






















































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright 2002-2004, Instant802 Networks, Inc.
 * Copyright 2008, Jouni Malinen <j@w1.fi>
 * Copyright (C) 2016-2017 Intel Deutschland GmbH
 * Copyright (C) 2020-2021 Intel Corporation
 */

#include <linux/netdevice.h>
#include <linux/types.h>
#include <linux/skbuff.h>
#include <linux/compiler.h>
#include <linux/ieee80211.h>
#include <linux/gfp.h>
#include <asm/unaligned.h>
#include <net/mac80211.h>
#include <crypto/aes.h>
#include <crypto/algapi.h>

#include "ieee80211_i.h"
#include "michael.h"
#include "tkip.h"
#include "aes_ccm.h"
#include "aes_cmac.h"
#include "aes_gmac.h"
#include "aes_gcm.h"
#include "wpa.h"

ieee80211_tx_result
ieee80211_tx_h_michael_mic_add(struct ieee80211_tx_data *tx)
{
        u8 *data, *key, *mic;
        size_t data_len;
        unsigned int hdrlen;
        struct ieee80211_hdr *hdr;
        struct sk_buff *skb = tx->skb;
        struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb);
        int tail;

        hdr = (struct ieee80211_hdr *)skb->data;
        if (!tx->key || tx->key->conf.cipher != WLAN_CIPHER_SUITE_TKIP ||
            skb->len < 24 || !ieee80211_is_data_present(hdr->frame_control))
                return TX_CONTINUE;

        hdrlen = ieee80211_hdrlen(hdr->frame_control);
        if (skb->len < hdrlen)
                return TX_DROP;

        data = skb->data + hdrlen;
        data_len = skb->len - hdrlen;

        if (unlikely(info->flags & IEEE80211_TX_INTFL_TKIP_MIC_FAILURE)) {
                /* Need to use software crypto for the test */
                info->control.hw_key = NULL;
        }

        if (info->control.hw_key &&
            (info->flags & IEEE80211_TX_CTL_DONTFRAG ||
             ieee80211_hw_check(&tx->local->hw, SUPPORTS_TX_FRAG)) &&
            !(tx->key->conf.flags & (IEEE80211_KEY_FLAG_GENERATE_MMIC |
                                     IEEE80211_KEY_FLAG_PUT_MIC_SPACE))) {
                /* hwaccel - with no need for SW-generated MMIC or MIC space */
                return TX_CONTINUE;
        }

        tail = MICHAEL_MIC_LEN;
        if (!info->control.hw_key)
                tail += IEEE80211_TKIP_ICV_LEN;

        if (WARN(skb_tailroom(skb) < tail ||
                 skb_headroom(skb) < IEEE80211_TKIP_IV_LEN,
                 "mmic: not enough head/tail (%d/%d,%d/%d)\n",
                 skb_headroom(skb), IEEE80211_TKIP_IV_LEN,
                 skb_tailroom(skb), tail))
                return TX_DROP;

        mic = skb_put(skb, MICHAEL_MIC_LEN);

        if (tx->key->conf.flags & IEEE80211_KEY_FLAG_PUT_MIC_SPACE) {
                /* Zeroed MIC can help with debug */
                memset(mic, 0, MICHAEL_MIC_LEN);
                return TX_CONTINUE;
        }

        key = &tx->key->conf.key[NL80211_TKIP_DATA_OFFSET_TX_MIC_KEY];
        michael_mic(key, hdr, data, data_len, mic);
        if (unlikely(info->flags & IEEE80211_TX_INTFL_TKIP_MIC_FAILURE))
                mic[0]++;

        return TX_CONTINUE;
}


ieee80211_rx_result
ieee80211_rx_h_michael_mic_verify(struct ieee80211_rx_data *rx)
{
        u8 *data, *key = NULL;
        size_t data_len;
        unsigned int hdrlen;
        u8 mic[MICHAEL_MIC_LEN];
        struct sk_buff *skb = rx->skb;
        struct ieee80211_rx_status *status = IEEE80211_SKB_RXCB(skb);
        struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data;

        /*
         * it makes no sense to check for MIC errors on anything other
         * than data frames.
         */
        if (!ieee80211_is_data_present(hdr->frame_control))
                return RX_CONTINUE;

        /*
         * No way to verify the MIC if the hardware stripped it or
         * the IV with the key index. In this case we have solely rely
         * on the driver to set RX_FLAG_MMIC_ERROR in the event of a
         * MIC failure report.
         */
        if (status->flag & (RX_FLAG_MMIC_STRIPPED | RX_FLAG_IV_STRIPPED)) {
                if (status->flag & RX_FLAG_MMIC_ERROR)
                        goto mic_fail_no_key;

                if (!(status->flag & RX_FLAG_IV_STRIPPED) && rx->key &&
                    rx->key->conf.cipher == WLAN_CIPHER_SUITE_TKIP)
                        goto update_iv;

                return RX_CONTINUE;
        }

        /*
         * Some hardware seems to generate Michael MIC failure reports; even
         * though, the frame was not encrypted with TKIP and therefore has no
         * MIC. Ignore the flag them to avoid triggering countermeasures.
         */
        if (!rx->key || rx->key->conf.cipher != WLAN_CIPHER_SUITE_TKIP ||
            !(status->flag & RX_FLAG_DECRYPTED))
                return RX_CONTINUE;

        if (rx->sdata->vif.type == NL80211_IFTYPE_AP && rx->key->conf.keyidx) {
                /*
                 * APs with pairwise keys should never receive Michael MIC
                 * errors for non-zero keyidx because these are reserved for
                 * group keys and only the AP is sending real multicast
                 * frames in the BSS.
                 */
                return RX_DROP_UNUSABLE;
        }

        if (status->flag & RX_FLAG_MMIC_ERROR)
                goto mic_fail;

        hdrlen = ieee80211_hdrlen(hdr->frame_control);
        if (skb->len < hdrlen + MICHAEL_MIC_LEN)
                return RX_DROP_UNUSABLE;

        if (skb_linearize(rx->skb))
                return RX_DROP_UNUSABLE;
        hdr = (void *)skb->data;

        data = skb->data + hdrlen;
        data_len = skb->len - hdrlen - MICHAEL_MIC_LEN;
        key = &rx->key->conf.key[NL80211_TKIP_DATA_OFFSET_RX_MIC_KEY];
        michael_mic(key, hdr, data, data_len, mic);
        if (crypto_memneq(mic, data + data_len, MICHAEL_MIC_LEN))
                goto mic_fail;

        /* remove Michael MIC from payload */
        skb_trim(skb, skb->len - MICHAEL_MIC_LEN);

update_iv:
        /* update IV in key information to be able to detect replays */
        rx->key->u.tkip.rx[rx->security_idx].iv32 = rx->tkip.iv32;
        rx->key->u.tkip.rx[rx->security_idx].iv16 = rx->tkip.iv16;

        return RX_CONTINUE;

mic_fail:
        rx->key->u.tkip.mic_failures++;

mic_fail_no_key:
        /*
         * In some cases the key can be unset - e.g. a multicast packet, in
         * a driver that supports HW encryption. Send up the key idx only if
         * the key is set.
         */
        cfg80211_michael_mic_failure(rx->sdata->dev, hdr->addr2,
                                     is_multicast_ether_addr(hdr->addr1) ?
                                     NL80211_KEYTYPE_GROUP :
                                     NL80211_KEYTYPE_PAIRWISE,
                                     rx->key ? rx->key->conf.keyidx : -1,
                                     NULL, GFP_ATOMIC);
        return RX_DROP_UNUSABLE;
}

static int tkip_encrypt_skb(struct ieee80211_tx_data *tx, struct sk_buff *skb)
{
        struct ieee80211_hdr *hdr = (struct ieee80211_hdr *) skb->data;
        struct ieee80211_key *key = tx->key;
        struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb);
        unsigned int hdrlen;
        int len, tail;
        u64 pn;
        u8 *pos;

        if (info->control.hw_key &&
            !(info->control.hw_key->flags & IEEE80211_KEY_FLAG_GENERATE_IV) &&
            !(info->control.hw_key->flags & IEEE80211_KEY_FLAG_PUT_IV_SPACE)) {
                /* hwaccel - with no need for software-generated IV */
                return 0;
        }

        hdrlen = ieee80211_hdrlen(hdr->frame_control);
        len = skb->len - hdrlen;

        if (info->control.hw_key)
                tail = 0;
        else
                tail = IEEE80211_TKIP_ICV_LEN;

        if (WARN_ON(skb_tailroom(skb) < tail ||
                    skb_headroom(skb) < IEEE80211_TKIP_IV_LEN))
                return -1;

        pos = skb_push(skb, IEEE80211_TKIP_IV_LEN);
        memmove(pos, pos + IEEE80211_TKIP_IV_LEN, hdrlen);
        pos += hdrlen;

        /* the HW only needs room for the IV, but not the actual IV */
        if (info->control.hw_key &&
            (info->control.hw_key->flags & IEEE80211_KEY_FLAG_PUT_IV_SPACE))
                return 0;

        /* Increase IV for the frame */
        pn = atomic64_inc_return(&key->conf.tx_pn);
        pos = ieee80211_tkip_add_iv(pos, &key->conf, pn);

        /* hwaccel - with software IV */
        if (info->control.hw_key)
                return 0;

        /* Add room for ICV */
        skb_put(skb, IEEE80211_TKIP_ICV_LEN);

        return ieee80211_tkip_encrypt_data(&tx->local->wep_tx_ctx,
                                           key, skb, pos, len);
}


ieee80211_tx_result
ieee80211_crypto_tkip_encrypt(struct ieee80211_tx_data *tx)
{
        struct sk_buff *skb;

        ieee80211_tx_set_protected(tx);

        skb_queue_walk(&tx->skbs, skb) {
                if (tkip_encrypt_skb(tx, skb) < 0)
                        return TX_DROP;
        }

        return TX_CONTINUE;
}


ieee80211_rx_result
ieee80211_crypto_tkip_decrypt(struct ieee80211_rx_data *rx)
{
        struct ieee80211_hdr *hdr = (struct ieee80211_hdr *) rx->skb->data;
        int hdrlen, res, hwaccel = 0;
        struct ieee80211_key *key = rx->key;
        struct sk_buff *skb = rx->skb;
        struct ieee80211_rx_status *status = IEEE80211_SKB_RXCB(skb);

        hdrlen = ieee80211_hdrlen(hdr->frame_control);

        if (!ieee80211_is_data(hdr->frame_control))
                return RX_CONTINUE;

        if (!rx->sta || skb->len - hdrlen < 12)
                return RX_DROP_UNUSABLE;

        /* it may be possible to optimize this a bit more */
        if (skb_linearize(rx->skb))
                return RX_DROP_UNUSABLE;
        hdr = (void *)skb->data;

        /*
         * Let TKIP code verify IV, but skip decryption.
         * In the case where hardware checks the IV as well,
         * we don't even get here, see ieee80211_rx_h_decrypt()
         */
        if (status->flag & RX_FLAG_DECRYPTED)
                hwaccel = 1;

        res = ieee80211_tkip_decrypt_data(&rx->local->wep_rx_ctx,
                                          key, skb->data + hdrlen,
                                          skb->len - hdrlen, rx->sta->sta.addr,
                                          hdr->addr1, hwaccel, rx->security_idx,
                                          &rx->tkip.iv32,
                                          &rx->tkip.iv16);
        if (res != TKIP_DECRYPT_OK)
                return RX_DROP_UNUSABLE;

        /* Trim ICV */
        if (!(status->flag & RX_FLAG_ICV_STRIPPED))
                skb_trim(skb, skb->len - IEEE80211_TKIP_ICV_LEN);

        /* Remove IV */
        memmove(skb->data + IEEE80211_TKIP_IV_LEN, skb->data, hdrlen);
        skb_pull(skb, IEEE80211_TKIP_IV_LEN);

        return RX_CONTINUE;
}


static void ccmp_special_blocks(struct sk_buff *skb, u8 *pn, u8 *b_0, u8 *aad)
{
        __le16 mask_fc;
        int a4_included, mgmt;
        u8 qos_tid;
        u16 len_a;
        unsigned int hdrlen;
        struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data;

        /*
         * Mask FC: zero subtype b4 b5 b6 (if not mgmt)
         * Retry, PwrMgt, MoreData; set Protected
         */
        mgmt = ieee80211_is_mgmt(hdr->frame_control);
        mask_fc = hdr->frame_control;
        mask_fc &= ~cpu_to_le16(IEEE80211_FCTL_RETRY |
                                IEEE80211_FCTL_PM | IEEE80211_FCTL_MOREDATA);
        if (!mgmt)
                mask_fc &= ~cpu_to_le16(0x0070);
        mask_fc |= cpu_to_le16(IEEE80211_FCTL_PROTECTED);

        hdrlen = ieee80211_hdrlen(hdr->frame_control);
        len_a = hdrlen - 2;
        a4_included = ieee80211_has_a4(hdr->frame_control);

        if (ieee80211_is_data_qos(hdr->frame_control))
                qos_tid = ieee80211_get_tid(hdr);
        else
                qos_tid = 0;

        /* In CCM, the initial vectors (IV) used for CTR mode encryption and CBC
         * mode authentication are not allowed to collide, yet both are derived
         * from this vector b_0. We only set L := 1 here to indicate that the
         * data size can be represented in (L+1) bytes. The CCM layer will take
         * care of storing the data length in the top (L+1) bytes and setting
         * and clearing the other bits as is required to derive the two IVs.
         */
        b_0[0] = 0x1;

        /* Nonce: Nonce Flags | A2 | PN
         * Nonce Flags: Priority (b0..b3) | Management (b4) | Reserved (b5..b7)
         */
        b_0[1] = qos_tid | (mgmt << 4);
        memcpy(&b_0[2], hdr->addr2, ETH_ALEN);
        memcpy(&b_0[8], pn, IEEE80211_CCMP_PN_LEN);

        /* AAD (extra authenticate-only data) / masked 802.11 header
         * FC | A1 | A2 | A3 | SC | [A4] | [QC] */
        put_unaligned_be16(len_a, &aad[0]);
        put_unaligned(mask_fc, (__le16 *)&aad[2]);
        memcpy(&aad[4], &hdr->addr1, 3 * ETH_ALEN);

        /* Mask Seq#, leave Frag# */
        aad[22] = *((u8 *) &hdr->seq_ctrl) & 0x0f;
        aad[23] = 0;

        if (a4_included) {
                memcpy(&aad[24], hdr->addr4, ETH_ALEN);
                aad[30] = qos_tid;
                aad[31] = 0;
        } else {
                memset(&aad[24], 0, ETH_ALEN + IEEE80211_QOS_CTL_LEN);
                aad[24] = qos_tid;
        }
}


static inline void ccmp_pn2hdr(u8 *hdr, u8 *pn, int key_id)
{
        hdr[0] = pn[5];
        hdr[1] = pn[4];
        hdr[2] = 0;
        hdr[3] = 0x20 | (key_id << 6);
        hdr[4] = pn[3];
        hdr[5] = pn[2];
        hdr[6] = pn[1];
        hdr[7] = pn[0];
}


static inline void ccmp_hdr2pn(u8 *pn, u8 *hdr)
{
        pn[0] = hdr[7];
        pn[1] = hdr[6];
        pn[2] = hdr[5];
        pn[3] = hdr[4];
        pn[4] = hdr[1];
        pn[5] = hdr[0];
}


static int ccmp_encrypt_skb(struct ieee80211_tx_data *tx, struct sk_buff *skb,
                            unsigned int mic_len)
{
        struct ieee80211_hdr *hdr = (struct ieee80211_hdr *) skb->data;
        struct ieee80211_key *key = tx->key;
        struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb);
        int hdrlen, len, tail;
        u8 *pos;
        u8 pn[6];
        u64 pn64;
        u8 aad[CCM_AAD_LEN];
        u8 b_0[AES_BLOCK_SIZE];

        if (info->control.hw_key &&
            !(info->control.hw_key->flags & IEEE80211_KEY_FLAG_GENERATE_IV) &&
            !(info->control.hw_key->flags & IEEE80211_KEY_FLAG_PUT_IV_SPACE) &&
            !((info->control.hw_key->flags &
               IEEE80211_KEY_FLAG_GENERATE_IV_MGMT) &&
              ieee80211_is_mgmt(hdr->frame_control))) {
                /*
                 * hwaccel has no need for preallocated room for CCMP
                 * header or MIC fields
                 */
                return 0;
        }

        hdrlen = ieee80211_hdrlen(hdr->frame_control);
        len = skb->len - hdrlen;

        if (info->control.hw_key)
                tail = 0;
        else
                tail = mic_len;

        if (WARN_ON(skb_tailroom(skb) < tail ||
                    skb_headroom(skb) < IEEE80211_CCMP_HDR_LEN))
                return -1;

        pos = skb_push(skb, IEEE80211_CCMP_HDR_LEN);
        memmove(pos, pos + IEEE80211_CCMP_HDR_LEN, hdrlen);

        /* the HW only needs room for the IV, but not the actual IV */
        if (info->control.hw_key &&
            (info->control.hw_key->flags & IEEE80211_KEY_FLAG_PUT_IV_SPACE))
                return 0;

        hdr = (struct ieee80211_hdr *) pos;
        pos += hdrlen;

        pn64 = atomic64_inc_return(&key->conf.tx_pn);

        pn[5] = pn64;
        pn[4] = pn64 >> 8;
        pn[3] = pn64 >> 16;
        pn[2] = pn64 >> 24;
        pn[1] = pn64 >> 32;
        pn[0] = pn64 >> 40;

        ccmp_pn2hdr(pos, pn, key->conf.keyidx);

        /* hwaccel - with software CCMP header */
        if (info->control.hw_key)
                return 0;

        pos += IEEE80211_CCMP_HDR_LEN;
        ccmp_special_blocks(skb, pn, b_0, aad);
        return ieee80211_aes_ccm_encrypt(key->u.ccmp.tfm, b_0, aad, pos, len,
                                         skb_put(skb, mic_len));
}


ieee80211_tx_result
ieee80211_crypto_ccmp_encrypt(struct ieee80211_tx_data *tx,
                              unsigned int mic_len)
{
        struct sk_buff *skb;

        ieee80211_tx_set_protected(tx);

        skb_queue_walk(&tx->skbs, skb) {
                if (ccmp_encrypt_skb(tx, skb, mic_len) < 0)
                        return TX_DROP;
        }

        return TX_CONTINUE;
}


ieee80211_rx_result
ieee80211_crypto_ccmp_decrypt(struct ieee80211_rx_data *rx,
                              unsigned int mic_len)
{
        struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)rx->skb->data;
        int hdrlen;
        struct ieee80211_key *key = rx->key;
        struct sk_buff *skb = rx->skb;
        struct ieee80211_rx_status *status = IEEE80211_SKB_RXCB(skb);
        u8 pn[IEEE80211_CCMP_PN_LEN];
        int data_len;
        int queue;

        hdrlen = ieee80211_hdrlen(hdr->frame_control);

        if (!ieee80211_is_data(hdr->frame_control) &&
            !ieee80211_is_robust_mgmt_frame(skb))
                return RX_CONTINUE;

        if (status->flag & RX_FLAG_DECRYPTED) {
                if (!pskb_may_pull(rx->skb, hdrlen + IEEE80211_CCMP_HDR_LEN))
                        return RX_DROP_UNUSABLE;
                if (status->flag & RX_FLAG_MIC_STRIPPED)
                        mic_len = 0;
        } else {
                if (skb_linearize(rx->skb))
                        return RX_DROP_UNUSABLE;
        }

        /* reload hdr - skb might have been reallocated */
        hdr = (void *)rx->skb->data;

        data_len = skb->len - hdrlen - IEEE80211_CCMP_HDR_LEN - mic_len;
        if (!rx->sta || data_len < 0)
                return RX_DROP_UNUSABLE;

        if (!(status->flag & RX_FLAG_PN_VALIDATED)) {
                int res;

                ccmp_hdr2pn(pn, skb->data + hdrlen);

                queue = rx->security_idx;

                res = memcmp(pn, key->u.ccmp.rx_pn[queue],
                             IEEE80211_CCMP_PN_LEN);
                if (res < 0 ||
                    (!res && !(status->flag & RX_FLAG_ALLOW_SAME_PN))) {
                        key->u.ccmp.replays++;
                        return RX_DROP_UNUSABLE;
                }

                if (!(status->flag & RX_FLAG_DECRYPTED)) {
                        u8 aad[2 * AES_BLOCK_SIZE];
                        u8 b_0[AES_BLOCK_SIZE];
                        /* hardware didn't decrypt/verify MIC */
                        ccmp_special_blocks(skb, pn, b_0, aad);

                        if (ieee80211_aes_ccm_decrypt(
                                    key->u.ccmp.tfm, b_0, aad,
                                    skb->data + hdrlen + IEEE80211_CCMP_HDR_LEN,
                                    data_len,
                                    skb->data + skb->len - mic_len))
                                return RX_DROP_UNUSABLE;
                }

                memcpy(key->u.ccmp.rx_pn[queue], pn, IEEE80211_CCMP_PN_LEN);
                if (unlikely(ieee80211_is_frag(hdr)))
                        memcpy(rx->ccm_gcm.pn, pn, IEEE80211_CCMP_PN_LEN);
        }

        /* Remove CCMP header and MIC */
        if (pskb_trim(skb, skb->len - mic_len))
                return RX_DROP_UNUSABLE;
        memmove(skb->data + IEEE80211_CCMP_HDR_LEN, skb->data, hdrlen);
        skb_pull(skb, IEEE80211_CCMP_HDR_LEN);

        return RX_CONTINUE;
}

static void gcmp_special_blocks(struct sk_buff *skb, u8 *pn, u8 *j_0, u8 *aad)
{
        __le16 mask_fc;
        u8 qos_tid;
        struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data;

        memcpy(j_0, hdr->addr2, ETH_ALEN);
        memcpy(&j_0[ETH_ALEN], pn, IEEE80211_GCMP_PN_LEN);
        j_0[13] = 0;
        j_0[14] = 0;
        j_0[AES_BLOCK_SIZE - 1] = 0x01;

        /* AAD (extra authenticate-only data) / masked 802.11 header
         * FC | A1 | A2 | A3 | SC | [A4] | [QC]
         */
        put_unaligned_be16(ieee80211_hdrlen(hdr->frame_control) - 2, &aad[0]);
        /* Mask FC: zero subtype b4 b5 b6 (if not mgmt)
         * Retry, PwrMgt, MoreData; set Protected
         */
        mask_fc = hdr->frame_control;
        mask_fc &= ~cpu_to_le16(IEEE80211_FCTL_RETRY |
                                IEEE80211_FCTL_PM | IEEE80211_FCTL_MOREDATA);
        if (!ieee80211_is_mgmt(hdr->frame_control))
                mask_fc &= ~cpu_to_le16(0x0070);
        mask_fc |= cpu_to_le16(IEEE80211_FCTL_PROTECTED);

        put_unaligned(mask_fc, (__le16 *)&aad[2]);
        memcpy(&aad[4], &hdr->addr1, 3 * ETH_ALEN);

        /* Mask Seq#, leave Frag# */
        aad[22] = *((u8 *)&hdr->seq_ctrl) & 0x0f;
        aad[23] = 0;

        if (ieee80211_is_data_qos(hdr->frame_control))
                qos_tid = ieee80211_get_tid(hdr);
        else
                qos_tid = 0;

        if (ieee80211_has_a4(hdr->frame_control)) {
                memcpy(&aad[24], hdr->addr4, ETH_ALEN);
                aad[30] = qos_tid;
                aad[31] = 0;
        } else {
                memset(&aad[24], 0, ETH_ALEN + IEEE80211_QOS_CTL_LEN);
                aad[24] = qos_tid;
        }
}

static inline void gcmp_pn2hdr(u8 *hdr, const u8 *pn, int key_id)
{
        hdr[0] = pn[5];
        hdr[1] = pn[4];
        hdr[2] = 0;
        hdr[3] = 0x20 | (key_id << 6);
        hdr[4] = pn[3];
        hdr[5] = pn[2];
        hdr[6] = pn[1];
        hdr[7] = pn[0];
}

static inline void gcmp_hdr2pn(u8 *pn, const u8 *hdr)
{
        pn[0] = hdr[7];
        pn[1] = hdr[6];
        pn[2] = hdr[5];
        pn[3] = hdr[4];
        pn[4] = hdr[1];
        pn[5] = hdr[0];
}

static int gcmp_encrypt_skb(struct ieee80211_tx_data *tx, struct sk_buff *skb)
{
        struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data;
        struct ieee80211_key *key = tx->key;
        struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb);
        int hdrlen, len, tail;
        u8 *pos;
        u8 pn[6];
        u64 pn64;
        u8 aad[GCM_AAD_LEN];
        u8 j_0[AES_BLOCK_SIZE];

        if (info->control.hw_key &&
            !(info->control.hw_key->flags & IEEE80211_KEY_FLAG_GENERATE_IV) &&
            !(info->control.hw_key->flags & IEEE80211_KEY_FLAG_PUT_IV_SPACE) &&
            !((info->control.hw_key->flags &
               IEEE80211_KEY_FLAG_GENERATE_IV_MGMT) &&
              ieee80211_is_mgmt(hdr->frame_control))) {
                /* hwaccel has no need for preallocated room for GCMP
                 * header or MIC fields
                 */
                return 0;
        }

        hdrlen = ieee80211_hdrlen(hdr->frame_control);
        len = skb->len - hdrlen;

        if (info->control.hw_key)
                tail = 0;
        else
                tail = IEEE80211_GCMP_MIC_LEN;

        if (WARN_ON(skb_tailroom(skb) < tail ||
                    skb_headroom(skb) < IEEE80211_GCMP_HDR_LEN))
                return -1;

        pos = skb_push(skb, IEEE80211_GCMP_HDR_LEN);
        memmove(pos, pos + IEEE80211_GCMP_HDR_LEN, hdrlen);
        skb_set_network_header(skb, skb_network_offset(skb) +
                                    IEEE80211_GCMP_HDR_LEN);

        /* the HW only needs room for the IV, but not the actual IV */
        if (info->control.hw_key &&
            (info->control.hw_key->flags & IEEE80211_KEY_FLAG_PUT_IV_SPACE))
                return 0;

        hdr = (struct ieee80211_hdr *)pos;
        pos += hdrlen;

        pn64 = atomic64_inc_return(&key->conf.tx_pn);

        pn[5] = pn64;
        pn[4] = pn64 >> 8;
        pn[3] = pn64 >> 16;
        pn[2] = pn64 >> 24;
        pn[1] = pn64 >> 32;
        pn[0] = pn64 >> 40;

        gcmp_pn2hdr(pos, pn, key->conf.keyidx);

        /* hwaccel - with software GCMP header */
        if (info->control.hw_key)
                return 0;

        pos += IEEE80211_GCMP_HDR_LEN;
        gcmp_special_blocks(skb, pn, j_0, aad);
        return ieee80211_aes_gcm_encrypt(key->u.gcmp.tfm, j_0, aad, pos, len,
                                         skb_put(skb, IEEE80211_GCMP_MIC_LEN));
}

ieee80211_tx_result
ieee80211_crypto_gcmp_encrypt(struct ieee80211_tx_data *tx)
{
        struct sk_buff *skb;

        ieee80211_tx_set_protected(tx);

        skb_queue_walk(&tx->skbs, skb) {
                if (gcmp_encrypt_skb(tx, skb) < 0)
                        return TX_DROP;
        }

        return TX_CONTINUE;
}

ieee80211_rx_result
ieee80211_crypto_gcmp_decrypt(struct ieee80211_rx_data *rx)
{
        struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)rx->skb->data;
        int hdrlen;
        struct ieee80211_key *key = rx->key;
        struct sk_buff *skb = rx->skb;
        struct ieee80211_rx_status *status = IEEE80211_SKB_RXCB(skb);
        u8 pn[IEEE80211_GCMP_PN_LEN];
        int data_len, queue, mic_len = IEEE80211_GCMP_MIC_LEN;

        hdrlen = ieee80211_hdrlen(hdr->frame_control);

        if (!ieee80211_is_data(hdr->frame_control) &&
            !ieee80211_is_robust_mgmt_frame(skb))
                return RX_CONTINUE;

        if (status->flag & RX_FLAG_DECRYPTED) {
                if (!pskb_may_pull(rx->skb, hdrlen + IEEE80211_GCMP_HDR_LEN))
                        return RX_DROP_UNUSABLE;
                if (status->flag & RX_FLAG_MIC_STRIPPED)
                        mic_len = 0;
        } else {
                if (skb_linearize(rx->skb))
                        return RX_DROP_UNUSABLE;
        }

        /* reload hdr - skb might have been reallocated */
        hdr = (void *)rx->skb->data;

        data_len = skb->len - hdrlen - IEEE80211_GCMP_HDR_LEN - mic_len;
        if (!rx->sta || data_len < 0)
                return RX_DROP_UNUSABLE;

        if (!(status->flag & RX_FLAG_PN_VALIDATED)) {
                int res;

                gcmp_hdr2pn(pn, skb->data + hdrlen);

                queue = rx->security_idx;

                res = memcmp(pn, key->u.gcmp.rx_pn[queue],
                             IEEE80211_GCMP_PN_LEN);
                if (res < 0 ||
                    (!res && !(status->flag & RX_FLAG_ALLOW_SAME_PN))) {
                        key->u.gcmp.replays++;
                        return RX_DROP_UNUSABLE;
                }

                if (!(status->flag & RX_FLAG_DECRYPTED)) {
                        u8 aad[2 * AES_BLOCK_SIZE];
                        u8 j_0[AES_BLOCK_SIZE];
                        /* hardware didn't decrypt/verify MIC */
                        gcmp_special_blocks(skb, pn, j_0, aad);

                        if (ieee80211_aes_gcm_decrypt(
                                    key->u.gcmp.tfm, j_0, aad,
                                    skb->data + hdrlen + IEEE80211_GCMP_HDR_LEN,
                                    data_len,
                                    skb->data + skb->len -
                                    IEEE80211_GCMP_MIC_LEN))
                                return RX_DROP_UNUSABLE;
                }

                memcpy(key->u.gcmp.rx_pn[queue], pn, IEEE80211_GCMP_PN_LEN);
                if (unlikely(ieee80211_is_frag(hdr)))
                        memcpy(rx->ccm_gcm.pn, pn, IEEE80211_CCMP_PN_LEN);
        }

        /* Remove GCMP header and MIC */
        if (pskb_trim(skb, skb->len - mic_len))
                return RX_DROP_UNUSABLE;
        memmove(skb->data + IEEE80211_GCMP_HDR_LEN, skb->data, hdrlen);
        skb_pull(skb, IEEE80211_GCMP_HDR_LEN);

        return RX_CONTINUE;
}

static ieee80211_tx_result
ieee80211_crypto_cs_encrypt(struct ieee80211_tx_data *tx,
                            struct sk_buff *skb)
{
        struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data;
        struct ieee80211_key *key = tx->key;
        struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb);
        int hdrlen;
        u8 *pos, iv_len = key->conf.iv_len;

        if (info->control.hw_key &&
            !(info->control.hw_key->flags & IEEE80211_KEY_FLAG_PUT_IV_SPACE)) {
                /* hwaccel has no need for preallocated head room */
                return TX_CONTINUE;
        }

        if (unlikely(skb_headroom(skb) < iv_len &&
                     pskb_expand_head(skb, iv_len, 0, GFP_ATOMIC)))
                return TX_DROP;

        hdrlen = ieee80211_hdrlen(hdr->frame_control);

        pos = skb_push(skb, iv_len);
        memmove(pos, pos + iv_len, hdrlen);

        return TX_CONTINUE;
}

static inline int ieee80211_crypto_cs_pn_compare(u8 *pn1, u8 *pn2, int len)
{
        int i;

        /* pn is little endian */
        for (i = len - 1; i >= 0; i--) {
                if (pn1[i] < pn2[i])
                        return -1;
                else if (pn1[i] > pn2[i])
                        return 1;
        }

        return 0;
}

static ieee80211_rx_result
ieee80211_crypto_cs_decrypt(struct ieee80211_rx_data *rx)
{
        struct ieee80211_key *key = rx->key;
        struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)rx->skb->data;
        const struct ieee80211_cipher_scheme *cs = NULL;
        int hdrlen = ieee80211_hdrlen(hdr->frame_control);
        struct ieee80211_rx_status *status = IEEE80211_SKB_RXCB(rx->skb);
        int data_len;
        u8 *rx_pn;
        u8 *skb_pn;
        u8 qos_tid;

        if (!rx->sta || !rx->sta->cipher_scheme ||
            !(status->flag & RX_FLAG_DECRYPTED))
                return RX_DROP_UNUSABLE;

        if (!ieee80211_is_data(hdr->frame_control))
                return RX_CONTINUE;

        cs = rx->sta->cipher_scheme;

        data_len = rx->skb->len - hdrlen - cs->hdr_len;

        if (data_len < 0)
                return RX_DROP_UNUSABLE;

        if (ieee80211_is_data_qos(hdr->frame_control))
                qos_tid = ieee80211_get_tid(hdr);
        else
                qos_tid = 0;

        if (skb_linearize(rx->skb))
                return RX_DROP_UNUSABLE;

        hdr = (struct ieee80211_hdr *)rx->skb->data;

        rx_pn = key->u.gen.rx_pn[qos_tid];
        skb_pn = rx->skb->data + hdrlen + cs->pn_off;

        if (ieee80211_crypto_cs_pn_compare(skb_pn, rx_pn, cs->pn_len) <= 0)
                return RX_DROP_UNUSABLE;

        memcpy(rx_pn, skb_pn, cs->pn_len);

        /* remove security header and MIC */
        if (pskb_trim(rx->skb, rx->skb->len - cs->mic_len))
                return RX_DROP_UNUSABLE;

        memmove(rx->skb->data + cs->hdr_len, rx->skb->data, hdrlen);
        skb_pull(rx->skb, cs->hdr_len);

        return RX_CONTINUE;
}

static void bip_aad(struct sk_buff *skb, u8 *aad)
{
        __le16 mask_fc;
        struct ieee80211_hdr *hdr = (struct ieee80211_hdr *) skb->data;

        /* BIP AAD: FC(masked) || A1 || A2 || A3 */

        /* FC type/subtype */
        /* Mask FC Retry, PwrMgt, MoreData flags to zero */
        mask_fc = hdr->frame_control;
        mask_fc &= ~cpu_to_le16(IEEE80211_FCTL_RETRY | IEEE80211_FCTL_PM |
                                IEEE80211_FCTL_MOREDATA);
        put_unaligned(mask_fc, (__le16 *) &aad[0]);
        /* A1 || A2 || A3 */
        memcpy(aad + 2, &hdr->addr1, 3 * ETH_ALEN);
}


static inline void bip_ipn_set64(u8 *d, u64 pn)
{
        *d++ = pn;
        *d++ = pn >> 8;
        *d++ = pn >> 16;
        *d++ = pn >> 24;
        *d++ = pn >> 32;
        *d = pn >> 40;
}

static inline void bip_ipn_swap(u8 *d, const u8 *s)
{
        *d++ = s[5];
        *d++ = s[4];
        *d++ = s[3];
        *d++ = s[2];
        *d++ = s[1];
        *d = s[0];
}


ieee80211_tx_result
ieee80211_crypto_aes_cmac_encrypt(struct ieee80211_tx_data *tx)
{
        struct sk_buff *skb;
        struct ieee80211_tx_info *info;
        struct ieee80211_key *key = tx->key;
        struct ieee80211_mmie *mmie;
        u8 aad[20];
        u64 pn64;

        if (WARN_ON(skb_queue_len(&tx->skbs) != 1))
                return TX_DROP;

        skb = skb_peek(&tx->skbs);

        info = IEEE80211_SKB_CB(skb);

        if (info->control.hw_key &&
            !(key->conf.flags & IEEE80211_KEY_FLAG_GENERATE_MMIE))
                return TX_CONTINUE;

        if (WARN_ON(skb_tailroom(skb) < sizeof(*mmie)))
                return TX_DROP;

        mmie = skb_put(skb, sizeof(*mmie));
        mmie->element_id = WLAN_EID_MMIE;
        mmie->length = sizeof(*mmie) - 2;
        mmie->key_id = cpu_to_le16(key->conf.keyidx);

        /* PN = PN + 1 */
        pn64 = atomic64_inc_return(&key->conf.tx_pn);

        bip_ipn_set64(mmie->sequence_number, pn64);

        if (info->control.hw_key)
                return TX_CONTINUE;

        bip_aad(skb, aad);

        /*
         * MIC = AES-128-CMAC(IGTK, AAD || Management Frame Body || MMIE, 64)
         */
        ieee80211_aes_cmac(key->u.aes_cmac.tfm, aad,
                           skb->data + 24, skb->len - 24, mmie->mic);

        return TX_CONTINUE;
}

ieee80211_tx_result
ieee80211_crypto_aes_cmac_256_encrypt(struct ieee80211_tx_data *tx)
{
        struct sk_buff *skb;
        struct ieee80211_tx_info *info;
        struct ieee80211_key *key = tx->key;
        struct ieee80211_mmie_16 *mmie;
        u8 aad[20];
        u64 pn64;

        if (WARN_ON(skb_queue_len(&tx->skbs) != 1))
                return TX_DROP;

        skb = skb_peek(&tx->skbs);

        info = IEEE80211_SKB_CB(skb);

        if (info->control.hw_key)
                return TX_CONTINUE;

        if (WARN_ON(skb_tailroom(skb) < sizeof(*mmie)))
                return TX_DROP;

        mmie = skb_put(skb, sizeof(*mmie));
        mmie->element_id = WLAN_EID_MMIE;
        mmie->length = sizeof(*mmie) - 2;
        mmie->key_id = cpu_to_le16(key->conf.keyidx);

        /* PN = PN + 1 */
        pn64 = atomic64_inc_return(&key->conf.tx_pn);

        bip_ipn_set64(mmie->sequence_number, pn64);

        bip_aad(skb, aad);

        /* MIC = AES-256-CMAC(IGTK, AAD || Management Frame Body || MMIE, 128)
         */
        ieee80211_aes_cmac_256(key->u.aes_cmac.tfm, aad,
                               skb->data + 24, skb->len - 24, mmie->mic);

        return TX_CONTINUE;
}

ieee80211_rx_result
ieee80211_crypto_aes_cmac_decrypt(struct ieee80211_rx_data *rx)
{
        struct sk_buff *skb = rx->skb;
        struct ieee80211_rx_status *status = IEEE80211_SKB_RXCB(skb);
        struct ieee80211_key *key = rx->key;
        struct ieee80211_mmie *mmie;
        u8 aad[20], mic[8], ipn[6];
        struct ieee80211_hdr *hdr = (struct ieee80211_hdr *) skb->data;

        if (!ieee80211_is_mgmt(hdr->frame_control))
                return RX_CONTINUE;

        /* management frames are already linear */

        if (skb->len < 24 + sizeof(*mmie))
                return RX_DROP_UNUSABLE;

        mmie = (struct ieee80211_mmie *)
                (skb->data + skb->len - sizeof(*mmie));
        if (mmie->element_id != WLAN_EID_MMIE ||
            mmie->length != sizeof(*mmie) - 2)
                return RX_DROP_UNUSABLE; /* Invalid MMIE */

        bip_ipn_swap(ipn, mmie->sequence_number);

        if (memcmp(ipn, key->u.aes_cmac.rx_pn, 6) <= 0) {
                key->u.aes_cmac.replays++;
                return RX_DROP_UNUSABLE;
        }

        if (!(status->flag & RX_FLAG_DECRYPTED)) {
                /* hardware didn't decrypt/verify MIC */
                bip_aad(skb, aad);
                ieee80211_aes_cmac(key->u.aes_cmac.tfm, aad,
                                   skb->data + 24, skb->len - 24, mic);
                if (crypto_memneq(mic, mmie->mic, sizeof(mmie->mic))) {
                        key->u.aes_cmac.icverrors++;
                        return RX_DROP_UNUSABLE;
                }
        }

        memcpy(key->u.aes_cmac.rx_pn, ipn, 6);

        /* Remove MMIE */
        skb_trim(skb, skb->len - sizeof(*mmie));

        return RX_CONTINUE;
}

ieee80211_rx_result
ieee80211_crypto_aes_cmac_256_decrypt(struct ieee80211_rx_data *rx)
{
        struct sk_buff *skb = rx->skb;
        struct ieee80211_rx_status *status = IEEE80211_SKB_RXCB(skb);
        struct ieee80211_key *key = rx->key;
        struct ieee80211_mmie_16 *mmie;
        u8 aad[20], mic[16], ipn[6];
        struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data;

        if (!ieee80211_is_mgmt(hdr->frame_control))
                return RX_CONTINUE;

        /* management frames are already linear */

        if (skb->len < 24 + sizeof(*mmie))
                return RX_DROP_UNUSABLE;

        mmie = (struct ieee80211_mmie_16 *)
                (skb->data + skb->len - sizeof(*mmie));
        if (mmie->element_id != WLAN_EID_MMIE ||
            mmie->length != sizeof(*mmie) - 2)
                return RX_DROP_UNUSABLE; /* Invalid MMIE */

        bip_ipn_swap(ipn, mmie->sequence_number);

        if (memcmp(ipn, key->u.aes_cmac.rx_pn, 6) <= 0) {
                key->u.aes_cmac.replays++;
                return RX_DROP_UNUSABLE;
        }

        if (!(status->flag & RX_FLAG_DECRYPTED)) {
                /* hardware didn't decrypt/verify MIC */
                bip_aad(skb, aad);
                ieee80211_aes_cmac_256(key->u.aes_cmac.tfm, aad,
                                       skb->data + 24, skb->len - 24, mic);
                if (crypto_memneq(mic, mmie->mic, sizeof(mmie->mic))) {
                        key->u.aes_cmac.icverrors++;
                        return RX_DROP_UNUSABLE;
                }
        }

        memcpy(key->u.aes_cmac.rx_pn, ipn, 6);

        /* Remove MMIE */
        skb_trim(skb, skb->len - sizeof(*mmie));

        return RX_CONTINUE;
}

ieee80211_tx_result
ieee80211_crypto_aes_gmac_encrypt(struct ieee80211_tx_data *tx)
{
        struct sk_buff *skb;
        struct ieee80211_tx_info *info;
        struct ieee80211_key *key = tx->key;
        struct ieee80211_mmie_16 *mmie;
        struct ieee80211_hdr *hdr;
        u8 aad[GMAC_AAD_LEN];
        u64 pn64;
        u8 nonce[GMAC_NONCE_LEN];

        if (WARN_ON(skb_queue_len(&tx->skbs) != 1))
                return TX_DROP;

        skb = skb_peek(&tx->skbs);

        info = IEEE80211_SKB_CB(skb);

        if (info->control.hw_key)
                return TX_CONTINUE;

        if (WARN_ON(skb_tailroom(skb) < sizeof(*mmie)))
                return TX_DROP;

        mmie = skb_put(skb, sizeof(*mmie));
        mmie->element_id = WLAN_EID_MMIE;
        mmie->length = sizeof(*mmie) - 2;
        mmie->key_id = cpu_to_le16(key->conf.keyidx);

        /* PN = PN + 1 */
        pn64 = atomic64_inc_return(&key->conf.tx_pn);

        bip_ipn_set64(mmie->sequence_number, pn64);

        bip_aad(skb, aad);

        hdr = (struct ieee80211_hdr *)skb->data;
        memcpy(nonce, hdr->addr2, ETH_ALEN);
        bip_ipn_swap(nonce + ETH_ALEN, mmie->sequence_number);

        /* MIC = AES-GMAC(IGTK, AAD || Management Frame Body || MMIE, 128) */
        if (ieee80211_aes_gmac(key->u.aes_gmac.tfm, aad, nonce,
                               skb->data + 24, skb->len - 24, mmie->mic) < 0)
                return TX_DROP;

        return TX_CONTINUE;
}

ieee80211_rx_result
ieee80211_crypto_aes_gmac_decrypt(struct ieee80211_rx_data *rx)
{
        struct sk_buff *skb = rx->skb;
        struct ieee80211_rx_status *status = IEEE80211_SKB_RXCB(skb);
        struct ieee80211_key *key = rx->key;
        struct ieee80211_mmie_16 *mmie;
        u8 aad[GMAC_AAD_LEN], *mic, ipn[6], nonce[GMAC_NONCE_LEN];
        struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data;

        if (!ieee80211_is_mgmt(hdr->frame_control))
                return RX_CONTINUE;

        /* management frames are already linear */

        if (skb->len < 24 + sizeof(*mmie))
                return RX_DROP_UNUSABLE;

        mmie = (struct ieee80211_mmie_16 *)
                (skb->data + skb->len - sizeof(*mmie));
        if (mmie->element_id != WLAN_EID_MMIE ||
            mmie->length != sizeof(*mmie) - 2)
                return RX_DROP_UNUSABLE; /* Invalid MMIE */

        bip_ipn_swap(ipn, mmie->sequence_number);

        if (memcmp(ipn, key->u.aes_gmac.rx_pn, 6) <= 0) {
                key->u.aes_gmac.replays++;
                return RX_DROP_UNUSABLE;
        }

        if (!(status->flag & RX_FLAG_DECRYPTED)) {
                /* hardware didn't decrypt/verify MIC */
                bip_aad(skb, aad);

                memcpy(nonce, hdr->addr2, ETH_ALEN);
                memcpy(nonce + ETH_ALEN, ipn, 6);

                mic = kmalloc(GMAC_MIC_LEN, GFP_ATOMIC);
                if (!mic)
                        return RX_DROP_UNUSABLE;
                if (ieee80211_aes_gmac(key->u.aes_gmac.tfm, aad, nonce,
                                       skb->data + 24, skb->len - 24,
                                       mic) < 0 ||
                    crypto_memneq(mic, mmie->mic, sizeof(mmie->mic))) {
                        key->u.aes_gmac.icverrors++;
                        kfree(mic);
                        return RX_DROP_UNUSABLE;
                }
                kfree(mic);
        }

        memcpy(key->u.aes_gmac.rx_pn, ipn, 6);

        /* Remove MMIE */
        skb_trim(skb, skb->len - sizeof(*mmie));

        return RX_CONTINUE;
}

ieee80211_tx_result
ieee80211_crypto_hw_encrypt(struct ieee80211_tx_data *tx)
{
        struct sk_buff *skb;
        struct ieee80211_tx_info *info = NULL;
        ieee80211_tx_result res;

        skb_queue_walk(&tx->skbs, skb) {
                info  = IEEE80211_SKB_CB(skb);

                /* handle hw-only algorithm */
                if (!info->control.hw_key)
                        return TX_DROP;

                if (tx->key->flags & KEY_FLAG_CIPHER_SCHEME) {
                        res = ieee80211_crypto_cs_encrypt(tx, skb);
                        if (res != TX_CONTINUE)
                                return res;
                }
        }

        ieee80211_tx_set_protected(tx);

        return TX_CONTINUE;
}

ieee80211_rx_result
ieee80211_crypto_hw_decrypt(struct ieee80211_rx_data *rx)
{
        if (rx->sta && rx->sta->cipher_scheme)
                return ieee80211_crypto_cs_decrypt(rx);

        return RX_DROP_UNUSABLE;
}

















    1 
    2 














































    7 




    1 

    3 


    3 




    3 





















    2 






















    5 


    3 














    9 



    3 




    4 
    5 





1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2020 Cloudflare Ltd https://cloudflare.com */

#include <linux/skmsg.h>
#include <net/sock.h>
#include <net/udp.h>
#include <net/inet_common.h>

#include "udp_impl.h"

static struct proto *udpv6_prot_saved __read_mostly;

static int sk_udp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len,
                          int noblock, int flags, int *addr_len)
{
#if IS_ENABLED(CONFIG_IPV6)
        if (sk->sk_family == AF_INET6)
                return udpv6_prot_saved->recvmsg(sk, msg, len, noblock, flags,
                                                 addr_len);
#endif
        return udp_prot.recvmsg(sk, msg, len, noblock, flags, addr_len);
}

static bool udp_sk_has_data(struct sock *sk)
{
        return !skb_queue_empty(&udp_sk(sk)->reader_queue) ||
               !skb_queue_empty(&sk->sk_receive_queue);
}

static bool psock_has_data(struct sk_psock *psock)
{
        return !skb_queue_empty(&psock->ingress_skb) ||
               !sk_psock_queue_empty(psock);
}

#define udp_msg_has_data(__sk, __psock)        \
                ({ udp_sk_has_data(__sk) || psock_has_data(__psock); })

static int udp_msg_wait_data(struct sock *sk, struct sk_psock *psock,
                             long timeo)
{
        DEFINE_WAIT_FUNC(wait, woken_wake_function);
        int ret = 0;

        if (sk->sk_shutdown & RCV_SHUTDOWN)
                return 1;

        if (!timeo)
                return ret;

        add_wait_queue(sk_sleep(sk), &wait);
        sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk);
        ret = udp_msg_has_data(sk, psock);
        if (!ret) {
                wait_woken(&wait, TASK_INTERRUPTIBLE, timeo);
                ret = udp_msg_has_data(sk, psock);
        }
        sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk);
        remove_wait_queue(sk_sleep(sk), &wait);
        return ret;
}

static int udp_bpf_recvmsg(struct sock *sk, struct msghdr *msg, size_t len,
                           int nonblock, int flags, int *addr_len)
{
        struct sk_psock *psock;
        int copied, ret;

        if (unlikely(flags & MSG_ERRQUEUE))
                return inet_recv_error(sk, msg, len, addr_len);

        if (!len)
                return 0;

        psock = sk_psock_get(sk);
        if (unlikely(!psock))
                return sk_udp_recvmsg(sk, msg, len, nonblock, flags, addr_len);

        if (!psock_has_data(psock)) {
                ret = sk_udp_recvmsg(sk, msg, len, nonblock, flags, addr_len);
                goto out;
        }

msg_bytes_ready:
        copied = sk_msg_recvmsg(sk, psock, msg, len, flags);
        if (!copied) {
                long timeo;
                int data;

                timeo = sock_rcvtimeo(sk, nonblock);
                data = udp_msg_wait_data(sk, psock, timeo);
                if (data) {
                        if (psock_has_data(psock))
                                goto msg_bytes_ready;
                        ret = sk_udp_recvmsg(sk, msg, len, nonblock, flags, addr_len);
                        goto out;
                }
                copied = -EAGAIN;
        }
        ret = copied;
out:
        sk_psock_put(sk, psock);
        return ret;
}

enum {
        UDP_BPF_IPV4,
        UDP_BPF_IPV6,
        UDP_BPF_NUM_PROTS,
};

static DEFINE_SPINLOCK(udpv6_prot_lock);
static struct proto udp_bpf_prots[UDP_BPF_NUM_PROTS];

static void udp_bpf_rebuild_protos(struct proto *prot, const struct proto *base)
{
        *prot        = *base;
        prot->close  = sock_map_close;
        prot->recvmsg = udp_bpf_recvmsg;
        prot->sock_is_readable = sk_msg_is_readable;
}

static void udp_bpf_check_v6_needs_rebuild(struct proto *ops)
{
        if (unlikely(ops != smp_load_acquire(&udpv6_prot_saved))) {
                spin_lock_bh(&udpv6_prot_lock);
                if (likely(ops != udpv6_prot_saved)) {
                        udp_bpf_rebuild_protos(&udp_bpf_prots[UDP_BPF_IPV6], ops);
                        smp_store_release(&udpv6_prot_saved, ops);
                }
                spin_unlock_bh(&udpv6_prot_lock);
        }
}

static int __init udp_bpf_v4_build_proto(void)
{
        udp_bpf_rebuild_protos(&udp_bpf_prots[UDP_BPF_IPV4], &udp_prot);
        return 0;
}
late_initcall(udp_bpf_v4_build_proto);

int udp_bpf_update_proto(struct sock *sk, struct sk_psock *psock, bool restore)
{
        int family = sk->sk_family == AF_INET ? UDP_BPF_IPV4 : UDP_BPF_IPV6;

        if (restore) {
                sk->sk_write_space = psock->saved_write_space;
                WRITE_ONCE(sk->sk_prot, psock->sk_proto);
                return 0;
        }

        if (sk->sk_family == AF_INET6)
                udp_bpf_check_v6_needs_rebuild(psock->sk_proto);

        WRITE_ONCE(sk->sk_prot, &udp_bpf_prots[family]);
        return 0;
}
EXPORT_SYMBOL_GPL(udp_bpf_update_proto);












































































    6 





    6 



    6 




















































































































































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
/*
 * Copyright (c) 2006, 2020 Oracle and/or its affiliates.
 *
 * This software is available to you under a choice of one of two
 * licenses.  You may choose to be licensed under the terms of the GNU
 * General Public License (GPL) Version 2, available from the file
 * COPYING in the main directory of this source tree, or the
 * OpenIB.org BSD license below:
 *
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
 *
 *      - Redistributions of source code must retain the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer.
 *
 *      - Redistributions in binary form must reproduce the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer in the documentation and/or other materials
 *        provided with the distribution.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 */
#include <linux/kernel.h>
#include <linux/slab.h>
#include <linux/export.h>
#include <linux/skbuff.h>
#include <linux/list.h>
#include <linux/errqueue.h>

#include "rds.h"

static unsigned int        rds_exthdr_size[__RDS_EXTHDR_MAX] = {
[RDS_EXTHDR_NONE]        = 0,
[RDS_EXTHDR_VERSION]        = sizeof(struct rds_ext_header_version),
[RDS_EXTHDR_RDMA]        = sizeof(struct rds_ext_header_rdma),
[RDS_EXTHDR_RDMA_DEST]        = sizeof(struct rds_ext_header_rdma_dest),
[RDS_EXTHDR_NPATHS]        = sizeof(u16),
[RDS_EXTHDR_GEN_NUM]        = sizeof(u32),
};

void rds_message_addref(struct rds_message *rm)
{
        rdsdebug("addref rm %p ref %d\n", rm, refcount_read(&rm->m_refcount));
        refcount_inc(&rm->m_refcount);
}
EXPORT_SYMBOL_GPL(rds_message_addref);

static inline bool rds_zcookie_add(struct rds_msg_zcopy_info *info, u32 cookie)
{
        struct rds_zcopy_cookies *ck = &info->zcookies;
        int ncookies = ck->num;

        if (ncookies == RDS_MAX_ZCOOKIES)
                return false;
        ck->cookies[ncookies] = cookie;
        ck->num =  ++ncookies;
        return true;
}

static struct rds_msg_zcopy_info *rds_info_from_znotifier(struct rds_znotifier *znotif)
{
        return container_of(znotif, struct rds_msg_zcopy_info, znotif);
}

void rds_notify_msg_zcopy_purge(struct rds_msg_zcopy_queue *q)
{
        unsigned long flags;
        LIST_HEAD(copy);
        struct rds_msg_zcopy_info *info, *tmp;

        spin_lock_irqsave(&q->lock, flags);
        list_splice(&q->zcookie_head, &copy);
        INIT_LIST_HEAD(&q->zcookie_head);
        spin_unlock_irqrestore(&q->lock, flags);

        list_for_each_entry_safe(info, tmp, &copy, rs_zcookie_next) {
                list_del(&info->rs_zcookie_next);
                kfree(info);
        }
}

static void rds_rm_zerocopy_callback(struct rds_sock *rs,
                                     struct rds_znotifier *znotif)
{
        struct rds_msg_zcopy_info *info;
        struct rds_msg_zcopy_queue *q;
        u32 cookie = znotif->z_cookie;
        struct rds_zcopy_cookies *ck;
        struct list_head *head;
        unsigned long flags;

        mm_unaccount_pinned_pages(&znotif->z_mmp);
        q = &rs->rs_zcookie_queue;
        spin_lock_irqsave(&q->lock, flags);
        head = &q->zcookie_head;
        if (!list_empty(head)) {
                info = list_first_entry(head, struct rds_msg_zcopy_info,
                                        rs_zcookie_next);
                if (rds_zcookie_add(info, cookie)) {
                        spin_unlock_irqrestore(&q->lock, flags);
                        kfree(rds_info_from_znotifier(znotif));
                        /* caller invokes rds_wake_sk_sleep() */
                        return;
                }
        }

        info = rds_info_from_znotifier(znotif);
        ck = &info->zcookies;
        memset(ck, 0, sizeof(*ck));
        WARN_ON(!rds_zcookie_add(info, cookie));
        list_add_tail(&info->rs_zcookie_next, &q->zcookie_head);

        spin_unlock_irqrestore(&q->lock, flags);
        /* caller invokes rds_wake_sk_sleep() */
}

/*
 * This relies on dma_map_sg() not touching sg[].page during merging.
 */
static void rds_message_purge(struct rds_message *rm)
{
        unsigned long i, flags;
        bool zcopy = false;

        if (unlikely(test_bit(RDS_MSG_PAGEVEC, &rm->m_flags)))
                return;

        spin_lock_irqsave(&rm->m_rs_lock, flags);
        if (rm->m_rs) {
                struct rds_sock *rs = rm->m_rs;

                if (rm->data.op_mmp_znotifier) {
                        zcopy = true;
                        rds_rm_zerocopy_callback(rs, rm->data.op_mmp_znotifier);
                        rds_wake_sk_sleep(rs);
                        rm->data.op_mmp_znotifier = NULL;
                }
                sock_put(rds_rs_to_sk(rs));
                rm->m_rs = NULL;
        }
        spin_unlock_irqrestore(&rm->m_rs_lock, flags);

        for (i = 0; i < rm->data.op_nents; i++) {
                /* XXX will have to put_page for page refs */
                if (!zcopy)
                        __free_page(sg_page(&rm->data.op_sg[i]));
                else
                        put_page(sg_page(&rm->data.op_sg[i]));
        }
        rm->data.op_nents = 0;

        if (rm->rdma.op_active)
                rds_rdma_free_op(&rm->rdma);
        if (rm->rdma.op_rdma_mr)
                kref_put(&rm->rdma.op_rdma_mr->r_kref, __rds_put_mr_final);

        if (rm->atomic.op_active)
                rds_atomic_free_op(&rm->atomic);
        if (rm->atomic.op_rdma_mr)
                kref_put(&rm->atomic.op_rdma_mr->r_kref, __rds_put_mr_final);
}

void rds_message_put(struct rds_message *rm)
{
        rdsdebug("put rm %p ref %d\n", rm, refcount_read(&rm->m_refcount));
        WARN(!refcount_read(&rm->m_refcount), "danger refcount zero on %p\n", rm);
        if (refcount_dec_and_test(&rm->m_refcount)) {
                BUG_ON(!list_empty(&rm->m_sock_item));
                BUG_ON(!list_empty(&rm->m_conn_item));
                rds_message_purge(rm);

                kfree(rm);
        }
}
EXPORT_SYMBOL_GPL(rds_message_put);

void rds_message_populate_header(struct rds_header *hdr, __be16 sport,
                                 __be16 dport, u64 seq)
{
        hdr->h_flags = 0;
        hdr->h_sport = sport;
        hdr->h_dport = dport;
        hdr->h_sequence = cpu_to_be64(seq);
        hdr->h_exthdr[0] = RDS_EXTHDR_NONE;
}
EXPORT_SYMBOL_GPL(rds_message_populate_header);

int rds_message_add_extension(struct rds_header *hdr, unsigned int type,
                              const void *data, unsigned int len)
{
        unsigned int ext_len = sizeof(u8) + len;
        unsigned char *dst;

        /* For now, refuse to add more than one extension header */
        if (hdr->h_exthdr[0] != RDS_EXTHDR_NONE)
                return 0;

        if (type >= __RDS_EXTHDR_MAX || len != rds_exthdr_size[type])
                return 0;

        if (ext_len >= RDS_HEADER_EXT_SPACE)
                return 0;
        dst = hdr->h_exthdr;

        *dst++ = type;
        memcpy(dst, data, len);

        dst[len] = RDS_EXTHDR_NONE;
        return 1;
}
EXPORT_SYMBOL_GPL(rds_message_add_extension);

/*
 * If a message has extension headers, retrieve them here.
 * Call like this:
 *
 * unsigned int pos = 0;
 *
 * while (1) {
 *        buflen = sizeof(buffer);
 *        type = rds_message_next_extension(hdr, &pos, buffer, &buflen);
 *        if (type == RDS_EXTHDR_NONE)
 *                break;
 *        ...
 * }
 */
int rds_message_next_extension(struct rds_header *hdr,
                unsigned int *pos, void *buf, unsigned int *buflen)
{
        unsigned int offset, ext_type, ext_len;
        u8 *src = hdr->h_exthdr;

        offset = *pos;
        if (offset >= RDS_HEADER_EXT_SPACE)
                goto none;

        /* Get the extension type and length. For now, the
         * length is implied by the extension type. */
        ext_type = src[offset++];

        if (ext_type == RDS_EXTHDR_NONE || ext_type >= __RDS_EXTHDR_MAX)
                goto none;
        ext_len = rds_exthdr_size[ext_type];
        if (offset + ext_len > RDS_HEADER_EXT_SPACE)
                goto none;

        *pos = offset + ext_len;
        if (ext_len < *buflen)
                *buflen = ext_len;
        memcpy(buf, src + offset, *buflen);
        return ext_type;

none:
        *pos = RDS_HEADER_EXT_SPACE;
        *buflen = 0;
        return RDS_EXTHDR_NONE;
}

int rds_message_add_rdma_dest_extension(struct rds_header *hdr, u32 r_key, u32 offset)
{
        struct rds_ext_header_rdma_dest ext_hdr;

        ext_hdr.h_rdma_rkey = cpu_to_be32(r_key);
        ext_hdr.h_rdma_offset = cpu_to_be32(offset);
        return rds_message_add_extension(hdr, RDS_EXTHDR_RDMA_DEST, &ext_hdr, sizeof(ext_hdr));
}
EXPORT_SYMBOL_GPL(rds_message_add_rdma_dest_extension);

/*
 * Each rds_message is allocated with extra space for the scatterlist entries
 * rds ops will need. This is to minimize memory allocation count. Then, each rds op
 * can grab SGs when initializing its part of the rds_message.
 */
struct rds_message *rds_message_alloc(unsigned int extra_len, gfp_t gfp)
{
        struct rds_message *rm;

        if (extra_len > KMALLOC_MAX_SIZE - sizeof(struct rds_message))
                return NULL;

        rm = kzalloc(sizeof(struct rds_message) + extra_len, gfp);
        if (!rm)
                goto out;

        rm->m_used_sgs = 0;
        rm->m_total_sgs = extra_len / sizeof(struct scatterlist);

        refcount_set(&rm->m_refcount, 1);
        INIT_LIST_HEAD(&rm->m_sock_item);
        INIT_LIST_HEAD(&rm->m_conn_item);
        spin_lock_init(&rm->m_rs_lock);
        init_waitqueue_head(&rm->m_flush_wait);

out:
        return rm;
}

/*
 * RDS ops use this to grab SG entries from the rm's sg pool.
 */
struct scatterlist *rds_message_alloc_sgs(struct rds_message *rm, int nents)
{
        struct scatterlist *sg_first = (struct scatterlist *) &rm[1];
        struct scatterlist *sg_ret;

        if (nents <= 0) {
                pr_warn("rds: alloc sgs failed! nents <= 0\n");
                return ERR_PTR(-EINVAL);
        }

        if (rm->m_used_sgs + nents > rm->m_total_sgs) {
                pr_warn("rds: alloc sgs failed! total %d used %d nents %d\n",
                        rm->m_total_sgs, rm->m_used_sgs, nents);
                return ERR_PTR(-ENOMEM);
        }

        sg_ret = &sg_first[rm->m_used_sgs];
        sg_init_table(sg_ret, nents);
        rm->m_used_sgs += nents;

        return sg_ret;
}

struct rds_message *rds_message_map_pages(unsigned long *page_addrs, unsigned int total_len)
{
        struct rds_message *rm;
        unsigned int i;
        int num_sgs = DIV_ROUND_UP(total_len, PAGE_SIZE);
        int extra_bytes = num_sgs * sizeof(struct scatterlist);

        rm = rds_message_alloc(extra_bytes, GFP_NOWAIT);
        if (!rm)
                return ERR_PTR(-ENOMEM);

        set_bit(RDS_MSG_PAGEVEC, &rm->m_flags);
        rm->m_inc.i_hdr.h_len = cpu_to_be32(total_len);
        rm->data.op_nents = DIV_ROUND_UP(total_len, PAGE_SIZE);
        rm->data.op_sg = rds_message_alloc_sgs(rm, num_sgs);
        if (IS_ERR(rm->data.op_sg)) {
                void *err = ERR_CAST(rm->data.op_sg);
                rds_message_put(rm);
                return err;
        }

        for (i = 0; i < rm->data.op_nents; ++i) {
                sg_set_page(&rm->data.op_sg[i],
                                virt_to_page(page_addrs[i]),
                                PAGE_SIZE, 0);
        }

        return rm;
}

static int rds_message_zcopy_from_user(struct rds_message *rm, struct iov_iter *from)
{
        struct scatterlist *sg;
        int ret = 0;
        int length = iov_iter_count(from);
        int total_copied = 0;
        struct rds_msg_zcopy_info *info;

        rm->m_inc.i_hdr.h_len = cpu_to_be32(iov_iter_count(from));

        /*
         * now allocate and copy in the data payload.
         */
        sg = rm->data.op_sg;

        info = kzalloc(sizeof(*info), GFP_KERNEL);
        if (!info)
                return -ENOMEM;
        INIT_LIST_HEAD(&info->rs_zcookie_next);
        rm->data.op_mmp_znotifier = &info->znotif;
        if (mm_account_pinned_pages(&rm->data.op_mmp_znotifier->z_mmp,
                                    length)) {
                ret = -ENOMEM;
                goto err;
        }
        while (iov_iter_count(from)) {
                struct page *pages;
                size_t start;
                ssize_t copied;

                copied = iov_iter_get_pages(from, &pages, PAGE_SIZE,
                                            1, &start);
                if (copied < 0) {
                        struct mmpin *mmp;
                        int i;

                        for (i = 0; i < rm->data.op_nents; i++)
                                put_page(sg_page(&rm->data.op_sg[i]));
                        mmp = &rm->data.op_mmp_znotifier->z_mmp;
                        mm_unaccount_pinned_pages(mmp);
                        ret = -EFAULT;
                        goto err;
                }
                total_copied += copied;
                iov_iter_advance(from, copied);
                length -= copied;
                sg_set_page(sg, pages, copied, start);
                rm->data.op_nents++;
                sg++;
        }
        WARN_ON_ONCE(length != 0);
        return ret;
err:
        kfree(info);
        rm->data.op_mmp_znotifier = NULL;
        return ret;
}

int rds_message_copy_from_user(struct rds_message *rm, struct iov_iter *from,
                               bool zcopy)
{
        unsigned long to_copy, nbytes;
        unsigned long sg_off;
        struct scatterlist *sg;
        int ret = 0;

        rm->m_inc.i_hdr.h_len = cpu_to_be32(iov_iter_count(from));

        /* now allocate and copy in the data payload.  */
        sg = rm->data.op_sg;
        sg_off = 0; /* Dear gcc, sg->page will be null from kzalloc. */

        if (zcopy)
                return rds_message_zcopy_from_user(rm, from);

        while (iov_iter_count(from)) {
                if (!sg_page(sg)) {
                        ret = rds_page_remainder_alloc(sg, iov_iter_count(from),
                                                       GFP_HIGHUSER);
                        if (ret)
                                return ret;
                        rm->data.op_nents++;
                        sg_off = 0;
                }

                to_copy = min_t(unsigned long, iov_iter_count(from),
                                sg->length - sg_off);

                rds_stats_add(s_copy_from_user, to_copy);
                nbytes = copy_page_from_iter(sg_page(sg), sg->offset + sg_off,
                                             to_copy, from);
                if (nbytes != to_copy)
                        return -EFAULT;

                sg_off += to_copy;

                if (sg_off == sg->length)
                        sg++;
        }

        return ret;
}

int rds_message_inc_copy_to_user(struct rds_incoming *inc, struct iov_iter *to)
{
        struct rds_message *rm;
        struct scatterlist *sg;
        unsigned long to_copy;
        unsigned long vec_off;
        int copied;
        int ret;
        u32 len;

        rm = container_of(inc, struct rds_message, m_inc);
        len = be32_to_cpu(rm->m_inc.i_hdr.h_len);

        sg = rm->data.op_sg;
        vec_off = 0;
        copied = 0;

        while (iov_iter_count(to) && copied < len) {
                to_copy = min_t(unsigned long, iov_iter_count(to),
                                sg->length - vec_off);
                to_copy = min_t(unsigned long, to_copy, len - copied);

                rds_stats_add(s_copy_to_user, to_copy);
                ret = copy_page_to_iter(sg_page(sg), sg->offset + vec_off,
                                        to_copy, to);
                if (ret != to_copy)
                        return -EFAULT;

                vec_off += to_copy;
                copied += to_copy;

                if (vec_off == sg->length) {
                        vec_off = 0;
                        sg++;
                }
        }

        return copied;
}

/*
 * If the message is still on the send queue, wait until the transport
 * is done with it. This is particularly important for RDMA operations.
 */
void rds_message_wait(struct rds_message *rm)
{
        wait_event_interruptible(rm->m_flush_wait,
                        !test_bit(RDS_MSG_MAPPED, &rm->m_flags));
}

void rds_message_unmapped(struct rds_message *rm)
{
        clear_bit(RDS_MSG_MAPPED, &rm->m_flags);
        wake_up_interruptible(&rm->m_flush_wait);
}
EXPORT_SYMBOL_GPL(rds_message_unmapped);





































  858 

























  184 
















  184 

  183 



  184 





  184 


















































  184 


   12 




  184 



  184 




  183 




  184 
  184 





















































  184 




  184 






























    3 

















    3 













   26 





   26 
















  817 














  804 

  810 











  635 
















  634 



  635 





  635 













  632 
  595 





















  205 
  188 


















  406 



    1 


   96 
  405 











  404 





   96 
  408 

  447 


















  189 
  192 



  181 
  170 





  190 


  181 

  170 












  795 

  834 












  675 



  673 









  193 



  173 
  167 



  193 










  193 





















   81 
   14 
   67 








  618 















  617 




  621 













  618 













  619 




















  620 


  619 














  621 




































































  617 




















  619 


  621 


    1 




  619 
    6 












  108 
  512 



















  754 




  755 







  756 
  753 



  680 
  753 





















































  553 




  556 

















































































































  571 















  571 








































  192 
















   16 
   48 

















  188 



  188 







  189 

















  160 






  160 




  108 



   98 













   97 










   98 




    2 


   96 




   96 









































































  744 



  705 



















  742 


  744 



  745 


  740 





  129 


  129 
















  625 





  627 
   19 


  618 
  618 









  192 


  192 






  191 


  192 



  192 
  190 
  192 



  192 







  192 








  192 







  191 




  192 








  192 









  162 




















  267 




  267 


















  266 










   96 
  198 


























































































  237 











  181 
  190 





















   26 











   26 




   26 











   26 








   26 
   26 









   26 





   26 




















































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
// SPDX-License-Identifier: GPL-2.0-only
/*
 * fs/kernfs/dir.c - kernfs directory implementation
 *
 * Copyright (c) 2001-3 Patrick Mochel
 * Copyright (c) 2007 SUSE Linux Products GmbH
 * Copyright (c) 2007, 2013 Tejun Heo <tj@kernel.org>
 */

#include <linux/sched.h>
#include <linux/fs.h>
#include <linux/namei.h>
#include <linux/idr.h>
#include <linux/slab.h>
#include <linux/security.h>
#include <linux/hash.h>

#include "kernfs-internal.h"

DECLARE_RWSEM(kernfs_rwsem);
static DEFINE_SPINLOCK(kernfs_rename_lock);        /* kn->parent and ->name */
/*
 * Don't use rename_lock to piggy back on pr_cont_buf. We don't want to
 * call pr_cont() while holding rename_lock. Because sometimes pr_cont()
 * will perform wakeups when releasing console_sem. Holding rename_lock
 * will introduce deadlock if the scheduler reads the kernfs_name in the
 * wakeup path.
 */
static DEFINE_SPINLOCK(kernfs_pr_cont_lock);
static char kernfs_pr_cont_buf[PATH_MAX];        /* protected by pr_cont_lock */
static DEFINE_SPINLOCK(kernfs_idr_lock);        /* root->ino_idr */

#define rb_to_kn(X) rb_entry((X), struct kernfs_node, rb)

static bool kernfs_active(struct kernfs_node *kn)
{
        lockdep_assert_held(&kernfs_rwsem);
        return atomic_read(&kn->active) >= 0;
}

static bool kernfs_lockdep(struct kernfs_node *kn)
{
#ifdef CONFIG_DEBUG_LOCK_ALLOC
        return kn->flags & KERNFS_LOCKDEP;
#else
        return false;
#endif
}

static int kernfs_name_locked(struct kernfs_node *kn, char *buf, size_t buflen)
{
        if (!kn)
                return strlcpy(buf, "(null)", buflen);

        return strlcpy(buf, kn->parent ? kn->name : "/", buflen);
}

/* kernfs_node_depth - compute depth from @from to @to */
static size_t kernfs_depth(struct kernfs_node *from, struct kernfs_node *to)
{
        size_t depth = 0;

        while (to->parent && to != from) {
                depth++;
                to = to->parent;
        }
        return depth;
}

static struct kernfs_node *kernfs_common_ancestor(struct kernfs_node *a,
                                                  struct kernfs_node *b)
{
        size_t da, db;
        struct kernfs_root *ra = kernfs_root(a), *rb = kernfs_root(b);

        if (ra != rb)
                return NULL;

        da = kernfs_depth(ra->kn, a);
        db = kernfs_depth(rb->kn, b);

        while (da > db) {
                a = a->parent;
                da--;
        }
        while (db > da) {
                b = b->parent;
                db--;
        }

        /* worst case b and a will be the same at root */
        while (b != a) {
                b = b->parent;
                a = a->parent;
        }

        return a;
}

/**
 * kernfs_path_from_node_locked - find a pseudo-absolute path to @kn_to,
 * where kn_from is treated as root of the path.
 * @kn_from: kernfs node which should be treated as root for the path
 * @kn_to: kernfs node to which path is needed
 * @buf: buffer to copy the path into
 * @buflen: size of @buf
 *
 * We need to handle couple of scenarios here:
 * [1] when @kn_from is an ancestor of @kn_to at some level
 * kn_from: /n1/n2/n3
 * kn_to:   /n1/n2/n3/n4/n5
 * result:  /n4/n5
 *
 * [2] when @kn_from is on a different hierarchy and we need to find common
 * ancestor between @kn_from and @kn_to.
 * kn_from: /n1/n2/n3/n4
 * kn_to:   /n1/n2/n5
 * result:  /../../n5
 * OR
 * kn_from: /n1/n2/n3/n4/n5   [depth=5]
 * kn_to:   /n1/n2/n3         [depth=3]
 * result:  /../..
 *
 * [3] when @kn_to is NULL result will be "(null)"
 *
 * Returns the length of the full path.  If the full length is equal to or
 * greater than @buflen, @buf contains the truncated path with the trailing
 * '\0'.  On error, -errno is returned.
 */
static int kernfs_path_from_node_locked(struct kernfs_node *kn_to,
                                        struct kernfs_node *kn_from,
                                        char *buf, size_t buflen)
{
        struct kernfs_node *kn, *common;
        const char parent_str[] = "/..";
        size_t depth_from, depth_to, len = 0;
        int i, j;

        if (!kn_to)
                return strlcpy(buf, "(null)", buflen);

        if (!kn_from)
                kn_from = kernfs_root(kn_to)->kn;

        if (kn_from == kn_to)
                return strlcpy(buf, "/", buflen);

        if (!buf)
                return -EINVAL;

        common = kernfs_common_ancestor(kn_from, kn_to);
        if (WARN_ON(!common))
                return -EINVAL;

        depth_to = kernfs_depth(common, kn_to);
        depth_from = kernfs_depth(common, kn_from);

        buf[0] = '\0';

        for (i = 0; i < depth_from; i++)
                len += strlcpy(buf + len, parent_str,
                               len < buflen ? buflen - len : 0);

        /* Calculate how many bytes we need for the rest */
        for (i = depth_to - 1; i >= 0; i--) {
                for (kn = kn_to, j = 0; j < i; j++)
                        kn = kn->parent;
                len += strlcpy(buf + len, "/",
                               len < buflen ? buflen - len : 0);
                len += strlcpy(buf + len, kn->name,
                               len < buflen ? buflen - len : 0);
        }

        return len;
}

/**
 * kernfs_name - obtain the name of a given node
 * @kn: kernfs_node of interest
 * @buf: buffer to copy @kn's name into
 * @buflen: size of @buf
 *
 * Copies the name of @kn into @buf of @buflen bytes.  The behavior is
 * similar to strlcpy().  It returns the length of @kn's name and if @buf
 * isn't long enough, it's filled upto @buflen-1 and nul terminated.
 *
 * Fills buffer with "(null)" if @kn is NULL.
 *
 * This function can be called from any context.
 */
int kernfs_name(struct kernfs_node *kn, char *buf, size_t buflen)
{
        unsigned long flags;
        int ret;

        spin_lock_irqsave(&kernfs_rename_lock, flags);
        ret = kernfs_name_locked(kn, buf, buflen);
        spin_unlock_irqrestore(&kernfs_rename_lock, flags);
        return ret;
}

/**
 * kernfs_path_from_node - build path of node @to relative to @from.
 * @from: parent kernfs_node relative to which we need to build the path
 * @to: kernfs_node of interest
 * @buf: buffer to copy @to's path into
 * @buflen: size of @buf
 *
 * Builds @to's path relative to @from in @buf. @from and @to must
 * be on the same kernfs-root. If @from is not parent of @to, then a relative
 * path (which includes '..'s) as needed to reach from @from to @to is
 * returned.
 *
 * Returns the length of the full path.  If the full length is equal to or
 * greater than @buflen, @buf contains the truncated path with the trailing
 * '\0'.  On error, -errno is returned.
 */
int kernfs_path_from_node(struct kernfs_node *to, struct kernfs_node *from,
                          char *buf, size_t buflen)
{
        unsigned long flags;
        int ret;

        spin_lock_irqsave(&kernfs_rename_lock, flags);
        ret = kernfs_path_from_node_locked(to, from, buf, buflen);
        spin_unlock_irqrestore(&kernfs_rename_lock, flags);
        return ret;
}
EXPORT_SYMBOL_GPL(kernfs_path_from_node);

/**
 * pr_cont_kernfs_name - pr_cont name of a kernfs_node
 * @kn: kernfs_node of interest
 *
 * This function can be called from any context.
 */
void pr_cont_kernfs_name(struct kernfs_node *kn)
{
        unsigned long flags;

        spin_lock_irqsave(&kernfs_pr_cont_lock, flags);

        kernfs_name(kn, kernfs_pr_cont_buf, sizeof(kernfs_pr_cont_buf));
        pr_cont("%s", kernfs_pr_cont_buf);

        spin_unlock_irqrestore(&kernfs_pr_cont_lock, flags);
}

/**
 * pr_cont_kernfs_path - pr_cont path of a kernfs_node
 * @kn: kernfs_node of interest
 *
 * This function can be called from any context.
 */
void pr_cont_kernfs_path(struct kernfs_node *kn)
{
        unsigned long flags;
        int sz;

        spin_lock_irqsave(&kernfs_pr_cont_lock, flags);

        sz = kernfs_path_from_node(kn, NULL, kernfs_pr_cont_buf,
                                   sizeof(kernfs_pr_cont_buf));
        if (sz < 0) {
                pr_cont("(error)");
                goto out;
        }

        if (sz >= sizeof(kernfs_pr_cont_buf)) {
                pr_cont("(name too long)");
                goto out;
        }

        pr_cont("%s", kernfs_pr_cont_buf);

out:
        spin_unlock_irqrestore(&kernfs_pr_cont_lock, flags);
}

/**
 * kernfs_get_parent - determine the parent node and pin it
 * @kn: kernfs_node of interest
 *
 * Determines @kn's parent, pins and returns it.  This function can be
 * called from any context.
 */
struct kernfs_node *kernfs_get_parent(struct kernfs_node *kn)
{
        struct kernfs_node *parent;
        unsigned long flags;

        spin_lock_irqsave(&kernfs_rename_lock, flags);
        parent = kn->parent;
        kernfs_get(parent);
        spin_unlock_irqrestore(&kernfs_rename_lock, flags);

        return parent;
}

/**
 *        kernfs_name_hash
 *        @name: Null terminated string to hash
 *        @ns:   Namespace tag to hash
 *
 *        Returns 31 bit hash of ns + name (so it fits in an off_t )
 */
static unsigned int kernfs_name_hash(const char *name, const void *ns)
{
        unsigned long hash = init_name_hash(ns);
        unsigned int len = strlen(name);
        while (len--)
                hash = partial_name_hash(*name++, hash);
        hash = end_name_hash(hash);
        hash &= 0x7fffffffU;
        /* Reserve hash numbers 0, 1 and INT_MAX for magic directory entries */
        if (hash < 2)
                hash += 2;
        if (hash >= INT_MAX)
                hash = INT_MAX - 1;
        return hash;
}

static int kernfs_name_compare(unsigned int hash, const char *name,
                               const void *ns, const struct kernfs_node *kn)
{
        if (hash < kn->hash)
                return -1;
        if (hash > kn->hash)
                return 1;
        if (ns < kn->ns)
                return -1;
        if (ns > kn->ns)
                return 1;
        return strcmp(name, kn->name);
}

static int kernfs_sd_compare(const struct kernfs_node *left,
                             const struct kernfs_node *right)
{
        return kernfs_name_compare(left->hash, left->name, left->ns, right);
}

/**
 *        kernfs_link_sibling - link kernfs_node into sibling rbtree
 *        @kn: kernfs_node of interest
 *
 *        Link @kn into its sibling rbtree which starts from
 *        @kn->parent->dir.children.
 *
 *        Locking:
 *        kernfs_rwsem held exclusive
 *
 *        RETURNS:
 *        0 on susccess -EEXIST on failure.
 */
static int kernfs_link_sibling(struct kernfs_node *kn)
{
        struct rb_node **node = &kn->parent->dir.children.rb_node;
        struct rb_node *parent = NULL;

        while (*node) {
                struct kernfs_node *pos;
                int result;

                pos = rb_to_kn(*node);
                parent = *node;
                result = kernfs_sd_compare(kn, pos);
                if (result < 0)
                        node = &pos->rb.rb_left;
                else if (result > 0)
                        node = &pos->rb.rb_right;
                else
                        return -EEXIST;
        }

        /* add new node and rebalance the tree */
        rb_link_node(&kn->rb, parent, node);
        rb_insert_color(&kn->rb, &kn->parent->dir.children);

        /* successfully added, account subdir number */
        if (kernfs_type(kn) == KERNFS_DIR)
                kn->parent->dir.subdirs++;
        kernfs_inc_rev(kn->parent);

        return 0;
}

/**
 *        kernfs_unlink_sibling - unlink kernfs_node from sibling rbtree
 *        @kn: kernfs_node of interest
 *
 *        Try to unlink @kn from its sibling rbtree which starts from
 *        kn->parent->dir.children.  Returns %true if @kn was actually
 *        removed, %false if @kn wasn't on the rbtree.
 *
 *        Locking:
 *        kernfs_rwsem held exclusive
 */
static bool kernfs_unlink_sibling(struct kernfs_node *kn)
{
        if (RB_EMPTY_NODE(&kn->rb))
                return false;

        if (kernfs_type(kn) == KERNFS_DIR)
                kn->parent->dir.subdirs--;
        kernfs_inc_rev(kn->parent);

        rb_erase(&kn->rb, &kn->parent->dir.children);
        RB_CLEAR_NODE(&kn->rb);
        return true;
}

/**
 *        kernfs_get_active - get an active reference to kernfs_node
 *        @kn: kernfs_node to get an active reference to
 *
 *        Get an active reference of @kn.  This function is noop if @kn
 *        is NULL.
 *
 *        RETURNS:
 *        Pointer to @kn on success, NULL on failure.
 */
struct kernfs_node *kernfs_get_active(struct kernfs_node *kn)
{
        if (unlikely(!kn))
                return NULL;

        if (!atomic_inc_unless_negative(&kn->active))
                return NULL;

        if (kernfs_lockdep(kn))
                rwsem_acquire_read(&kn->dep_map, 0, 1, _RET_IP_);
        return kn;
}

/**
 *        kernfs_put_active - put an active reference to kernfs_node
 *        @kn: kernfs_node to put an active reference to
 *
 *        Put an active reference to @kn.  This function is noop if @kn
 *        is NULL.
 */
void kernfs_put_active(struct kernfs_node *kn)
{
        int v;

        if (unlikely(!kn))
                return;

        if (kernfs_lockdep(kn))
                rwsem_release(&kn->dep_map, _RET_IP_);
        v = atomic_dec_return(&kn->active);
        if (likely(v != KN_DEACTIVATED_BIAS))
                return;

        wake_up_all(&kernfs_root(kn)->deactivate_waitq);
}

/**
 * kernfs_drain - drain kernfs_node
 * @kn: kernfs_node to drain
 *
 * Drain existing usages and nuke all existing mmaps of @kn.  Mutiple
 * removers may invoke this function concurrently on @kn and all will
 * return after draining is complete.
 */
static void kernfs_drain(struct kernfs_node *kn)
        __releases(&kernfs_rwsem) __acquires(&kernfs_rwsem)
{
        struct kernfs_root *root = kernfs_root(kn);

        lockdep_assert_held_write(&kernfs_rwsem);
        WARN_ON_ONCE(kernfs_active(kn));

        up_write(&kernfs_rwsem);

        if (kernfs_lockdep(kn)) {
                rwsem_acquire(&kn->dep_map, 0, 0, _RET_IP_);
                if (atomic_read(&kn->active) != KN_DEACTIVATED_BIAS)
                        lock_contended(&kn->dep_map, _RET_IP_);
        }

        /* but everyone should wait for draining */
        wait_event(root->deactivate_waitq,
                   atomic_read(&kn->active) == KN_DEACTIVATED_BIAS);

        if (kernfs_lockdep(kn)) {
                lock_acquired(&kn->dep_map, _RET_IP_);
                rwsem_release(&kn->dep_map, _RET_IP_);
        }

        kernfs_drain_open_files(kn);

        down_write(&kernfs_rwsem);
}

/**
 * kernfs_get - get a reference count on a kernfs_node
 * @kn: the target kernfs_node
 */
void kernfs_get(struct kernfs_node *kn)
{
        if (kn) {
                WARN_ON(!atomic_read(&kn->count));
                atomic_inc(&kn->count);
        }
}
EXPORT_SYMBOL_GPL(kernfs_get);

/**
 * kernfs_put - put a reference count on a kernfs_node
 * @kn: the target kernfs_node
 *
 * Put a reference count of @kn and destroy it if it reached zero.
 */
void kernfs_put(struct kernfs_node *kn)
{
        struct kernfs_node *parent;
        struct kernfs_root *root;

        if (!kn || !atomic_dec_and_test(&kn->count))
                return;
        root = kernfs_root(kn);
 repeat:
        /*
         * Moving/renaming is always done while holding reference.
         * kn->parent won't change beneath us.
         */
        parent = kn->parent;

        WARN_ONCE(atomic_read(&kn->active) != KN_DEACTIVATED_BIAS,
                  "kernfs_put: %s/%s: released with incorrect active_ref %d\n",
                  parent ? parent->name : "", kn->name, atomic_read(&kn->active));

        if (kernfs_type(kn) == KERNFS_LINK)
                kernfs_put(kn->symlink.target_kn);

        kfree_const(kn->name);

        if (kn->iattr) {
                simple_xattrs_free(&kn->iattr->xattrs);
                kmem_cache_free(kernfs_iattrs_cache, kn->iattr);
        }
        spin_lock(&kernfs_idr_lock);
        idr_remove(&root->ino_idr, (u32)kernfs_ino(kn));
        spin_unlock(&kernfs_idr_lock);
        kmem_cache_free(kernfs_node_cache, kn);

        kn = parent;
        if (kn) {
                if (atomic_dec_and_test(&kn->count))
                        goto repeat;
        } else {
                /* just released the root kn, free @root too */
                idr_destroy(&root->ino_idr);
                kfree(root);
        }
}
EXPORT_SYMBOL_GPL(kernfs_put);

/**
 * kernfs_node_from_dentry - determine kernfs_node associated with a dentry
 * @dentry: the dentry in question
 *
 * Return the kernfs_node associated with @dentry.  If @dentry is not a
 * kernfs one, %NULL is returned.
 *
 * While the returned kernfs_node will stay accessible as long as @dentry
 * is accessible, the returned node can be in any state and the caller is
 * fully responsible for determining what's accessible.
 */
struct kernfs_node *kernfs_node_from_dentry(struct dentry *dentry)
{
        if (dentry->d_sb->s_op == &kernfs_sops)
                return kernfs_dentry_node(dentry);
        return NULL;
}

static struct kernfs_node *__kernfs_new_node(struct kernfs_root *root,
                                             struct kernfs_node *parent,
                                             const char *name, umode_t mode,
                                             kuid_t uid, kgid_t gid,
                                             unsigned flags)
{
        struct kernfs_node *kn;
        u32 id_highbits;
        int ret;

        name = kstrdup_const(name, GFP_KERNEL);
        if (!name)
                return NULL;

        kn = kmem_cache_zalloc(kernfs_node_cache, GFP_KERNEL);
        if (!kn)
                goto err_out1;

        idr_preload(GFP_KERNEL);
        spin_lock(&kernfs_idr_lock);
        ret = idr_alloc_cyclic(&root->ino_idr, kn, 1, 0, GFP_ATOMIC);
        if (ret >= 0 && ret < root->last_id_lowbits)
                root->id_highbits++;
        id_highbits = root->id_highbits;
        root->last_id_lowbits = ret;
        spin_unlock(&kernfs_idr_lock);
        idr_preload_end();
        if (ret < 0)
                goto err_out2;

        kn->id = (u64)id_highbits << 32 | ret;

        atomic_set(&kn->count, 1);
        atomic_set(&kn->active, KN_DEACTIVATED_BIAS);
        RB_CLEAR_NODE(&kn->rb);

        kn->name = name;
        kn->mode = mode;
        kn->flags = flags;

        if (!uid_eq(uid, GLOBAL_ROOT_UID) || !gid_eq(gid, GLOBAL_ROOT_GID)) {
                struct iattr iattr = {
                        .ia_valid = ATTR_UID | ATTR_GID,
                        .ia_uid = uid,
                        .ia_gid = gid,
                };

                ret = __kernfs_setattr(kn, &iattr);
                if (ret < 0)
                        goto err_out3;
        }

        if (parent) {
                ret = security_kernfs_init_security(parent, kn);
                if (ret)
                        goto err_out3;
        }

        return kn;

 err_out3:
        spin_lock(&kernfs_idr_lock);
        idr_remove(&root->ino_idr, (u32)kernfs_ino(kn));
        spin_unlock(&kernfs_idr_lock);
 err_out2:
        kmem_cache_free(kernfs_node_cache, kn);
 err_out1:
        kfree_const(name);
        return NULL;
}

struct kernfs_node *kernfs_new_node(struct kernfs_node *parent,
                                    const char *name, umode_t mode,
                                    kuid_t uid, kgid_t gid,
                                    unsigned flags)
{
        struct kernfs_node *kn;

        if (parent->mode & S_ISGID) {
                /* this code block imitates inode_init_owner() for
                 * kernfs
                 */

                if (parent->iattr)
                        gid = parent->iattr->ia_gid;

                if (flags & KERNFS_DIR)
                        mode |= S_ISGID;
        }

        kn = __kernfs_new_node(kernfs_root(parent), parent,
                               name, mode, uid, gid, flags);
        if (kn) {
                kernfs_get(parent);
                kn->parent = parent;
        }
        return kn;
}

/*
 * kernfs_find_and_get_node_by_id - get kernfs_node from node id
 * @root: the kernfs root
 * @id: the target node id
 *
 * @id's lower 32bits encode ino and upper gen.  If the gen portion is
 * zero, all generations are matched.
 *
 * RETURNS:
 * NULL on failure. Return a kernfs node with reference counter incremented
 */
struct kernfs_node *kernfs_find_and_get_node_by_id(struct kernfs_root *root,
                                                   u64 id)
{
        struct kernfs_node *kn;
        ino_t ino = kernfs_id_ino(id);
        u32 gen = kernfs_id_gen(id);

        spin_lock(&kernfs_idr_lock);

        kn = idr_find(&root->ino_idr, (u32)ino);
        if (!kn)
                goto err_unlock;

        if (sizeof(ino_t) >= sizeof(u64)) {
                /* we looked up with the low 32bits, compare the whole */
                if (kernfs_ino(kn) != ino)
                        goto err_unlock;
        } else {
                /* 0 matches all generations */
                if (unlikely(gen && kernfs_gen(kn) != gen))
                        goto err_unlock;
        }

        /*
         * ACTIVATED is protected with kernfs_mutex but it was clear when
         * @kn was added to idr and we just wanna see it set.  No need to
         * grab kernfs_mutex.
         */
        if (unlikely(!(kn->flags & KERNFS_ACTIVATED) ||
                     !atomic_inc_not_zero(&kn->count)))
                goto err_unlock;

        spin_unlock(&kernfs_idr_lock);
        return kn;
err_unlock:
        spin_unlock(&kernfs_idr_lock);
        return NULL;
}

/**
 *        kernfs_add_one - add kernfs_node to parent without warning
 *        @kn: kernfs_node to be added
 *
 *        The caller must already have initialized @kn->parent.  This
 *        function increments nlink of the parent's inode if @kn is a
 *        directory and link into the children list of the parent.
 *
 *        RETURNS:
 *        0 on success, -EEXIST if entry with the given name already
 *        exists.
 */
int kernfs_add_one(struct kernfs_node *kn)
{
        struct kernfs_node *parent = kn->parent;
        struct kernfs_iattrs *ps_iattr;
        bool has_ns;
        int ret;

        down_write(&kernfs_rwsem);

        ret = -EINVAL;
        has_ns = kernfs_ns_enabled(parent);
        if (WARN(has_ns != (bool)kn->ns, KERN_WARNING "kernfs: ns %s in '%s' for '%s'\n",
                 has_ns ? "required" : "invalid", parent->name, kn->name))
                goto out_unlock;

        if (kernfs_type(parent) != KERNFS_DIR)
                goto out_unlock;

        ret = -ENOENT;
        if (parent->flags & KERNFS_EMPTY_DIR)
                goto out_unlock;

        if ((parent->flags & KERNFS_ACTIVATED) && !kernfs_active(parent))
                goto out_unlock;

        kn->hash = kernfs_name_hash(kn->name, kn->ns);

        ret = kernfs_link_sibling(kn);
        if (ret)
                goto out_unlock;

        /* Update timestamps on the parent */
        ps_iattr = parent->iattr;
        if (ps_iattr) {
                ktime_get_real_ts64(&ps_iattr->ia_ctime);
                ps_iattr->ia_mtime = ps_iattr->ia_ctime;
        }

        up_write(&kernfs_rwsem);

        /*
         * Activate the new node unless CREATE_DEACTIVATED is requested.
         * If not activated here, the kernfs user is responsible for
         * activating the node with kernfs_activate().  A node which hasn't
         * been activated is not visible to userland and its removal won't
         * trigger deactivation.
         */
        if (!(kernfs_root(kn)->flags & KERNFS_ROOT_CREATE_DEACTIVATED))
                kernfs_activate(kn);
        return 0;

out_unlock:
        up_write(&kernfs_rwsem);
        return ret;
}

/**
 * kernfs_find_ns - find kernfs_node with the given name
 * @parent: kernfs_node to search under
 * @name: name to look for
 * @ns: the namespace tag to use
 *
 * Look for kernfs_node with name @name under @parent.  Returns pointer to
 * the found kernfs_node on success, %NULL on failure.
 */
static struct kernfs_node *kernfs_find_ns(struct kernfs_node *parent,
                                          const unsigned char *name,
                                          const void *ns)
{
        struct rb_node *node = parent->dir.children.rb_node;
        bool has_ns = kernfs_ns_enabled(parent);
        unsigned int hash;

        lockdep_assert_held(&kernfs_rwsem);

        if (has_ns != (bool)ns) {
                WARN(1, KERN_WARNING "kernfs: ns %s in '%s' for '%s'\n",
                     has_ns ? "required" : "invalid", parent->name, name);
                return NULL;
        }

        hash = kernfs_name_hash(name, ns);
        while (node) {
                struct kernfs_node *kn;
                int result;

                kn = rb_to_kn(node);
                result = kernfs_name_compare(hash, name, ns, kn);
                if (result < 0)
                        node = node->rb_left;
                else if (result > 0)
                        node = node->rb_right;
                else
                        return kn;
        }
        return NULL;
}

static struct kernfs_node *kernfs_walk_ns(struct kernfs_node *parent,
                                          const unsigned char *path,
                                          const void *ns)
{
        size_t len;
        char *p, *name;

        lockdep_assert_held_read(&kernfs_rwsem);

        spin_lock_irq(&kernfs_pr_cont_lock);

        len = strlcpy(kernfs_pr_cont_buf, path, sizeof(kernfs_pr_cont_buf));

        if (len >= sizeof(kernfs_pr_cont_buf)) {
                spin_unlock_irq(&kernfs_pr_cont_lock);
                return NULL;
        }

        p = kernfs_pr_cont_buf;

        while ((name = strsep(&p, "/")) && parent) {
                if (*name == '\0')
                        continue;
                parent = kernfs_find_ns(parent, name, ns);
        }

        spin_unlock_irq(&kernfs_pr_cont_lock);

        return parent;
}

/**
 * kernfs_find_and_get_ns - find and get kernfs_node with the given name
 * @parent: kernfs_node to search under
 * @name: name to look for
 * @ns: the namespace tag to use
 *
 * Look for kernfs_node with name @name under @parent and get a reference
 * if found.  This function may sleep and returns pointer to the found
 * kernfs_node on success, %NULL on failure.
 */
struct kernfs_node *kernfs_find_and_get_ns(struct kernfs_node *parent,
                                           const char *name, const void *ns)
{
        struct kernfs_node *kn;

        down_read(&kernfs_rwsem);
        kn = kernfs_find_ns(parent, name, ns);
        kernfs_get(kn);
        up_read(&kernfs_rwsem);

        return kn;
}
EXPORT_SYMBOL_GPL(kernfs_find_and_get_ns);

/**
 * kernfs_walk_and_get_ns - find and get kernfs_node with the given path
 * @parent: kernfs_node to search under
 * @path: path to look for
 * @ns: the namespace tag to use
 *
 * Look for kernfs_node with path @path under @parent and get a reference
 * if found.  This function may sleep and returns pointer to the found
 * kernfs_node on success, %NULL on failure.
 */
struct kernfs_node *kernfs_walk_and_get_ns(struct kernfs_node *parent,
                                           const char *path, const void *ns)
{
        struct kernfs_node *kn;

        down_read(&kernfs_rwsem);
        kn = kernfs_walk_ns(parent, path, ns);
        kernfs_get(kn);
        up_read(&kernfs_rwsem);

        return kn;
}

/**
 * kernfs_create_root - create a new kernfs hierarchy
 * @scops: optional syscall operations for the hierarchy
 * @flags: KERNFS_ROOT_* flags
 * @priv: opaque data associated with the new directory
 *
 * Returns the root of the new hierarchy on success, ERR_PTR() value on
 * failure.
 */
struct kernfs_root *kernfs_create_root(struct kernfs_syscall_ops *scops,
                                       unsigned int flags, void *priv)
{
        struct kernfs_root *root;
        struct kernfs_node *kn;

        root = kzalloc(sizeof(*root), GFP_KERNEL);
        if (!root)
                return ERR_PTR(-ENOMEM);

        idr_init(&root->ino_idr);
        INIT_LIST_HEAD(&root->supers);

        /*
         * On 64bit ino setups, id is ino.  On 32bit, low 32bits are ino.
         * High bits generation.  The starting value for both ino and
         * genenration is 1.  Initialize upper 32bit allocation
         * accordingly.
         */
        if (sizeof(ino_t) >= sizeof(u64))
                root->id_highbits = 0;
        else
                root->id_highbits = 1;

        kn = __kernfs_new_node(root, NULL, "", S_IFDIR | S_IRUGO | S_IXUGO,
                               GLOBAL_ROOT_UID, GLOBAL_ROOT_GID,
                               KERNFS_DIR);
        if (!kn) {
                idr_destroy(&root->ino_idr);
                kfree(root);
                return ERR_PTR(-ENOMEM);
        }

        kn->priv = priv;
        kn->dir.root = root;

        root->syscall_ops = scops;
        root->flags = flags;
        root->kn = kn;
        init_waitqueue_head(&root->deactivate_waitq);

        if (!(root->flags & KERNFS_ROOT_CREATE_DEACTIVATED))
                kernfs_activate(kn);

        return root;
}

/**
 * kernfs_destroy_root - destroy a kernfs hierarchy
 * @root: root of the hierarchy to destroy
 *
 * Destroy the hierarchy anchored at @root by removing all existing
 * directories and destroying @root.
 */
void kernfs_destroy_root(struct kernfs_root *root)
{
        kernfs_remove(root->kn);        /* will also free @root */
}

/**
 * kernfs_create_dir_ns - create a directory
 * @parent: parent in which to create a new directory
 * @name: name of the new directory
 * @mode: mode of the new directory
 * @uid: uid of the new directory
 * @gid: gid of the new directory
 * @priv: opaque data associated with the new directory
 * @ns: optional namespace tag of the directory
 *
 * Returns the created node on success, ERR_PTR() value on failure.
 */
struct kernfs_node *kernfs_create_dir_ns(struct kernfs_node *parent,
                                         const char *name, umode_t mode,
                                         kuid_t uid, kgid_t gid,
                                         void *priv, const void *ns)
{
        struct kernfs_node *kn;
        int rc;

        /* allocate */
        kn = kernfs_new_node(parent, name, mode | S_IFDIR,
                             uid, gid, KERNFS_DIR);
        if (!kn)
                return ERR_PTR(-ENOMEM);

        kn->dir.root = parent->dir.root;
        kn->ns = ns;
        kn->priv = priv;

        /* link in */
        rc = kernfs_add_one(kn);
        if (!rc)
                return kn;

        kernfs_put(kn);
        return ERR_PTR(rc);
}

/**
 * kernfs_create_empty_dir - create an always empty directory
 * @parent: parent in which to create a new directory
 * @name: name of the new directory
 *
 * Returns the created node on success, ERR_PTR() value on failure.
 */
struct kernfs_node *kernfs_create_empty_dir(struct kernfs_node *parent,
                                            const char *name)
{
        struct kernfs_node *kn;
        int rc;

        /* allocate */
        kn = kernfs_new_node(parent, name, S_IRUGO|S_IXUGO|S_IFDIR,
                             GLOBAL_ROOT_UID, GLOBAL_ROOT_GID, KERNFS_DIR);
        if (!kn)
                return ERR_PTR(-ENOMEM);

        kn->flags |= KERNFS_EMPTY_DIR;
        kn->dir.root = parent->dir.root;
        kn->ns = NULL;
        kn->priv = NULL;

        /* link in */
        rc = kernfs_add_one(kn);
        if (!rc)
                return kn;

        kernfs_put(kn);
        return ERR_PTR(rc);
}

static int kernfs_dop_revalidate(struct dentry *dentry, unsigned int flags)
{
        struct kernfs_node *kn;

        if (flags & LOOKUP_RCU)
                return -ECHILD;

        /* Negative hashed dentry? */
        if (d_really_is_negative(dentry)) {
                struct kernfs_node *parent;

                /* If the kernfs parent node has changed discard and
                 * proceed to ->lookup.
                 */
                down_read(&kernfs_rwsem);
                spin_lock(&dentry->d_lock);
                parent = kernfs_dentry_node(dentry->d_parent);
                if (parent) {
                        if (kernfs_dir_changed(parent, dentry)) {
                                spin_unlock(&dentry->d_lock);
                                up_read(&kernfs_rwsem);
                                return 0;
                        }
                }
                spin_unlock(&dentry->d_lock);
                up_read(&kernfs_rwsem);

                /* The kernfs parent node hasn't changed, leave the
                 * dentry negative and return success.
                 */
                return 1;
        }

        kn = kernfs_dentry_node(dentry);
        down_read(&kernfs_rwsem);

        /* The kernfs node has been deactivated */
        if (!kernfs_active(kn))
                goto out_bad;

        /* The kernfs node has been moved? */
        if (kernfs_dentry_node(dentry->d_parent) != kn->parent)
                goto out_bad;

        /* The kernfs node has been renamed */
        if (strcmp(dentry->d_name.name, kn->name) != 0)
                goto out_bad;

        /* The kernfs node has been moved to a different namespace */
        if (kn->parent && kernfs_ns_enabled(kn->parent) &&
            kernfs_info(dentry->d_sb)->ns != kn->ns)
                goto out_bad;

        up_read(&kernfs_rwsem);
        return 1;
out_bad:
        up_read(&kernfs_rwsem);
        return 0;
}

const struct dentry_operations kernfs_dops = {
        .d_revalidate        = kernfs_dop_revalidate,
};

static struct dentry *kernfs_iop_lookup(struct inode *dir,
                                        struct dentry *dentry,
                                        unsigned int flags)
{
        struct kernfs_node *parent = dir->i_private;
        struct kernfs_node *kn;
        struct inode *inode = NULL;
        const void *ns = NULL;

        down_read(&kernfs_rwsem);
        if (kernfs_ns_enabled(parent))
                ns = kernfs_info(dir->i_sb)->ns;

        kn = kernfs_find_ns(parent, dentry->d_name.name, ns);
        /* attach dentry and inode */
        if (kn) {
                /* Inactive nodes are invisible to the VFS so don't
                 * create a negative.
                 */
                if (!kernfs_active(kn)) {
                        up_read(&kernfs_rwsem);
                        return NULL;
                }
                inode = kernfs_get_inode(dir->i_sb, kn);
                if (!inode)
                        inode = ERR_PTR(-ENOMEM);
        }
        /*
         * Needed for negative dentry validation.
         * The negative dentry can be created in kernfs_iop_lookup()
         * or transforms from positive dentry in dentry_unlink_inode()
         * called from vfs_rmdir().
         */
        if (!IS_ERR(inode))
                kernfs_set_rev(parent, dentry);
        up_read(&kernfs_rwsem);

        /* instantiate and hash (possibly negative) dentry */
        return d_splice_alias(inode, dentry);
}

static int kernfs_iop_mkdir(struct user_namespace *mnt_userns,
                            struct inode *dir, struct dentry *dentry,
                            umode_t mode)
{
        struct kernfs_node *parent = dir->i_private;
        struct kernfs_syscall_ops *scops = kernfs_root(parent)->syscall_ops;
        int ret;

        if (!scops || !scops->mkdir)
                return -EPERM;

        if (!kernfs_get_active(parent))
                return -ENODEV;

        ret = scops->mkdir(parent, dentry->d_name.name, mode);

        kernfs_put_active(parent);
        return ret;
}

static int kernfs_iop_rmdir(struct inode *dir, struct dentry *dentry)
{
        struct kernfs_node *kn  = kernfs_dentry_node(dentry);
        struct kernfs_syscall_ops *scops = kernfs_root(kn)->syscall_ops;
        int ret;

        if (!scops || !scops->rmdir)
                return -EPERM;

        if (!kernfs_get_active(kn))
                return -ENODEV;

        ret = scops->rmdir(kn);

        kernfs_put_active(kn);
        return ret;
}

static int kernfs_iop_rename(struct user_namespace *mnt_userns,
                             struct inode *old_dir, struct dentry *old_dentry,
                             struct inode *new_dir, struct dentry *new_dentry,
                             unsigned int flags)
{
        struct kernfs_node *kn = kernfs_dentry_node(old_dentry);
        struct kernfs_node *new_parent = new_dir->i_private;
        struct kernfs_syscall_ops *scops = kernfs_root(kn)->syscall_ops;
        int ret;

        if (flags)
                return -EINVAL;

        if (!scops || !scops->rename)
                return -EPERM;

        if (!kernfs_get_active(kn))
                return -ENODEV;

        if (!kernfs_get_active(new_parent)) {
                kernfs_put_active(kn);
                return -ENODEV;
        }

        ret = scops->rename(kn, new_parent, new_dentry->d_name.name);

        kernfs_put_active(new_parent);
        kernfs_put_active(kn);
        return ret;
}

const struct inode_operations kernfs_dir_iops = {
        .lookup                = kernfs_iop_lookup,
        .permission        = kernfs_iop_permission,
        .setattr        = kernfs_iop_setattr,
        .getattr        = kernfs_iop_getattr,
        .listxattr        = kernfs_iop_listxattr,

        .mkdir                = kernfs_iop_mkdir,
        .rmdir                = kernfs_iop_rmdir,
        .rename                = kernfs_iop_rename,
};

static struct kernfs_node *kernfs_leftmost_descendant(struct kernfs_node *pos)
{
        struct kernfs_node *last;

        while (true) {
                struct rb_node *rbn;

                last = pos;

                if (kernfs_type(pos) != KERNFS_DIR)
                        break;

                rbn = rb_first(&pos->dir.children);
                if (!rbn)
                        break;

                pos = rb_to_kn(rbn);
        }

        return last;
}

/**
 * kernfs_next_descendant_post - find the next descendant for post-order walk
 * @pos: the current position (%NULL to initiate traversal)
 * @root: kernfs_node whose descendants to walk
 *
 * Find the next descendant to visit for post-order traversal of @root's
 * descendants.  @root is included in the iteration and the last node to be
 * visited.
 */
static struct kernfs_node *kernfs_next_descendant_post(struct kernfs_node *pos,
                                                       struct kernfs_node *root)
{
        struct rb_node *rbn;

        lockdep_assert_held_write(&kernfs_rwsem);

        /* if first iteration, visit leftmost descendant which may be root */
        if (!pos)
                return kernfs_leftmost_descendant(root);

        /* if we visited @root, we're done */
        if (pos == root)
                return NULL;

        /* if there's an unvisited sibling, visit its leftmost descendant */
        rbn = rb_next(&pos->rb);
        if (rbn)
                return kernfs_leftmost_descendant(rb_to_kn(rbn));

        /* no sibling left, visit parent */
        return pos->parent;
}

/**
 * kernfs_activate - activate a node which started deactivated
 * @kn: kernfs_node whose subtree is to be activated
 *
 * If the root has KERNFS_ROOT_CREATE_DEACTIVATED set, a newly created node
 * needs to be explicitly activated.  A node which hasn't been activated
 * isn't visible to userland and deactivation is skipped during its
 * removal.  This is useful to construct atomic init sequences where
 * creation of multiple nodes should either succeed or fail atomically.
 *
 * The caller is responsible for ensuring that this function is not called
 * after kernfs_remove*() is invoked on @kn.
 */
void kernfs_activate(struct kernfs_node *kn)
{
        struct kernfs_node *pos;

        down_write(&kernfs_rwsem);

        pos = NULL;
        while ((pos = kernfs_next_descendant_post(pos, kn))) {
                if (pos->flags & KERNFS_ACTIVATED)
                        continue;

                WARN_ON_ONCE(pos->parent && RB_EMPTY_NODE(&pos->rb));
                WARN_ON_ONCE(atomic_read(&pos->active) != KN_DEACTIVATED_BIAS);

                atomic_sub(KN_DEACTIVATED_BIAS, &pos->active);
                pos->flags |= KERNFS_ACTIVATED;
        }

        up_write(&kernfs_rwsem);
}

static void __kernfs_remove(struct kernfs_node *kn)
{
        struct kernfs_node *pos;

        lockdep_assert_held_write(&kernfs_rwsem);

        /*
         * Short-circuit if non-root @kn has already finished removal.
         * This is for kernfs_remove_self() which plays with active ref
         * after removal.
         */
        if (!kn || (kn->parent && RB_EMPTY_NODE(&kn->rb)))
                return;

        pr_debug("kernfs %s: removing\n", kn->name);

        /* prevent any new usage under @kn by deactivating all nodes */
        pos = NULL;
        while ((pos = kernfs_next_descendant_post(pos, kn)))
                if (kernfs_active(pos))
                        atomic_add(KN_DEACTIVATED_BIAS, &pos->active);

        /* deactivate and unlink the subtree node-by-node */
        do {
                pos = kernfs_leftmost_descendant(kn);

                /*
                 * kernfs_drain() drops kernfs_rwsem temporarily and @pos's
                 * base ref could have been put by someone else by the time
                 * the function returns.  Make sure it doesn't go away
                 * underneath us.
                 */
                kernfs_get(pos);

                /*
                 * Drain iff @kn was activated.  This avoids draining and
                 * its lockdep annotations for nodes which have never been
                 * activated and allows embedding kernfs_remove() in create
                 * error paths without worrying about draining.
                 */
                if (kn->flags & KERNFS_ACTIVATED)
                        kernfs_drain(pos);
                else
                        WARN_ON_ONCE(atomic_read(&kn->active) != KN_DEACTIVATED_BIAS);

                /*
                 * kernfs_unlink_sibling() succeeds once per node.  Use it
                 * to decide who's responsible for cleanups.
                 */
                if (!pos->parent || kernfs_unlink_sibling(pos)) {
                        struct kernfs_iattrs *ps_iattr =
                                pos->parent ? pos->parent->iattr : NULL;

                        /* update timestamps on the parent */
                        if (ps_iattr) {
                                ktime_get_real_ts64(&ps_iattr->ia_ctime);
                                ps_iattr->ia_mtime = ps_iattr->ia_ctime;
                        }

                        kernfs_put(pos);
                }

                kernfs_put(pos);
        } while (pos != kn);
}

/**
 * kernfs_remove - remove a kernfs_node recursively
 * @kn: the kernfs_node to remove
 *
 * Remove @kn along with all its subdirectories and files.
 */
void kernfs_remove(struct kernfs_node *kn)
{
        down_write(&kernfs_rwsem);
        __kernfs_remove(kn);
        up_write(&kernfs_rwsem);
}

/**
 * kernfs_break_active_protection - break out of active protection
 * @kn: the self kernfs_node
 *
 * The caller must be running off of a kernfs operation which is invoked
 * with an active reference - e.g. one of kernfs_ops.  Each invocation of
 * this function must also be matched with an invocation of
 * kernfs_unbreak_active_protection().
 *
 * This function releases the active reference of @kn the caller is
 * holding.  Once this function is called, @kn may be removed at any point
 * and the caller is solely responsible for ensuring that the objects it
 * dereferences are accessible.
 */
void kernfs_break_active_protection(struct kernfs_node *kn)
{
        /*
         * Take out ourself out of the active ref dependency chain.  If
         * we're called without an active ref, lockdep will complain.
         */
        kernfs_put_active(kn);
}

/**
 * kernfs_unbreak_active_protection - undo kernfs_break_active_protection()
 * @kn: the self kernfs_node
 *
 * If kernfs_break_active_protection() was called, this function must be
 * invoked before finishing the kernfs operation.  Note that while this
 * function restores the active reference, it doesn't and can't actually
 * restore the active protection - @kn may already or be in the process of
 * being removed.  Once kernfs_break_active_protection() is invoked, that
 * protection is irreversibly gone for the kernfs operation instance.
 *
 * While this function may be called at any point after
 * kernfs_break_active_protection() is invoked, its most useful location
 * would be right before the enclosing kernfs operation returns.
 */
void kernfs_unbreak_active_protection(struct kernfs_node *kn)
{
        /*
         * @kn->active could be in any state; however, the increment we do
         * here will be undone as soon as the enclosing kernfs operation
         * finishes and this temporary bump can't break anything.  If @kn
         * is alive, nothing changes.  If @kn is being deactivated, the
         * soon-to-follow put will either finish deactivation or restore
         * deactivated state.  If @kn is already removed, the temporary
         * bump is guaranteed to be gone before @kn is released.
         */
        atomic_inc(&kn->active);
        if (kernfs_lockdep(kn))
                rwsem_acquire(&kn->dep_map, 0, 1, _RET_IP_);
}

/**
 * kernfs_remove_self - remove a kernfs_node from its own method
 * @kn: the self kernfs_node to remove
 *
 * The caller must be running off of a kernfs operation which is invoked
 * with an active reference - e.g. one of kernfs_ops.  This can be used to
 * implement a file operation which deletes itself.
 *
 * For example, the "delete" file for a sysfs device directory can be
 * implemented by invoking kernfs_remove_self() on the "delete" file
 * itself.  This function breaks the circular dependency of trying to
 * deactivate self while holding an active ref itself.  It isn't necessary
 * to modify the usual removal path to use kernfs_remove_self().  The
 * "delete" implementation can simply invoke kernfs_remove_self() on self
 * before proceeding with the usual removal path.  kernfs will ignore later
 * kernfs_remove() on self.
 *
 * kernfs_remove_self() can be called multiple times concurrently on the
 * same kernfs_node.  Only the first one actually performs removal and
 * returns %true.  All others will wait until the kernfs operation which
 * won self-removal finishes and return %false.  Note that the losers wait
 * for the completion of not only the winning kernfs_remove_self() but also
 * the whole kernfs_ops which won the arbitration.  This can be used to
 * guarantee, for example, all concurrent writes to a "delete" file to
 * finish only after the whole operation is complete.
 */
bool kernfs_remove_self(struct kernfs_node *kn)
{
        bool ret;

        down_write(&kernfs_rwsem);
        kernfs_break_active_protection(kn);

        /*
         * SUICIDAL is used to arbitrate among competing invocations.  Only
         * the first one will actually perform removal.  When the removal
         * is complete, SUICIDED is set and the active ref is restored
         * while kernfs_rwsem for held exclusive.  The ones which lost
         * arbitration waits for SUICIDED && drained which can happen only
         * after the enclosing kernfs operation which executed the winning
         * instance of kernfs_remove_self() finished.
         */
        if (!(kn->flags & KERNFS_SUICIDAL)) {
                kn->flags |= KERNFS_SUICIDAL;
                __kernfs_remove(kn);
                kn->flags |= KERNFS_SUICIDED;
                ret = true;
        } else {
                wait_queue_head_t *waitq = &kernfs_root(kn)->deactivate_waitq;
                DEFINE_WAIT(wait);

                while (true) {
                        prepare_to_wait(waitq, &wait, TASK_UNINTERRUPTIBLE);

                        if ((kn->flags & KERNFS_SUICIDED) &&
                            atomic_read(&kn->active) == KN_DEACTIVATED_BIAS)
                                break;

                        up_write(&kernfs_rwsem);
                        schedule();
                        down_write(&kernfs_rwsem);
                }
                finish_wait(waitq, &wait);
                WARN_ON_ONCE(!RB_EMPTY_NODE(&kn->rb));
                ret = false;
        }

        /*
         * This must be done while kernfs_rwsem held exclusive; otherwise,
         * waiting for SUICIDED && deactivated could finish prematurely.
         */
        kernfs_unbreak_active_protection(kn);

        up_write(&kernfs_rwsem);
        return ret;
}

/**
 * kernfs_remove_by_name_ns - find a kernfs_node by name and remove it
 * @parent: parent of the target
 * @name: name of the kernfs_node to remove
 * @ns: namespace tag of the kernfs_node to remove
 *
 * Look for the kernfs_node with @name and @ns under @parent and remove it.
 * Returns 0 on success, -ENOENT if such entry doesn't exist.
 */
int kernfs_remove_by_name_ns(struct kernfs_node *parent, const char *name,
                             const void *ns)
{
        struct kernfs_node *kn;

        if (!parent) {
                WARN(1, KERN_WARNING "kernfs: can not remove '%s', no directory\n",
                        name);
                return -ENOENT;
        }

        down_write(&kernfs_rwsem);

        kn = kernfs_find_ns(parent, name, ns);
        if (kn) {
                kernfs_get(kn);
                __kernfs_remove(kn);
                kernfs_put(kn);
        }

        up_write(&kernfs_rwsem);

        if (kn)
                return 0;
        else
                return -ENOENT;
}

/**
 * kernfs_rename_ns - move and rename a kernfs_node
 * @kn: target node
 * @new_parent: new parent to put @sd under
 * @new_name: new name
 * @new_ns: new namespace tag
 */
int kernfs_rename_ns(struct kernfs_node *kn, struct kernfs_node *new_parent,
                     const char *new_name, const void *new_ns)
{
        struct kernfs_node *old_parent;
        const char *old_name = NULL;
        int error;

        /* can't move or rename root */
        if (!kn->parent)
                return -EINVAL;

        down_write(&kernfs_rwsem);

        error = -ENOENT;
        if (!kernfs_active(kn) || !kernfs_active(new_parent) ||
            (new_parent->flags & KERNFS_EMPTY_DIR))
                goto out;

        error = 0;
        if ((kn->parent == new_parent) && (kn->ns == new_ns) &&
            (strcmp(kn->name, new_name) == 0))
                goto out;        /* nothing to rename */

        error = -EEXIST;
        if (kernfs_find_ns(new_parent, new_name, new_ns))
                goto out;

        /* rename kernfs_node */
        if (strcmp(kn->name, new_name) != 0) {
                error = -ENOMEM;
                new_name = kstrdup_const(new_name, GFP_KERNEL);
                if (!new_name)
                        goto out;
        } else {
                new_name = NULL;
        }

        /*
         * Move to the appropriate place in the appropriate directories rbtree.
         */
        kernfs_unlink_sibling(kn);
        kernfs_get(new_parent);

        /* rename_lock protects ->parent and ->name accessors */
        spin_lock_irq(&kernfs_rename_lock);

        old_parent = kn->parent;
        kn->parent = new_parent;

        kn->ns = new_ns;
        if (new_name) {
                old_name = kn->name;
                kn->name = new_name;
        }

        spin_unlock_irq(&kernfs_rename_lock);

        kn->hash = kernfs_name_hash(kn->name, kn->ns);
        kernfs_link_sibling(kn);

        kernfs_put(old_parent);
        kfree_const(old_name);

        error = 0;
 out:
        up_write(&kernfs_rwsem);
        return error;
}

/* Relationship between mode and the DT_xxx types */
static inline unsigned char dt_type(struct kernfs_node *kn)
{
        return (kn->mode >> 12) & 15;
}

static int kernfs_dir_fop_release(struct inode *inode, struct file *filp)
{
        kernfs_put(filp->private_data);
        return 0;
}

static struct kernfs_node *kernfs_dir_pos(const void *ns,
        struct kernfs_node *parent, loff_t hash, struct kernfs_node *pos)
{
        if (pos) {
                int valid = kernfs_active(pos) &&
                        pos->parent == parent && hash == pos->hash;
                kernfs_put(pos);
                if (!valid)
                        pos = NULL;
        }
        if (!pos && (hash > 1) && (hash < INT_MAX)) {
                struct rb_node *node = parent->dir.children.rb_node;
                while (node) {
                        pos = rb_to_kn(node);

                        if (hash < pos->hash)
                                node = node->rb_left;
                        else if (hash > pos->hash)
                                node = node->rb_right;
                        else
                                break;
                }
        }
        /* Skip over entries which are dying/dead or in the wrong namespace */
        while (pos && (!kernfs_active(pos) || pos->ns != ns)) {
                struct rb_node *node = rb_next(&pos->rb);
                if (!node)
                        pos = NULL;
                else
                        pos = rb_to_kn(node);
        }
        return pos;
}

static struct kernfs_node *kernfs_dir_next_pos(const void *ns,
        struct kernfs_node *parent, ino_t ino, struct kernfs_node *pos)
{
        pos = kernfs_dir_pos(ns, parent, ino, pos);
        if (pos) {
                do {
                        struct rb_node *node = rb_next(&pos->rb);
                        if (!node)
                                pos = NULL;
                        else
                                pos = rb_to_kn(node);
                } while (pos && (!kernfs_active(pos) || pos->ns != ns));
        }
        return pos;
}

static int kernfs_fop_readdir(struct file *file, struct dir_context *ctx)
{
        struct dentry *dentry = file->f_path.dentry;
        struct kernfs_node *parent = kernfs_dentry_node(dentry);
        struct kernfs_node *pos = file->private_data;
        const void *ns = NULL;

        if (!dir_emit_dots(file, ctx))
                return 0;
        down_read(&kernfs_rwsem);

        if (kernfs_ns_enabled(parent))
                ns = kernfs_info(dentry->d_sb)->ns;

        for (pos = kernfs_dir_pos(ns, parent, ctx->pos, pos);
             pos;
             pos = kernfs_dir_next_pos(ns, parent, ctx->pos, pos)) {
                const char *name = pos->name;
                unsigned int type = dt_type(pos);
                int len = strlen(name);
                ino_t ino = kernfs_ino(pos);

                ctx->pos = pos->hash;
                file->private_data = pos;
                kernfs_get(pos);

                up_read(&kernfs_rwsem);
                if (!dir_emit(ctx, name, len, ino, type))
                        return 0;
                down_read(&kernfs_rwsem);
        }
        up_read(&kernfs_rwsem);
        file->private_data = NULL;
        ctx->pos = INT_MAX;
        return 0;
}

const struct file_operations kernfs_dir_fops = {
        .read                = generic_read_dir,
        .iterate_shared        = kernfs_fop_readdir,
        .release        = kernfs_dir_fop_release,
        .llseek                = generic_file_llseek,
};




















   56 














   40 
   40 


















   39 
























































































































































































































































































































































































































































   60 













































   40 
   40 

   40 





   40 









   40 
   40 




























   40 










   40 








   40 
   40 




























































   40 


















































    1 








    1 





   12 







    7 


    8 


    8 








   12 



























































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * drivers/net/bond/bond_netlink.c - Netlink interface for bonding
 * Copyright (c) 2013 Jiri Pirko <jiri@resnulli.us>
 * Copyright (c) 2013 Scott Feldman <sfeldma@cumulusnetworks.com>
 */

#include <linux/module.h>
#include <linux/errno.h>
#include <linux/netdevice.h>
#include <linux/etherdevice.h>
#include <linux/if_link.h>
#include <linux/if_ether.h>
#include <net/netlink.h>
#include <net/rtnetlink.h>
#include <net/bonding.h>

static size_t bond_get_slave_size(const struct net_device *bond_dev,
                                  const struct net_device *slave_dev)
{
        return nla_total_size(sizeof(u8)) +        /* IFLA_BOND_SLAVE_STATE */
                nla_total_size(sizeof(u8)) +        /* IFLA_BOND_SLAVE_MII_STATUS */
                nla_total_size(sizeof(u32)) +        /* IFLA_BOND_SLAVE_LINK_FAILURE_COUNT */
                nla_total_size(MAX_ADDR_LEN) +        /* IFLA_BOND_SLAVE_PERM_HWADDR */
                nla_total_size(sizeof(u16)) +        /* IFLA_BOND_SLAVE_QUEUE_ID */
                nla_total_size(sizeof(u16)) +        /* IFLA_BOND_SLAVE_AD_AGGREGATOR_ID */
                nla_total_size(sizeof(u8)) +        /* IFLA_BOND_SLAVE_AD_ACTOR_OPER_PORT_STATE */
                nla_total_size(sizeof(u16)) +        /* IFLA_BOND_SLAVE_AD_PARTNER_OPER_PORT_STATE */
                0;
}

static int bond_fill_slave_info(struct sk_buff *skb,
                                const struct net_device *bond_dev,
                                const struct net_device *slave_dev)
{
        struct slave *slave = bond_slave_get_rtnl(slave_dev);

        if (nla_put_u8(skb, IFLA_BOND_SLAVE_STATE, bond_slave_state(slave)))
                goto nla_put_failure;

        if (nla_put_u8(skb, IFLA_BOND_SLAVE_MII_STATUS, slave->link))
                goto nla_put_failure;

        if (nla_put_u32(skb, IFLA_BOND_SLAVE_LINK_FAILURE_COUNT,
                        slave->link_failure_count))
                goto nla_put_failure;

        if (nla_put(skb, IFLA_BOND_SLAVE_PERM_HWADDR,
                    slave_dev->addr_len, slave->perm_hwaddr))
                goto nla_put_failure;

        if (nla_put_u16(skb, IFLA_BOND_SLAVE_QUEUE_ID, slave->queue_id))
                goto nla_put_failure;

        if (BOND_MODE(slave->bond) == BOND_MODE_8023AD) {
                const struct aggregator *agg;
                const struct port *ad_port;

                ad_port = &SLAVE_AD_INFO(slave)->port;
                agg = SLAVE_AD_INFO(slave)->port.aggregator;
                if (agg) {
                        if (nla_put_u16(skb, IFLA_BOND_SLAVE_AD_AGGREGATOR_ID,
                                        agg->aggregator_identifier))
                                goto nla_put_failure;
                        if (nla_put_u8(skb,
                                       IFLA_BOND_SLAVE_AD_ACTOR_OPER_PORT_STATE,
                                       ad_port->actor_oper_port_state))
                                goto nla_put_failure;
                        if (nla_put_u16(skb,
                                        IFLA_BOND_SLAVE_AD_PARTNER_OPER_PORT_STATE,
                                        ad_port->partner_oper.port_state))
                                goto nla_put_failure;
                }
        }

        return 0;

nla_put_failure:
        return -EMSGSIZE;
}

/* Limit the max delay range to 300s */
static struct netlink_range_validation delay_range = {
        .max = 300000,
};

static const struct nla_policy bond_policy[IFLA_BOND_MAX + 1] = {
        [IFLA_BOND_MODE]                = { .type = NLA_U8 },
        [IFLA_BOND_ACTIVE_SLAVE]        = { .type = NLA_U32 },
        [IFLA_BOND_MIIMON]                = { .type = NLA_U32 },
        [IFLA_BOND_UPDELAY]                = { .type = NLA_U32 },
        [IFLA_BOND_DOWNDELAY]                = { .type = NLA_U32 },
        [IFLA_BOND_USE_CARRIER]                = { .type = NLA_U8 },
        [IFLA_BOND_ARP_INTERVAL]        = { .type = NLA_U32 },
        [IFLA_BOND_ARP_IP_TARGET]        = { .type = NLA_NESTED },
        [IFLA_BOND_ARP_VALIDATE]        = { .type = NLA_U32 },
        [IFLA_BOND_ARP_ALL_TARGETS]        = { .type = NLA_U32 },
        [IFLA_BOND_PRIMARY]                = { .type = NLA_U32 },
        [IFLA_BOND_PRIMARY_RESELECT]        = { .type = NLA_U8 },
        [IFLA_BOND_FAIL_OVER_MAC]        = { .type = NLA_U8 },
        [IFLA_BOND_XMIT_HASH_POLICY]        = { .type = NLA_U8 },
        [IFLA_BOND_RESEND_IGMP]                = { .type = NLA_U32 },
        [IFLA_BOND_NUM_PEER_NOTIF]        = { .type = NLA_U8 },
        [IFLA_BOND_ALL_SLAVES_ACTIVE]        = { .type = NLA_U8 },
        [IFLA_BOND_MIN_LINKS]                = { .type = NLA_U32 },
        [IFLA_BOND_LP_INTERVAL]                = { .type = NLA_U32 },
        [IFLA_BOND_PACKETS_PER_SLAVE]        = { .type = NLA_U32 },
        [IFLA_BOND_AD_LACP_ACTIVE]        = { .type = NLA_U8 },
        [IFLA_BOND_AD_LACP_RATE]        = { .type = NLA_U8 },
        [IFLA_BOND_AD_SELECT]                = { .type = NLA_U8 },
        [IFLA_BOND_AD_INFO]                = { .type = NLA_NESTED },
        [IFLA_BOND_AD_ACTOR_SYS_PRIO]        = { .type = NLA_U16 },
        [IFLA_BOND_AD_USER_PORT_KEY]        = { .type = NLA_U16 },
        [IFLA_BOND_AD_ACTOR_SYSTEM]        = { .type = NLA_BINARY,
                                            .len  = ETH_ALEN },
        [IFLA_BOND_TLB_DYNAMIC_LB]        = { .type = NLA_U8 },
        [IFLA_BOND_PEER_NOTIF_DELAY]    = NLA_POLICY_FULL_RANGE(NLA_U32, &delay_range),
        [IFLA_BOND_MISSED_MAX]                = { .type = NLA_U8 },
};

static const struct nla_policy bond_slave_policy[IFLA_BOND_SLAVE_MAX + 1] = {
        [IFLA_BOND_SLAVE_QUEUE_ID]        = { .type = NLA_U16 },
};

static int bond_validate(struct nlattr *tb[], struct nlattr *data[],
                         struct netlink_ext_ack *extack)
{
        if (tb[IFLA_ADDRESS]) {
                if (nla_len(tb[IFLA_ADDRESS]) != ETH_ALEN)
                        return -EINVAL;
                if (!is_valid_ether_addr(nla_data(tb[IFLA_ADDRESS])))
                        return -EADDRNOTAVAIL;
        }
        return 0;
}

static int bond_slave_changelink(struct net_device *bond_dev,
                                 struct net_device *slave_dev,
                                 struct nlattr *tb[], struct nlattr *data[],
                                 struct netlink_ext_ack *extack)
{
        struct bonding *bond = netdev_priv(bond_dev);
        struct bond_opt_value newval;
        int err;

        if (!data)
                return 0;

        if (data[IFLA_BOND_SLAVE_QUEUE_ID]) {
                u16 queue_id = nla_get_u16(data[IFLA_BOND_SLAVE_QUEUE_ID]);
                char queue_id_str[IFNAMSIZ + 7];

                /* queue_id option setting expects slave_name:queue_id */
                snprintf(queue_id_str, sizeof(queue_id_str), "%s:%u\n",
                         slave_dev->name, queue_id);
                bond_opt_initstr(&newval, queue_id_str);
                err = __bond_opt_set(bond, BOND_OPT_QUEUE_ID, &newval);
                if (err)
                        return err;
        }

        return 0;
}

static int bond_changelink(struct net_device *bond_dev, struct nlattr *tb[],
                           struct nlattr *data[],
                           struct netlink_ext_ack *extack)
{
        struct bonding *bond = netdev_priv(bond_dev);
        struct bond_opt_value newval;
        int miimon = 0;
        int err;

        if (!data)
                return 0;

        if (data[IFLA_BOND_MODE]) {
                int mode = nla_get_u8(data[IFLA_BOND_MODE]);

                bond_opt_initval(&newval, mode);
                err = __bond_opt_set(bond, BOND_OPT_MODE, &newval);
                if (err)
                        return err;
        }
        if (data[IFLA_BOND_ACTIVE_SLAVE]) {
                int ifindex = nla_get_u32(data[IFLA_BOND_ACTIVE_SLAVE]);
                struct net_device *slave_dev;
                char *active_slave = "";

                if (ifindex != 0) {
                        slave_dev = __dev_get_by_index(dev_net(bond_dev),
                                                       ifindex);
                        if (!slave_dev)
                                return -ENODEV;
                        active_slave = slave_dev->name;
                }
                bond_opt_initstr(&newval, active_slave);
                err = __bond_opt_set(bond, BOND_OPT_ACTIVE_SLAVE, &newval);
                if (err)
                        return err;
        }
        if (data[IFLA_BOND_MIIMON]) {
                miimon = nla_get_u32(data[IFLA_BOND_MIIMON]);

                bond_opt_initval(&newval, miimon);
                err = __bond_opt_set(bond, BOND_OPT_MIIMON, &newval);
                if (err)
                        return err;
        }
        if (data[IFLA_BOND_UPDELAY]) {
                int updelay = nla_get_u32(data[IFLA_BOND_UPDELAY]);

                bond_opt_initval(&newval, updelay);
                err = __bond_opt_set(bond, BOND_OPT_UPDELAY, &newval);
                if (err)
                        return err;
        }
        if (data[IFLA_BOND_DOWNDELAY]) {
                int downdelay = nla_get_u32(data[IFLA_BOND_DOWNDELAY]);

                bond_opt_initval(&newval, downdelay);
                err = __bond_opt_set(bond, BOND_OPT_DOWNDELAY, &newval);
                if (err)
                        return err;
        }
        if (data[IFLA_BOND_PEER_NOTIF_DELAY]) {
                int delay = nla_get_u32(data[IFLA_BOND_PEER_NOTIF_DELAY]);

                bond_opt_initval(&newval, delay);
                err = __bond_opt_set(bond, BOND_OPT_PEER_NOTIF_DELAY, &newval);
                if (err)
                        return err;
        }
        if (data[IFLA_BOND_USE_CARRIER]) {
                int use_carrier = nla_get_u8(data[IFLA_BOND_USE_CARRIER]);

                bond_opt_initval(&newval, use_carrier);
                err = __bond_opt_set(bond, BOND_OPT_USE_CARRIER, &newval);
                if (err)
                        return err;
        }
        if (data[IFLA_BOND_ARP_INTERVAL]) {
                int arp_interval = nla_get_u32(data[IFLA_BOND_ARP_INTERVAL]);

                if (arp_interval && miimon) {
                        netdev_err(bond->dev, "ARP monitoring cannot be used with MII monitoring\n");
                        return -EINVAL;
                }

                bond_opt_initval(&newval, arp_interval);
                err = __bond_opt_set(bond, BOND_OPT_ARP_INTERVAL, &newval);
                if (err)
                        return err;
        }
        if (data[IFLA_BOND_ARP_IP_TARGET]) {
                struct nlattr *attr;
                int i = 0, rem;

                bond_option_arp_ip_targets_clear(bond);
                nla_for_each_nested(attr, data[IFLA_BOND_ARP_IP_TARGET], rem) {
                        __be32 target;

                        if (nla_len(attr) < sizeof(target))
                                return -EINVAL;

                        target = nla_get_be32(attr);

                        bond_opt_initval(&newval, (__force u64)target);
                        err = __bond_opt_set(bond, BOND_OPT_ARP_TARGETS,
                                             &newval);
                        if (err)
                                break;
                        i++;
                }
                if (i == 0 && bond->params.arp_interval)
                        netdev_warn(bond->dev, "Removing last arp target with arp_interval on\n");
                if (err)
                        return err;
        }
        if (data[IFLA_BOND_ARP_VALIDATE]) {
                int arp_validate = nla_get_u32(data[IFLA_BOND_ARP_VALIDATE]);

                if (arp_validate && miimon) {
                        netdev_err(bond->dev, "ARP validating cannot be used with MII monitoring\n");
                        return -EINVAL;
                }

                bond_opt_initval(&newval, arp_validate);
                err = __bond_opt_set(bond, BOND_OPT_ARP_VALIDATE, &newval);
                if (err)
                        return err;
        }
        if (data[IFLA_BOND_ARP_ALL_TARGETS]) {
                int arp_all_targets =
                        nla_get_u32(data[IFLA_BOND_ARP_ALL_TARGETS]);

                bond_opt_initval(&newval, arp_all_targets);
                err = __bond_opt_set(bond, BOND_OPT_ARP_ALL_TARGETS, &newval);
                if (err)
                        return err;
        }
        if (data[IFLA_BOND_PRIMARY]) {
                int ifindex = nla_get_u32(data[IFLA_BOND_PRIMARY]);
                struct net_device *dev;
                char *primary = "";

                dev = __dev_get_by_index(dev_net(bond_dev), ifindex);
                if (dev)
                        primary = dev->name;

                bond_opt_initstr(&newval, primary);
                err = __bond_opt_set(bond, BOND_OPT_PRIMARY, &newval);
                if (err)
                        return err;
        }
        if (data[IFLA_BOND_PRIMARY_RESELECT]) {
                int primary_reselect =
                        nla_get_u8(data[IFLA_BOND_PRIMARY_RESELECT]);

                bond_opt_initval(&newval, primary_reselect);
                err = __bond_opt_set(bond, BOND_OPT_PRIMARY_RESELECT, &newval);
                if (err)
                        return err;
        }
        if (data[IFLA_BOND_FAIL_OVER_MAC]) {
                int fail_over_mac =
                        nla_get_u8(data[IFLA_BOND_FAIL_OVER_MAC]);

                bond_opt_initval(&newval, fail_over_mac);
                err = __bond_opt_set(bond, BOND_OPT_FAIL_OVER_MAC, &newval);
                if (err)
                        return err;
        }
        if (data[IFLA_BOND_XMIT_HASH_POLICY]) {
                int xmit_hash_policy =
                        nla_get_u8(data[IFLA_BOND_XMIT_HASH_POLICY]);

                bond_opt_initval(&newval, xmit_hash_policy);
                err = __bond_opt_set(bond, BOND_OPT_XMIT_HASH, &newval);
                if (err)
                        return err;
        }
        if (data[IFLA_BOND_RESEND_IGMP]) {
                int resend_igmp =
                        nla_get_u32(data[IFLA_BOND_RESEND_IGMP]);

                bond_opt_initval(&newval, resend_igmp);
                err = __bond_opt_set(bond, BOND_OPT_RESEND_IGMP, &newval);
                if (err)
                        return err;
        }
        if (data[IFLA_BOND_NUM_PEER_NOTIF]) {
                int num_peer_notif =
                        nla_get_u8(data[IFLA_BOND_NUM_PEER_NOTIF]);

                bond_opt_initval(&newval, num_peer_notif);
                err = __bond_opt_set(bond, BOND_OPT_NUM_PEER_NOTIF, &newval);
                if (err)
                        return err;
        }
        if (data[IFLA_BOND_ALL_SLAVES_ACTIVE]) {
                int all_slaves_active =
                        nla_get_u8(data[IFLA_BOND_ALL_SLAVES_ACTIVE]);

                bond_opt_initval(&newval, all_slaves_active);
                err = __bond_opt_set(bond, BOND_OPT_ALL_SLAVES_ACTIVE, &newval);
                if (err)
                        return err;
        }
        if (data[IFLA_BOND_MIN_LINKS]) {
                int min_links =
                        nla_get_u32(data[IFLA_BOND_MIN_LINKS]);

                bond_opt_initval(&newval, min_links);
                err = __bond_opt_set(bond, BOND_OPT_MINLINKS, &newval);
                if (err)
                        return err;
        }
        if (data[IFLA_BOND_LP_INTERVAL]) {
                int lp_interval =
                        nla_get_u32(data[IFLA_BOND_LP_INTERVAL]);

                bond_opt_initval(&newval, lp_interval);
                err = __bond_opt_set(bond, BOND_OPT_LP_INTERVAL, &newval);
                if (err)
                        return err;
        }
        if (data[IFLA_BOND_PACKETS_PER_SLAVE]) {
                int packets_per_slave =
                        nla_get_u32(data[IFLA_BOND_PACKETS_PER_SLAVE]);

                bond_opt_initval(&newval, packets_per_slave);
                err = __bond_opt_set(bond, BOND_OPT_PACKETS_PER_SLAVE, &newval);
                if (err)
                        return err;
        }

        if (data[IFLA_BOND_AD_LACP_ACTIVE]) {
                int lacp_active = nla_get_u8(data[IFLA_BOND_AD_LACP_ACTIVE]);

                bond_opt_initval(&newval, lacp_active);
                err = __bond_opt_set(bond, BOND_OPT_LACP_ACTIVE, &newval);
                if (err)
                        return err;
        }

        if (data[IFLA_BOND_AD_LACP_RATE]) {
                int lacp_rate =
                        nla_get_u8(data[IFLA_BOND_AD_LACP_RATE]);

                bond_opt_initval(&newval, lacp_rate);
                err = __bond_opt_set(bond, BOND_OPT_LACP_RATE, &newval);
                if (err)
                        return err;
        }
        if (data[IFLA_BOND_AD_SELECT]) {
                int ad_select =
                        nla_get_u8(data[IFLA_BOND_AD_SELECT]);

                bond_opt_initval(&newval, ad_select);
                err = __bond_opt_set(bond, BOND_OPT_AD_SELECT, &newval);
                if (err)
                        return err;
        }
        if (data[IFLA_BOND_AD_ACTOR_SYS_PRIO]) {
                int actor_sys_prio =
                        nla_get_u16(data[IFLA_BOND_AD_ACTOR_SYS_PRIO]);

                bond_opt_initval(&newval, actor_sys_prio);
                err = __bond_opt_set(bond, BOND_OPT_AD_ACTOR_SYS_PRIO, &newval);
                if (err)
                        return err;
        }
        if (data[IFLA_BOND_AD_USER_PORT_KEY]) {
                int port_key =
                        nla_get_u16(data[IFLA_BOND_AD_USER_PORT_KEY]);

                bond_opt_initval(&newval, port_key);
                err = __bond_opt_set(bond, BOND_OPT_AD_USER_PORT_KEY, &newval);
                if (err)
                        return err;
        }
        if (data[IFLA_BOND_AD_ACTOR_SYSTEM]) {
                if (nla_len(data[IFLA_BOND_AD_ACTOR_SYSTEM]) != ETH_ALEN)
                        return -EINVAL;

                bond_opt_initval(&newval,
                                 nla_get_u64(data[IFLA_BOND_AD_ACTOR_SYSTEM]));
                err = __bond_opt_set(bond, BOND_OPT_AD_ACTOR_SYSTEM, &newval);
                if (err)
                        return err;
        }
        if (data[IFLA_BOND_TLB_DYNAMIC_LB]) {
                int dynamic_lb = nla_get_u8(data[IFLA_BOND_TLB_DYNAMIC_LB]);

                bond_opt_initval(&newval, dynamic_lb);
                err = __bond_opt_set(bond, BOND_OPT_TLB_DYNAMIC_LB, &newval);
                if (err)
                        return err;
        }

        if (data[IFLA_BOND_MISSED_MAX]) {
                int missed_max = nla_get_u8(data[IFLA_BOND_MISSED_MAX]);

                bond_opt_initval(&newval, missed_max);
                err = __bond_opt_set(bond, BOND_OPT_MISSED_MAX, &newval);
                if (err)
                        return err;
        }

        return 0;
}

static int bond_newlink(struct net *src_net, struct net_device *bond_dev,
                        struct nlattr *tb[], struct nlattr *data[],
                        struct netlink_ext_ack *extack)
{
        int err;

        err = bond_changelink(bond_dev, tb, data, extack);
        if (err < 0)
                return err;

        err = register_netdevice(bond_dev);
        if (!err) {
                struct bonding *bond = netdev_priv(bond_dev);

                netif_carrier_off(bond_dev);
                bond_work_init_all(bond);
        }

        return err;
}

static size_t bond_get_size(const struct net_device *bond_dev)
{
        return nla_total_size(sizeof(u8)) +        /* IFLA_BOND_MODE */
                nla_total_size(sizeof(u32)) +        /* IFLA_BOND_ACTIVE_SLAVE */
                nla_total_size(sizeof(u32)) +        /* IFLA_BOND_MIIMON */
                nla_total_size(sizeof(u32)) +        /* IFLA_BOND_UPDELAY */
                nla_total_size(sizeof(u32)) +        /* IFLA_BOND_DOWNDELAY */
                nla_total_size(sizeof(u8)) +        /* IFLA_BOND_USE_CARRIER */
                nla_total_size(sizeof(u32)) +        /* IFLA_BOND_ARP_INTERVAL */
                                                /* IFLA_BOND_ARP_IP_TARGET */
                nla_total_size(sizeof(struct nlattr)) +
                nla_total_size(sizeof(u32)) * BOND_MAX_ARP_TARGETS +
                nla_total_size(sizeof(u32)) +        /* IFLA_BOND_ARP_VALIDATE */
                nla_total_size(sizeof(u32)) +        /* IFLA_BOND_ARP_ALL_TARGETS */
                nla_total_size(sizeof(u32)) +        /* IFLA_BOND_PRIMARY */
                nla_total_size(sizeof(u8)) +        /* IFLA_BOND_PRIMARY_RESELECT */
                nla_total_size(sizeof(u8)) +        /* IFLA_BOND_FAIL_OVER_MAC */
                nla_total_size(sizeof(u8)) +        /* IFLA_BOND_XMIT_HASH_POLICY */
                nla_total_size(sizeof(u32)) +        /* IFLA_BOND_RESEND_IGMP */
                nla_total_size(sizeof(u8)) +        /* IFLA_BOND_NUM_PEER_NOTIF */
                nla_total_size(sizeof(u8)) +   /* IFLA_BOND_ALL_SLAVES_ACTIVE */
                nla_total_size(sizeof(u32)) +        /* IFLA_BOND_MIN_LINKS */
                nla_total_size(sizeof(u32)) +        /* IFLA_BOND_LP_INTERVAL */
                nla_total_size(sizeof(u32)) +  /* IFLA_BOND_PACKETS_PER_SLAVE */
                nla_total_size(sizeof(u8)) +        /* IFLA_BOND_AD_LACP_ACTIVE */
                nla_total_size(sizeof(u8)) +        /* IFLA_BOND_AD_LACP_RATE */
                nla_total_size(sizeof(u8)) +        /* IFLA_BOND_AD_SELECT */
                nla_total_size(sizeof(struct nlattr)) + /* IFLA_BOND_AD_INFO */
                nla_total_size(sizeof(u16)) + /* IFLA_BOND_AD_INFO_AGGREGATOR */
                nla_total_size(sizeof(u16)) + /* IFLA_BOND_AD_INFO_NUM_PORTS */
                nla_total_size(sizeof(u16)) + /* IFLA_BOND_AD_INFO_ACTOR_KEY */
                nla_total_size(sizeof(u16)) + /* IFLA_BOND_AD_INFO_PARTNER_KEY*/
                nla_total_size(ETH_ALEN) +    /* IFLA_BOND_AD_INFO_PARTNER_MAC*/
                nla_total_size(sizeof(u16)) + /* IFLA_BOND_AD_ACTOR_SYS_PRIO */
                nla_total_size(sizeof(u16)) + /* IFLA_BOND_AD_USER_PORT_KEY */
                nla_total_size(ETH_ALEN) + /* IFLA_BOND_AD_ACTOR_SYSTEM */
                nla_total_size(sizeof(u8)) + /* IFLA_BOND_TLB_DYNAMIC_LB */
                nla_total_size(sizeof(u32)) +        /* IFLA_BOND_PEER_NOTIF_DELAY */
                nla_total_size(sizeof(u8)) +        /* IFLA_BOND_MISSED_MAX */
                0;
}

static int bond_option_active_slave_get_ifindex(struct bonding *bond)
{
        const struct net_device *slave;
        int ifindex;

        rcu_read_lock();
        slave = bond_option_active_slave_get_rcu(bond);
        ifindex = slave ? slave->ifindex : 0;
        rcu_read_unlock();
        return ifindex;
}

static int bond_fill_info(struct sk_buff *skb,
                          const struct net_device *bond_dev)
{
        struct bonding *bond = netdev_priv(bond_dev);
        unsigned int packets_per_slave;
        int ifindex, i, targets_added;
        struct nlattr *targets;
        struct slave *primary;

        if (nla_put_u8(skb, IFLA_BOND_MODE, BOND_MODE(bond)))
                goto nla_put_failure;

        ifindex = bond_option_active_slave_get_ifindex(bond);
        if (ifindex && nla_put_u32(skb, IFLA_BOND_ACTIVE_SLAVE, ifindex))
                goto nla_put_failure;

        if (nla_put_u32(skb, IFLA_BOND_MIIMON, bond->params.miimon))
                goto nla_put_failure;

        if (nla_put_u32(skb, IFLA_BOND_UPDELAY,
                        bond->params.updelay * bond->params.miimon))
                goto nla_put_failure;

        if (nla_put_u32(skb, IFLA_BOND_DOWNDELAY,
                        bond->params.downdelay * bond->params.miimon))
                goto nla_put_failure;

        if (nla_put_u32(skb, IFLA_BOND_PEER_NOTIF_DELAY,
                        bond->params.peer_notif_delay * bond->params.miimon))
                goto nla_put_failure;

        if (nla_put_u8(skb, IFLA_BOND_USE_CARRIER, bond->params.use_carrier))
                goto nla_put_failure;

        if (nla_put_u32(skb, IFLA_BOND_ARP_INTERVAL, bond->params.arp_interval))
                goto nla_put_failure;

        targets = nla_nest_start_noflag(skb, IFLA_BOND_ARP_IP_TARGET);
        if (!targets)
                goto nla_put_failure;

        targets_added = 0;
        for (i = 0; i < BOND_MAX_ARP_TARGETS; i++) {
                if (bond->params.arp_targets[i]) {
                        if (nla_put_be32(skb, i, bond->params.arp_targets[i]))
                                goto nla_put_failure;
                        targets_added = 1;
                }
        }

        if (targets_added)
                nla_nest_end(skb, targets);
        else
                nla_nest_cancel(skb, targets);

        if (nla_put_u32(skb, IFLA_BOND_ARP_VALIDATE, bond->params.arp_validate))
                goto nla_put_failure;

        if (nla_put_u32(skb, IFLA_BOND_ARP_ALL_TARGETS,
                        bond->params.arp_all_targets))
                goto nla_put_failure;

        primary = rtnl_dereference(bond->primary_slave);
        if (primary &&
            nla_put_u32(skb, IFLA_BOND_PRIMARY, primary->dev->ifindex))
                goto nla_put_failure;

        if (nla_put_u8(skb, IFLA_BOND_PRIMARY_RESELECT,
                       bond->params.primary_reselect))
                goto nla_put_failure;

        if (nla_put_u8(skb, IFLA_BOND_FAIL_OVER_MAC,
                       bond->params.fail_over_mac))
                goto nla_put_failure;

        if (nla_put_u8(skb, IFLA_BOND_XMIT_HASH_POLICY,
                       bond->params.xmit_policy))
                goto nla_put_failure;

        if (nla_put_u32(skb, IFLA_BOND_RESEND_IGMP,
                        bond->params.resend_igmp))
                goto nla_put_failure;

        if (nla_put_u8(skb, IFLA_BOND_NUM_PEER_NOTIF,
                       bond->params.num_peer_notif))
                goto nla_put_failure;

        if (nla_put_u8(skb, IFLA_BOND_ALL_SLAVES_ACTIVE,
                       bond->params.all_slaves_active))
                goto nla_put_failure;

        if (nla_put_u32(skb, IFLA_BOND_MIN_LINKS,
                        bond->params.min_links))
                goto nla_put_failure;

        if (nla_put_u32(skb, IFLA_BOND_LP_INTERVAL,
                        bond->params.lp_interval))
                goto nla_put_failure;

        packets_per_slave = bond->params.packets_per_slave;
        if (nla_put_u32(skb, IFLA_BOND_PACKETS_PER_SLAVE,
                        packets_per_slave))
                goto nla_put_failure;

        if (nla_put_u8(skb, IFLA_BOND_AD_LACP_ACTIVE,
                       bond->params.lacp_active))
                goto nla_put_failure;

        if (nla_put_u8(skb, IFLA_BOND_AD_LACP_RATE,
                       bond->params.lacp_fast))
                goto nla_put_failure;

        if (nla_put_u8(skb, IFLA_BOND_AD_SELECT,
                       bond->params.ad_select))
                goto nla_put_failure;

        if (nla_put_u8(skb, IFLA_BOND_TLB_DYNAMIC_LB,
                       bond->params.tlb_dynamic_lb))
                goto nla_put_failure;

        if (nla_put_u8(skb, IFLA_BOND_MISSED_MAX,
                       bond->params.missed_max))
                goto nla_put_failure;

        if (BOND_MODE(bond) == BOND_MODE_8023AD) {
                struct ad_info info;

                if (capable(CAP_NET_ADMIN)) {
                        if (nla_put_u16(skb, IFLA_BOND_AD_ACTOR_SYS_PRIO,
                                        bond->params.ad_actor_sys_prio))
                                goto nla_put_failure;

                        if (nla_put_u16(skb, IFLA_BOND_AD_USER_PORT_KEY,
                                        bond->params.ad_user_port_key))
                                goto nla_put_failure;

                        if (nla_put(skb, IFLA_BOND_AD_ACTOR_SYSTEM,
                                    ETH_ALEN, &bond->params.ad_actor_system))
                                goto nla_put_failure;
                }
                if (!bond_3ad_get_active_agg_info(bond, &info)) {
                        struct nlattr *nest;

                        nest = nla_nest_start_noflag(skb, IFLA_BOND_AD_INFO);
                        if (!nest)
                                goto nla_put_failure;

                        if (nla_put_u16(skb, IFLA_BOND_AD_INFO_AGGREGATOR,
                                        info.aggregator_id))
                                goto nla_put_failure;
                        if (nla_put_u16(skb, IFLA_BOND_AD_INFO_NUM_PORTS,
                                        info.ports))
                                goto nla_put_failure;
                        if (nla_put_u16(skb, IFLA_BOND_AD_INFO_ACTOR_KEY,
                                        info.actor_key))
                                goto nla_put_failure;
                        if (nla_put_u16(skb, IFLA_BOND_AD_INFO_PARTNER_KEY,
                                        info.partner_key))
                                goto nla_put_failure;
                        if (nla_put(skb, IFLA_BOND_AD_INFO_PARTNER_MAC,
                                    sizeof(info.partner_system),
                                    &info.partner_system))
                                goto nla_put_failure;

                        nla_nest_end(skb, nest);
                }
        }

        return 0;

nla_put_failure:
        return -EMSGSIZE;
}

static size_t bond_get_linkxstats_size(const struct net_device *dev, int attr)
{
        switch (attr) {
        case IFLA_STATS_LINK_XSTATS:
        case IFLA_STATS_LINK_XSTATS_SLAVE:
                break;
        default:
                return 0;
        }

        return bond_3ad_stats_size() + nla_total_size(0);
}

static int bond_fill_linkxstats(struct sk_buff *skb,
                                const struct net_device *dev,
                                int *prividx, int attr)
{
        struct nlattr *nla __maybe_unused;
        struct slave *slave = NULL;
        struct nlattr *nest, *nest2;
        struct bonding *bond;

        switch (attr) {
        case IFLA_STATS_LINK_XSTATS:
                bond = netdev_priv(dev);
                break;
        case IFLA_STATS_LINK_XSTATS_SLAVE:
                slave = bond_slave_get_rtnl(dev);
                if (!slave)
                        return 0;
                bond = slave->bond;
                break;
        default:
                return -EINVAL;
        }

        nest = nla_nest_start_noflag(skb, LINK_XSTATS_TYPE_BOND);
        if (!nest)
                return -EMSGSIZE;
        if (BOND_MODE(bond) == BOND_MODE_8023AD) {
                struct bond_3ad_stats *stats;

                if (slave)
                        stats = &SLAVE_AD_INFO(slave)->stats;
                else
                        stats = &BOND_AD_INFO(bond).stats;

                nest2 = nla_nest_start_noflag(skb, BOND_XSTATS_3AD);
                if (!nest2) {
                        nla_nest_end(skb, nest);
                        return -EMSGSIZE;
                }

                if (bond_3ad_stats_fill(skb, stats)) {
                        nla_nest_cancel(skb, nest2);
                        nla_nest_end(skb, nest);
                        return -EMSGSIZE;
                }
                nla_nest_end(skb, nest2);
        }
        nla_nest_end(skb, nest);

        return 0;
}

struct rtnl_link_ops bond_link_ops __read_mostly = {
        .kind                        = "bond",
        .priv_size                = sizeof(struct bonding),
        .setup                        = bond_setup,
        .maxtype                = IFLA_BOND_MAX,
        .policy                        = bond_policy,
        .validate                = bond_validate,
        .newlink                = bond_newlink,
        .changelink                = bond_changelink,
        .get_size                = bond_get_size,
        .fill_info                = bond_fill_info,
        .get_num_tx_queues        = bond_get_num_tx_queues,
        .get_num_rx_queues        = bond_get_num_tx_queues, /* Use the same number
                                                             as for TX queues */
        .fill_linkxstats        = bond_fill_linkxstats,
        .get_linkxstats_size    = bond_get_linkxstats_size,
        .slave_maxtype                = IFLA_BOND_SLAVE_MAX,
        .slave_policy                = bond_slave_policy,
        .slave_changelink        = bond_slave_changelink,
        .get_slave_size                = bond_get_slave_size,
        .fill_slave_info        = bond_fill_slave_info,
};

int __init bond_netlink_init(void)
{
        return rtnl_link_register(&bond_link_ops);
}

void bond_netlink_fini(void)
{
        rtnl_link_unregister(&bond_link_ops);
}

MODULE_ALIAS_RTNL_LINK("bond");









































































































































































































































































































































































































































































































































































































































































  374 
  107 
  357 
   59 
  356 
  373 
  374 








































































   58 

   60 









  981 

















  521 


  526 









   71 






    6 
   77 





  106 
















  374 

















  386 



  158 


  106 


















  234 



  234 









  126 













  319 























































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
/*
 * Copyright (c) 2015, Mellanox Technologies inc.  All rights reserved.
 *
 * This software is available to you under a choice of one of two
 * licenses.  You may choose to be licensed under the terms of the GNU
 * General Public License (GPL) Version 2, available from the file
 * COPYING in the main directory of this source tree, or the
 * OpenIB.org BSD license below:
 *
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
 *
 *      - Redistributions of source code must retain the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer.
 *
 *      - Redistributions in binary form must reproduce the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer in the documentation and/or other materials
 *        provided with the distribution.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

#include "core_priv.h"

#include <linux/in.h>
#include <linux/in6.h>

/* For in6_dev_get/in6_dev_put */
#include <net/addrconf.h>
#include <net/bonding.h>

#include <rdma/ib_cache.h>
#include <rdma/ib_addr.h>

static struct workqueue_struct *gid_cache_wq;

enum gid_op_type {
        GID_DEL = 0,
        GID_ADD
};

struct update_gid_event_work {
        struct work_struct work;
        union ib_gid       gid;
        struct ib_gid_attr gid_attr;
        enum gid_op_type gid_op;
};

#define ROCE_NETDEV_CALLBACK_SZ                3
struct netdev_event_work_cmd {
        roce_netdev_callback        cb;
        roce_netdev_filter        filter;
        struct net_device        *ndev;
        struct net_device        *filter_ndev;
};

struct netdev_event_work {
        struct work_struct                work;
        struct netdev_event_work_cmd        cmds[ROCE_NETDEV_CALLBACK_SZ];
};

static const struct {
        bool (*is_supported)(const struct ib_device *device, u32 port_num);
        enum ib_gid_type gid_type;
} PORT_CAP_TO_GID_TYPE[] = {
        {rdma_protocol_roce_eth_encap, IB_GID_TYPE_ROCE},
        {rdma_protocol_roce_udp_encap, IB_GID_TYPE_ROCE_UDP_ENCAP},
};

#define CAP_TO_GID_TABLE_SIZE        ARRAY_SIZE(PORT_CAP_TO_GID_TYPE)

unsigned long roce_gid_type_mask_support(struct ib_device *ib_dev, u32 port)
{
        int i;
        unsigned int ret_flags = 0;

        if (!rdma_protocol_roce(ib_dev, port))
                return 1UL << IB_GID_TYPE_IB;

        for (i = 0; i < CAP_TO_GID_TABLE_SIZE; i++)
                if (PORT_CAP_TO_GID_TYPE[i].is_supported(ib_dev, port))
                        ret_flags |= 1UL << PORT_CAP_TO_GID_TYPE[i].gid_type;

        return ret_flags;
}
EXPORT_SYMBOL(roce_gid_type_mask_support);

static void update_gid(enum gid_op_type gid_op, struct ib_device *ib_dev,
                       u32 port, union ib_gid *gid,
                       struct ib_gid_attr *gid_attr)
{
        int i;
        unsigned long gid_type_mask = roce_gid_type_mask_support(ib_dev, port);

        for (i = 0; i < IB_GID_TYPE_SIZE; i++) {
                if ((1UL << i) & gid_type_mask) {
                        gid_attr->gid_type = i;
                        switch (gid_op) {
                        case GID_ADD:
                                ib_cache_gid_add(ib_dev, port,
                                                 gid, gid_attr);
                                break;
                        case GID_DEL:
                                ib_cache_gid_del(ib_dev, port,
                                                 gid, gid_attr);
                                break;
                        }
                }
        }
}

enum bonding_slave_state {
        BONDING_SLAVE_STATE_ACTIVE        = 1UL << 0,
        BONDING_SLAVE_STATE_INACTIVE        = 1UL << 1,
        /* No primary slave or the device isn't a slave in bonding */
        BONDING_SLAVE_STATE_NA                = 1UL << 2,
};

static enum bonding_slave_state is_eth_active_slave_of_bonding_rcu(struct net_device *dev,
                                                                   struct net_device *upper)
{
        if (upper && netif_is_bond_master(upper)) {
                struct net_device *pdev =
                        bond_option_active_slave_get_rcu(netdev_priv(upper));

                if (pdev)
                        return dev == pdev ? BONDING_SLAVE_STATE_ACTIVE :
                                BONDING_SLAVE_STATE_INACTIVE;
        }

        return BONDING_SLAVE_STATE_NA;
}

#define REQUIRED_BOND_STATES                (BONDING_SLAVE_STATE_ACTIVE |        \
                                         BONDING_SLAVE_STATE_NA)
static bool
is_eth_port_of_netdev_filter(struct ib_device *ib_dev, u32 port,
                             struct net_device *rdma_ndev, void *cookie)
{
        struct net_device *real_dev;
        bool res;

        if (!rdma_ndev)
                return false;

        rcu_read_lock();
        real_dev = rdma_vlan_dev_real_dev(cookie);
        if (!real_dev)
                real_dev = cookie;

        res = ((rdma_is_upper_dev_rcu(rdma_ndev, cookie) &&
               (is_eth_active_slave_of_bonding_rcu(rdma_ndev, real_dev) &
                REQUIRED_BOND_STATES)) ||
               real_dev == rdma_ndev);

        rcu_read_unlock();
        return res;
}

static bool
is_eth_port_inactive_slave_filter(struct ib_device *ib_dev, u32 port,
                                  struct net_device *rdma_ndev, void *cookie)
{
        struct net_device *master_dev;
        bool res;

        if (!rdma_ndev)
                return false;

        rcu_read_lock();
        master_dev = netdev_master_upper_dev_get_rcu(rdma_ndev);
        res = is_eth_active_slave_of_bonding_rcu(rdma_ndev, master_dev) ==
                BONDING_SLAVE_STATE_INACTIVE;
        rcu_read_unlock();

        return res;
}

/**
 * is_ndev_for_default_gid_filter - Check if a given netdevice
 * can be considered for default GIDs or not.
 * @ib_dev:                IB device to check
 * @port:                Port to consider for adding default GID
 * @rdma_ndev:                rdma netdevice pointer
 * @cookie:             Netdevice to consider to form a default GID
 *
 * is_ndev_for_default_gid_filter() returns true if a given netdevice can be
 * considered for deriving default RoCE GID, returns false otherwise.
 */
static bool
is_ndev_for_default_gid_filter(struct ib_device *ib_dev, u32 port,
                               struct net_device *rdma_ndev, void *cookie)
{
        struct net_device *cookie_ndev = cookie;
        bool res;

        if (!rdma_ndev)
                return false;

        rcu_read_lock();

        /*
         * When rdma netdevice is used in bonding, bonding master netdevice
         * should be considered for default GIDs. Therefore, ignore slave rdma
         * netdevices when bonding is considered.
         * Additionally when event(cookie) netdevice is bond master device,
         * make sure that it the upper netdevice of rdma netdevice.
         */
        res = ((cookie_ndev == rdma_ndev && !netif_is_bond_slave(rdma_ndev)) ||
               (netif_is_bond_master(cookie_ndev) &&
                rdma_is_upper_dev_rcu(rdma_ndev, cookie_ndev)));

        rcu_read_unlock();
        return res;
}

static bool pass_all_filter(struct ib_device *ib_dev, u32 port,
                            struct net_device *rdma_ndev, void *cookie)
{
        return true;
}

static bool upper_device_filter(struct ib_device *ib_dev, u32 port,
                                struct net_device *rdma_ndev, void *cookie)
{
        bool res;

        if (!rdma_ndev)
                return false;

        if (rdma_ndev == cookie)
                return true;

        rcu_read_lock();
        res = rdma_is_upper_dev_rcu(rdma_ndev, cookie);
        rcu_read_unlock();

        return res;
}

/**
 * is_upper_ndev_bond_master_filter - Check if a given netdevice
 * is bond master device of netdevice of the the RDMA device of port.
 * @ib_dev:                IB device to check
 * @port:                Port to consider for adding default GID
 * @rdma_ndev:                Pointer to rdma netdevice
 * @cookie:                Netdevice to consider to form a default GID
 *
 * is_upper_ndev_bond_master_filter() returns true if a cookie_netdev
 * is bond master device and rdma_ndev is its lower netdevice. It might
 * not have been established as slave device yet.
 */
static bool
is_upper_ndev_bond_master_filter(struct ib_device *ib_dev, u32 port,
                                 struct net_device *rdma_ndev,
                                 void *cookie)
{
        struct net_device *cookie_ndev = cookie;
        bool match = false;

        if (!rdma_ndev)
                return false;

        rcu_read_lock();
        if (netif_is_bond_master(cookie_ndev) &&
            rdma_is_upper_dev_rcu(rdma_ndev, cookie_ndev))
                match = true;
        rcu_read_unlock();
        return match;
}

static void update_gid_ip(enum gid_op_type gid_op,
                          struct ib_device *ib_dev,
                          u32 port, struct net_device *ndev,
                          struct sockaddr *addr)
{
        union ib_gid gid;
        struct ib_gid_attr gid_attr;

        rdma_ip2gid(addr, &gid);
        memset(&gid_attr, 0, sizeof(gid_attr));
        gid_attr.ndev = ndev;

        update_gid(gid_op, ib_dev, port, &gid, &gid_attr);
}

static void bond_delete_netdev_default_gids(struct ib_device *ib_dev,
                                            u32 port,
                                            struct net_device *rdma_ndev,
                                            struct net_device *event_ndev)
{
        struct net_device *real_dev = rdma_vlan_dev_real_dev(event_ndev);
        unsigned long gid_type_mask;

        if (!rdma_ndev)
                return;

        if (!real_dev)
                real_dev = event_ndev;

        rcu_read_lock();

        if (((rdma_ndev != event_ndev &&
              !rdma_is_upper_dev_rcu(rdma_ndev, event_ndev)) ||
             is_eth_active_slave_of_bonding_rcu(rdma_ndev, real_dev)
                                                 ==
             BONDING_SLAVE_STATE_INACTIVE)) {
                rcu_read_unlock();
                return;
        }

        rcu_read_unlock();

        gid_type_mask = roce_gid_type_mask_support(ib_dev, port);

        ib_cache_gid_set_default_gid(ib_dev, port, rdma_ndev,
                                     gid_type_mask,
                                     IB_CACHE_GID_DEFAULT_MODE_DELETE);
}

static void enum_netdev_ipv4_ips(struct ib_device *ib_dev,
                                 u32 port, struct net_device *ndev)
{
        const struct in_ifaddr *ifa;
        struct in_device *in_dev;
        struct sin_list {
                struct list_head        list;
                struct sockaddr_in        ip;
        };
        struct sin_list *sin_iter;
        struct sin_list *sin_temp;

        LIST_HEAD(sin_list);
        if (ndev->reg_state >= NETREG_UNREGISTERING)
                return;

        rcu_read_lock();
        in_dev = __in_dev_get_rcu(ndev);
        if (!in_dev) {
                rcu_read_unlock();
                return;
        }

        in_dev_for_each_ifa_rcu(ifa, in_dev) {
                struct sin_list *entry = kzalloc(sizeof(*entry), GFP_ATOMIC);

                if (!entry)
                        continue;

                entry->ip.sin_family = AF_INET;
                entry->ip.sin_addr.s_addr = ifa->ifa_address;
                list_add_tail(&entry->list, &sin_list);
        }

        rcu_read_unlock();

        list_for_each_entry_safe(sin_iter, sin_temp, &sin_list, list) {
                update_gid_ip(GID_ADD, ib_dev, port, ndev,
                              (struct sockaddr *)&sin_iter->ip);
                list_del(&sin_iter->list);
                kfree(sin_iter);
        }
}

static void enum_netdev_ipv6_ips(struct ib_device *ib_dev,
                                 u32 port, struct net_device *ndev)
{
        struct inet6_ifaddr *ifp;
        struct inet6_dev *in6_dev;
        struct sin6_list {
                struct list_head        list;
                struct sockaddr_in6        sin6;
        };
        struct sin6_list *sin6_iter;
        struct sin6_list *sin6_temp;
        struct ib_gid_attr gid_attr = {.ndev = ndev};
        LIST_HEAD(sin6_list);

        if (ndev->reg_state >= NETREG_UNREGISTERING)
                return;

        in6_dev = in6_dev_get(ndev);
        if (!in6_dev)
                return;

        read_lock_bh(&in6_dev->lock);
        list_for_each_entry(ifp, &in6_dev->addr_list, if_list) {
                struct sin6_list *entry = kzalloc(sizeof(*entry), GFP_ATOMIC);

                if (!entry)
                        continue;

                entry->sin6.sin6_family = AF_INET6;
                entry->sin6.sin6_addr = ifp->addr;
                list_add_tail(&entry->list, &sin6_list);
        }
        read_unlock_bh(&in6_dev->lock);

        in6_dev_put(in6_dev);

        list_for_each_entry_safe(sin6_iter, sin6_temp, &sin6_list, list) {
                union ib_gid        gid;

                rdma_ip2gid((struct sockaddr *)&sin6_iter->sin6, &gid);
                update_gid(GID_ADD, ib_dev, port, &gid, &gid_attr);
                list_del(&sin6_iter->list);
                kfree(sin6_iter);
        }
}

static void _add_netdev_ips(struct ib_device *ib_dev, u32 port,
                            struct net_device *ndev)
{
        enum_netdev_ipv4_ips(ib_dev, port, ndev);
        if (IS_ENABLED(CONFIG_IPV6))
                enum_netdev_ipv6_ips(ib_dev, port, ndev);
}

static void add_netdev_ips(struct ib_device *ib_dev, u32 port,
                           struct net_device *rdma_ndev, void *cookie)
{
        _add_netdev_ips(ib_dev, port, cookie);
}

static void del_netdev_ips(struct ib_device *ib_dev, u32 port,
                           struct net_device *rdma_ndev, void *cookie)
{
        ib_cache_gid_del_all_netdev_gids(ib_dev, port, cookie);
}

/**
 * del_default_gids - Delete default GIDs of the event/cookie netdevice
 * @ib_dev:        RDMA device pointer
 * @port:        Port of the RDMA device whose GID table to consider
 * @rdma_ndev:        Unused rdma netdevice
 * @cookie:        Pointer to event netdevice
 *
 * del_default_gids() deletes the default GIDs of the event/cookie netdevice.
 */
static void del_default_gids(struct ib_device *ib_dev, u32 port,
                             struct net_device *rdma_ndev, void *cookie)
{
        struct net_device *cookie_ndev = cookie;
        unsigned long gid_type_mask;

        gid_type_mask = roce_gid_type_mask_support(ib_dev, port);

        ib_cache_gid_set_default_gid(ib_dev, port, cookie_ndev, gid_type_mask,
                                     IB_CACHE_GID_DEFAULT_MODE_DELETE);
}

static void add_default_gids(struct ib_device *ib_dev, u32 port,
                             struct net_device *rdma_ndev, void *cookie)
{
        struct net_device *event_ndev = cookie;
        unsigned long gid_type_mask;

        gid_type_mask = roce_gid_type_mask_support(ib_dev, port);
        ib_cache_gid_set_default_gid(ib_dev, port, event_ndev, gid_type_mask,
                                     IB_CACHE_GID_DEFAULT_MODE_SET);
}

static void enum_all_gids_of_dev_cb(struct ib_device *ib_dev,
                                    u32 port,
                                    struct net_device *rdma_ndev,
                                    void *cookie)
{
        struct net *net;
        struct net_device *ndev;

        /* Lock the rtnl to make sure the netdevs does not move under
         * our feet
         */
        rtnl_lock();
        down_read(&net_rwsem);
        for_each_net(net)
                for_each_netdev(net, ndev) {
                        /*
                         * Filter and add default GIDs of the primary netdevice
                         * when not in bonding mode, or add default GIDs
                         * of bond master device, when in bonding mode.
                         */
                        if (is_ndev_for_default_gid_filter(ib_dev, port,
                                                           rdma_ndev, ndev))
                                add_default_gids(ib_dev, port, rdma_ndev, ndev);

                        if (is_eth_port_of_netdev_filter(ib_dev, port,
                                                         rdma_ndev, ndev))
                                _add_netdev_ips(ib_dev, port, ndev);
                }
        up_read(&net_rwsem);
        rtnl_unlock();
}

/**
 * rdma_roce_rescan_device - Rescan all of the network devices in the system
 * and add their gids, as needed, to the relevant RoCE devices.
 *
 * @ib_dev:         the rdma device
 */
void rdma_roce_rescan_device(struct ib_device *ib_dev)
{
        ib_enum_roce_netdev(ib_dev, pass_all_filter, NULL,
                            enum_all_gids_of_dev_cb, NULL);
}
EXPORT_SYMBOL(rdma_roce_rescan_device);

static void callback_for_addr_gid_device_scan(struct ib_device *device,
                                              u32 port,
                                              struct net_device *rdma_ndev,
                                              void *cookie)
{
        struct update_gid_event_work *parsed = cookie;

        return update_gid(parsed->gid_op, device,
                          port, &parsed->gid,
                          &parsed->gid_attr);
}

struct upper_list {
        struct list_head list;
        struct net_device *upper;
};

static int netdev_upper_walk(struct net_device *upper,
                             struct netdev_nested_priv *priv)
{
        struct upper_list *entry = kmalloc(sizeof(*entry), GFP_ATOMIC);
        struct list_head *upper_list = (struct list_head *)priv->data;

        if (!entry)
                return 0;

        list_add_tail(&entry->list, upper_list);
        dev_hold(upper);
        entry->upper = upper;

        return 0;
}

static void handle_netdev_upper(struct ib_device *ib_dev, u32 port,
                                void *cookie,
                                void (*handle_netdev)(struct ib_device *ib_dev,
                                                      u32 port,
                                                      struct net_device *ndev))
{
        struct net_device *ndev = cookie;
        struct netdev_nested_priv priv;
        struct upper_list *upper_iter;
        struct upper_list *upper_temp;
        LIST_HEAD(upper_list);

        priv.data = &upper_list;
        rcu_read_lock();
        netdev_walk_all_upper_dev_rcu(ndev, netdev_upper_walk, &priv);
        rcu_read_unlock();

        handle_netdev(ib_dev, port, ndev);
        list_for_each_entry_safe(upper_iter, upper_temp, &upper_list,
                                 list) {
                handle_netdev(ib_dev, port, upper_iter->upper);
                dev_put(upper_iter->upper);
                list_del(&upper_iter->list);
                kfree(upper_iter);
        }
}

static void _roce_del_all_netdev_gids(struct ib_device *ib_dev, u32 port,
                                      struct net_device *event_ndev)
{
        ib_cache_gid_del_all_netdev_gids(ib_dev, port, event_ndev);
}

static void del_netdev_upper_ips(struct ib_device *ib_dev, u32 port,
                                 struct net_device *rdma_ndev, void *cookie)
{
        handle_netdev_upper(ib_dev, port, cookie, _roce_del_all_netdev_gids);
}

static void add_netdev_upper_ips(struct ib_device *ib_dev, u32 port,
                                 struct net_device *rdma_ndev, void *cookie)
{
        handle_netdev_upper(ib_dev, port, cookie, _add_netdev_ips);
}

static void del_netdev_default_ips_join(struct ib_device *ib_dev, u32 port,
                                        struct net_device *rdma_ndev,
                                        void *cookie)
{
        struct net_device *master_ndev;

        rcu_read_lock();
        master_ndev = netdev_master_upper_dev_get_rcu(rdma_ndev);
        if (master_ndev)
                dev_hold(master_ndev);
        rcu_read_unlock();

        if (master_ndev) {
                bond_delete_netdev_default_gids(ib_dev, port, rdma_ndev,
                                                master_ndev);
                dev_put(master_ndev);
        }
}

/* The following functions operate on all IB devices. netdevice_event and
 * addr_event execute ib_enum_all_roce_netdevs through a work.
 * ib_enum_all_roce_netdevs iterates through all IB devices.
 */

static void netdevice_event_work_handler(struct work_struct *_work)
{
        struct netdev_event_work *work =
                container_of(_work, struct netdev_event_work, work);
        unsigned int i;

        for (i = 0; i < ARRAY_SIZE(work->cmds) && work->cmds[i].cb; i++) {
                ib_enum_all_roce_netdevs(work->cmds[i].filter,
                                         work->cmds[i].filter_ndev,
                                         work->cmds[i].cb,
                                         work->cmds[i].ndev);
                dev_put(work->cmds[i].ndev);
                dev_put(work->cmds[i].filter_ndev);
        }

        kfree(work);
}

static int netdevice_queue_work(struct netdev_event_work_cmd *cmds,
                                struct net_device *ndev)
{
        unsigned int i;
        struct netdev_event_work *ndev_work =
                kmalloc(sizeof(*ndev_work), GFP_KERNEL);

        if (!ndev_work)
                return NOTIFY_DONE;

        memcpy(ndev_work->cmds, cmds, sizeof(ndev_work->cmds));
        for (i = 0; i < ARRAY_SIZE(ndev_work->cmds) && ndev_work->cmds[i].cb; i++) {
                if (!ndev_work->cmds[i].ndev)
                        ndev_work->cmds[i].ndev = ndev;
                if (!ndev_work->cmds[i].filter_ndev)
                        ndev_work->cmds[i].filter_ndev = ndev;
                dev_hold(ndev_work->cmds[i].ndev);
                dev_hold(ndev_work->cmds[i].filter_ndev);
        }
        INIT_WORK(&ndev_work->work, netdevice_event_work_handler);

        queue_work(gid_cache_wq, &ndev_work->work);

        return NOTIFY_DONE;
}

static const struct netdev_event_work_cmd add_cmd = {
        .cb        = add_netdev_ips,
        .filter        = is_eth_port_of_netdev_filter
};

static const struct netdev_event_work_cmd add_cmd_upper_ips = {
        .cb        = add_netdev_upper_ips,
        .filter = is_eth_port_of_netdev_filter
};

static void
ndev_event_unlink(struct netdev_notifier_changeupper_info *changeupper_info,
                  struct netdev_event_work_cmd *cmds)
{
        static const struct netdev_event_work_cmd
                        upper_ips_del_cmd = {
                                .cb        = del_netdev_upper_ips,
                                .filter        = upper_device_filter
        };

        cmds[0] = upper_ips_del_cmd;
        cmds[0].ndev = changeupper_info->upper_dev;
        cmds[1] = add_cmd;
}

static const struct netdev_event_work_cmd bonding_default_add_cmd = {
        .cb        = add_default_gids,
        .filter        = is_upper_ndev_bond_master_filter
};

static void
ndev_event_link(struct net_device *event_ndev,
                struct netdev_notifier_changeupper_info *changeupper_info,
                struct netdev_event_work_cmd *cmds)
{
        static const struct netdev_event_work_cmd
                        bonding_default_del_cmd = {
                                .cb        = del_default_gids,
                                .filter        = is_upper_ndev_bond_master_filter
                        };
        /*
         * When a lower netdev is linked to its upper bonding
         * netdev, delete lower slave netdev's default GIDs.
         */
        cmds[0] = bonding_default_del_cmd;
        cmds[0].ndev = event_ndev;
        cmds[0].filter_ndev = changeupper_info->upper_dev;

        /* Now add bonding upper device default GIDs */
        cmds[1] = bonding_default_add_cmd;
        cmds[1].ndev = changeupper_info->upper_dev;
        cmds[1].filter_ndev = changeupper_info->upper_dev;

        /* Now add bonding upper device IP based GIDs */
        cmds[2] = add_cmd_upper_ips;
        cmds[2].ndev = changeupper_info->upper_dev;
        cmds[2].filter_ndev = changeupper_info->upper_dev;
}

static void netdevice_event_changeupper(struct net_device *event_ndev,
                struct netdev_notifier_changeupper_info *changeupper_info,
                struct netdev_event_work_cmd *cmds)
{
        if (changeupper_info->linking)
                ndev_event_link(event_ndev, changeupper_info, cmds);
        else
                ndev_event_unlink(changeupper_info, cmds);
}

static const struct netdev_event_work_cmd add_default_gid_cmd = {
        .cb        = add_default_gids,
        .filter        = is_ndev_for_default_gid_filter,
};

static int netdevice_event(struct notifier_block *this, unsigned long event,
                           void *ptr)
{
        static const struct netdev_event_work_cmd del_cmd = {
                .cb = del_netdev_ips, .filter = pass_all_filter};
        static const struct netdev_event_work_cmd
                        bonding_default_del_cmd_join = {
                                .cb        = del_netdev_default_ips_join,
                                .filter        = is_eth_port_inactive_slave_filter
                        };
        static const struct netdev_event_work_cmd
                        netdev_del_cmd = {
                                .cb        = del_netdev_ips,
                                .filter = is_eth_port_of_netdev_filter
                        };
        static const struct netdev_event_work_cmd bonding_event_ips_del_cmd = {
                .cb = del_netdev_upper_ips, .filter = upper_device_filter};
        struct net_device *ndev = netdev_notifier_info_to_dev(ptr);
        struct netdev_event_work_cmd cmds[ROCE_NETDEV_CALLBACK_SZ] = { {NULL} };

        if (ndev->type != ARPHRD_ETHER)
                return NOTIFY_DONE;

        switch (event) {
        case NETDEV_REGISTER:
        case NETDEV_UP:
                cmds[0] = bonding_default_del_cmd_join;
                cmds[1] = add_default_gid_cmd;
                cmds[2] = add_cmd;
                break;

        case NETDEV_UNREGISTER:
                if (ndev->reg_state < NETREG_UNREGISTERED)
                        cmds[0] = del_cmd;
                else
                        return NOTIFY_DONE;
                break;

        case NETDEV_CHANGEADDR:
                cmds[0] = netdev_del_cmd;
                if (ndev->reg_state == NETREG_REGISTERED) {
                        cmds[1] = add_default_gid_cmd;
                        cmds[2] = add_cmd;
                }
                break;

        case NETDEV_CHANGEUPPER:
                netdevice_event_changeupper(ndev,
                        container_of(ptr, struct netdev_notifier_changeupper_info, info),
                        cmds);
                break;

        case NETDEV_BONDING_FAILOVER:
                cmds[0] = bonding_event_ips_del_cmd;
                /* Add default GIDs of the bond device */
                cmds[1] = bonding_default_add_cmd;
                /* Add IP based GIDs of the bond device */
                cmds[2] = add_cmd_upper_ips;
                break;

        default:
                return NOTIFY_DONE;
        }

        return netdevice_queue_work(cmds, ndev);
}

static void update_gid_event_work_handler(struct work_struct *_work)
{
        struct update_gid_event_work *work =
                container_of(_work, struct update_gid_event_work, work);

        ib_enum_all_roce_netdevs(is_eth_port_of_netdev_filter,
                                 work->gid_attr.ndev,
                                 callback_for_addr_gid_device_scan, work);

        dev_put(work->gid_attr.ndev);
        kfree(work);
}

static int addr_event(struct notifier_block *this, unsigned long event,
                      struct sockaddr *sa, struct net_device *ndev)
{
        struct update_gid_event_work *work;
        enum gid_op_type gid_op;

        if (ndev->type != ARPHRD_ETHER)
                return NOTIFY_DONE;

        switch (event) {
        case NETDEV_UP:
                gid_op = GID_ADD;
                break;

        case NETDEV_DOWN:
                gid_op = GID_DEL;
                break;

        default:
                return NOTIFY_DONE;
        }

        work = kmalloc(sizeof(*work), GFP_ATOMIC);
        if (!work)
                return NOTIFY_DONE;

        INIT_WORK(&work->work, update_gid_event_work_handler);

        rdma_ip2gid(sa, &work->gid);
        work->gid_op = gid_op;

        memset(&work->gid_attr, 0, sizeof(work->gid_attr));
        dev_hold(ndev);
        work->gid_attr.ndev   = ndev;

        queue_work(gid_cache_wq, &work->work);

        return NOTIFY_DONE;
}

static int inetaddr_event(struct notifier_block *this, unsigned long event,
                          void *ptr)
{
        struct sockaddr_in        in;
        struct net_device        *ndev;
        struct in_ifaddr        *ifa = ptr;

        in.sin_family = AF_INET;
        in.sin_addr.s_addr = ifa->ifa_address;
        ndev = ifa->ifa_dev->dev;

        return addr_event(this, event, (struct sockaddr *)&in, ndev);
}

static int inet6addr_event(struct notifier_block *this, unsigned long event,
                           void *ptr)
{
        struct sockaddr_in6        in6;
        struct net_device        *ndev;
        struct inet6_ifaddr        *ifa6 = ptr;

        in6.sin6_family = AF_INET6;
        in6.sin6_addr = ifa6->addr;
        ndev = ifa6->idev->dev;

        return addr_event(this, event, (struct sockaddr *)&in6, ndev);
}

static struct notifier_block nb_netdevice = {
        .notifier_call = netdevice_event
};

static struct notifier_block nb_inetaddr = {
        .notifier_call = inetaddr_event
};

static struct notifier_block nb_inet6addr = {
        .notifier_call = inet6addr_event
};

int __init roce_gid_mgmt_init(void)
{
        gid_cache_wq = alloc_ordered_workqueue("gid-cache-wq", 0);
        if (!gid_cache_wq)
                return -ENOMEM;

        register_inetaddr_notifier(&nb_inetaddr);
        if (IS_ENABLED(CONFIG_IPV6))
                register_inet6addr_notifier(&nb_inet6addr);
        /* We relay on the netdevice notifier to enumerate all
         * existing devices in the system. Register to this notifier
         * last to make sure we will not miss any IP add/del
         * callbacks.
         */
        register_netdevice_notifier(&nb_netdevice);

        return 0;
}

void __exit roce_gid_mgmt_cleanup(void)
{
        if (IS_ENABLED(CONFIG_IPV6))
                unregister_inet6addr_notifier(&nb_inet6addr);
        unregister_inetaddr_notifier(&nb_inetaddr);
        unregister_netdevice_notifier(&nb_netdevice);
        /* Ensure all gid deletion tasks complete before we go down,
         * to avoid any reference to free'd memory. By the time
         * ib-core is removed, all physical devices have been removed,
         * so no issue with remaining hardware contexts.
         */
        destroy_workqueue(gid_cache_wq);
}














































































































































































    1 






    1 






    1 











































































































































































































































































    1 






    1 





















    1 
    1 












































































    9 

































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * Asynchronous Cryptographic Hash operations.
 *
 * This is the asynchronous version of hash.c with notification of
 * completion via a callback.
 *
 * Copyright (c) 2008 Loc Ho <lho@amcc.com>
 */

#include <crypto/internal/hash.h>
#include <crypto/scatterwalk.h>
#include <linux/err.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/sched.h>
#include <linux/slab.h>
#include <linux/seq_file.h>
#include <linux/cryptouser.h>
#include <linux/compiler.h>
#include <net/netlink.h>

#include "internal.h"

static const struct crypto_type crypto_ahash_type;

struct ahash_request_priv {
        crypto_completion_t complete;
        void *data;
        u8 *result;
        u32 flags;
        void *ubuf[] CRYPTO_MINALIGN_ATTR;
};

static inline struct ahash_alg *crypto_ahash_alg(struct crypto_ahash *hash)
{
        return container_of(crypto_hash_alg_common(hash), struct ahash_alg,
                            halg);
}

static int hash_walk_next(struct crypto_hash_walk *walk)
{
        unsigned int alignmask = walk->alignmask;
        unsigned int offset = walk->offset;
        unsigned int nbytes = min(walk->entrylen,
                                  ((unsigned int)(PAGE_SIZE)) - offset);

        walk->data = kmap_atomic(walk->pg);
        walk->data += offset;

        if (offset & alignmask) {
                unsigned int unaligned = alignmask + 1 - (offset & alignmask);

                if (nbytes > unaligned)
                        nbytes = unaligned;
        }

        walk->entrylen -= nbytes;
        return nbytes;
}

static int hash_walk_new_entry(struct crypto_hash_walk *walk)
{
        struct scatterlist *sg;

        sg = walk->sg;
        walk->offset = sg->offset;
        walk->pg = sg_page(walk->sg) + (walk->offset >> PAGE_SHIFT);
        walk->offset = offset_in_page(walk->offset);
        walk->entrylen = sg->length;

        if (walk->entrylen > walk->total)
                walk->entrylen = walk->total;
        walk->total -= walk->entrylen;

        return hash_walk_next(walk);
}

int crypto_hash_walk_done(struct crypto_hash_walk *walk, int err)
{
        unsigned int alignmask = walk->alignmask;

        walk->data -= walk->offset;

        if (walk->entrylen && (walk->offset & alignmask) && !err) {
                unsigned int nbytes;

                walk->offset = ALIGN(walk->offset, alignmask + 1);
                nbytes = min(walk->entrylen,
                             (unsigned int)(PAGE_SIZE - walk->offset));
                if (nbytes) {
                        walk->entrylen -= nbytes;
                        walk->data += walk->offset;
                        return nbytes;
                }
        }

        kunmap_atomic(walk->data);
        crypto_yield(walk->flags);

        if (err)
                return err;

        if (walk->entrylen) {
                walk->offset = 0;
                walk->pg++;
                return hash_walk_next(walk);
        }

        if (!walk->total)
                return 0;

        walk->sg = sg_next(walk->sg);

        return hash_walk_new_entry(walk);
}
EXPORT_SYMBOL_GPL(crypto_hash_walk_done);

int crypto_hash_walk_first(struct ahash_request *req,
                           struct crypto_hash_walk *walk)
{
        walk->total = req->nbytes;

        if (!walk->total) {
                walk->entrylen = 0;
                return 0;
        }

        walk->alignmask = crypto_ahash_alignmask(crypto_ahash_reqtfm(req));
        walk->sg = req->src;
        walk->flags = req->base.flags;

        return hash_walk_new_entry(walk);
}
EXPORT_SYMBOL_GPL(crypto_hash_walk_first);

static int ahash_setkey_unaligned(struct crypto_ahash *tfm, const u8 *key,
                                unsigned int keylen)
{
        unsigned long alignmask = crypto_ahash_alignmask(tfm);
        int ret;
        u8 *buffer, *alignbuffer;
        unsigned long absize;

        absize = keylen + alignmask;
        buffer = kmalloc(absize, GFP_KERNEL);
        if (!buffer)
                return -ENOMEM;

        alignbuffer = (u8 *)ALIGN((unsigned long)buffer, alignmask + 1);
        memcpy(alignbuffer, key, keylen);
        ret = tfm->setkey(tfm, alignbuffer, keylen);
        kfree_sensitive(buffer);
        return ret;
}

static int ahash_nosetkey(struct crypto_ahash *tfm, const u8 *key,
                          unsigned int keylen)
{
        return -ENOSYS;
}

static void ahash_set_needkey(struct crypto_ahash *tfm)
{
        const struct hash_alg_common *alg = crypto_hash_alg_common(tfm);

        if (tfm->setkey != ahash_nosetkey &&
            !(alg->base.cra_flags & CRYPTO_ALG_OPTIONAL_KEY))
                crypto_ahash_set_flags(tfm, CRYPTO_TFM_NEED_KEY);
}

int crypto_ahash_setkey(struct crypto_ahash *tfm, const u8 *key,
                        unsigned int keylen)
{
        unsigned long alignmask = crypto_ahash_alignmask(tfm);
        int err;

        if ((unsigned long)key & alignmask)
                err = ahash_setkey_unaligned(tfm, key, keylen);
        else
                err = tfm->setkey(tfm, key, keylen);

        if (unlikely(err)) {
                ahash_set_needkey(tfm);
                return err;
        }

        crypto_ahash_clear_flags(tfm, CRYPTO_TFM_NEED_KEY);
        return 0;
}
EXPORT_SYMBOL_GPL(crypto_ahash_setkey);

static inline unsigned int ahash_align_buffer_size(unsigned len,
                                                   unsigned long mask)
{
        return len + (mask & ~(crypto_tfm_ctx_alignment() - 1));
}

static int ahash_save_req(struct ahash_request *req, crypto_completion_t cplt)
{
        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
        unsigned long alignmask = crypto_ahash_alignmask(tfm);
        unsigned int ds = crypto_ahash_digestsize(tfm);
        struct ahash_request_priv *priv;

        priv = kmalloc(sizeof(*priv) + ahash_align_buffer_size(ds, alignmask),
                       (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
                       GFP_KERNEL : GFP_ATOMIC);
        if (!priv)
                return -ENOMEM;

        /*
         * WARNING: Voodoo programming below!
         *
         * The code below is obscure and hard to understand, thus explanation
         * is necessary. See include/crypto/hash.h and include/linux/crypto.h
         * to understand the layout of structures used here!
         *
         * The code here will replace portions of the ORIGINAL request with
         * pointers to new code and buffers so the hashing operation can store
         * the result in aligned buffer. We will call the modified request
         * an ADJUSTED request.
         *
         * The newly mangled request will look as such:
         *
         * req {
         *   .result        = ADJUSTED[new aligned buffer]
         *   .base.complete = ADJUSTED[pointer to completion function]
         *   .base.data     = ADJUSTED[*req (pointer to self)]
         *   .priv          = ADJUSTED[new priv] {
         *           .result   = ORIGINAL(result)
         *           .complete = ORIGINAL(base.complete)
         *           .data     = ORIGINAL(base.data)
         *   }
         */

        priv->result = req->result;
        priv->complete = req->base.complete;
        priv->data = req->base.data;
        priv->flags = req->base.flags;

        /*
         * WARNING: We do not backup req->priv here! The req->priv
         *          is for internal use of the Crypto API and the
         *          user must _NOT_ _EVER_ depend on it's content!
         */

        req->result = PTR_ALIGN((u8 *)priv->ubuf, alignmask + 1);
        req->base.complete = cplt;
        req->base.data = req;
        req->priv = priv;

        return 0;
}

static void ahash_restore_req(struct ahash_request *req, int err)
{
        struct ahash_request_priv *priv = req->priv;

        if (!err)
                memcpy(priv->result, req->result,
                       crypto_ahash_digestsize(crypto_ahash_reqtfm(req)));

        /* Restore the original crypto request. */
        req->result = priv->result;

        ahash_request_set_callback(req, priv->flags,
                                   priv->complete, priv->data);
        req->priv = NULL;

        /* Free the req->priv.priv from the ADJUSTED request. */
        kfree_sensitive(priv);
}

static void ahash_notify_einprogress(struct ahash_request *req)
{
        struct ahash_request_priv *priv = req->priv;
        struct crypto_async_request oreq;

        oreq.data = priv->data;

        priv->complete(&oreq, -EINPROGRESS);
}

static void ahash_op_unaligned_done(struct crypto_async_request *req, int err)
{
        struct ahash_request *areq = req->data;

        if (err == -EINPROGRESS) {
                ahash_notify_einprogress(areq);
                return;
        }

        /*
         * Restore the original request, see ahash_op_unaligned() for what
         * goes where.
         *
         * The "struct ahash_request *req" here is in fact the "req.base"
         * from the ADJUSTED request from ahash_op_unaligned(), thus as it
         * is a pointer to self, it is also the ADJUSTED "req" .
         */

        /* First copy req->result into req->priv.result */
        ahash_restore_req(areq, err);

        /* Complete the ORIGINAL request. */
        areq->base.complete(&areq->base, err);
}

static int ahash_op_unaligned(struct ahash_request *req,
                              int (*op)(struct ahash_request *))
{
        int err;

        err = ahash_save_req(req, ahash_op_unaligned_done);
        if (err)
                return err;

        err = op(req);
        if (err == -EINPROGRESS || err == -EBUSY)
                return err;

        ahash_restore_req(req, err);

        return err;
}

static int crypto_ahash_op(struct ahash_request *req,
                           int (*op)(struct ahash_request *))
{
        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
        unsigned long alignmask = crypto_ahash_alignmask(tfm);

        if ((unsigned long)req->result & alignmask)
                return ahash_op_unaligned(req, op);

        return op(req);
}

int crypto_ahash_final(struct ahash_request *req)
{
        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
        struct crypto_alg *alg = tfm->base.__crt_alg;
        unsigned int nbytes = req->nbytes;
        int ret;

        crypto_stats_get(alg);
        ret = crypto_ahash_op(req, crypto_ahash_reqtfm(req)->final);
        crypto_stats_ahash_final(nbytes, ret, alg);
        return ret;
}
EXPORT_SYMBOL_GPL(crypto_ahash_final);

int crypto_ahash_finup(struct ahash_request *req)
{
        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
        struct crypto_alg *alg = tfm->base.__crt_alg;
        unsigned int nbytes = req->nbytes;
        int ret;

        crypto_stats_get(alg);
        ret = crypto_ahash_op(req, crypto_ahash_reqtfm(req)->finup);
        crypto_stats_ahash_final(nbytes, ret, alg);
        return ret;
}
EXPORT_SYMBOL_GPL(crypto_ahash_finup);

int crypto_ahash_digest(struct ahash_request *req)
{
        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
        struct crypto_alg *alg = tfm->base.__crt_alg;
        unsigned int nbytes = req->nbytes;
        int ret;

        crypto_stats_get(alg);
        if (crypto_ahash_get_flags(tfm) & CRYPTO_TFM_NEED_KEY)
                ret = -ENOKEY;
        else
                ret = crypto_ahash_op(req, tfm->digest);
        crypto_stats_ahash_final(nbytes, ret, alg);
        return ret;
}
EXPORT_SYMBOL_GPL(crypto_ahash_digest);

static void ahash_def_finup_done2(struct crypto_async_request *req, int err)
{
        struct ahash_request *areq = req->data;

        if (err == -EINPROGRESS)
                return;

        ahash_restore_req(areq, err);

        areq->base.complete(&areq->base, err);
}

static int ahash_def_finup_finish1(struct ahash_request *req, int err)
{
        if (err)
                goto out;

        req->base.complete = ahash_def_finup_done2;

        err = crypto_ahash_reqtfm(req)->final(req);
        if (err == -EINPROGRESS || err == -EBUSY)
                return err;

out:
        ahash_restore_req(req, err);
        return err;
}

static void ahash_def_finup_done1(struct crypto_async_request *req, int err)
{
        struct ahash_request *areq = req->data;

        if (err == -EINPROGRESS) {
                ahash_notify_einprogress(areq);
                return;
        }

        areq->base.flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;

        err = ahash_def_finup_finish1(areq, err);
        if (areq->priv)
                return;

        areq->base.complete(&areq->base, err);
}

static int ahash_def_finup(struct ahash_request *req)
{
        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
        int err;

        err = ahash_save_req(req, ahash_def_finup_done1);
        if (err)
                return err;

        err = tfm->update(req);
        if (err == -EINPROGRESS || err == -EBUSY)
                return err;

        return ahash_def_finup_finish1(req, err);
}

static void crypto_ahash_exit_tfm(struct crypto_tfm *tfm)
{
        struct crypto_ahash *hash = __crypto_ahash_cast(tfm);
        struct ahash_alg *alg = crypto_ahash_alg(hash);

        alg->exit_tfm(hash);
}

static int crypto_ahash_init_tfm(struct crypto_tfm *tfm)
{
        struct crypto_ahash *hash = __crypto_ahash_cast(tfm);
        struct ahash_alg *alg = crypto_ahash_alg(hash);

        hash->setkey = ahash_nosetkey;

        if (tfm->__crt_alg->cra_type != &crypto_ahash_type)
                return crypto_init_shash_ops_async(tfm);

        hash->init = alg->init;
        hash->update = alg->update;
        hash->final = alg->final;
        hash->finup = alg->finup ?: ahash_def_finup;
        hash->digest = alg->digest;
        hash->export = alg->export;
        hash->import = alg->import;

        if (alg->setkey) {
                hash->setkey = alg->setkey;
                ahash_set_needkey(hash);
        }

        if (alg->exit_tfm)
                tfm->exit = crypto_ahash_exit_tfm;

        return alg->init_tfm ? alg->init_tfm(hash) : 0;
}

static unsigned int crypto_ahash_extsize(struct crypto_alg *alg)
{
        if (alg->cra_type != &crypto_ahash_type)
                return sizeof(struct crypto_shash *);

        return crypto_alg_extsize(alg);
}

static void crypto_ahash_free_instance(struct crypto_instance *inst)
{
        struct ahash_instance *ahash = ahash_instance(inst);

        ahash->free(ahash);
}

#ifdef CONFIG_NET
static int crypto_ahash_report(struct sk_buff *skb, struct crypto_alg *alg)
{
        struct crypto_report_hash rhash;

        memset(&rhash, 0, sizeof(rhash));

        strscpy(rhash.type, "ahash", sizeof(rhash.type));

        rhash.blocksize = alg->cra_blocksize;
        rhash.digestsize = __crypto_hash_alg_common(alg)->digestsize;

        return nla_put(skb, CRYPTOCFGA_REPORT_HASH, sizeof(rhash), &rhash);
}
#else
static int crypto_ahash_report(struct sk_buff *skb, struct crypto_alg *alg)
{
        return -ENOSYS;
}
#endif

static void crypto_ahash_show(struct seq_file *m, struct crypto_alg *alg)
        __maybe_unused;
static void crypto_ahash_show(struct seq_file *m, struct crypto_alg *alg)
{
        seq_printf(m, "type         : ahash\n");
        seq_printf(m, "async        : %s\n", alg->cra_flags & CRYPTO_ALG_ASYNC ?
                                             "yes" : "no");
        seq_printf(m, "blocksize    : %u\n", alg->cra_blocksize);
        seq_printf(m, "digestsize   : %u\n",
                   __crypto_hash_alg_common(alg)->digestsize);
}

static const struct crypto_type crypto_ahash_type = {
        .extsize = crypto_ahash_extsize,
        .init_tfm = crypto_ahash_init_tfm,
        .free = crypto_ahash_free_instance,
#ifdef CONFIG_PROC_FS
        .show = crypto_ahash_show,
#endif
        .report = crypto_ahash_report,
        .maskclear = ~CRYPTO_ALG_TYPE_MASK,
        .maskset = CRYPTO_ALG_TYPE_AHASH_MASK,
        .type = CRYPTO_ALG_TYPE_AHASH,
        .tfmsize = offsetof(struct crypto_ahash, base),
};

int crypto_grab_ahash(struct crypto_ahash_spawn *spawn,
                      struct crypto_instance *inst,
                      const char *name, u32 type, u32 mask)
{
        spawn->base.frontend = &crypto_ahash_type;
        return crypto_grab_spawn(&spawn->base, inst, name, type, mask);
}
EXPORT_SYMBOL_GPL(crypto_grab_ahash);

struct crypto_ahash *crypto_alloc_ahash(const char *alg_name, u32 type,
                                        u32 mask)
{
        return crypto_alloc_tfm(alg_name, &crypto_ahash_type, type, mask);
}
EXPORT_SYMBOL_GPL(crypto_alloc_ahash);

int crypto_has_ahash(const char *alg_name, u32 type, u32 mask)
{
        return crypto_type_has_alg(alg_name, &crypto_ahash_type, type, mask);
}
EXPORT_SYMBOL_GPL(crypto_has_ahash);

static int ahash_prepare_alg(struct ahash_alg *alg)
{
        struct crypto_alg *base = &alg->halg.base;

        if (alg->halg.digestsize > HASH_MAX_DIGESTSIZE ||
            alg->halg.statesize > HASH_MAX_STATESIZE ||
            alg->halg.statesize == 0)
                return -EINVAL;

        base->cra_type = &crypto_ahash_type;
        base->cra_flags &= ~CRYPTO_ALG_TYPE_MASK;
        base->cra_flags |= CRYPTO_ALG_TYPE_AHASH;

        return 0;
}

int crypto_register_ahash(struct ahash_alg *alg)
{
        struct crypto_alg *base = &alg->halg.base;
        int err;

        err = ahash_prepare_alg(alg);
        if (err)
                return err;

        return crypto_register_alg(base);
}
EXPORT_SYMBOL_GPL(crypto_register_ahash);

void crypto_unregister_ahash(struct ahash_alg *alg)
{
        crypto_unregister_alg(&alg->halg.base);
}
EXPORT_SYMBOL_GPL(crypto_unregister_ahash);

int crypto_register_ahashes(struct ahash_alg *algs, int count)
{
        int i, ret;

        for (i = 0; i < count; i++) {
                ret = crypto_register_ahash(&algs[i]);
                if (ret)
                        goto err;
        }

        return 0;

err:
        for (--i; i >= 0; --i)
                crypto_unregister_ahash(&algs[i]);

        return ret;
}
EXPORT_SYMBOL_GPL(crypto_register_ahashes);

void crypto_unregister_ahashes(struct ahash_alg *algs, int count)
{
        int i;

        for (i = count - 1; i >= 0; --i)
                crypto_unregister_ahash(&algs[i]);
}
EXPORT_SYMBOL_GPL(crypto_unregister_ahashes);

int ahash_register_instance(struct crypto_template *tmpl,
                            struct ahash_instance *inst)
{
        int err;

        if (WARN_ON(!inst->free))
                return -EINVAL;

        err = ahash_prepare_alg(&inst->alg);
        if (err)
                return err;

        return crypto_register_instance(tmpl, ahash_crypto_instance(inst));
}
EXPORT_SYMBOL_GPL(ahash_register_instance);

bool crypto_hash_alg_has_setkey(struct hash_alg_common *halg)
{
        struct crypto_alg *alg = &halg->base;

        if (alg->cra_type != &crypto_ahash_type)
                return crypto_shash_alg_has_setkey(__crypto_shash_alg(alg));

        return __crypto_ahash_alg(alg)->setkey != NULL;
}
EXPORT_SYMBOL_GPL(crypto_hash_alg_has_setkey);

MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("Asynchronous cryptographic hash type");





































   21 




   22 
    3 

   22 



   22 


   22 


   22 


   22 


   22 


   22 


   22 





   22 





   22 

   22 























    1 






   64 

   63 





    1 


   64 



   64 
    1 





   64 











   64 



   64 
   63 
   52 



   16 



   64 

   59 
   12 







   63 





   64 




   22 
   60 






















   63 





   63 



   18 










   17 






















   17 





































































































  126 













  120 


    8 






    2 


    1 







    1 
    1 

    1 
    1 






    1 












  323 









  322 








   15 


    3 

























































































































   31 










   31 






   31 



































   31 



















    4 
   27 













   31 

   31 















































































































   31 





   31 












































   11 


































   20 







































































   31 







   31 




   30 





   31 



   31 





   31 




   26 









    4 




    4 










   31 













   30 



   31 


   31 





   31 









    4 
   27 









   31 
   31 

   31 




   31 



   31 



   31 


















   31 








   31 





   31 
   31 










   31 

   31 










   31 





   31 












   31 






   31 




















   31 





    4 

   27 











   31 







   31 



    4 
   27 



   31 



   31 








   31 









   31 



   31 


















   31 






   31 


   31 























   31 












   31 






   31 





   30 



































   26 

















   26 



































    5 
































    4 




















    2 






    2 













    2 

















































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright 2002-2005, Instant802 Networks, Inc.
 * Copyright 2005-2006, Devicescape Software, Inc.
 * Copyright 2006-2007        Jiri Benc <jbenc@suse.cz>
 * Copyright 2013-2014  Intel Mobile Communications GmbH
 * Copyright (C) 2017     Intel Deutschland GmbH
 * Copyright (C) 2018-2021 Intel Corporation
 */

#include <net/mac80211.h>
#include <linux/module.h>
#include <linux/fips.h>
#include <linux/init.h>
#include <linux/netdevice.h>
#include <linux/types.h>
#include <linux/slab.h>
#include <linux/skbuff.h>
#include <linux/etherdevice.h>
#include <linux/if_arp.h>
#include <linux/rtnetlink.h>
#include <linux/bitmap.h>
#include <linux/inetdevice.h>
#include <net/net_namespace.h>
#include <net/cfg80211.h>
#include <net/addrconf.h>

#include "ieee80211_i.h"
#include "driver-ops.h"
#include "rate.h"
#include "mesh.h"
#include "wep.h"
#include "led.h"
#include "debugfs.h"

void ieee80211_configure_filter(struct ieee80211_local *local)
{
        u64 mc;
        unsigned int changed_flags;
        unsigned int new_flags = 0;

        if (atomic_read(&local->iff_allmultis))
                new_flags |= FIF_ALLMULTI;

        if (local->monitors || test_bit(SCAN_SW_SCANNING, &local->scanning) ||
            test_bit(SCAN_ONCHANNEL_SCANNING, &local->scanning))
                new_flags |= FIF_BCN_PRBRESP_PROMISC;

        if (local->fif_probe_req || local->probe_req_reg)
                new_flags |= FIF_PROBE_REQ;

        if (local->fif_fcsfail)
                new_flags |= FIF_FCSFAIL;

        if (local->fif_plcpfail)
                new_flags |= FIF_PLCPFAIL;

        if (local->fif_control)
                new_flags |= FIF_CONTROL;

        if (local->fif_other_bss)
                new_flags |= FIF_OTHER_BSS;

        if (local->fif_pspoll)
                new_flags |= FIF_PSPOLL;

        if (local->rx_mcast_action_reg)
                new_flags |= FIF_MCAST_ACTION;

        spin_lock_bh(&local->filter_lock);
        changed_flags = local->filter_flags ^ new_flags;

        mc = drv_prepare_multicast(local, &local->mc_list);
        spin_unlock_bh(&local->filter_lock);

        /* be a bit nasty */
        new_flags |= (1<<31);

        drv_configure_filter(local, changed_flags, &new_flags, mc);

        WARN_ON(new_flags & (1<<31));

        local->filter_flags = new_flags & ~(1<<31);
}

static void ieee80211_reconfig_filter(struct work_struct *work)
{
        struct ieee80211_local *local =
                container_of(work, struct ieee80211_local, reconfig_filter);

        ieee80211_configure_filter(local);
}

static u32 ieee80211_hw_conf_chan(struct ieee80211_local *local)
{
        struct ieee80211_sub_if_data *sdata;
        struct cfg80211_chan_def chandef = {};
        u32 changed = 0;
        int power;
        u32 offchannel_flag;

        offchannel_flag = local->hw.conf.flags & IEEE80211_CONF_OFFCHANNEL;

        if (local->scan_chandef.chan) {
                chandef = local->scan_chandef;
        } else if (local->tmp_channel) {
                chandef.chan = local->tmp_channel;
                chandef.width = NL80211_CHAN_WIDTH_20_NOHT;
                chandef.center_freq1 = chandef.chan->center_freq;
                chandef.freq1_offset = chandef.chan->freq_offset;
        } else
                chandef = local->_oper_chandef;

        WARN(!cfg80211_chandef_valid(&chandef),
             "control:%d.%03d MHz width:%d center: %d.%03d/%d MHz",
             chandef.chan->center_freq, chandef.chan->freq_offset,
             chandef.width, chandef.center_freq1, chandef.freq1_offset,
             chandef.center_freq2);

        if (!cfg80211_chandef_identical(&chandef, &local->_oper_chandef))
                local->hw.conf.flags |= IEEE80211_CONF_OFFCHANNEL;
        else
                local->hw.conf.flags &= ~IEEE80211_CONF_OFFCHANNEL;

        offchannel_flag ^= local->hw.conf.flags & IEEE80211_CONF_OFFCHANNEL;

        if (offchannel_flag ||
            !cfg80211_chandef_identical(&local->hw.conf.chandef,
                                        &local->_oper_chandef)) {
                local->hw.conf.chandef = chandef;
                changed |= IEEE80211_CONF_CHANGE_CHANNEL;
        }

        if (!conf_is_ht(&local->hw.conf)) {
                /*
                 * mac80211.h documents that this is only valid
                 * when the channel is set to an HT type, and
                 * that otherwise STATIC is used.
                 */
                local->hw.conf.smps_mode = IEEE80211_SMPS_STATIC;
        } else if (local->hw.conf.smps_mode != local->smps_mode) {
                local->hw.conf.smps_mode = local->smps_mode;
                changed |= IEEE80211_CONF_CHANGE_SMPS;
        }

        power = ieee80211_chandef_max_power(&chandef);
        if (local->user_power_level != IEEE80211_UNSET_POWER_LEVEL)
                power = min(local->user_power_level, power);

        rcu_read_lock();
        list_for_each_entry_rcu(sdata, &local->interfaces, list) {
                if (!rcu_access_pointer(sdata->vif.chanctx_conf))
                        continue;
                if (sdata->vif.type == NL80211_IFTYPE_AP_VLAN)
                        continue;
                if (sdata->vif.bss_conf.txpower == INT_MIN)
                        continue;
                power = min(power, sdata->vif.bss_conf.txpower);
        }
        rcu_read_unlock();

        if (local->hw.conf.power_level != power) {
                changed |= IEEE80211_CONF_CHANGE_POWER;
                local->hw.conf.power_level = power;
        }

        return changed;
}

int ieee80211_hw_config(struct ieee80211_local *local, u32 changed)
{
        int ret = 0;

        might_sleep();

        if (!local->use_chanctx)
                changed |= ieee80211_hw_conf_chan(local);
        else
                changed &= ~(IEEE80211_CONF_CHANGE_CHANNEL |
                             IEEE80211_CONF_CHANGE_POWER);

        if (changed && local->open_count) {
                ret = drv_config(local, changed);
                /*
                 * Goal:
                 * HW reconfiguration should never fail, the driver has told
                 * us what it can support so it should live up to that promise.
                 *
                 * Current status:
                 * rfkill is not integrated with mac80211 and a
                 * configuration command can thus fail if hardware rfkill
                 * is enabled
                 *
                 * FIXME: integrate rfkill with mac80211 and then add this
                 * WARN_ON() back
                 *
                 */
                /* WARN_ON(ret); */
        }

        return ret;
}

void ieee80211_bss_info_change_notify(struct ieee80211_sub_if_data *sdata,
                                      u32 changed)
{
        struct ieee80211_local *local = sdata->local;

        if (!changed || sdata->vif.type == NL80211_IFTYPE_AP_VLAN)
                return;

        drv_bss_info_changed(local, sdata, &sdata->vif.bss_conf, changed);
}

u32 ieee80211_reset_erp_info(struct ieee80211_sub_if_data *sdata)
{
        sdata->vif.bss_conf.use_cts_prot = false;
        sdata->vif.bss_conf.use_short_preamble = false;
        sdata->vif.bss_conf.use_short_slot = false;
        return BSS_CHANGED_ERP_CTS_PROT |
               BSS_CHANGED_ERP_PREAMBLE |
               BSS_CHANGED_ERP_SLOT;
}

/* context: requires softirqs disabled */
void ieee80211_handle_queued_frames(struct ieee80211_local *local)
{
        struct sk_buff *skb;

        while ((skb = skb_dequeue(&local->skb_queue)) ||
               (skb = skb_dequeue(&local->skb_queue_unreliable))) {
                switch (skb->pkt_type) {
                case IEEE80211_RX_MSG:
                        /* Clear skb->pkt_type in order to not confuse kernel
                         * netstack. */
                        skb->pkt_type = 0;
                        ieee80211_rx(&local->hw, skb);
                        break;
                case IEEE80211_TX_STATUS_MSG:
                        skb->pkt_type = 0;
                        ieee80211_tx_status(&local->hw, skb);
                        break;
                default:
                        WARN(1, "mac80211: Packet is of unknown type %d\n",
                             skb->pkt_type);
                        dev_kfree_skb(skb);
                        break;
                }
        }
}

static void ieee80211_tasklet_handler(struct tasklet_struct *t)
{
        struct ieee80211_local *local = from_tasklet(local, t, tasklet);

        ieee80211_handle_queued_frames(local);
}

static void ieee80211_restart_work(struct work_struct *work)
{
        struct ieee80211_local *local =
                container_of(work, struct ieee80211_local, restart_work);
        struct ieee80211_sub_if_data *sdata;
        int ret;

        /* wait for scan work complete */
        flush_workqueue(local->workqueue);
        flush_work(&local->sched_scan_stopped_work);
        flush_work(&local->radar_detected_work);

        rtnl_lock();
        /* we might do interface manipulations, so need both */
        wiphy_lock(local->hw.wiphy);

        WARN(test_bit(SCAN_HW_SCANNING, &local->scanning),
             "%s called with hardware scan in progress\n", __func__);

        list_for_each_entry(sdata, &local->interfaces, list) {
                /*
                 * XXX: there may be more work for other vif types and even
                 * for station mode: a good thing would be to run most of
                 * the iface type's dependent _stop (ieee80211_mg_stop,
                 * ieee80211_ibss_stop) etc...
                 * For now, fix only the specific bug that was seen: race
                 * between csa_connection_drop_work and us.
                 */
                if (sdata->vif.type == NL80211_IFTYPE_STATION) {
                        /*
                         * This worker is scheduled from the iface worker that
                         * runs on mac80211's workqueue, so we can't be
                         * scheduling this worker after the cancel right here.
                         * The exception is ieee80211_chswitch_done.
                         * Then we can have a race...
                         */
                        cancel_work_sync(&sdata->u.mgd.csa_connection_drop_work);
                        if (sdata->vif.csa_active) {
                                sdata_lock(sdata);
                                ieee80211_sta_connection_lost(sdata,
                                                              sdata->u.mgd.associated->bssid,
                                                              WLAN_REASON_UNSPECIFIED, false);
                                sdata_unlock(sdata);
                        }
                }
                flush_delayed_work(&sdata->dec_tailroom_needed_wk);
        }
        ieee80211_scan_cancel(local);

        /* make sure any new ROC will consider local->in_reconfig */
        flush_delayed_work(&local->roc_work);
        flush_work(&local->hw_roc_done);

        /* wait for all packet processing to be done */
        synchronize_net();

        ret = ieee80211_reconfig(local);
        wiphy_unlock(local->hw.wiphy);

        if (ret)
                cfg80211_shutdown_all_interfaces(local->hw.wiphy);

        rtnl_unlock();
}

void ieee80211_restart_hw(struct ieee80211_hw *hw)
{
        struct ieee80211_local *local = hw_to_local(hw);

        trace_api_restart_hw(local);

        wiphy_info(hw->wiphy,
                   "Hardware restart was requested\n");

        /* use this reason, ieee80211_reconfig will unblock it */
        ieee80211_stop_queues_by_reason(hw, IEEE80211_MAX_QUEUE_MAP,
                                        IEEE80211_QUEUE_STOP_REASON_SUSPEND,
                                        false);

        /*
         * Stop all Rx during the reconfig. We don't want state changes
         * or driver callbacks while this is in progress.
         */
        local->in_reconfig = true;
        barrier();

        queue_work(system_freezable_wq, &local->restart_work);
}
EXPORT_SYMBOL(ieee80211_restart_hw);

#ifdef CONFIG_INET
static int ieee80211_ifa_changed(struct notifier_block *nb,
                                 unsigned long data, void *arg)
{
        struct in_ifaddr *ifa = arg;
        struct ieee80211_local *local =
                container_of(nb, struct ieee80211_local,
                             ifa_notifier);
        struct net_device *ndev = ifa->ifa_dev->dev;
        struct wireless_dev *wdev = ndev->ieee80211_ptr;
        struct in_device *idev;
        struct ieee80211_sub_if_data *sdata;
        struct ieee80211_bss_conf *bss_conf;
        struct ieee80211_if_managed *ifmgd;
        int c = 0;

        /* Make sure it's our interface that got changed */
        if (!wdev)
                return NOTIFY_DONE;

        if (wdev->wiphy != local->hw.wiphy)
                return NOTIFY_DONE;

        sdata = IEEE80211_DEV_TO_SUB_IF(ndev);
        bss_conf = &sdata->vif.bss_conf;

        /* ARP filtering is only supported in managed mode */
        if (sdata->vif.type != NL80211_IFTYPE_STATION)
                return NOTIFY_DONE;

        idev = __in_dev_get_rtnl(sdata->dev);
        if (!idev)
                return NOTIFY_DONE;

        ifmgd = &sdata->u.mgd;
        sdata_lock(sdata);

        /* Copy the addresses to the bss_conf list */
        ifa = rtnl_dereference(idev->ifa_list);
        while (ifa) {
                if (c < IEEE80211_BSS_ARP_ADDR_LIST_LEN)
                        bss_conf->arp_addr_list[c] = ifa->ifa_address;
                ifa = rtnl_dereference(ifa->ifa_next);
                c++;
        }

        bss_conf->arp_addr_cnt = c;

        /* Configure driver only if associated (which also implies it is up) */
        if (ifmgd->associated)
                ieee80211_bss_info_change_notify(sdata,
                                                 BSS_CHANGED_ARP_FILTER);

        sdata_unlock(sdata);

        return NOTIFY_OK;
}
#endif

#if IS_ENABLED(CONFIG_IPV6)
static int ieee80211_ifa6_changed(struct notifier_block *nb,
                                  unsigned long data, void *arg)
{
        struct inet6_ifaddr *ifa = (struct inet6_ifaddr *)arg;
        struct inet6_dev *idev = ifa->idev;
        struct net_device *ndev = ifa->idev->dev;
        struct ieee80211_local *local =
                container_of(nb, struct ieee80211_local, ifa6_notifier);
        struct wireless_dev *wdev = ndev->ieee80211_ptr;
        struct ieee80211_sub_if_data *sdata;

        /* Make sure it's our interface that got changed */
        if (!wdev || wdev->wiphy != local->hw.wiphy)
                return NOTIFY_DONE;

        sdata = IEEE80211_DEV_TO_SUB_IF(ndev);

        /*
         * For now only support station mode. This is mostly because
         * doing AP would have to handle AP_VLAN in some way ...
         */
        if (sdata->vif.type != NL80211_IFTYPE_STATION)
                return NOTIFY_DONE;

        drv_ipv6_addr_change(local, sdata, idev);

        return NOTIFY_OK;
}
#endif

/* There isn't a lot of sense in it, but you can transmit anything you like */
static const struct ieee80211_txrx_stypes
ieee80211_default_mgmt_stypes[NUM_NL80211_IFTYPES] = {
        [NL80211_IFTYPE_ADHOC] = {
                .tx = 0xffff,
                .rx = BIT(IEEE80211_STYPE_ACTION >> 4) |
                        BIT(IEEE80211_STYPE_AUTH >> 4) |
                        BIT(IEEE80211_STYPE_DEAUTH >> 4) |
                        BIT(IEEE80211_STYPE_PROBE_REQ >> 4),
        },
        [NL80211_IFTYPE_STATION] = {
                .tx = 0xffff,
                /*
                 * To support Pre Association Security Negotiation (PASN) while
                 * already associated to one AP, allow user space to register to
                 * Rx authentication frames, so that the user space logic would
                 * be able to receive/handle authentication frames from a
                 * different AP as part of PASN.
                 * It is expected that user space would intelligently register
                 * for Rx authentication frames, i.e., only when PASN is used
                 * and configure a match filter only for PASN authentication
                 * algorithm, as otherwise the MLME functionality of mac80211
                 * would be broken.
                 */
                .rx = BIT(IEEE80211_STYPE_ACTION >> 4) |
                        BIT(IEEE80211_STYPE_AUTH >> 4) |
                        BIT(IEEE80211_STYPE_PROBE_REQ >> 4),
        },
        [NL80211_IFTYPE_AP] = {
                .tx = 0xffff,
                .rx = BIT(IEEE80211_STYPE_ASSOC_REQ >> 4) |
                        BIT(IEEE80211_STYPE_REASSOC_REQ >> 4) |
                        BIT(IEEE80211_STYPE_PROBE_REQ >> 4) |
                        BIT(IEEE80211_STYPE_DISASSOC >> 4) |
                        BIT(IEEE80211_STYPE_AUTH >> 4) |
                        BIT(IEEE80211_STYPE_DEAUTH >> 4) |
                        BIT(IEEE80211_STYPE_ACTION >> 4),
        },
        [NL80211_IFTYPE_AP_VLAN] = {
                /* copy AP */
                .tx = 0xffff,
                .rx = BIT(IEEE80211_STYPE_ASSOC_REQ >> 4) |
                        BIT(IEEE80211_STYPE_REASSOC_REQ >> 4) |
                        BIT(IEEE80211_STYPE_PROBE_REQ >> 4) |
                        BIT(IEEE80211_STYPE_DISASSOC >> 4) |
                        BIT(IEEE80211_STYPE_AUTH >> 4) |
                        BIT(IEEE80211_STYPE_DEAUTH >> 4) |
                        BIT(IEEE80211_STYPE_ACTION >> 4),
        },
        [NL80211_IFTYPE_P2P_CLIENT] = {
                .tx = 0xffff,
                .rx = BIT(IEEE80211_STYPE_ACTION >> 4) |
                        BIT(IEEE80211_STYPE_PROBE_REQ >> 4),
        },
        [NL80211_IFTYPE_P2P_GO] = {
                .tx = 0xffff,
                .rx = BIT(IEEE80211_STYPE_ASSOC_REQ >> 4) |
                        BIT(IEEE80211_STYPE_REASSOC_REQ >> 4) |
                        BIT(IEEE80211_STYPE_PROBE_REQ >> 4) |
                        BIT(IEEE80211_STYPE_DISASSOC >> 4) |
                        BIT(IEEE80211_STYPE_AUTH >> 4) |
                        BIT(IEEE80211_STYPE_DEAUTH >> 4) |
                        BIT(IEEE80211_STYPE_ACTION >> 4),
        },
        [NL80211_IFTYPE_MESH_POINT] = {
                .tx = 0xffff,
                .rx = BIT(IEEE80211_STYPE_ACTION >> 4) |
                        BIT(IEEE80211_STYPE_AUTH >> 4) |
                        BIT(IEEE80211_STYPE_DEAUTH >> 4),
        },
        [NL80211_IFTYPE_P2P_DEVICE] = {
                .tx = 0xffff,
                .rx = BIT(IEEE80211_STYPE_ACTION >> 4) |
                        BIT(IEEE80211_STYPE_PROBE_REQ >> 4),
        },
};

static const struct ieee80211_ht_cap mac80211_ht_capa_mod_mask = {
        .ampdu_params_info = IEEE80211_HT_AMPDU_PARM_FACTOR |
                             IEEE80211_HT_AMPDU_PARM_DENSITY,

        .cap_info = cpu_to_le16(IEEE80211_HT_CAP_SUP_WIDTH_20_40 |
                                IEEE80211_HT_CAP_MAX_AMSDU |
                                IEEE80211_HT_CAP_SGI_20 |
                                IEEE80211_HT_CAP_SGI_40 |
                                IEEE80211_HT_CAP_TX_STBC |
                                IEEE80211_HT_CAP_RX_STBC |
                                IEEE80211_HT_CAP_LDPC_CODING |
                                IEEE80211_HT_CAP_40MHZ_INTOLERANT),
        .mcs = {
                .rx_mask = { 0xff, 0xff, 0xff, 0xff, 0xff,
                             0xff, 0xff, 0xff, 0xff, 0xff, },
        },
};

static const struct ieee80211_vht_cap mac80211_vht_capa_mod_mask = {
        .vht_cap_info =
                cpu_to_le32(IEEE80211_VHT_CAP_RXLDPC |
                            IEEE80211_VHT_CAP_SHORT_GI_80 |
                            IEEE80211_VHT_CAP_SHORT_GI_160 |
                            IEEE80211_VHT_CAP_RXSTBC_MASK |
                            IEEE80211_VHT_CAP_TXSTBC |
                            IEEE80211_VHT_CAP_SU_BEAMFORMER_CAPABLE |
                            IEEE80211_VHT_CAP_SU_BEAMFORMEE_CAPABLE |
                            IEEE80211_VHT_CAP_TX_ANTENNA_PATTERN |
                            IEEE80211_VHT_CAP_RX_ANTENNA_PATTERN |
                            IEEE80211_VHT_CAP_MAX_A_MPDU_LENGTH_EXPONENT_MASK),
        .supp_mcs = {
                .rx_mcs_map = cpu_to_le16(~0),
                .tx_mcs_map = cpu_to_le16(~0),
        },
};

struct ieee80211_hw *ieee80211_alloc_hw_nm(size_t priv_data_len,
                                           const struct ieee80211_ops *ops,
                                           const char *requested_name)
{
        struct ieee80211_local *local;
        int priv_size, i;
        struct wiphy *wiphy;
        bool use_chanctx;

        if (WARN_ON(!ops->tx || !ops->start || !ops->stop || !ops->config ||
                    !ops->add_interface || !ops->remove_interface ||
                    !ops->configure_filter))
                return NULL;

        if (WARN_ON(ops->sta_state && (ops->sta_add || ops->sta_remove)))
                return NULL;

        /* check all or no channel context operations exist */
        i = !!ops->add_chanctx + !!ops->remove_chanctx +
            !!ops->change_chanctx + !!ops->assign_vif_chanctx +
            !!ops->unassign_vif_chanctx;
        if (WARN_ON(i != 0 && i != 5))
                return NULL;
        use_chanctx = i == 5;

        /* Ensure 32-byte alignment of our private data and hw private data.
         * We use the wiphy priv data for both our ieee80211_local and for
         * the driver's private data
         *
         * In memory it'll be like this:
         *
         * +-------------------------+
         * | struct wiphy            |
         * +-------------------------+
         * | struct ieee80211_local  |
         * +-------------------------+
         * | driver's private data   |
         * +-------------------------+
         *
         */
        priv_size = ALIGN(sizeof(*local), NETDEV_ALIGN) + priv_data_len;

        wiphy = wiphy_new_nm(&mac80211_config_ops, priv_size, requested_name);

        if (!wiphy)
                return NULL;

        wiphy->mgmt_stypes = ieee80211_default_mgmt_stypes;

        wiphy->privid = mac80211_wiphy_privid;

        wiphy->flags |= WIPHY_FLAG_NETNS_OK |
                        WIPHY_FLAG_4ADDR_AP |
                        WIPHY_FLAG_4ADDR_STATION |
                        WIPHY_FLAG_REPORTS_OBSS |
                        WIPHY_FLAG_OFFCHAN_TX;

        if (!use_chanctx || ops->remain_on_channel)
                wiphy->flags |= WIPHY_FLAG_HAS_REMAIN_ON_CHANNEL;

        wiphy->features |= NL80211_FEATURE_SK_TX_STATUS |
                           NL80211_FEATURE_SAE |
                           NL80211_FEATURE_HT_IBSS |
                           NL80211_FEATURE_VIF_TXPOWER |
                           NL80211_FEATURE_MAC_ON_CREATE |
                           NL80211_FEATURE_USERSPACE_MPM |
                           NL80211_FEATURE_FULL_AP_CLIENT_STATE;
        wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_FILS_STA);
        wiphy_ext_feature_set(wiphy,
                              NL80211_EXT_FEATURE_CONTROL_PORT_OVER_NL80211);
        wiphy_ext_feature_set(wiphy,
                              NL80211_EXT_FEATURE_CONTROL_PORT_NO_PREAUTH);
        wiphy_ext_feature_set(wiphy,
                              NL80211_EXT_FEATURE_CONTROL_PORT_OVER_NL80211_TX_STATUS);
        wiphy_ext_feature_set(wiphy,
                              NL80211_EXT_FEATURE_SCAN_FREQ_KHZ);

        if (!ops->hw_scan) {
                wiphy->features |= NL80211_FEATURE_LOW_PRIORITY_SCAN |
                                   NL80211_FEATURE_AP_SCAN;
                /*
                 * if the driver behaves correctly using the probe request
                 * (template) from mac80211, then both of these should be
                 * supported even with hw scan - but let drivers opt in.
                 */
                wiphy_ext_feature_set(wiphy,
                                      NL80211_EXT_FEATURE_SCAN_RANDOM_SN);
                wiphy_ext_feature_set(wiphy,
                                      NL80211_EXT_FEATURE_SCAN_MIN_PREQ_CONTENT);
        }

        if (!ops->set_key)
                wiphy->flags |= WIPHY_FLAG_IBSS_RSN;

        if (ops->wake_tx_queue)
                wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_TXQS);

        wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_RRM);

        wiphy->bss_priv_size = sizeof(struct ieee80211_bss);

        local = wiphy_priv(wiphy);

        if (sta_info_init(local))
                goto err_free;

        local->hw.wiphy = wiphy;

        local->hw.priv = (char *)local + ALIGN(sizeof(*local), NETDEV_ALIGN);

        local->ops = ops;
        local->use_chanctx = use_chanctx;

        /*
         * We need a bit of data queued to build aggregates properly, so
         * instruct the TCP stack to allow more than a single ms of data
         * to be queued in the stack. The value is a bit-shift of 1
         * second, so 7 is ~8ms of queued data. Only affects local TCP
         * sockets.
         * This is the default, anyhow - drivers may need to override it
         * for local reasons (longer buffers, longer completion time, or
         * similar).
         */
        local->hw.tx_sk_pacing_shift = 7;

        /* set up some defaults */
        local->hw.queues = 1;
        local->hw.max_rates = 1;
        local->hw.max_report_rates = 0;
        local->hw.max_rx_aggregation_subframes = IEEE80211_MAX_AMPDU_BUF_HT;
        local->hw.max_tx_aggregation_subframes = IEEE80211_MAX_AMPDU_BUF_HT;
        local->hw.offchannel_tx_hw_queue = IEEE80211_INVAL_HW_QUEUE;
        local->hw.conf.long_frame_max_tx_count = wiphy->retry_long;
        local->hw.conf.short_frame_max_tx_count = wiphy->retry_short;
        local->hw.radiotap_mcs_details = IEEE80211_RADIOTAP_MCS_HAVE_MCS |
                                         IEEE80211_RADIOTAP_MCS_HAVE_GI |
                                         IEEE80211_RADIOTAP_MCS_HAVE_BW;
        local->hw.radiotap_vht_details = IEEE80211_RADIOTAP_VHT_KNOWN_GI |
                                         IEEE80211_RADIOTAP_VHT_KNOWN_BANDWIDTH;
        local->hw.uapsd_queues = IEEE80211_DEFAULT_UAPSD_QUEUES;
        local->hw.uapsd_max_sp_len = IEEE80211_DEFAULT_MAX_SP_LEN;
        local->hw.max_mtu = IEEE80211_MAX_DATA_LEN;
        local->user_power_level = IEEE80211_UNSET_POWER_LEVEL;
        wiphy->ht_capa_mod_mask = &mac80211_ht_capa_mod_mask;
        wiphy->vht_capa_mod_mask = &mac80211_vht_capa_mod_mask;

        local->ext_capa[7] = WLAN_EXT_CAPA8_OPMODE_NOTIF;

        wiphy->extended_capabilities = local->ext_capa;
        wiphy->extended_capabilities_mask = local->ext_capa;
        wiphy->extended_capabilities_len =
                ARRAY_SIZE(local->ext_capa);

        INIT_LIST_HEAD(&local->interfaces);
        INIT_LIST_HEAD(&local->mon_list);

        __hw_addr_init(&local->mc_list);

        mutex_init(&local->iflist_mtx);
        mutex_init(&local->mtx);

        mutex_init(&local->key_mtx);
        spin_lock_init(&local->filter_lock);
        spin_lock_init(&local->rx_path_lock);
        spin_lock_init(&local->queue_stop_reason_lock);

        for (i = 0; i < IEEE80211_NUM_ACS; i++) {
                struct airtime_sched_info *air_sched = &local->airtime[i];

                air_sched->active_txqs = RB_ROOT_CACHED;
                INIT_LIST_HEAD(&air_sched->active_list);
                spin_lock_init(&air_sched->lock);
                air_sched->aql_txq_limit_low = IEEE80211_DEFAULT_AQL_TXQ_LIMIT_L;
                air_sched->aql_txq_limit_high =
                        IEEE80211_DEFAULT_AQL_TXQ_LIMIT_H;
        }

        local->airtime_flags = AIRTIME_USE_TX | AIRTIME_USE_RX;
        local->aql_threshold = IEEE80211_AQL_THRESHOLD;
        atomic_set(&local->aql_total_pending_airtime, 0);

        INIT_LIST_HEAD(&local->chanctx_list);
        mutex_init(&local->chanctx_mtx);

        INIT_DELAYED_WORK(&local->scan_work, ieee80211_scan_work);

        INIT_WORK(&local->restart_work, ieee80211_restart_work);

        INIT_WORK(&local->radar_detected_work,
                  ieee80211_dfs_radar_detected_work);

        INIT_WORK(&local->reconfig_filter, ieee80211_reconfig_filter);
        local->smps_mode = IEEE80211_SMPS_OFF;

        INIT_WORK(&local->dynamic_ps_enable_work,
                  ieee80211_dynamic_ps_enable_work);
        INIT_WORK(&local->dynamic_ps_disable_work,
                  ieee80211_dynamic_ps_disable_work);
        timer_setup(&local->dynamic_ps_timer, ieee80211_dynamic_ps_timer, 0);

        INIT_WORK(&local->sched_scan_stopped_work,
                  ieee80211_sched_scan_stopped_work);

        spin_lock_init(&local->ack_status_lock);
        idr_init(&local->ack_status_frames);

        for (i = 0; i < IEEE80211_MAX_QUEUES; i++) {
                skb_queue_head_init(&local->pending[i]);
                atomic_set(&local->agg_queue_stop[i], 0);
        }
        tasklet_setup(&local->tx_pending_tasklet, ieee80211_tx_pending);

        if (ops->wake_tx_queue)
                tasklet_setup(&local->wake_txqs_tasklet, ieee80211_wake_txqs);

        tasklet_setup(&local->tasklet, ieee80211_tasklet_handler);

        skb_queue_head_init(&local->skb_queue);
        skb_queue_head_init(&local->skb_queue_unreliable);

        ieee80211_alloc_led_names(local);

        ieee80211_roc_setup(local);

        local->hw.radiotap_timestamp.units_pos = -1;
        local->hw.radiotap_timestamp.accuracy = -1;

        return &local->hw;
 err_free:
        wiphy_free(wiphy);
        return NULL;
}
EXPORT_SYMBOL(ieee80211_alloc_hw_nm);

static int ieee80211_init_cipher_suites(struct ieee80211_local *local)
{
        bool have_wep = !fips_enabled; /* FIPS does not permit the use of RC4 */
        bool have_mfp = ieee80211_hw_check(&local->hw, MFP_CAPABLE);
        int n_suites = 0, r = 0, w = 0;
        u32 *suites;
        static const u32 cipher_suites[] = {
                /* keep WEP first, it may be removed below */
                WLAN_CIPHER_SUITE_WEP40,
                WLAN_CIPHER_SUITE_WEP104,
                WLAN_CIPHER_SUITE_TKIP,
                WLAN_CIPHER_SUITE_CCMP,
                WLAN_CIPHER_SUITE_CCMP_256,
                WLAN_CIPHER_SUITE_GCMP,
                WLAN_CIPHER_SUITE_GCMP_256,

                /* keep last -- depends on hw flags! */
                WLAN_CIPHER_SUITE_AES_CMAC,
                WLAN_CIPHER_SUITE_BIP_CMAC_256,
                WLAN_CIPHER_SUITE_BIP_GMAC_128,
                WLAN_CIPHER_SUITE_BIP_GMAC_256,
        };

        if (ieee80211_hw_check(&local->hw, SW_CRYPTO_CONTROL) ||
            local->hw.wiphy->cipher_suites) {
                /* If the driver advertises, or doesn't support SW crypto,
                 * we only need to remove WEP if necessary.
                 */
                if (have_wep)
                        return 0;

                /* well if it has _no_ ciphers ... fine */
                if (!local->hw.wiphy->n_cipher_suites)
                        return 0;

                /* Driver provides cipher suites, but we need to exclude WEP */
                suites = kmemdup(local->hw.wiphy->cipher_suites,
                                 sizeof(u32) * local->hw.wiphy->n_cipher_suites,
                                 GFP_KERNEL);
                if (!suites)
                        return -ENOMEM;

                for (r = 0; r < local->hw.wiphy->n_cipher_suites; r++) {
                        u32 suite = local->hw.wiphy->cipher_suites[r];

                        if (suite == WLAN_CIPHER_SUITE_WEP40 ||
                            suite == WLAN_CIPHER_SUITE_WEP104)
                                continue;
                        suites[w++] = suite;
                }
        } else if (!local->hw.cipher_schemes) {
                /* If the driver doesn't have cipher schemes, there's nothing
                 * else to do other than assign the (software supported and
                 * perhaps offloaded) cipher suites.
                 */
                local->hw.wiphy->cipher_suites = cipher_suites;
                local->hw.wiphy->n_cipher_suites = ARRAY_SIZE(cipher_suites);

                if (!have_mfp)
                        local->hw.wiphy->n_cipher_suites -= 4;

                if (!have_wep) {
                        local->hw.wiphy->cipher_suites += 2;
                        local->hw.wiphy->n_cipher_suites -= 2;
                }

                /* not dynamically allocated, so just return */
                return 0;
        } else {
                const struct ieee80211_cipher_scheme *cs;

                cs = local->hw.cipher_schemes;

                /* Driver specifies cipher schemes only (but not cipher suites
                 * including the schemes)
                 *
                 * We start counting ciphers defined by schemes, TKIP, CCMP,
                 * CCMP-256, GCMP, and GCMP-256
                 */
                n_suites = local->hw.n_cipher_schemes + 5;

                /* check if we have WEP40 and WEP104 */
                if (have_wep)
                        n_suites += 2;

                /* check if we have AES_CMAC, BIP-CMAC-256, BIP-GMAC-128,
                 * BIP-GMAC-256
                 */
                if (have_mfp)
                        n_suites += 4;

                suites = kmalloc_array(n_suites, sizeof(u32), GFP_KERNEL);
                if (!suites)
                        return -ENOMEM;

                suites[w++] = WLAN_CIPHER_SUITE_CCMP;
                suites[w++] = WLAN_CIPHER_SUITE_CCMP_256;
                suites[w++] = WLAN_CIPHER_SUITE_TKIP;
                suites[w++] = WLAN_CIPHER_SUITE_GCMP;
                suites[w++] = WLAN_CIPHER_SUITE_GCMP_256;

                if (have_wep) {
                        suites[w++] = WLAN_CIPHER_SUITE_WEP40;
                        suites[w++] = WLAN_CIPHER_SUITE_WEP104;
                }

                if (have_mfp) {
                        suites[w++] = WLAN_CIPHER_SUITE_AES_CMAC;
                        suites[w++] = WLAN_CIPHER_SUITE_BIP_CMAC_256;
                        suites[w++] = WLAN_CIPHER_SUITE_BIP_GMAC_128;
                        suites[w++] = WLAN_CIPHER_SUITE_BIP_GMAC_256;
                }

                for (r = 0; r < local->hw.n_cipher_schemes; r++) {
                        suites[w++] = cs[r].cipher;
                        if (WARN_ON(cs[r].pn_len > IEEE80211_MAX_PN_LEN)) {
                                kfree(suites);
                                return -EINVAL;
                        }
                }
        }

        local->hw.wiphy->cipher_suites = suites;
        local->hw.wiphy->n_cipher_suites = w;
        local->wiphy_ciphers_allocated = true;

        return 0;
}

int ieee80211_register_hw(struct ieee80211_hw *hw)
{
        struct ieee80211_local *local = hw_to_local(hw);
        int result, i;
        enum nl80211_band band;
        int channels, max_bitrates;
        bool supp_ht, supp_vht, supp_he;
        struct cfg80211_chan_def dflt_chandef = {};

        if (ieee80211_hw_check(hw, QUEUE_CONTROL) &&
            (local->hw.offchannel_tx_hw_queue == IEEE80211_INVAL_HW_QUEUE ||
             local->hw.offchannel_tx_hw_queue >= local->hw.queues))
                return -EINVAL;

        if ((hw->wiphy->features & NL80211_FEATURE_TDLS_CHANNEL_SWITCH) &&
            (!local->ops->tdls_channel_switch ||
             !local->ops->tdls_cancel_channel_switch ||
             !local->ops->tdls_recv_channel_switch))
                return -EOPNOTSUPP;

        if (WARN_ON(ieee80211_hw_check(hw, SUPPORTS_TX_FRAG) &&
                    !local->ops->set_frag_threshold))
                return -EINVAL;

        if (WARN_ON(local->hw.wiphy->interface_modes &
                        BIT(NL80211_IFTYPE_NAN) &&
                    (!local->ops->start_nan || !local->ops->stop_nan)))
                return -EINVAL;

#ifdef CONFIG_PM
        if (hw->wiphy->wowlan && (!local->ops->suspend || !local->ops->resume))
                return -EINVAL;
#endif

        if (!local->use_chanctx) {
                for (i = 0; i < local->hw.wiphy->n_iface_combinations; i++) {
                        const struct ieee80211_iface_combination *comb;

                        comb = &local->hw.wiphy->iface_combinations[i];

                        if (comb->num_different_channels > 1)
                                return -EINVAL;
                }
        } else {
                /* DFS is not supported with multi-channel combinations yet */
                for (i = 0; i < local->hw.wiphy->n_iface_combinations; i++) {
                        const struct ieee80211_iface_combination *comb;

                        comb = &local->hw.wiphy->iface_combinations[i];

                        if (comb->radar_detect_widths &&
                            comb->num_different_channels > 1)
                                return -EINVAL;
                }
        }

        /* Only HW csum features are currently compatible with mac80211 */
        if (WARN_ON(hw->netdev_features & ~MAC80211_SUPPORTED_FEATURES))
                return -EINVAL;

        if (hw->max_report_rates == 0)
                hw->max_report_rates = hw->max_rates;

        local->rx_chains = 1;

        /*
         * generic code guarantees at least one band,
         * set this very early because much code assumes
         * that hw.conf.channel is assigned
         */
        channels = 0;
        max_bitrates = 0;
        supp_ht = false;
        supp_vht = false;
        supp_he = false;
        for (band = 0; band < NUM_NL80211_BANDS; band++) {
                struct ieee80211_supported_band *sband;

                sband = local->hw.wiphy->bands[band];
                if (!sband)
                        continue;

                if (!dflt_chandef.chan) {
                        /*
                         * Assign the first enabled channel to dflt_chandef
                         * from the list of channels
                         */
                        for (i = 0; i < sband->n_channels; i++)
                                if (!(sband->channels[i].flags &
                                                IEEE80211_CHAN_DISABLED))
                                        break;
                        /* if none found then use the first anyway */
                        if (i == sband->n_channels)
                                i = 0;
                        cfg80211_chandef_create(&dflt_chandef,
                                                &sband->channels[i],
                                                NL80211_CHAN_NO_HT);
                        /* init channel we're on */
                        if (!local->use_chanctx && !local->_oper_chandef.chan) {
                                local->hw.conf.chandef = dflt_chandef;
                                local->_oper_chandef = dflt_chandef;
                        }
                        local->monitor_chandef = dflt_chandef;
                }

                channels += sband->n_channels;

                if (max_bitrates < sband->n_bitrates)
                        max_bitrates = sband->n_bitrates;
                supp_ht = supp_ht || sband->ht_cap.ht_supported;
                supp_vht = supp_vht || sband->vht_cap.vht_supported;

                for (i = 0; i < sband->n_iftype_data; i++) {
                        const struct ieee80211_sband_iftype_data *iftd;

                        iftd = &sband->iftype_data[i];

                        supp_he = supp_he || iftd->he_cap.has_he;
                }

                /* HT, VHT, HE require QoS, thus >= 4 queues */
                if (WARN_ON(local->hw.queues < IEEE80211_NUM_ACS &&
                            (supp_ht || supp_vht || supp_he)))
                        return -EINVAL;

                if (!sband->ht_cap.ht_supported)
                        continue;

                /* TODO: consider VHT for RX chains, hopefully it's the same */
                local->rx_chains =
                        max(ieee80211_mcs_to_chains(&sband->ht_cap.mcs),
                            local->rx_chains);

                /* no need to mask, SM_PS_DISABLED has all bits set */
                sband->ht_cap.cap |= WLAN_HT_CAP_SM_PS_DISABLED <<
                                     IEEE80211_HT_CAP_SM_PS_SHIFT;
        }

        /* if low-level driver supports AP, we also support VLAN.
         * drivers advertising SW_CRYPTO_CONTROL should enable AP_VLAN
         * based on their support to transmit SW encrypted packets.
         */
        if (local->hw.wiphy->interface_modes & BIT(NL80211_IFTYPE_AP) &&
            !ieee80211_hw_check(&local->hw, SW_CRYPTO_CONTROL)) {
                hw->wiphy->interface_modes |= BIT(NL80211_IFTYPE_AP_VLAN);
                hw->wiphy->software_iftypes |= BIT(NL80211_IFTYPE_AP_VLAN);
        }

        /* mac80211 always supports monitor */
        hw->wiphy->interface_modes |= BIT(NL80211_IFTYPE_MONITOR);
        hw->wiphy->software_iftypes |= BIT(NL80211_IFTYPE_MONITOR);

        /* mac80211 doesn't support more than one IBSS interface right now */
        for (i = 0; i < hw->wiphy->n_iface_combinations; i++) {
                const struct ieee80211_iface_combination *c;
                int j;

                c = &hw->wiphy->iface_combinations[i];

                for (j = 0; j < c->n_limits; j++)
                        if ((c->limits[j].types & BIT(NL80211_IFTYPE_ADHOC)) &&
                            c->limits[j].max > 1)
                                return -EINVAL;
        }

        local->int_scan_req = kzalloc(sizeof(*local->int_scan_req) +
                                      sizeof(void *) * channels, GFP_KERNEL);
        if (!local->int_scan_req)
                return -ENOMEM;

        for (band = 0; band < NUM_NL80211_BANDS; band++) {
                if (!local->hw.wiphy->bands[band])
                        continue;
                local->int_scan_req->rates[band] = (u32) -1;
        }

#ifndef CONFIG_MAC80211_MESH
        /* mesh depends on Kconfig, but drivers should set it if they want */
        local->hw.wiphy->interface_modes &= ~BIT(NL80211_IFTYPE_MESH_POINT);
#endif

        /* if the underlying driver supports mesh, mac80211 will (at least)
         * provide routing of mesh authentication frames to userspace */
        if (local->hw.wiphy->interface_modes & BIT(NL80211_IFTYPE_MESH_POINT))
                local->hw.wiphy->flags |= WIPHY_FLAG_MESH_AUTH;

        /* mac80211 supports control port protocol changing */
        local->hw.wiphy->flags |= WIPHY_FLAG_CONTROL_PORT_PROTOCOL;

        if (ieee80211_hw_check(&local->hw, SIGNAL_DBM)) {
                local->hw.wiphy->signal_type = CFG80211_SIGNAL_TYPE_MBM;
        } else if (ieee80211_hw_check(&local->hw, SIGNAL_UNSPEC)) {
                local->hw.wiphy->signal_type = CFG80211_SIGNAL_TYPE_UNSPEC;
                if (hw->max_signal <= 0) {
                        result = -EINVAL;
                        goto fail_workqueue;
                }
        }

        /* Mac80211 and therefore all drivers using SW crypto only
         * are able to handle PTK rekeys and Extended Key ID.
         */
        if (!local->ops->set_key) {
                wiphy_ext_feature_set(local->hw.wiphy,
                                      NL80211_EXT_FEATURE_CAN_REPLACE_PTK0);
                wiphy_ext_feature_set(local->hw.wiphy,
                                      NL80211_EXT_FEATURE_EXT_KEY_ID);
        }

        if (local->hw.wiphy->interface_modes & BIT(NL80211_IFTYPE_ADHOC))
                wiphy_ext_feature_set(local->hw.wiphy,
                                      NL80211_EXT_FEATURE_DEL_IBSS_STA);

        /*
         * Calculate scan IE length -- we need this to alloc
         * memory and to subtract from the driver limit. It
         * includes the DS Params, (extended) supported rates, and HT
         * information -- SSID is the driver's responsibility.
         */
        local->scan_ies_len = 4 + max_bitrates /* (ext) supp rates */ +
                3 /* DS Params */;
        if (supp_ht)
                local->scan_ies_len += 2 + sizeof(struct ieee80211_ht_cap);

        if (supp_vht)
                local->scan_ies_len +=
                        2 + sizeof(struct ieee80211_vht_cap);

        /*
         * HE cap element is variable in size - set len to allow max size */
        if (supp_he) {
                local->scan_ies_len +=
                        3 + sizeof(struct ieee80211_he_cap_elem) +
                        sizeof(struct ieee80211_he_mcs_nss_supp) +
                        IEEE80211_HE_PPE_THRES_MAX_LEN;
        }

        if (!local->ops->hw_scan) {
                /* For hw_scan, driver needs to set these up. */
                local->hw.wiphy->max_scan_ssids = 4;
                local->hw.wiphy->max_scan_ie_len = IEEE80211_MAX_DATA_LEN;
        }

        /*
         * If the driver supports any scan IEs, then assume the
         * limit includes the IEs mac80211 will add, otherwise
         * leave it at zero and let the driver sort it out; we
         * still pass our IEs to the driver but userspace will
         * not be allowed to in that case.
         */
        if (local->hw.wiphy->max_scan_ie_len)
                local->hw.wiphy->max_scan_ie_len -= local->scan_ies_len;

        if (WARN_ON(!ieee80211_cs_list_valid(local->hw.cipher_schemes,
                                             local->hw.n_cipher_schemes))) {
                result = -EINVAL;
                goto fail_workqueue;
        }

        result = ieee80211_init_cipher_suites(local);
        if (result < 0)
                goto fail_workqueue;

        if (!local->ops->remain_on_channel)
                local->hw.wiphy->max_remain_on_channel_duration = 5000;

        /* mac80211 based drivers don't support internal TDLS setup */
        if (local->hw.wiphy->flags & WIPHY_FLAG_SUPPORTS_TDLS)
                local->hw.wiphy->flags |= WIPHY_FLAG_TDLS_EXTERNAL_SETUP;

        /* mac80211 supports eCSA, if the driver supports STA CSA at all */
        if (ieee80211_hw_check(&local->hw, CHANCTX_STA_CSA))
                local->ext_capa[0] |= WLAN_EXT_CAPA1_EXT_CHANNEL_SWITCHING;

        /* mac80211 supports multi BSSID, if the driver supports it */
        if (ieee80211_hw_check(&local->hw, SUPPORTS_MULTI_BSSID)) {
                local->hw.wiphy->support_mbssid = true;
                if (ieee80211_hw_check(&local->hw,
                                       SUPPORTS_ONLY_HE_MULTI_BSSID))
                        local->hw.wiphy->support_only_he_mbssid = true;
                else
                        local->ext_capa[2] |=
                                WLAN_EXT_CAPA3_MULTI_BSSID_SUPPORT;
        }

        local->hw.wiphy->max_num_csa_counters = IEEE80211_MAX_CNTDWN_COUNTERS_NUM;

        /*
         * We use the number of queues for feature tests (QoS, HT) internally
         * so restrict them appropriately.
         */
        if (hw->queues > IEEE80211_MAX_QUEUES)
                hw->queues = IEEE80211_MAX_QUEUES;

        local->workqueue =
                alloc_ordered_workqueue("%s", 0, wiphy_name(local->hw.wiphy));
        if (!local->workqueue) {
                result = -ENOMEM;
                goto fail_workqueue;
        }

        /*
         * The hardware needs headroom for sending the frame,
         * and we need some headroom for passing the frame to monitor
         * interfaces, but never both at the same time.
         */
        local->tx_headroom = max_t(unsigned int , local->hw.extra_tx_headroom,
                                   IEEE80211_TX_STATUS_HEADROOM);

        /*
         * if the driver doesn't specify a max listen interval we
         * use 5 which should be a safe default
         */
        if (local->hw.max_listen_interval == 0)
                local->hw.max_listen_interval = 5;

        local->hw.conf.listen_interval = local->hw.max_listen_interval;

        local->dynamic_ps_forced_timeout = -1;

        if (!local->hw.max_nan_de_entries)
                local->hw.max_nan_de_entries = IEEE80211_MAX_NAN_INSTANCE_ID;

        if (!local->hw.weight_multiplier)
                local->hw.weight_multiplier = 1;

        ieee80211_wep_init(local);

        local->hw.conf.flags = IEEE80211_CONF_IDLE;

        ieee80211_led_init(local);

        result = ieee80211_txq_setup_flows(local);
        if (result)
                goto fail_flows;

        rtnl_lock();
        result = ieee80211_init_rate_ctrl_alg(local,
                                              hw->rate_control_algorithm);
        rtnl_unlock();
        if (result < 0) {
                wiphy_debug(local->hw.wiphy,
                            "Failed to initialize rate control algorithm\n");
                goto fail_rate;
        }

        if (local->rate_ctrl) {
                clear_bit(IEEE80211_HW_SUPPORTS_VHT_EXT_NSS_BW, hw->flags);
                if (local->rate_ctrl->ops->capa & RATE_CTRL_CAPA_VHT_EXT_NSS_BW)
                        ieee80211_hw_set(hw, SUPPORTS_VHT_EXT_NSS_BW);
        }

        /*
         * If the VHT capabilities don't have IEEE80211_VHT_EXT_NSS_BW_CAPABLE,
         * or have it when we don't, copy the sband structure and set/clear it.
         * This is necessary because rate scaling algorithms could be switched
         * and have different support values.
         * Print a message so that in the common case the reallocation can be
         * avoided.
         */
        BUILD_BUG_ON(NUM_NL80211_BANDS > 8 * sizeof(local->sband_allocated));
        for (band = 0; band < NUM_NL80211_BANDS; band++) {
                struct ieee80211_supported_band *sband;
                bool local_cap, ie_cap;

                local_cap = ieee80211_hw_check(hw, SUPPORTS_VHT_EXT_NSS_BW);

                sband = local->hw.wiphy->bands[band];
                if (!sband || !sband->vht_cap.vht_supported)
                        continue;

                ie_cap = !!(sband->vht_cap.vht_mcs.tx_highest &
                            cpu_to_le16(IEEE80211_VHT_EXT_NSS_BW_CAPABLE));

                if (local_cap == ie_cap)
                        continue;

                sband = kmemdup(sband, sizeof(*sband), GFP_KERNEL);
                if (!sband) {
                        result = -ENOMEM;
                        goto fail_rate;
                }

                wiphy_dbg(hw->wiphy, "copying sband (band %d) due to VHT EXT NSS BW flag\n",
                          band);

                sband->vht_cap.vht_mcs.tx_highest ^=
                        cpu_to_le16(IEEE80211_VHT_EXT_NSS_BW_CAPABLE);

                local->hw.wiphy->bands[band] = sband;
                local->sband_allocated |= BIT(band);
        }

        result = wiphy_register(local->hw.wiphy);
        if (result < 0)
                goto fail_wiphy_register;

        debugfs_hw_add(local);
        rate_control_add_debugfs(local);

        rtnl_lock();
        wiphy_lock(hw->wiphy);

        /* add one default STA interface if supported */
        if (local->hw.wiphy->interface_modes & BIT(NL80211_IFTYPE_STATION) &&
            !ieee80211_hw_check(hw, NO_AUTO_VIF)) {
                struct vif_params params = {0};

                result = ieee80211_if_add(local, "wlan%d", NET_NAME_ENUM, NULL,
                                          NL80211_IFTYPE_STATION, &params);
                if (result)
                        wiphy_warn(local->hw.wiphy,
                                   "Failed to add default virtual iface\n");
        }

        wiphy_unlock(hw->wiphy);
        rtnl_unlock();

#ifdef CONFIG_INET
        local->ifa_notifier.notifier_call = ieee80211_ifa_changed;
        result = register_inetaddr_notifier(&local->ifa_notifier);
        if (result)
                goto fail_ifa;
#endif

#if IS_ENABLED(CONFIG_IPV6)
        local->ifa6_notifier.notifier_call = ieee80211_ifa6_changed;
        result = register_inet6addr_notifier(&local->ifa6_notifier);
        if (result)
                goto fail_ifa6;
#endif

        return 0;

#if IS_ENABLED(CONFIG_IPV6)
 fail_ifa6:
#ifdef CONFIG_INET
        unregister_inetaddr_notifier(&local->ifa_notifier);
#endif
#endif
#if defined(CONFIG_INET) || defined(CONFIG_IPV6)
 fail_ifa:
#endif
        wiphy_unregister(local->hw.wiphy);
 fail_wiphy_register:
        rtnl_lock();
        rate_control_deinitialize(local);
        ieee80211_remove_interfaces(local);
        rtnl_unlock();
 fail_rate:
 fail_flows:
        ieee80211_led_exit(local);
        destroy_workqueue(local->workqueue);
 fail_workqueue:
        if (local->wiphy_ciphers_allocated) {
                kfree(local->hw.wiphy->cipher_suites);
                local->wiphy_ciphers_allocated = false;
        }
        kfree(local->int_scan_req);
        return result;
}
EXPORT_SYMBOL(ieee80211_register_hw);

void ieee80211_unregister_hw(struct ieee80211_hw *hw)
{
        struct ieee80211_local *local = hw_to_local(hw);

        tasklet_kill(&local->tx_pending_tasklet);
        tasklet_kill(&local->tasklet);

#ifdef CONFIG_INET
        unregister_inetaddr_notifier(&local->ifa_notifier);
#endif
#if IS_ENABLED(CONFIG_IPV6)
        unregister_inet6addr_notifier(&local->ifa6_notifier);
#endif

        rtnl_lock();

        /*
         * At this point, interface list manipulations are fine
         * because the driver cannot be handing us frames any
         * more and the tasklet is killed.
         */
        ieee80211_remove_interfaces(local);

        rtnl_unlock();

        cancel_delayed_work_sync(&local->roc_work);
        cancel_work_sync(&local->restart_work);
        cancel_work_sync(&local->reconfig_filter);
        flush_work(&local->sched_scan_stopped_work);
        flush_work(&local->radar_detected_work);

        ieee80211_clear_tx_pending(local);
        rate_control_deinitialize(local);

        if (skb_queue_len(&local->skb_queue) ||
            skb_queue_len(&local->skb_queue_unreliable))
                wiphy_warn(local->hw.wiphy, "skb_queue not empty\n");
        skb_queue_purge(&local->skb_queue);
        skb_queue_purge(&local->skb_queue_unreliable);

        wiphy_unregister(local->hw.wiphy);
        destroy_workqueue(local->workqueue);
        ieee80211_led_exit(local);
        kfree(local->int_scan_req);
}
EXPORT_SYMBOL(ieee80211_unregister_hw);

static int ieee80211_free_ack_frame(int id, void *p, void *data)
{
        WARN_ONCE(1, "Have pending ack frames!\n");
        kfree_skb(p);
        return 0;
}

void ieee80211_free_hw(struct ieee80211_hw *hw)
{
        struct ieee80211_local *local = hw_to_local(hw);
        enum nl80211_band band;

        mutex_destroy(&local->iflist_mtx);
        mutex_destroy(&local->mtx);

        if (local->wiphy_ciphers_allocated) {
                kfree(local->hw.wiphy->cipher_suites);
                local->wiphy_ciphers_allocated = false;
        }

        idr_for_each(&local->ack_status_frames,
                     ieee80211_free_ack_frame, NULL);
        idr_destroy(&local->ack_status_frames);

        sta_info_stop(local);

        ieee80211_free_led_names(local);

        for (band = 0; band < NUM_NL80211_BANDS; band++) {
                if (!(local->sband_allocated & BIT(band)))
                        continue;
                kfree(local->hw.wiphy->bands[band]);
        }

        wiphy_free(local->hw.wiphy);
}
EXPORT_SYMBOL(ieee80211_free_hw);

static int __init ieee80211_init(void)
{
        struct sk_buff *skb;
        int ret;

        BUILD_BUG_ON(sizeof(struct ieee80211_tx_info) > sizeof(skb->cb));
        BUILD_BUG_ON(offsetof(struct ieee80211_tx_info, driver_data) +
                     IEEE80211_TX_INFO_DRIVER_DATA_SIZE > sizeof(skb->cb));

        ret = rc80211_minstrel_init();
        if (ret)
                return ret;

        ret = ieee80211_iface_init();
        if (ret)
                goto err_netdev;

        return 0;
 err_netdev:
        rc80211_minstrel_exit();

        return ret;
}

static void __exit ieee80211_exit(void)
{
        rc80211_minstrel_exit();

        ieee80211s_stop();

        ieee80211_iface_exit();

        rcu_barrier();
}


subsys_initcall(ieee80211_init);
module_exit(ieee80211_exit);

MODULE_DESCRIPTION("IEEE 802.11 subsystem");
MODULE_LICENSE("GPL");














































   49 






   49 





























































































































































































































































































































































































































   49 


























   49 





































   49 







   49 





























1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
// SPDX-License-Identifier: GPL-2.0-only
#include <linux/kernel.h>
#include <linux/init.h>
#include <linux/module.h>
#include <linux/proc_fs.h>
#include <linux/skbuff.h>
#include <linux/netfilter.h>
#include <linux/seq_file.h>
#include <net/protocol.h>
#include <net/netfilter/nf_log.h>

#include "nf_internals.h"

/* Internal logging interface, which relies on the real
   LOG target modules */

#define NFLOGGER_NAME_LEN                64

int sysctl_nf_log_all_netns __read_mostly;
EXPORT_SYMBOL(sysctl_nf_log_all_netns);

static struct nf_logger __rcu *loggers[NFPROTO_NUMPROTO][NF_LOG_TYPE_MAX] __read_mostly;
static DEFINE_MUTEX(nf_log_mutex);

#define nft_log_dereference(logger) \
        rcu_dereference_protected(logger, lockdep_is_held(&nf_log_mutex))

static struct nf_logger *__find_logger(int pf, const char *str_logger)
{
        struct nf_logger *log;
        int i;

        for (i = 0; i < NF_LOG_TYPE_MAX; i++) {
                if (loggers[pf][i] == NULL)
                        continue;

                log = nft_log_dereference(loggers[pf][i]);
                if (!strncasecmp(str_logger, log->name, strlen(log->name)))
                        return log;
        }

        return NULL;
}

int nf_log_set(struct net *net, u_int8_t pf, const struct nf_logger *logger)
{
        const struct nf_logger *log;

        if (pf == NFPROTO_UNSPEC || pf >= ARRAY_SIZE(net->nf.nf_loggers))
                return -EOPNOTSUPP;

        mutex_lock(&nf_log_mutex);
        log = nft_log_dereference(net->nf.nf_loggers[pf]);
        if (log == NULL)
                rcu_assign_pointer(net->nf.nf_loggers[pf], logger);

        mutex_unlock(&nf_log_mutex);

        return 0;
}
EXPORT_SYMBOL(nf_log_set);

void nf_log_unset(struct net *net, const struct nf_logger *logger)
{
        int i;
        const struct nf_logger *log;

        mutex_lock(&nf_log_mutex);
        for (i = 0; i < NFPROTO_NUMPROTO; i++) {
                log = nft_log_dereference(net->nf.nf_loggers[i]);
                if (log == logger)
                        RCU_INIT_POINTER(net->nf.nf_loggers[i], NULL);
        }
        mutex_unlock(&nf_log_mutex);
}
EXPORT_SYMBOL(nf_log_unset);

/* return EEXIST if the same logger is registered, 0 on success. */
int nf_log_register(u_int8_t pf, struct nf_logger *logger)
{
        int i;
        int ret = 0;

        if (pf >= ARRAY_SIZE(init_net.nf.nf_loggers))
                return -EINVAL;

        mutex_lock(&nf_log_mutex);

        if (pf == NFPROTO_UNSPEC) {
                for (i = NFPROTO_UNSPEC; i < NFPROTO_NUMPROTO; i++) {
                        if (rcu_access_pointer(loggers[i][logger->type])) {
                                ret = -EEXIST;
                                goto unlock;
                        }
                }
                for (i = NFPROTO_UNSPEC; i < NFPROTO_NUMPROTO; i++)
                        rcu_assign_pointer(loggers[i][logger->type], logger);
        } else {
                if (rcu_access_pointer(loggers[pf][logger->type])) {
                        ret = -EEXIST;
                        goto unlock;
                }
                rcu_assign_pointer(loggers[pf][logger->type], logger);
        }

unlock:
        mutex_unlock(&nf_log_mutex);
        return ret;
}
EXPORT_SYMBOL(nf_log_register);

void nf_log_unregister(struct nf_logger *logger)
{
        const struct nf_logger *log;
        int i;

        mutex_lock(&nf_log_mutex);
        for (i = 0; i < NFPROTO_NUMPROTO; i++) {
                log = nft_log_dereference(loggers[i][logger->type]);
                if (log == logger)
                        RCU_INIT_POINTER(loggers[i][logger->type], NULL);
        }
        mutex_unlock(&nf_log_mutex);
        synchronize_rcu();
}
EXPORT_SYMBOL(nf_log_unregister);

int nf_log_bind_pf(struct net *net, u_int8_t pf,
                   const struct nf_logger *logger)
{
        if (pf >= ARRAY_SIZE(net->nf.nf_loggers))
                return -EINVAL;
        mutex_lock(&nf_log_mutex);
        if (__find_logger(pf, logger->name) == NULL) {
                mutex_unlock(&nf_log_mutex);
                return -ENOENT;
        }
        rcu_assign_pointer(net->nf.nf_loggers[pf], logger);
        mutex_unlock(&nf_log_mutex);
        return 0;
}
EXPORT_SYMBOL(nf_log_bind_pf);

void nf_log_unbind_pf(struct net *net, u_int8_t pf)
{
        if (pf >= ARRAY_SIZE(net->nf.nf_loggers))
                return;
        mutex_lock(&nf_log_mutex);
        RCU_INIT_POINTER(net->nf.nf_loggers[pf], NULL);
        mutex_unlock(&nf_log_mutex);
}
EXPORT_SYMBOL(nf_log_unbind_pf);

int nf_logger_find_get(int pf, enum nf_log_type type)
{
        struct nf_logger *logger;
        int ret = -ENOENT;

        if (pf == NFPROTO_INET) {
                ret = nf_logger_find_get(NFPROTO_IPV4, type);
                if (ret < 0)
                        return ret;

                ret = nf_logger_find_get(NFPROTO_IPV6, type);
                if (ret < 0) {
                        nf_logger_put(NFPROTO_IPV4, type);
                        return ret;
                }

                return 0;
        }

        rcu_read_lock();
        logger = rcu_dereference(loggers[pf][type]);
        if (logger == NULL)
                goto out;

        if (try_module_get(logger->me))
                ret = 0;
out:
        rcu_read_unlock();
        return ret;
}
EXPORT_SYMBOL_GPL(nf_logger_find_get);

void nf_logger_put(int pf, enum nf_log_type type)
{
        struct nf_logger *logger;

        if (pf == NFPROTO_INET) {
                nf_logger_put(NFPROTO_IPV4, type);
                nf_logger_put(NFPROTO_IPV6, type);
                return;
        }

        rcu_read_lock();
        logger = rcu_dereference(loggers[pf][type]);
        if (!logger)
                WARN_ON_ONCE(1);
        else
                module_put(logger->me);
        rcu_read_unlock();
}
EXPORT_SYMBOL_GPL(nf_logger_put);

void nf_log_packet(struct net *net,
                   u_int8_t pf,
                   unsigned int hooknum,
                   const struct sk_buff *skb,
                   const struct net_device *in,
                   const struct net_device *out,
                   const struct nf_loginfo *loginfo,
                   const char *fmt, ...)
{
        va_list args;
        char prefix[NF_LOG_PREFIXLEN];
        const struct nf_logger *logger;

        rcu_read_lock();
        if (loginfo != NULL)
                logger = rcu_dereference(loggers[pf][loginfo->type]);
        else
                logger = rcu_dereference(net->nf.nf_loggers[pf]);

        if (logger) {
                va_start(args, fmt);
                vsnprintf(prefix, sizeof(prefix), fmt, args);
                va_end(args);
                logger->logfn(net, pf, hooknum, skb, in, out, loginfo, prefix);
        }
        rcu_read_unlock();
}
EXPORT_SYMBOL(nf_log_packet);

void nf_log_trace(struct net *net,
                  u_int8_t pf,
                  unsigned int hooknum,
                  const struct sk_buff *skb,
                  const struct net_device *in,
                  const struct net_device *out,
                  const struct nf_loginfo *loginfo, const char *fmt, ...)
{
        va_list args;
        char prefix[NF_LOG_PREFIXLEN];
        const struct nf_logger *logger;

        rcu_read_lock();
        logger = rcu_dereference(net->nf.nf_loggers[pf]);
        if (logger) {
                va_start(args, fmt);
                vsnprintf(prefix, sizeof(prefix), fmt, args);
                va_end(args);
                logger->logfn(net, pf, hooknum, skb, in, out, loginfo, prefix);
        }
        rcu_read_unlock();
}
EXPORT_SYMBOL(nf_log_trace);

#define S_SIZE (1024 - (sizeof(unsigned int) + 1))

struct nf_log_buf {
        unsigned int        count;
        char                buf[S_SIZE + 1];
};
static struct nf_log_buf emergency, *emergency_ptr = &emergency;

__printf(2, 3) int nf_log_buf_add(struct nf_log_buf *m, const char *f, ...)
{
        va_list args;
        int len;

        if (likely(m->count < S_SIZE)) {
                va_start(args, f);
                len = vsnprintf(m->buf + m->count, S_SIZE - m->count, f, args);
                va_end(args);
                if (likely(m->count + len < S_SIZE)) {
                        m->count += len;
                        return 0;
                }
        }
        m->count = S_SIZE;
        printk_once(KERN_ERR KBUILD_MODNAME " please increase S_SIZE\n");
        return -1;
}
EXPORT_SYMBOL_GPL(nf_log_buf_add);

struct nf_log_buf *nf_log_buf_open(void)
{
        struct nf_log_buf *m = kmalloc(sizeof(*m), GFP_ATOMIC);

        if (unlikely(!m)) {
                local_bh_disable();
                do {
                        m = xchg(&emergency_ptr, NULL);
                } while (!m);
        }
        m->count = 0;
        return m;
}
EXPORT_SYMBOL_GPL(nf_log_buf_open);

void nf_log_buf_close(struct nf_log_buf *m)
{
        m->buf[m->count] = 0;
        printk("%s\n", m->buf);

        if (likely(m != &emergency))
                kfree(m);
        else {
                emergency_ptr = m;
                local_bh_enable();
        }
}
EXPORT_SYMBOL_GPL(nf_log_buf_close);

#ifdef CONFIG_PROC_FS
static void *seq_start(struct seq_file *seq, loff_t *pos)
{
        struct net *net = seq_file_net(seq);

        mutex_lock(&nf_log_mutex);

        if (*pos >= ARRAY_SIZE(net->nf.nf_loggers))
                return NULL;

        return pos;
}

static void *seq_next(struct seq_file *s, void *v, loff_t *pos)
{
        struct net *net = seq_file_net(s);

        (*pos)++;

        if (*pos >= ARRAY_SIZE(net->nf.nf_loggers))
                return NULL;

        return pos;
}

static void seq_stop(struct seq_file *s, void *v)
{
        mutex_unlock(&nf_log_mutex);
}

static int seq_show(struct seq_file *s, void *v)
{
        loff_t *pos = v;
        const struct nf_logger *logger;
        int i;
        struct net *net = seq_file_net(s);

        logger = nft_log_dereference(net->nf.nf_loggers[*pos]);

        if (!logger)
                seq_printf(s, "%2lld NONE (", *pos);
        else
                seq_printf(s, "%2lld %s (", *pos, logger->name);

        if (seq_has_overflowed(s))
                return -ENOSPC;

        for (i = 0; i < NF_LOG_TYPE_MAX; i++) {
                if (loggers[*pos][i] == NULL)
                        continue;

                logger = nft_log_dereference(loggers[*pos][i]);
                seq_puts(s, logger->name);
                if (i == 0 && loggers[*pos][i + 1] != NULL)
                        seq_puts(s, ",");

                if (seq_has_overflowed(s))
                        return -ENOSPC;
        }

        seq_puts(s, ")\n");

        if (seq_has_overflowed(s))
                return -ENOSPC;
        return 0;
}

static const struct seq_operations nflog_seq_ops = {
        .start        = seq_start,
        .next        = seq_next,
        .stop        = seq_stop,
        .show        = seq_show,
};
#endif /* PROC_FS */

#ifdef CONFIG_SYSCTL
static char nf_log_sysctl_fnames[NFPROTO_NUMPROTO-NFPROTO_UNSPEC][3];
static struct ctl_table nf_log_sysctl_table[NFPROTO_NUMPROTO+1];
static struct ctl_table_header *nf_log_sysctl_fhdr;

static struct ctl_table nf_log_sysctl_ftable[] = {
        {
                .procname        = "nf_log_all_netns",
                .data                = &sysctl_nf_log_all_netns,
                .maxlen                = sizeof(sysctl_nf_log_all_netns),
                .mode                = 0644,
                .proc_handler        = proc_dointvec,
        },
        { }
};

static int nf_log_proc_dostring(struct ctl_table *table, int write,
                         void *buffer, size_t *lenp, loff_t *ppos)
{
        const struct nf_logger *logger;
        char buf[NFLOGGER_NAME_LEN];
        int r = 0;
        int tindex = (unsigned long)table->extra1;
        struct net *net = table->extra2;

        if (write) {
                struct ctl_table tmp = *table;

                /* proc_dostring() can append to existing strings, so we need to
                 * initialize it as an empty string.
                 */
                buf[0] = '\0';
                tmp.data = buf;
                r = proc_dostring(&tmp, write, buffer, lenp, ppos);
                if (r)
                        return r;

                if (!strcmp(buf, "NONE")) {
                        nf_log_unbind_pf(net, tindex);
                        return 0;
                }
                mutex_lock(&nf_log_mutex);
                logger = __find_logger(tindex, buf);
                if (logger == NULL) {
                        mutex_unlock(&nf_log_mutex);
                        return -ENOENT;
                }
                rcu_assign_pointer(net->nf.nf_loggers[tindex], logger);
                mutex_unlock(&nf_log_mutex);
        } else {
                struct ctl_table tmp = *table;

                tmp.data = buf;
                mutex_lock(&nf_log_mutex);
                logger = nft_log_dereference(net->nf.nf_loggers[tindex]);
                if (!logger)
                        strlcpy(buf, "NONE", sizeof(buf));
                else
                        strlcpy(buf, logger->name, sizeof(buf));
                mutex_unlock(&nf_log_mutex);
                r = proc_dostring(&tmp, write, buffer, lenp, ppos);
        }

        return r;
}

static int netfilter_log_sysctl_init(struct net *net)
{
        int i;
        struct ctl_table *table;

        table = nf_log_sysctl_table;
        if (!net_eq(net, &init_net)) {
                table = kmemdup(nf_log_sysctl_table,
                                 sizeof(nf_log_sysctl_table),
                                 GFP_KERNEL);
                if (!table)
                        goto err_alloc;
        } else {
                for (i = NFPROTO_UNSPEC; i < NFPROTO_NUMPROTO; i++) {
                        snprintf(nf_log_sysctl_fnames[i],
                                 3, "%d", i);
                        nf_log_sysctl_table[i].procname        =
                                nf_log_sysctl_fnames[i];
                        nf_log_sysctl_table[i].maxlen = NFLOGGER_NAME_LEN;
                        nf_log_sysctl_table[i].mode = 0644;
                        nf_log_sysctl_table[i].proc_handler =
                                nf_log_proc_dostring;
                        nf_log_sysctl_table[i].extra1 =
                                (void *)(unsigned long) i;
                }
                nf_log_sysctl_fhdr = register_net_sysctl(net, "net/netfilter",
                                                         nf_log_sysctl_ftable);
                if (!nf_log_sysctl_fhdr)
                        goto err_freg;
        }

        for (i = NFPROTO_UNSPEC; i < NFPROTO_NUMPROTO; i++)
                table[i].extra2 = net;

        net->nf.nf_log_dir_header = register_net_sysctl(net,
                                                "net/netfilter/nf_log",
                                                table);
        if (!net->nf.nf_log_dir_header)
                goto err_reg;

        return 0;

err_reg:
        if (!net_eq(net, &init_net))
                kfree(table);
        else
                unregister_net_sysctl_table(nf_log_sysctl_fhdr);
err_freg:
err_alloc:
        return -ENOMEM;
}

static void netfilter_log_sysctl_exit(struct net *net)
{
        struct ctl_table *table;

        table = net->nf.nf_log_dir_header->ctl_table_arg;
        unregister_net_sysctl_table(net->nf.nf_log_dir_header);
        if (!net_eq(net, &init_net))
                kfree(table);
        else
                unregister_net_sysctl_table(nf_log_sysctl_fhdr);
}
#else
static int netfilter_log_sysctl_init(struct net *net)
{
        return 0;
}

static void netfilter_log_sysctl_exit(struct net *net)
{
}
#endif /* CONFIG_SYSCTL */

static int __net_init nf_log_net_init(struct net *net)
{
        int ret = -ENOMEM;

#ifdef CONFIG_PROC_FS
        if (!proc_create_net("nf_log", 0444, net->nf.proc_netfilter,
                        &nflog_seq_ops, sizeof(struct seq_net_private)))
                return ret;
#endif
        ret = netfilter_log_sysctl_init(net);
        if (ret < 0)
                goto out_sysctl;

        return 0;

out_sysctl:
#ifdef CONFIG_PROC_FS
        remove_proc_entry("nf_log", net->nf.proc_netfilter);
#endif
        return ret;
}

static void __net_exit nf_log_net_exit(struct net *net)
{
        netfilter_log_sysctl_exit(net);
#ifdef CONFIG_PROC_FS
        remove_proc_entry("nf_log", net->nf.proc_netfilter);
#endif
}

static struct pernet_operations nf_log_net_ops = {
        .init = nf_log_net_init,
        .exit = nf_log_net_exit,
};

int __init netfilter_log_init(void)
{
        return register_pernet_subsys(&nf_log_net_ops);
}












































  414 















































































































































































































































































































































































































  416 
  414 



































  415 



  413 























  131 



  133 































































































































































































































































































































































































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
// SPDX-License-Identifier: GPL-2.0
/*
 * bus.c - bus driver management
 *
 * Copyright (c) 2002-3 Patrick Mochel
 * Copyright (c) 2002-3 Open Source Development Labs
 * Copyright (c) 2007 Greg Kroah-Hartman <gregkh@suse.de>
 * Copyright (c) 2007 Novell Inc.
 */

#include <linux/async.h>
#include <linux/device/bus.h>
#include <linux/device.h>
#include <linux/module.h>
#include <linux/errno.h>
#include <linux/slab.h>
#include <linux/init.h>
#include <linux/string.h>
#include <linux/mutex.h>
#include <linux/sysfs.h>
#include "base.h"
#include "power/power.h"

/* /sys/devices/system */
static struct kset *system_kset;

#define to_bus_attr(_attr) container_of(_attr, struct bus_attribute, attr)

/*
 * sysfs bindings for drivers
 */

#define to_drv_attr(_attr) container_of(_attr, struct driver_attribute, attr)

#define DRIVER_ATTR_IGNORE_LOCKDEP(_name, _mode, _show, _store) \
        struct driver_attribute driver_attr_##_name =                \
                __ATTR_IGNORE_LOCKDEP(_name, _mode, _show, _store)

static int __must_check bus_rescan_devices_helper(struct device *dev,
                                                void *data);

static struct bus_type *bus_get(struct bus_type *bus)
{
        if (bus) {
                kset_get(&bus->p->subsys);
                return bus;
        }
        return NULL;
}

static void bus_put(struct bus_type *bus)
{
        if (bus)
                kset_put(&bus->p->subsys);
}

static ssize_t drv_attr_show(struct kobject *kobj, struct attribute *attr,
                             char *buf)
{
        struct driver_attribute *drv_attr = to_drv_attr(attr);
        struct driver_private *drv_priv = to_driver(kobj);
        ssize_t ret = -EIO;

        if (drv_attr->show)
                ret = drv_attr->show(drv_priv->driver, buf);
        return ret;
}

static ssize_t drv_attr_store(struct kobject *kobj, struct attribute *attr,
                              const char *buf, size_t count)
{
        struct driver_attribute *drv_attr = to_drv_attr(attr);
        struct driver_private *drv_priv = to_driver(kobj);
        ssize_t ret = -EIO;

        if (drv_attr->store)
                ret = drv_attr->store(drv_priv->driver, buf, count);
        return ret;
}

static const struct sysfs_ops driver_sysfs_ops = {
        .show        = drv_attr_show,
        .store        = drv_attr_store,
};

static void driver_release(struct kobject *kobj)
{
        struct driver_private *drv_priv = to_driver(kobj);

        pr_debug("driver: '%s': %s\n", kobject_name(kobj), __func__);
        kfree(drv_priv);
}

static struct kobj_type driver_ktype = {
        .sysfs_ops        = &driver_sysfs_ops,
        .release        = driver_release,
};

/*
 * sysfs bindings for buses
 */
static ssize_t bus_attr_show(struct kobject *kobj, struct attribute *attr,
                             char *buf)
{
        struct bus_attribute *bus_attr = to_bus_attr(attr);
        struct subsys_private *subsys_priv = to_subsys_private(kobj);
        /* return -EIO for reading a bus attribute without show() */
        ssize_t ret = -EIO;

        if (bus_attr->show)
                ret = bus_attr->show(subsys_priv->bus, buf);
        return ret;
}

static ssize_t bus_attr_store(struct kobject *kobj, struct attribute *attr,
                              const char *buf, size_t count)
{
        struct bus_attribute *bus_attr = to_bus_attr(attr);
        struct subsys_private *subsys_priv = to_subsys_private(kobj);
        /* return -EIO for writing a bus attribute without store() */
        ssize_t ret = -EIO;

        if (bus_attr->store)
                ret = bus_attr->store(subsys_priv->bus, buf, count);
        return ret;
}

static const struct sysfs_ops bus_sysfs_ops = {
        .show        = bus_attr_show,
        .store        = bus_attr_store,
};

int bus_create_file(struct bus_type *bus, struct bus_attribute *attr)
{
        int error;
        if (bus_get(bus)) {
                error = sysfs_create_file(&bus->p->subsys.kobj, &attr->attr);
                bus_put(bus);
        } else
                error = -EINVAL;
        return error;
}
EXPORT_SYMBOL_GPL(bus_create_file);

void bus_remove_file(struct bus_type *bus, struct bus_attribute *attr)
{
        if (bus_get(bus)) {
                sysfs_remove_file(&bus->p->subsys.kobj, &attr->attr);
                bus_put(bus);
        }
}
EXPORT_SYMBOL_GPL(bus_remove_file);

static void bus_release(struct kobject *kobj)
{
        struct subsys_private *priv = to_subsys_private(kobj);
        struct bus_type *bus = priv->bus;

        kfree(priv);
        bus->p = NULL;
}

static struct kobj_type bus_ktype = {
        .sysfs_ops        = &bus_sysfs_ops,
        .release        = bus_release,
};

static int bus_uevent_filter(struct kset *kset, struct kobject *kobj)
{
        struct kobj_type *ktype = get_ktype(kobj);

        if (ktype == &bus_ktype)
                return 1;
        return 0;
}

static const struct kset_uevent_ops bus_uevent_ops = {
        .filter = bus_uevent_filter,
};

static struct kset *bus_kset;

/* Manually detach a device from its associated driver. */
static ssize_t unbind_store(struct device_driver *drv, const char *buf,
                            size_t count)
{
        struct bus_type *bus = bus_get(drv->bus);
        struct device *dev;
        int err = -ENODEV;

        dev = bus_find_device_by_name(bus, NULL, buf);
        if (dev && dev->driver == drv) {
                device_driver_detach(dev);
                err = count;
        }
        put_device(dev);
        bus_put(bus);
        return err;
}
static DRIVER_ATTR_IGNORE_LOCKDEP(unbind, 0200, NULL, unbind_store);

/*
 * Manually attach a device to a driver.
 * Note: the driver must want to bind to the device,
 * it is not possible to override the driver's id table.
 */
static ssize_t bind_store(struct device_driver *drv, const char *buf,
                          size_t count)
{
        struct bus_type *bus = bus_get(drv->bus);
        struct device *dev;
        int err = -ENODEV;

        dev = bus_find_device_by_name(bus, NULL, buf);
        if (dev && driver_match_device(drv, dev)) {
                err = device_driver_attach(drv, dev);
                if (!err) {
                        /* success */
                        err = count;
                }
        }
        put_device(dev);
        bus_put(bus);
        return err;
}
static DRIVER_ATTR_IGNORE_LOCKDEP(bind, 0200, NULL, bind_store);

static ssize_t drivers_autoprobe_show(struct bus_type *bus, char *buf)
{
        return sysfs_emit(buf, "%d\n", bus->p->drivers_autoprobe);
}

static ssize_t drivers_autoprobe_store(struct bus_type *bus,
                                       const char *buf, size_t count)
{
        if (buf[0] == '0')
                bus->p->drivers_autoprobe = 0;
        else
                bus->p->drivers_autoprobe = 1;
        return count;
}

static ssize_t drivers_probe_store(struct bus_type *bus,
                                   const char *buf, size_t count)
{
        struct device *dev;
        int err = -EINVAL;

        dev = bus_find_device_by_name(bus, NULL, buf);
        if (!dev)
                return -ENODEV;
        if (bus_rescan_devices_helper(dev, NULL) == 0)
                err = count;
        put_device(dev);
        return err;
}

static struct device *next_device(struct klist_iter *i)
{
        struct klist_node *n = klist_next(i);
        struct device *dev = NULL;
        struct device_private *dev_prv;

        if (n) {
                dev_prv = to_device_private_bus(n);
                dev = dev_prv->device;
        }
        return dev;
}

/**
 * bus_for_each_dev - device iterator.
 * @bus: bus type.
 * @start: device to start iterating from.
 * @data: data for the callback.
 * @fn: function to be called for each device.
 *
 * Iterate over @bus's list of devices, and call @fn for each,
 * passing it @data. If @start is not NULL, we use that device to
 * begin iterating from.
 *
 * We check the return of @fn each time. If it returns anything
 * other than 0, we break out and return that value.
 *
 * NOTE: The device that returns a non-zero value is not retained
 * in any way, nor is its refcount incremented. If the caller needs
 * to retain this data, it should do so, and increment the reference
 * count in the supplied callback.
 */
int bus_for_each_dev(struct bus_type *bus, struct device *start,
                     void *data, int (*fn)(struct device *, void *))
{
        struct klist_iter i;
        struct device *dev;
        int error = 0;

        if (!bus || !bus->p)
                return -EINVAL;

        klist_iter_init_node(&bus->p->klist_devices, &i,
                             (start ? &start->p->knode_bus : NULL));
        while (!error && (dev = next_device(&i)))
                error = fn(dev, data);
        klist_iter_exit(&i);
        return error;
}
EXPORT_SYMBOL_GPL(bus_for_each_dev);

/**
 * bus_find_device - device iterator for locating a particular device.
 * @bus: bus type
 * @start: Device to begin with
 * @data: Data to pass to match function
 * @match: Callback function to check device
 *
 * This is similar to the bus_for_each_dev() function above, but it
 * returns a reference to a device that is 'found' for later use, as
 * determined by the @match callback.
 *
 * The callback should return 0 if the device doesn't match and non-zero
 * if it does.  If the callback returns non-zero, this function will
 * return to the caller and not iterate over any more devices.
 */
struct device *bus_find_device(struct bus_type *bus,
                               struct device *start, const void *data,
                               int (*match)(struct device *dev, const void *data))
{
        struct klist_iter i;
        struct device *dev;

        if (!bus || !bus->p)
                return NULL;

        klist_iter_init_node(&bus->p->klist_devices, &i,
                             (start ? &start->p->knode_bus : NULL));
        while ((dev = next_device(&i)))
                if (match(dev, data) && get_device(dev))
                        break;
        klist_iter_exit(&i);
        return dev;
}
EXPORT_SYMBOL_GPL(bus_find_device);

/**
 * subsys_find_device_by_id - find a device with a specific enumeration number
 * @subsys: subsystem
 * @id: index 'id' in struct device
 * @hint: device to check first
 *
 * Check the hint's next object and if it is a match return it directly,
 * otherwise, fall back to a full list search. Either way a reference for
 * the returned object is taken.
 */
struct device *subsys_find_device_by_id(struct bus_type *subsys, unsigned int id,
                                        struct device *hint)
{
        struct klist_iter i;
        struct device *dev;

        if (!subsys)
                return NULL;

        if (hint) {
                klist_iter_init_node(&subsys->p->klist_devices, &i, &hint->p->knode_bus);
                dev = next_device(&i);
                if (dev && dev->id == id && get_device(dev)) {
                        klist_iter_exit(&i);
                        return dev;
                }
                klist_iter_exit(&i);
        }

        klist_iter_init_node(&subsys->p->klist_devices, &i, NULL);
        while ((dev = next_device(&i))) {
                if (dev->id == id && get_device(dev)) {
                        klist_iter_exit(&i);
                        return dev;
                }
        }
        klist_iter_exit(&i);
        return NULL;
}
EXPORT_SYMBOL_GPL(subsys_find_device_by_id);

static struct device_driver *next_driver(struct klist_iter *i)
{
        struct klist_node *n = klist_next(i);
        struct driver_private *drv_priv;

        if (n) {
                drv_priv = container_of(n, struct driver_private, knode_bus);
                return drv_priv->driver;
        }
        return NULL;
}

/**
 * bus_for_each_drv - driver iterator
 * @bus: bus we're dealing with.
 * @start: driver to start iterating on.
 * @data: data to pass to the callback.
 * @fn: function to call for each driver.
 *
 * This is nearly identical to the device iterator above.
 * We iterate over each driver that belongs to @bus, and call
 * @fn for each. If @fn returns anything but 0, we break out
 * and return it. If @start is not NULL, we use it as the head
 * of the list.
 *
 * NOTE: we don't return the driver that returns a non-zero
 * value, nor do we leave the reference count incremented for that
 * driver. If the caller needs to know that info, it must set it
 * in the callback. It must also be sure to increment the refcount
 * so it doesn't disappear before returning to the caller.
 */
int bus_for_each_drv(struct bus_type *bus, struct device_driver *start,
                     void *data, int (*fn)(struct device_driver *, void *))
{
        struct klist_iter i;
        struct device_driver *drv;
        int error = 0;

        if (!bus)
                return -EINVAL;

        klist_iter_init_node(&bus->p->klist_drivers, &i,
                             start ? &start->p->knode_bus : NULL);
        while ((drv = next_driver(&i)) && !error)
                error = fn(drv, data);
        klist_iter_exit(&i);
        return error;
}
EXPORT_SYMBOL_GPL(bus_for_each_drv);

/**
 * bus_add_device - add device to bus
 * @dev: device being added
 *
 * - Add device's bus attributes.
 * - Create links to device's bus.
 * - Add the device to its bus's list of devices.
 */
int bus_add_device(struct device *dev)
{
        struct bus_type *bus = bus_get(dev->bus);
        int error = 0;

        if (bus) {
                pr_debug("bus: '%s': add device %s\n", bus->name, dev_name(dev));
                error = device_add_groups(dev, bus->dev_groups);
                if (error)
                        goto out_put;
                error = sysfs_create_link(&bus->p->devices_kset->kobj,
                                                &dev->kobj, dev_name(dev));
                if (error)
                        goto out_groups;
                error = sysfs_create_link(&dev->kobj,
                                &dev->bus->p->subsys.kobj, "subsystem");
                if (error)
                        goto out_subsys;
                klist_add_tail(&dev->p->knode_bus, &bus->p->klist_devices);
        }
        return 0;

out_subsys:
        sysfs_remove_link(&bus->p->devices_kset->kobj, dev_name(dev));
out_groups:
        device_remove_groups(dev, bus->dev_groups);
out_put:
        bus_put(dev->bus);
        return error;
}

/**
 * bus_probe_device - probe drivers for a new device
 * @dev: device to probe
 *
 * - Automatically probe for a driver if the bus allows it.
 */
void bus_probe_device(struct device *dev)
{
        struct bus_type *bus = dev->bus;
        struct subsys_interface *sif;

        if (!bus)
                return;

        if (bus->p->drivers_autoprobe)
                device_initial_probe(dev);

        mutex_lock(&bus->p->mutex);
        list_for_each_entry(sif, &bus->p->interfaces, node)
                if (sif->add_dev)
                        sif->add_dev(dev, sif);
        mutex_unlock(&bus->p->mutex);
}

/**
 * bus_remove_device - remove device from bus
 * @dev: device to be removed
 *
 * - Remove device from all interfaces.
 * - Remove symlink from bus' directory.
 * - Delete device from bus's list.
 * - Detach from its driver.
 * - Drop reference taken in bus_add_device().
 */
void bus_remove_device(struct device *dev)
{
        struct bus_type *bus = dev->bus;
        struct subsys_interface *sif;

        if (!bus)
                return;

        mutex_lock(&bus->p->mutex);
        list_for_each_entry(sif, &bus->p->interfaces, node)
                if (sif->remove_dev)
                        sif->remove_dev(dev, sif);
        mutex_unlock(&bus->p->mutex);

        sysfs_remove_link(&dev->kobj, "subsystem");
        sysfs_remove_link(&dev->bus->p->devices_kset->kobj,
                          dev_name(dev));
        device_remove_groups(dev, dev->bus->dev_groups);
        if (klist_node_attached(&dev->p->knode_bus))
                klist_del(&dev->p->knode_bus);

        pr_debug("bus: '%s': remove device %s\n",
                 dev->bus->name, dev_name(dev));
        device_release_driver(dev);
        bus_put(dev->bus);
}

static int __must_check add_bind_files(struct device_driver *drv)
{
        int ret;

        ret = driver_create_file(drv, &driver_attr_unbind);
        if (ret == 0) {
                ret = driver_create_file(drv, &driver_attr_bind);
                if (ret)
                        driver_remove_file(drv, &driver_attr_unbind);
        }
        return ret;
}

static void remove_bind_files(struct device_driver *drv)
{
        driver_remove_file(drv, &driver_attr_bind);
        driver_remove_file(drv, &driver_attr_unbind);
}

static BUS_ATTR_WO(drivers_probe);
static BUS_ATTR_RW(drivers_autoprobe);

static int add_probe_files(struct bus_type *bus)
{
        int retval;

        retval = bus_create_file(bus, &bus_attr_drivers_probe);
        if (retval)
                goto out;

        retval = bus_create_file(bus, &bus_attr_drivers_autoprobe);
        if (retval)
                bus_remove_file(bus, &bus_attr_drivers_probe);
out:
        return retval;
}

static void remove_probe_files(struct bus_type *bus)
{
        bus_remove_file(bus, &bus_attr_drivers_autoprobe);
        bus_remove_file(bus, &bus_attr_drivers_probe);
}

static ssize_t uevent_store(struct device_driver *drv, const char *buf,
                            size_t count)
{
        int rc;

        rc = kobject_synth_uevent(&drv->p->kobj, buf, count);
        return rc ? rc : count;
}
static DRIVER_ATTR_WO(uevent);

/**
 * bus_add_driver - Add a driver to the bus.
 * @drv: driver.
 */
int bus_add_driver(struct device_driver *drv)
{
        struct bus_type *bus;
        struct driver_private *priv;
        int error = 0;

        bus = bus_get(drv->bus);
        if (!bus)
                return -EINVAL;

        pr_debug("bus: '%s': add driver %s\n", bus->name, drv->name);

        priv = kzalloc(sizeof(*priv), GFP_KERNEL);
        if (!priv) {
                error = -ENOMEM;
                goto out_put_bus;
        }
        klist_init(&priv->klist_devices, NULL, NULL);
        priv->driver = drv;
        drv->p = priv;
        priv->kobj.kset = bus->p->drivers_kset;
        error = kobject_init_and_add(&priv->kobj, &driver_ktype, NULL,
                                     "%s", drv->name);
        if (error)
                goto out_unregister;

        klist_add_tail(&priv->knode_bus, &bus->p->klist_drivers);
        if (drv->bus->p->drivers_autoprobe) {
                error = driver_attach(drv);
                if (error)
                        goto out_del_list;
        }
        module_add_driver(drv->owner, drv);

        error = driver_create_file(drv, &driver_attr_uevent);
        if (error) {
                printk(KERN_ERR "%s: uevent attr (%s) failed\n",
                        __func__, drv->name);
        }
        error = driver_add_groups(drv, bus->drv_groups);
        if (error) {
                /* How the hell do we get out of this pickle? Give up */
                printk(KERN_ERR "%s: driver_add_groups(%s) failed\n",
                        __func__, drv->name);
        }

        if (!drv->suppress_bind_attrs) {
                error = add_bind_files(drv);
                if (error) {
                        /* Ditto */
                        printk(KERN_ERR "%s: add_bind_files(%s) failed\n",
                                __func__, drv->name);
                }
        }

        return 0;

out_del_list:
        klist_del(&priv->knode_bus);
out_unregister:
        kobject_put(&priv->kobj);
        /* drv->p is freed in driver_release()  */
        drv->p = NULL;
out_put_bus:
        bus_put(bus);
        return error;
}

/**
 * bus_remove_driver - delete driver from bus's knowledge.
 * @drv: driver.
 *
 * Detach the driver from the devices it controls, and remove
 * it from its bus's list of drivers. Finally, we drop the reference
 * to the bus we took in bus_add_driver().
 */
void bus_remove_driver(struct device_driver *drv)
{
        if (!drv->bus)
                return;

        if (!drv->suppress_bind_attrs)
                remove_bind_files(drv);
        driver_remove_groups(drv, drv->bus->drv_groups);
        driver_remove_file(drv, &driver_attr_uevent);
        klist_remove(&drv->p->knode_bus);
        pr_debug("bus: '%s': remove driver %s\n", drv->bus->name, drv->name);
        driver_detach(drv);
        module_remove_driver(drv);
        kobject_put(&drv->p->kobj);
        bus_put(drv->bus);
}

/* Helper for bus_rescan_devices's iter */
static int __must_check bus_rescan_devices_helper(struct device *dev,
                                                  void *data)
{
        int ret = 0;

        if (!dev->driver) {
                if (dev->parent && dev->bus->need_parent_lock)
                        device_lock(dev->parent);
                ret = device_attach(dev);
                if (dev->parent && dev->bus->need_parent_lock)
                        device_unlock(dev->parent);
        }
        return ret < 0 ? ret : 0;
}

/**
 * bus_rescan_devices - rescan devices on the bus for possible drivers
 * @bus: the bus to scan.
 *
 * This function will look for devices on the bus with no driver
 * attached and rescan it against existing drivers to see if it matches
 * any by calling device_attach() for the unbound devices.
 */
int bus_rescan_devices(struct bus_type *bus)
{
        return bus_for_each_dev(bus, NULL, NULL, bus_rescan_devices_helper);
}
EXPORT_SYMBOL_GPL(bus_rescan_devices);

/**
 * device_reprobe - remove driver for a device and probe for a new driver
 * @dev: the device to reprobe
 *
 * This function detaches the attached driver (if any) for the given
 * device and restarts the driver probing process.  It is intended
 * to use if probing criteria changed during a devices lifetime and
 * driver attachment should change accordingly.
 */
int device_reprobe(struct device *dev)
{
        if (dev->driver)
                device_driver_detach(dev);
        return bus_rescan_devices_helper(dev, NULL);
}
EXPORT_SYMBOL_GPL(device_reprobe);

static int bus_add_groups(struct bus_type *bus,
                          const struct attribute_group **groups)
{
        return sysfs_create_groups(&bus->p->subsys.kobj, groups);
}

static void bus_remove_groups(struct bus_type *bus,
                              const struct attribute_group **groups)
{
        sysfs_remove_groups(&bus->p->subsys.kobj, groups);
}

static void klist_devices_get(struct klist_node *n)
{
        struct device_private *dev_prv = to_device_private_bus(n);
        struct device *dev = dev_prv->device;

        get_device(dev);
}

static void klist_devices_put(struct klist_node *n)
{
        struct device_private *dev_prv = to_device_private_bus(n);
        struct device *dev = dev_prv->device;

        put_device(dev);
}

static ssize_t bus_uevent_store(struct bus_type *bus,
                                const char *buf, size_t count)
{
        int rc;

        rc = kobject_synth_uevent(&bus->p->subsys.kobj, buf, count);
        return rc ? rc : count;
}
/*
 * "open code" the old BUS_ATTR() macro here.  We want to use BUS_ATTR_WO()
 * here, but can not use it as earlier in the file we have
 * DEVICE_ATTR_WO(uevent), which would cause a clash with the with the store
 * function name.
 */
static struct bus_attribute bus_attr_uevent = __ATTR(uevent, 0200, NULL,
                                                     bus_uevent_store);

/**
 * bus_register - register a driver-core subsystem
 * @bus: bus to register
 *
 * Once we have that, we register the bus with the kobject
 * infrastructure, then register the children subsystems it has:
 * the devices and drivers that belong to the subsystem.
 */
int bus_register(struct bus_type *bus)
{
        int retval;
        struct subsys_private *priv;
        struct lock_class_key *key = &bus->lock_key;

        priv = kzalloc(sizeof(struct subsys_private), GFP_KERNEL);
        if (!priv)
                return -ENOMEM;

        priv->bus = bus;
        bus->p = priv;

        BLOCKING_INIT_NOTIFIER_HEAD(&priv->bus_notifier);

        retval = kobject_set_name(&priv->subsys.kobj, "%s", bus->name);
        if (retval)
                goto out;

        priv->subsys.kobj.kset = bus_kset;
        priv->subsys.kobj.ktype = &bus_ktype;
        priv->drivers_autoprobe = 1;

        retval = kset_register(&priv->subsys);
        if (retval)
                goto out;

        retval = bus_create_file(bus, &bus_attr_uevent);
        if (retval)
                goto bus_uevent_fail;

        priv->devices_kset = kset_create_and_add("devices", NULL,
                                                 &priv->subsys.kobj);
        if (!priv->devices_kset) {
                retval = -ENOMEM;
                goto bus_devices_fail;
        }

        priv->drivers_kset = kset_create_and_add("drivers", NULL,
                                                 &priv->subsys.kobj);
        if (!priv->drivers_kset) {
                retval = -ENOMEM;
                goto bus_drivers_fail;
        }

        INIT_LIST_HEAD(&priv->interfaces);
        __mutex_init(&priv->mutex, "subsys mutex", key);
        klist_init(&priv->klist_devices, klist_devices_get, klist_devices_put);
        klist_init(&priv->klist_drivers, NULL, NULL);

        retval = add_probe_files(bus);
        if (retval)
                goto bus_probe_files_fail;

        retval = bus_add_groups(bus, bus->bus_groups);
        if (retval)
                goto bus_groups_fail;

        pr_debug("bus: '%s': registered\n", bus->name);
        return 0;

bus_groups_fail:
        remove_probe_files(bus);
bus_probe_files_fail:
        kset_unregister(bus->p->drivers_kset);
bus_drivers_fail:
        kset_unregister(bus->p->devices_kset);
bus_devices_fail:
        bus_remove_file(bus, &bus_attr_uevent);
bus_uevent_fail:
        kset_unregister(&bus->p->subsys);
        /* Above kset_unregister() will kfree @bus->p */
        bus->p = NULL;
out:
        kfree(bus->p);
        bus->p = NULL;
        return retval;
}
EXPORT_SYMBOL_GPL(bus_register);

/**
 * bus_unregister - remove a bus from the system
 * @bus: bus.
 *
 * Unregister the child subsystems and the bus itself.
 * Finally, we call bus_put() to release the refcount
 */
void bus_unregister(struct bus_type *bus)
{
        pr_debug("bus: '%s': unregistering\n", bus->name);
        if (bus->dev_root)
                device_unregister(bus->dev_root);
        bus_remove_groups(bus, bus->bus_groups);
        remove_probe_files(bus);
        kset_unregister(bus->p->drivers_kset);
        kset_unregister(bus->p->devices_kset);
        bus_remove_file(bus, &bus_attr_uevent);
        kset_unregister(&bus->p->subsys);
}
EXPORT_SYMBOL_GPL(bus_unregister);

int bus_register_notifier(struct bus_type *bus, struct notifier_block *nb)
{
        return blocking_notifier_chain_register(&bus->p->bus_notifier, nb);
}
EXPORT_SYMBOL_GPL(bus_register_notifier);

int bus_unregister_notifier(struct bus_type *bus, struct notifier_block *nb)
{
        return blocking_notifier_chain_unregister(&bus->p->bus_notifier, nb);
}
EXPORT_SYMBOL_GPL(bus_unregister_notifier);

struct kset *bus_get_kset(struct bus_type *bus)
{
        return &bus->p->subsys;
}
EXPORT_SYMBOL_GPL(bus_get_kset);

struct klist *bus_get_device_klist(struct bus_type *bus)
{
        return &bus->p->klist_devices;
}
EXPORT_SYMBOL_GPL(bus_get_device_klist);

/*
 * Yes, this forcibly breaks the klist abstraction temporarily.  It
 * just wants to sort the klist, not change reference counts and
 * take/drop locks rapidly in the process.  It does all this while
 * holding the lock for the list, so objects can't otherwise be
 * added/removed while we're swizzling.
 */
static void device_insertion_sort_klist(struct device *a, struct list_head *list,
                                        int (*compare)(const struct device *a,
                                                        const struct device *b))
{
        struct klist_node *n;
        struct device_private *dev_prv;
        struct device *b;

        list_for_each_entry(n, list, n_node) {
                dev_prv = to_device_private_bus(n);
                b = dev_prv->device;
                if (compare(a, b) <= 0) {
                        list_move_tail(&a->p->knode_bus.n_node,
                                       &b->p->knode_bus.n_node);
                        return;
                }
        }
        list_move_tail(&a->p->knode_bus.n_node, list);
}

void bus_sort_breadthfirst(struct bus_type *bus,
                           int (*compare)(const struct device *a,
                                          const struct device *b))
{
        LIST_HEAD(sorted_devices);
        struct klist_node *n, *tmp;
        struct device_private *dev_prv;
        struct device *dev;
        struct klist *device_klist;

        device_klist = bus_get_device_klist(bus);

        spin_lock(&device_klist->k_lock);
        list_for_each_entry_safe(n, tmp, &device_klist->k_list, n_node) {
                dev_prv = to_device_private_bus(n);
                dev = dev_prv->device;
                device_insertion_sort_klist(dev, &sorted_devices, compare);
        }
        list_splice(&sorted_devices, &device_klist->k_list);
        spin_unlock(&device_klist->k_lock);
}
EXPORT_SYMBOL_GPL(bus_sort_breadthfirst);

/**
 * subsys_dev_iter_init - initialize subsys device iterator
 * @iter: subsys iterator to initialize
 * @subsys: the subsys we wanna iterate over
 * @start: the device to start iterating from, if any
 * @type: device_type of the devices to iterate over, NULL for all
 *
 * Initialize subsys iterator @iter such that it iterates over devices
 * of @subsys.  If @start is set, the list iteration will start there,
 * otherwise if it is NULL, the iteration starts at the beginning of
 * the list.
 */
void subsys_dev_iter_init(struct subsys_dev_iter *iter, struct bus_type *subsys,
                          struct device *start, const struct device_type *type)
{
        struct klist_node *start_knode = NULL;

        if (start)
                start_knode = &start->p->knode_bus;
        klist_iter_init_node(&subsys->p->klist_devices, &iter->ki, start_knode);
        iter->type = type;
}
EXPORT_SYMBOL_GPL(subsys_dev_iter_init);

/**
 * subsys_dev_iter_next - iterate to the next device
 * @iter: subsys iterator to proceed
 *
 * Proceed @iter to the next device and return it.  Returns NULL if
 * iteration is complete.
 *
 * The returned device is referenced and won't be released till
 * iterator is proceed to the next device or exited.  The caller is
 * free to do whatever it wants to do with the device including
 * calling back into subsys code.
 */
struct device *subsys_dev_iter_next(struct subsys_dev_iter *iter)
{
        struct klist_node *knode;
        struct device *dev;

        for (;;) {
                knode = klist_next(&iter->ki);
                if (!knode)
                        return NULL;
                dev = to_device_private_bus(knode)->device;
                if (!iter->type || iter->type == dev->type)
                        return dev;
        }
}
EXPORT_SYMBOL_GPL(subsys_dev_iter_next);

/**
 * subsys_dev_iter_exit - finish iteration
 * @iter: subsys iterator to finish
 *
 * Finish an iteration.  Always call this function after iteration is
 * complete whether the iteration ran till the end or not.
 */
void subsys_dev_iter_exit(struct subsys_dev_iter *iter)
{
        klist_iter_exit(&iter->ki);
}
EXPORT_SYMBOL_GPL(subsys_dev_iter_exit);

int subsys_interface_register(struct subsys_interface *sif)
{
        struct bus_type *subsys;
        struct subsys_dev_iter iter;
        struct device *dev;

        if (!sif || !sif->subsys)
                return -ENODEV;

        subsys = bus_get(sif->subsys);
        if (!subsys)
                return -EINVAL;

        mutex_lock(&subsys->p->mutex);
        list_add_tail(&sif->node, &subsys->p->interfaces);
        if (sif->add_dev) {
                subsys_dev_iter_init(&iter, subsys, NULL, NULL);
                while ((dev = subsys_dev_iter_next(&iter)))
                        sif->add_dev(dev, sif);
                subsys_dev_iter_exit(&iter);
        }
        mutex_unlock(&subsys->p->mutex);

        return 0;
}
EXPORT_SYMBOL_GPL(subsys_interface_register);

void subsys_interface_unregister(struct subsys_interface *sif)
{
        struct bus_type *subsys;
        struct subsys_dev_iter iter;
        struct device *dev;

        if (!sif || !sif->subsys)
                return;

        subsys = sif->subsys;

        mutex_lock(&subsys->p->mutex);
        list_del_init(&sif->node);
        if (sif->remove_dev) {
                subsys_dev_iter_init(&iter, subsys, NULL, NULL);
                while ((dev = subsys_dev_iter_next(&iter)))
                        sif->remove_dev(dev, sif);
                subsys_dev_iter_exit(&iter);
        }
        mutex_unlock(&subsys->p->mutex);

        bus_put(subsys);
}
EXPORT_SYMBOL_GPL(subsys_interface_unregister);

static void system_root_device_release(struct device *dev)
{
        kfree(dev);
}

static int subsys_register(struct bus_type *subsys,
                           const struct attribute_group **groups,
                           struct kobject *parent_of_root)
{
        struct device *dev;
        int err;

        err = bus_register(subsys);
        if (err < 0)
                return err;

        dev = kzalloc(sizeof(struct device), GFP_KERNEL);
        if (!dev) {
                err = -ENOMEM;
                goto err_dev;
        }

        err = dev_set_name(dev, "%s", subsys->name);
        if (err < 0)
                goto err_name;

        dev->kobj.parent = parent_of_root;
        dev->groups = groups;
        dev->release = system_root_device_release;

        err = device_register(dev);
        if (err < 0)
                goto err_dev_reg;

        subsys->dev_root = dev;
        return 0;

err_dev_reg:
        put_device(dev);
        dev = NULL;
err_name:
        kfree(dev);
err_dev:
        bus_unregister(subsys);
        return err;
}

/**
 * subsys_system_register - register a subsystem at /sys/devices/system/
 * @subsys: system subsystem
 * @groups: default attributes for the root device
 *
 * All 'system' subsystems have a /sys/devices/system/<name> root device
 * with the name of the subsystem. The root device can carry subsystem-
 * wide attributes. All registered devices are below this single root
 * device and are named after the subsystem with a simple enumeration
 * number appended. The registered devices are not explicitly named;
 * only 'id' in the device needs to be set.
 *
 * Do not use this interface for anything new, it exists for compatibility
 * with bad ideas only. New subsystems should use plain subsystems; and
 * add the subsystem-wide attributes should be added to the subsystem
 * directory itself and not some create fake root-device placed in
 * /sys/devices/system/<name>.
 */
int subsys_system_register(struct bus_type *subsys,
                           const struct attribute_group **groups)
{
        return subsys_register(subsys, groups, &system_kset->kobj);
}
EXPORT_SYMBOL_GPL(subsys_system_register);

/**
 * subsys_virtual_register - register a subsystem at /sys/devices/virtual/
 * @subsys: virtual subsystem
 * @groups: default attributes for the root device
 *
 * All 'virtual' subsystems have a /sys/devices/system/<name> root device
 * with the name of the subystem.  The root device can carry subsystem-wide
 * attributes.  All registered devices are below this single root device.
 * There's no restriction on device naming.  This is for kernel software
 * constructs which need sysfs interface.
 */
int subsys_virtual_register(struct bus_type *subsys,
                            const struct attribute_group **groups)
{
        struct kobject *virtual_dir;

        virtual_dir = virtual_device_parent(NULL);
        if (!virtual_dir)
                return -ENOMEM;

        return subsys_register(subsys, groups, virtual_dir);
}
EXPORT_SYMBOL_GPL(subsys_virtual_register);

int __init buses_init(void)
{
        bus_kset = kset_create_and_add("bus", &bus_uevent_ops, NULL);
        if (!bus_kset)
                return -ENOMEM;

        system_kset = kset_create_and_add("system", NULL, &devices_kset->kobj);
        if (!system_kset)
                return -ENOMEM;

        return 0;
}






































































































    2 
    2 

    2 



    2 
    2 
    2 


    2 









    2 








    2 






    2 













    2 
    2 
    2 



    2 
    2 
    2 







    2 























































































































  256 
  259 





















   69 




   69 


















































































































































































































































































































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
// SPDX-License-Identifier: GPL-2.0
/*
 *  linux/mm/swap_state.c
 *
 *  Copyright (C) 1991, 1992, 1993, 1994  Linus Torvalds
 *  Swap reorganised 29.12.95, Stephen Tweedie
 *
 *  Rewritten to use page cache, (C) 1998 Stephen Tweedie
 */
#include <linux/mm.h>
#include <linux/gfp.h>
#include <linux/kernel_stat.h>
#include <linux/swap.h>
#include <linux/swapops.h>
#include <linux/init.h>
#include <linux/pagemap.h>
#include <linux/backing-dev.h>
#include <linux/blkdev.h>
#include <linux/pagevec.h>
#include <linux/migrate.h>
#include <linux/vmalloc.h>
#include <linux/swap_slots.h>
#include <linux/huge_mm.h>
#include <linux/shmem_fs.h>
#include "internal.h"

/*
 * swapper_space is a fiction, retained to simplify the path through
 * vmscan's shrink_page_list.
 */
static const struct address_space_operations swap_aops = {
        .writepage        = swap_writepage,
        .set_page_dirty        = swap_set_page_dirty,
#ifdef CONFIG_MIGRATION
        .migratepage        = migrate_page,
#endif
};

struct address_space *swapper_spaces[MAX_SWAPFILES] __read_mostly;
static unsigned int nr_swapper_spaces[MAX_SWAPFILES] __read_mostly;
static bool enable_vma_readahead __read_mostly = true;

#define SWAP_RA_WIN_SHIFT        (PAGE_SHIFT / 2)
#define SWAP_RA_HITS_MASK        ((1UL << SWAP_RA_WIN_SHIFT) - 1)
#define SWAP_RA_HITS_MAX        SWAP_RA_HITS_MASK
#define SWAP_RA_WIN_MASK        (~PAGE_MASK & ~SWAP_RA_HITS_MASK)

#define SWAP_RA_HITS(v)                ((v) & SWAP_RA_HITS_MASK)
#define SWAP_RA_WIN(v)                (((v) & SWAP_RA_WIN_MASK) >> SWAP_RA_WIN_SHIFT)
#define SWAP_RA_ADDR(v)                ((v) & PAGE_MASK)

#define SWAP_RA_VAL(addr, win, hits)                                \
        (((addr) & PAGE_MASK) |                                        \
         (((win) << SWAP_RA_WIN_SHIFT) & SWAP_RA_WIN_MASK) |        \
         ((hits) & SWAP_RA_HITS_MASK))

/* Initial readahead hits is 4 to start up with a small window */
#define GET_SWAP_RA_VAL(vma)                                        \
        (atomic_long_read(&(vma)->swap_readahead_info) ? : 4)

#define INC_CACHE_INFO(x)        data_race(swap_cache_info.x++)
#define ADD_CACHE_INFO(x, nr)        data_race(swap_cache_info.x += (nr))

static struct {
        unsigned long add_total;
        unsigned long del_total;
        unsigned long find_success;
        unsigned long find_total;
} swap_cache_info;

static atomic_t swapin_readahead_hits = ATOMIC_INIT(4);

void show_swap_cache_info(void)
{
        printk("%lu pages in swap cache\n", total_swapcache_pages());
        printk("Swap cache stats: add %lu, delete %lu, find %lu/%lu\n",
                swap_cache_info.add_total, swap_cache_info.del_total,
                swap_cache_info.find_success, swap_cache_info.find_total);
        printk("Free swap  = %ldkB\n",
                get_nr_swap_pages() << (PAGE_SHIFT - 10));
        printk("Total swap = %lukB\n", total_swap_pages << (PAGE_SHIFT - 10));
}

void *get_shadow_from_swap_cache(swp_entry_t entry)
{
        struct address_space *address_space = swap_address_space(entry);
        pgoff_t idx = swp_offset(entry);
        struct page *page;

        page = xa_load(&address_space->i_pages, idx);
        if (xa_is_value(page))
                return page;
        return NULL;
}

/*
 * add_to_swap_cache resembles add_to_page_cache_locked on swapper_space,
 * but sets SwapCache flag and private instead of mapping and index.
 */
int add_to_swap_cache(struct page *page, swp_entry_t entry,
                        gfp_t gfp, void **shadowp)
{
        struct address_space *address_space = swap_address_space(entry);
        pgoff_t idx = swp_offset(entry);
        XA_STATE_ORDER(xas, &address_space->i_pages, idx, compound_order(page));
        unsigned long i, nr = thp_nr_pages(page);
        void *old;

        VM_BUG_ON_PAGE(!PageLocked(page), page);
        VM_BUG_ON_PAGE(PageSwapCache(page), page);
        VM_BUG_ON_PAGE(!PageSwapBacked(page), page);

        page_ref_add(page, nr);
        SetPageSwapCache(page);

        do {
                xas_lock_irq(&xas);
                xas_create_range(&xas);
                if (xas_error(&xas))
                        goto unlock;
                for (i = 0; i < nr; i++) {
                        VM_BUG_ON_PAGE(xas.xa_index != idx + i, page);
                        old = xas_load(&xas);
                        if (xa_is_value(old)) {
                                if (shadowp)
                                        *shadowp = old;
                        }
                        set_page_private(page + i, entry.val + i);
                        xas_store(&xas, page);
                        xas_next(&xas);
                }
                address_space->nrpages += nr;
                __mod_node_page_state(page_pgdat(page), NR_FILE_PAGES, nr);
                __mod_lruvec_page_state(page, NR_SWAPCACHE, nr);
                ADD_CACHE_INFO(add_total, nr);
unlock:
                xas_unlock_irq(&xas);
        } while (xas_nomem(&xas, gfp));

        if (!xas_error(&xas))
                return 0;

        ClearPageSwapCache(page);
        page_ref_sub(page, nr);
        return xas_error(&xas);
}

/*
 * This must be called only on pages that have
 * been verified to be in the swap cache.
 */
void __delete_from_swap_cache(struct page *page,
                        swp_entry_t entry, void *shadow)
{
        struct address_space *address_space = swap_address_space(entry);
        int i, nr = thp_nr_pages(page);
        pgoff_t idx = swp_offset(entry);
        XA_STATE(xas, &address_space->i_pages, idx);

        VM_BUG_ON_PAGE(!PageLocked(page), page);
        VM_BUG_ON_PAGE(!PageSwapCache(page), page);
        VM_BUG_ON_PAGE(PageWriteback(page), page);

        for (i = 0; i < nr; i++) {
                void *entry = xas_store(&xas, shadow);
                VM_BUG_ON_PAGE(entry != page, entry);
                set_page_private(page + i, 0);
                xas_next(&xas);
        }
        ClearPageSwapCache(page);
        address_space->nrpages -= nr;
        __mod_node_page_state(page_pgdat(page), NR_FILE_PAGES, -nr);
        __mod_lruvec_page_state(page, NR_SWAPCACHE, -nr);
        ADD_CACHE_INFO(del_total, nr);
}

/**
 * add_to_swap - allocate swap space for a page
 * @page: page we want to move to swap
 *
 * Allocate swap space for the page and add the page to the
 * swap cache.  Caller needs to hold the page lock. 
 */
int add_to_swap(struct page *page)
{
        swp_entry_t entry;
        int err;

        VM_BUG_ON_PAGE(!PageLocked(page), page);
        VM_BUG_ON_PAGE(!PageUptodate(page), page);

        entry = get_swap_page(page);
        if (!entry.val)
                return 0;

        /*
         * XArray node allocations from PF_MEMALLOC contexts could
         * completely exhaust the page allocator. __GFP_NOMEMALLOC
         * stops emergency reserves from being allocated.
         *
         * TODO: this could cause a theoretical memory reclaim
         * deadlock in the swap out path.
         */
        /*
         * Add it to the swap cache.
         */
        err = add_to_swap_cache(page, entry,
                        __GFP_HIGH|__GFP_NOMEMALLOC|__GFP_NOWARN, NULL);
        if (err)
                /*
                 * add_to_swap_cache() doesn't return -EEXIST, so we can safely
                 * clear SWAP_HAS_CACHE flag.
                 */
                goto fail;
        /*
         * Normally the page will be dirtied in unmap because its pte should be
         * dirty. A special case is MADV_FREE page. The page's pte could have
         * dirty bit cleared but the page's SwapBacked bit is still set because
         * clearing the dirty bit and SwapBacked bit has no lock protected. For
         * such page, unmap will not set dirty bit for it, so page reclaim will
         * not write the page out. This can cause data corruption when the page
         * is swap in later. Always setting the dirty bit for the page solves
         * the problem.
         */
        set_page_dirty(page);

        return 1;

fail:
        put_swap_page(page, entry);
        return 0;
}

/*
 * This must be called only on pages that have
 * been verified to be in the swap cache and locked.
 * It will never put the page into the free list,
 * the caller has a reference on the page.
 */
void delete_from_swap_cache(struct page *page)
{
        swp_entry_t entry = { .val = page_private(page) };
        struct address_space *address_space = swap_address_space(entry);

        xa_lock_irq(&address_space->i_pages);
        __delete_from_swap_cache(page, entry, NULL);
        xa_unlock_irq(&address_space->i_pages);

        put_swap_page(page, entry);
        page_ref_sub(page, thp_nr_pages(page));
}

void clear_shadow_from_swap_cache(int type, unsigned long begin,
                                unsigned long end)
{
        unsigned long curr = begin;
        void *old;

        for (;;) {
                swp_entry_t entry = swp_entry(type, curr);
                struct address_space *address_space = swap_address_space(entry);
                XA_STATE(xas, &address_space->i_pages, curr);

                xa_lock_irq(&address_space->i_pages);
                xas_for_each(&xas, old, end) {
                        if (!xa_is_value(old))
                                continue;
                        xas_store(&xas, NULL);
                }
                xa_unlock_irq(&address_space->i_pages);

                /* search the next swapcache until we meet end */
                curr >>= SWAP_ADDRESS_SPACE_SHIFT;
                curr++;
                curr <<= SWAP_ADDRESS_SPACE_SHIFT;
                if (curr > end)
                        break;
        }
}

/* 
 * If we are the only user, then try to free up the swap cache. 
 * 
 * Its ok to check for PageSwapCache without the page lock
 * here because we are going to recheck again inside
 * try_to_free_swap() _with_ the lock.
 *                                         - Marcelo
 */
void free_swap_cache(struct page *page)
{
        if (PageSwapCache(page) && !page_mapped(page) && trylock_page(page)) {
                try_to_free_swap(page);
                unlock_page(page);
        }
}

/* 
 * Perform a free_page(), also freeing any swap cache associated with
 * this page if it is the last user of the page.
 */
void free_page_and_swap_cache(struct page *page)
{
        free_swap_cache(page);
        if (!is_huge_zero_page(page))
                put_page(page);
}

/*
 * Passed an array of pages, drop them all from swapcache and then release
 * them.  They are removed from the LRU and freed if this is their last use.
 */
void free_pages_and_swap_cache(struct page **pages, int nr)
{
        struct page **pagep = pages;
        int i;

        lru_add_drain();
        for (i = 0; i < nr; i++)
                free_swap_cache(pagep[i]);
        release_pages(pagep, nr);
}

static inline bool swap_use_vma_readahead(void)
{
        return READ_ONCE(enable_vma_readahead) && !atomic_read(&nr_rotate_swap);
}

/*
 * Lookup a swap entry in the swap cache. A found page will be returned
 * unlocked and with its refcount incremented - we rely on the kernel
 * lock getting page table operations atomic even if we drop the page
 * lock before returning.
 */
struct page *lookup_swap_cache(swp_entry_t entry, struct vm_area_struct *vma,
                               unsigned long addr)
{
        struct page *page;
        struct swap_info_struct *si;

        si = get_swap_device(entry);
        if (!si)
                return NULL;
        page = find_get_page(swap_address_space(entry), swp_offset(entry));
        put_swap_device(si);

        INC_CACHE_INFO(find_total);
        if (page) {
                bool vma_ra = swap_use_vma_readahead();
                bool readahead;

                INC_CACHE_INFO(find_success);
                /*
                 * At the moment, we don't support PG_readahead for anon THP
                 * so let's bail out rather than confusing the readahead stat.
                 */
                if (unlikely(PageTransCompound(page)))
                        return page;

                readahead = TestClearPageReadahead(page);
                if (vma && vma_ra) {
                        unsigned long ra_val;
                        int win, hits;

                        ra_val = GET_SWAP_RA_VAL(vma);
                        win = SWAP_RA_WIN(ra_val);
                        hits = SWAP_RA_HITS(ra_val);
                        if (readahead)
                                hits = min_t(int, hits + 1, SWAP_RA_HITS_MAX);
                        atomic_long_set(&vma->swap_readahead_info,
                                        SWAP_RA_VAL(addr, win, hits));
                }

                if (readahead) {
                        count_vm_event(SWAP_RA_HIT);
                        if (!vma || !vma_ra)
                                atomic_inc(&swapin_readahead_hits);
                }
        }

        return page;
}

/**
 * find_get_incore_page - Find and get a page from the page or swap caches.
 * @mapping: The address_space to search.
 * @index: The page cache index.
 *
 * This differs from find_get_page() in that it will also look for the
 * page in the swap cache.
 *
 * Return: The found page or %NULL.
 */
struct page *find_get_incore_page(struct address_space *mapping, pgoff_t index)
{
        swp_entry_t swp;
        struct swap_info_struct *si;
        struct page *page = pagecache_get_page(mapping, index,
                                                FGP_ENTRY | FGP_HEAD, 0);

        if (!page)
                return page;
        if (!xa_is_value(page))
                return find_subpage(page, index);
        if (!shmem_mapping(mapping))
                return NULL;

        swp = radix_to_swp_entry(page);
        /* Prevent swapoff from happening to us */
        si = get_swap_device(swp);
        if (!si)
                return NULL;
        page = find_get_page(swap_address_space(swp), swp_offset(swp));
        put_swap_device(si);
        return page;
}

struct page *__read_swap_cache_async(swp_entry_t entry, gfp_t gfp_mask,
                        struct vm_area_struct *vma, unsigned long addr,
                        bool *new_page_allocated)
{
        struct swap_info_struct *si;
        struct page *page;
        void *shadow = NULL;

        *new_page_allocated = false;

        for (;;) {
                int err;
                /*
                 * First check the swap cache.  Since this is normally
                 * called after lookup_swap_cache() failed, re-calling
                 * that would confuse statistics.
                 */
                si = get_swap_device(entry);
                if (!si)
                        return NULL;
                page = find_get_page(swap_address_space(entry),
                                     swp_offset(entry));
                put_swap_device(si);
                if (page)
                        return page;

                /*
                 * Just skip read ahead for unused swap slot.
                 * During swap_off when swap_slot_cache is disabled,
                 * we have to handle the race between putting
                 * swap entry in swap cache and marking swap slot
                 * as SWAP_HAS_CACHE.  That's done in later part of code or
                 * else swap_off will be aborted if we return NULL.
                 */
                if (!__swp_swapcount(entry) && swap_slot_cache_enabled)
                        return NULL;

                /*
                 * Get a new page to read into from swap.  Allocate it now,
                 * before marking swap_map SWAP_HAS_CACHE, when -EEXIST will
                 * cause any racers to loop around until we add it to cache.
                 */
                page = alloc_page_vma(gfp_mask, vma, addr);
                if (!page)
                        return NULL;

                /*
                 * Swap entry may have been freed since our caller observed it.
                 */
                err = swapcache_prepare(entry);
                if (!err)
                        break;

                put_page(page);
                if (err != -EEXIST)
                        return NULL;

                /*
                 * We might race against __delete_from_swap_cache(), and
                 * stumble across a swap_map entry whose SWAP_HAS_CACHE
                 * has not yet been cleared.  Or race against another
                 * __read_swap_cache_async(), which has set SWAP_HAS_CACHE
                 * in swap_map, but not yet added its page to swap cache.
                 */
                schedule_timeout_uninterruptible(1);
        }

        /*
         * The swap entry is ours to swap in. Prepare the new page.
         */

        __SetPageLocked(page);
        __SetPageSwapBacked(page);

        if (mem_cgroup_swapin_charge_page(page, NULL, gfp_mask, entry))
                goto fail_unlock;

        /* May fail (-ENOMEM) if XArray node allocation failed. */
        if (add_to_swap_cache(page, entry, gfp_mask & GFP_RECLAIM_MASK, &shadow))
                goto fail_unlock;

        mem_cgroup_swapin_uncharge_swap(entry);

        if (shadow)
                workingset_refault(page, shadow);

        /* Caller will initiate read into locked page */
        lru_cache_add(page);
        *new_page_allocated = true;
        return page;

fail_unlock:
        put_swap_page(page, entry);
        unlock_page(page);
        put_page(page);
        return NULL;
}

/*
 * Locate a page of swap in physical memory, reserving swap cache space
 * and reading the disk if it is not already cached.
 * A failure return means that either the page allocation failed or that
 * the swap entry is no longer in use.
 */
struct page *read_swap_cache_async(swp_entry_t entry, gfp_t gfp_mask,
                struct vm_area_struct *vma, unsigned long addr, bool do_poll)
{
        bool page_was_allocated;
        struct page *retpage = __read_swap_cache_async(entry, gfp_mask,
                        vma, addr, &page_was_allocated);

        if (page_was_allocated)
                swap_readpage(retpage, do_poll);

        return retpage;
}

static unsigned int __swapin_nr_pages(unsigned long prev_offset,
                                      unsigned long offset,
                                      int hits,
                                      int max_pages,
                                      int prev_win)
{
        unsigned int pages, last_ra;

        /*
         * This heuristic has been found to work well on both sequential and
         * random loads, swapping to hard disk or to SSD: please don't ask
         * what the "+ 2" means, it just happens to work well, that's all.
         */
        pages = hits + 2;
        if (pages == 2) {
                /*
                 * We can have no readahead hits to judge by: but must not get
                 * stuck here forever, so check for an adjacent offset instead
                 * (and don't even bother to check whether swap type is same).
                 */
                if (offset != prev_offset + 1 && offset != prev_offset - 1)
                        pages = 1;
        } else {
                unsigned int roundup = 4;
                while (roundup < pages)
                        roundup <<= 1;
                pages = roundup;
        }

        if (pages > max_pages)
                pages = max_pages;

        /* Don't shrink readahead too fast */
        last_ra = prev_win / 2;
        if (pages < last_ra)
                pages = last_ra;

        return pages;
}

static unsigned long swapin_nr_pages(unsigned long offset)
{
        static unsigned long prev_offset;
        unsigned int hits, pages, max_pages;
        static atomic_t last_readahead_pages;

        max_pages = 1 << READ_ONCE(page_cluster);
        if (max_pages <= 1)
                return 1;

        hits = atomic_xchg(&swapin_readahead_hits, 0);
        pages = __swapin_nr_pages(READ_ONCE(prev_offset), offset, hits,
                                  max_pages,
                                  atomic_read(&last_readahead_pages));
        if (!hits)
                WRITE_ONCE(prev_offset, offset);
        atomic_set(&last_readahead_pages, pages);

        return pages;
}

/**
 * swap_cluster_readahead - swap in pages in hope we need them soon
 * @entry: swap entry of this memory
 * @gfp_mask: memory allocation flags
 * @vmf: fault information
 *
 * Returns the struct page for entry and addr, after queueing swapin.
 *
 * Primitive swap readahead code. We simply read an aligned block of
 * (1 << page_cluster) entries in the swap area. This method is chosen
 * because it doesn't cost us any seek time.  We also make sure to queue
 * the 'original' request together with the readahead ones...
 *
 * This has been extended to use the NUMA policies from the mm triggering
 * the readahead.
 *
 * Caller must hold read mmap_lock if vmf->vma is not NULL.
 */
struct page *swap_cluster_readahead(swp_entry_t entry, gfp_t gfp_mask,
                                struct vm_fault *vmf)
{
        struct page *page;
        unsigned long entry_offset = swp_offset(entry);
        unsigned long offset = entry_offset;
        unsigned long start_offset, end_offset;
        unsigned long mask;
        struct swap_info_struct *si = swp_swap_info(entry);
        struct blk_plug plug;
        bool do_poll = true, page_allocated;
        struct vm_area_struct *vma = vmf->vma;
        unsigned long addr = vmf->address;

        mask = swapin_nr_pages(offset) - 1;
        if (!mask)
                goto skip;

        do_poll = false;
        /* Read a page_cluster sized and aligned cluster around offset. */
        start_offset = offset & ~mask;
        end_offset = offset | mask;
        if (!start_offset)        /* First page is swap header. */
                start_offset++;
        if (end_offset >= si->max)
                end_offset = si->max - 1;

        blk_start_plug(&plug);
        for (offset = start_offset; offset <= end_offset ; offset++) {
                /* Ok, do the async read-ahead now */
                page = __read_swap_cache_async(
                        swp_entry(swp_type(entry), offset),
                        gfp_mask, vma, addr, &page_allocated);
                if (!page)
                        continue;
                if (page_allocated) {
                        swap_readpage(page, false);
                        if (offset != entry_offset) {
                                SetPageReadahead(page);
                                count_vm_event(SWAP_RA);
                        }
                }
                put_page(page);
        }
        blk_finish_plug(&plug);

        lru_add_drain();        /* Push any new pages onto the LRU now */
skip:
        return read_swap_cache_async(entry, gfp_mask, vma, addr, do_poll);
}

int init_swap_address_space(unsigned int type, unsigned long nr_pages)
{
        struct address_space *spaces, *space;
        unsigned int i, nr;

        nr = DIV_ROUND_UP(nr_pages, SWAP_ADDRESS_SPACE_PAGES);
        spaces = kvcalloc(nr, sizeof(struct address_space), GFP_KERNEL);
        if (!spaces)
                return -ENOMEM;
        for (i = 0; i < nr; i++) {
                space = spaces + i;
                xa_init_flags(&space->i_pages, XA_FLAGS_LOCK_IRQ);
                atomic_set(&space->i_mmap_writable, 0);
                space->a_ops = &swap_aops;
                /* swap cache doesn't use writeback related tags */
                mapping_set_no_writeback_tags(space);
        }
        nr_swapper_spaces[type] = nr;
        swapper_spaces[type] = spaces;

        return 0;
}

void exit_swap_address_space(unsigned int type)
{
        int i;
        struct address_space *spaces = swapper_spaces[type];

        for (i = 0; i < nr_swapper_spaces[type]; i++)
                VM_WARN_ON_ONCE(!mapping_empty(&spaces[i]));
        kvfree(spaces);
        nr_swapper_spaces[type] = 0;
        swapper_spaces[type] = NULL;
}

static inline void swap_ra_clamp_pfn(struct vm_area_struct *vma,
                                     unsigned long faddr,
                                     unsigned long lpfn,
                                     unsigned long rpfn,
                                     unsigned long *start,
                                     unsigned long *end)
{
        *start = max3(lpfn, PFN_DOWN(vma->vm_start),
                      PFN_DOWN(faddr & PMD_MASK));
        *end = min3(rpfn, PFN_DOWN(vma->vm_end),
                    PFN_DOWN((faddr & PMD_MASK) + PMD_SIZE));
}

static void swap_ra_info(struct vm_fault *vmf,
                        struct vma_swap_readahead *ra_info)
{
        struct vm_area_struct *vma = vmf->vma;
        unsigned long ra_val;
        unsigned long faddr, pfn, fpfn;
        unsigned long start, end;
        pte_t *pte, *orig_pte;
        unsigned int max_win, hits, prev_win, win, left;
#ifndef CONFIG_64BIT
        pte_t *tpte;
#endif

        max_win = 1 << min_t(unsigned int, READ_ONCE(page_cluster),
                             SWAP_RA_ORDER_CEILING);
        if (max_win == 1) {
                ra_info->win = 1;
                return;
        }

        faddr = vmf->address;
        orig_pte = pte = pte_offset_map(vmf->pmd, faddr);

        fpfn = PFN_DOWN(faddr);
        ra_val = GET_SWAP_RA_VAL(vma);
        pfn = PFN_DOWN(SWAP_RA_ADDR(ra_val));
        prev_win = SWAP_RA_WIN(ra_val);
        hits = SWAP_RA_HITS(ra_val);
        ra_info->win = win = __swapin_nr_pages(pfn, fpfn, hits,
                                               max_win, prev_win);
        atomic_long_set(&vma->swap_readahead_info,
                        SWAP_RA_VAL(faddr, win, 0));

        if (win == 1) {
                pte_unmap(orig_pte);
                return;
        }

        /* Copy the PTEs because the page table may be unmapped */
        if (fpfn == pfn + 1)
                swap_ra_clamp_pfn(vma, faddr, fpfn, fpfn + win, &start, &end);
        else if (pfn == fpfn + 1)
                swap_ra_clamp_pfn(vma, faddr, fpfn - win + 1, fpfn + 1,
                                  &start, &end);
        else {
                left = (win - 1) / 2;
                swap_ra_clamp_pfn(vma, faddr, fpfn - left, fpfn + win - left,
                                  &start, &end);
        }
        ra_info->nr_pte = end - start;
        ra_info->offset = fpfn - start;
        pte -= ra_info->offset;
#ifdef CONFIG_64BIT
        ra_info->ptes = pte;
#else
        tpte = ra_info->ptes;
        for (pfn = start; pfn != end; pfn++)
                *tpte++ = *pte++;
#endif
        pte_unmap(orig_pte);
}

/**
 * swap_vma_readahead - swap in pages in hope we need them soon
 * @fentry: swap entry of this memory
 * @gfp_mask: memory allocation flags
 * @vmf: fault information
 *
 * Returns the struct page for entry and addr, after queueing swapin.
 *
 * Primitive swap readahead code. We simply read in a few pages whose
 * virtual addresses are around the fault address in the same vma.
 *
 * Caller must hold read mmap_lock if vmf->vma is not NULL.
 *
 */
static struct page *swap_vma_readahead(swp_entry_t fentry, gfp_t gfp_mask,
                                       struct vm_fault *vmf)
{
        struct blk_plug plug;
        struct vm_area_struct *vma = vmf->vma;
        struct page *page;
        pte_t *pte, pentry;
        swp_entry_t entry;
        unsigned int i;
        bool page_allocated;
        struct vma_swap_readahead ra_info = {
                .win = 1,
        };

        swap_ra_info(vmf, &ra_info);
        if (ra_info.win == 1)
                goto skip;

        blk_start_plug(&plug);
        for (i = 0, pte = ra_info.ptes; i < ra_info.nr_pte;
             i++, pte++) {
                pentry = *pte;
                if (pte_none(pentry))
                        continue;
                if (pte_present(pentry))
                        continue;
                entry = pte_to_swp_entry(pentry);
                if (unlikely(non_swap_entry(entry)))
                        continue;
                page = __read_swap_cache_async(entry, gfp_mask, vma,
                                               vmf->address, &page_allocated);
                if (!page)
                        continue;
                if (page_allocated) {
                        swap_readpage(page, false);
                        if (i != ra_info.offset) {
                                SetPageReadahead(page);
                                count_vm_event(SWAP_RA);
                        }
                }
                put_page(page);
        }
        blk_finish_plug(&plug);
        lru_add_drain();
skip:
        return read_swap_cache_async(fentry, gfp_mask, vma, vmf->address,
                                     ra_info.win == 1);
}

/**
 * swapin_readahead - swap in pages in hope we need them soon
 * @entry: swap entry of this memory
 * @gfp_mask: memory allocation flags
 * @vmf: fault information
 *
 * Returns the struct page for entry and addr, after queueing swapin.
 *
 * It's a main entry function for swap readahead. By the configuration,
 * it will read ahead blocks by cluster-based(ie, physical disk based)
 * or vma-based(ie, virtual address based on faulty address) readahead.
 */
struct page *swapin_readahead(swp_entry_t entry, gfp_t gfp_mask,
                                struct vm_fault *vmf)
{
        return swap_use_vma_readahead() ?
                        swap_vma_readahead(entry, gfp_mask, vmf) :
                        swap_cluster_readahead(entry, gfp_mask, vmf);
}

#ifdef CONFIG_SYSFS
static ssize_t vma_ra_enabled_show(struct kobject *kobj,
                                     struct kobj_attribute *attr, char *buf)
{
        return sysfs_emit(buf, "%s\n",
                          enable_vma_readahead ? "true" : "false");
}
static ssize_t vma_ra_enabled_store(struct kobject *kobj,
                                      struct kobj_attribute *attr,
                                      const char *buf, size_t count)
{
        if (!strncmp(buf, "true", 4) || !strncmp(buf, "1", 1))
                enable_vma_readahead = true;
        else if (!strncmp(buf, "false", 5) || !strncmp(buf, "0", 1))
                enable_vma_readahead = false;
        else
                return -EINVAL;

        return count;
}
static struct kobj_attribute vma_ra_enabled_attr =
        __ATTR(vma_ra_enabled, 0644, vma_ra_enabled_show,
               vma_ra_enabled_store);

static struct attribute *swap_attrs[] = {
        &vma_ra_enabled_attr.attr,
        NULL,
};

static const struct attribute_group swap_attr_group = {
        .attrs = swap_attrs,
};

static int __init swap_init_sysfs(void)
{
        int err;
        struct kobject *swap_kobj;

        swap_kobj = kobject_create_and_add("swap", mm_kobj);
        if (!swap_kobj) {
                pr_err("failed to create swap kobject\n");
                return -ENOMEM;
        }
        err = sysfs_create_group(swap_kobj, &swap_attr_group);
        if (err) {
                pr_err("failed to register swap group\n");
                goto delete_obj;
        }
        return 0;

delete_obj:
        kobject_put(swap_kobj);
        return err;
}
subsys_initcall(swap_init_sysfs);
#endif































































































































































































































































































































































































































   49 


   49 































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
/* Copyright (C) 2019 Netronome Systems, Inc. */

#include <linux/if_arp.h>
#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/mpls.h>
#include <linux/rtnetlink.h>
#include <linux/skbuff.h>
#include <linux/tc_act/tc_mpls.h>
#include <net/mpls.h>
#include <net/netlink.h>
#include <net/pkt_sched.h>
#include <net/pkt_cls.h>
#include <net/tc_act/tc_mpls.h>

static unsigned int mpls_net_id;
static struct tc_action_ops act_mpls_ops;

#define ACT_MPLS_TTL_DEFAULT        255

static __be32 tcf_mpls_get_lse(struct mpls_shim_hdr *lse,
                               struct tcf_mpls_params *p, bool set_bos)
{
        u32 new_lse = 0;

        if (lse)
                new_lse = be32_to_cpu(lse->label_stack_entry);

        if (p->tcfm_label != ACT_MPLS_LABEL_NOT_SET) {
                new_lse &= ~MPLS_LS_LABEL_MASK;
                new_lse |= p->tcfm_label << MPLS_LS_LABEL_SHIFT;
        }
        if (p->tcfm_ttl) {
                new_lse &= ~MPLS_LS_TTL_MASK;
                new_lse |= p->tcfm_ttl << MPLS_LS_TTL_SHIFT;
        }
        if (p->tcfm_tc != ACT_MPLS_TC_NOT_SET) {
                new_lse &= ~MPLS_LS_TC_MASK;
                new_lse |= p->tcfm_tc << MPLS_LS_TC_SHIFT;
        }
        if (p->tcfm_bos != ACT_MPLS_BOS_NOT_SET) {
                new_lse &= ~MPLS_LS_S_MASK;
                new_lse |= p->tcfm_bos << MPLS_LS_S_SHIFT;
        } else if (set_bos) {
                new_lse |= 1 << MPLS_LS_S_SHIFT;
        }

        return cpu_to_be32(new_lse);
}

static int tcf_mpls_act(struct sk_buff *skb, const struct tc_action *a,
                        struct tcf_result *res)
{
        struct tcf_mpls *m = to_mpls(a);
        struct tcf_mpls_params *p;
        __be32 new_lse;
        int ret, mac_len;

        tcf_lastuse_update(&m->tcf_tm);
        bstats_cpu_update(this_cpu_ptr(m->common.cpu_bstats), skb);

        /* Ensure 'data' points at mac_header prior calling mpls manipulating
         * functions.
         */
        if (skb_at_tc_ingress(skb)) {
                skb_push_rcsum(skb, skb->mac_len);
                mac_len = skb->mac_len;
        } else {
                mac_len = skb_network_header(skb) - skb_mac_header(skb);
        }

        ret = READ_ONCE(m->tcf_action);

        p = rcu_dereference_bh(m->mpls_p);

        switch (p->tcfm_action) {
        case TCA_MPLS_ACT_POP:
                if (skb_mpls_pop(skb, p->tcfm_proto, mac_len,
                                 skb->dev && skb->dev->type == ARPHRD_ETHER))
                        goto drop;
                break;
        case TCA_MPLS_ACT_PUSH:
                new_lse = tcf_mpls_get_lse(NULL, p, !eth_p_mpls(skb_protocol(skb, true)));
                if (skb_mpls_push(skb, new_lse, p->tcfm_proto, mac_len,
                                  skb->dev && skb->dev->type == ARPHRD_ETHER))
                        goto drop;
                break;
        case TCA_MPLS_ACT_MAC_PUSH:
                if (skb_vlan_tag_present(skb)) {
                        if (__vlan_insert_inner_tag(skb, skb->vlan_proto,
                                                    skb_vlan_tag_get(skb),
                                                    ETH_HLEN) < 0)
                                goto drop;

                        skb->protocol = skb->vlan_proto;
                        __vlan_hwaccel_clear_tag(skb);
                }

                new_lse = tcf_mpls_get_lse(NULL, p, mac_len ||
                                           !eth_p_mpls(skb->protocol));

                if (skb_mpls_push(skb, new_lse, p->tcfm_proto, 0, false))
                        goto drop;
                break;
        case TCA_MPLS_ACT_MODIFY:
                if (!pskb_may_pull(skb,
                                   skb_network_offset(skb) + MPLS_HLEN))
                        goto drop;
                new_lse = tcf_mpls_get_lse(mpls_hdr(skb), p, false);
                if (skb_mpls_update_lse(skb, new_lse))
                        goto drop;
                break;
        case TCA_MPLS_ACT_DEC_TTL:
                if (skb_mpls_dec_ttl(skb))
                        goto drop;
                break;
        }

        if (skb_at_tc_ingress(skb))
                skb_pull_rcsum(skb, skb->mac_len);

        return ret;

drop:
        qstats_drop_inc(this_cpu_ptr(m->common.cpu_qstats));
        return TC_ACT_SHOT;
}

static int valid_label(const struct nlattr *attr,
                       struct netlink_ext_ack *extack)
{
        const u32 *label = nla_data(attr);

        if (nla_len(attr) != sizeof(*label)) {
                NL_SET_ERR_MSG_MOD(extack, "Invalid MPLS label length");
                return -EINVAL;
        }

        if (*label & ~MPLS_LABEL_MASK || *label == MPLS_LABEL_IMPLNULL) {
                NL_SET_ERR_MSG_MOD(extack, "MPLS label out of range");
                return -EINVAL;
        }

        return 0;
}

static const struct nla_policy mpls_policy[TCA_MPLS_MAX + 1] = {
        [TCA_MPLS_PARMS]        = NLA_POLICY_EXACT_LEN(sizeof(struct tc_mpls)),
        [TCA_MPLS_PROTO]        = { .type = NLA_U16 },
        [TCA_MPLS_LABEL]        = NLA_POLICY_VALIDATE_FN(NLA_BINARY,
                                                         valid_label),
        [TCA_MPLS_TC]                = NLA_POLICY_RANGE(NLA_U8, 0, 7),
        [TCA_MPLS_TTL]                = NLA_POLICY_MIN(NLA_U8, 1),
        [TCA_MPLS_BOS]                = NLA_POLICY_RANGE(NLA_U8, 0, 1),
};

static int tcf_mpls_init(struct net *net, struct nlattr *nla,
                         struct nlattr *est, struct tc_action **a,
                         struct tcf_proto *tp, u32 flags,
                         struct netlink_ext_ack *extack)
{
        struct tc_action_net *tn = net_generic(net, mpls_net_id);
        bool bind = flags & TCA_ACT_FLAGS_BIND;
        struct nlattr *tb[TCA_MPLS_MAX + 1];
        struct tcf_chain *goto_ch = NULL;
        struct tcf_mpls_params *p;
        struct tc_mpls *parm;
        bool exists = false;
        struct tcf_mpls *m;
        int ret = 0, err;
        u8 mpls_ttl = 0;
        u32 index;

        if (!nla) {
                NL_SET_ERR_MSG_MOD(extack, "Missing netlink attributes");
                return -EINVAL;
        }

        err = nla_parse_nested(tb, TCA_MPLS_MAX, nla, mpls_policy, extack);
        if (err < 0)
                return err;

        if (!tb[TCA_MPLS_PARMS]) {
                NL_SET_ERR_MSG_MOD(extack, "No MPLS params");
                return -EINVAL;
        }
        parm = nla_data(tb[TCA_MPLS_PARMS]);
        index = parm->index;

        err = tcf_idr_check_alloc(tn, &index, a, bind);
        if (err < 0)
                return err;
        exists = err;
        if (exists && bind)
                return 0;

        if (!exists) {
                ret = tcf_idr_create(tn, index, est, a, &act_mpls_ops, bind,
                                     true, flags);
                if (ret) {
                        tcf_idr_cleanup(tn, index);
                        return ret;
                }

                ret = ACT_P_CREATED;
        } else if (!(flags & TCA_ACT_FLAGS_REPLACE)) {
                tcf_idr_release(*a, bind);
                return -EEXIST;
        }

        /* Verify parameters against action type. */
        switch (parm->m_action) {
        case TCA_MPLS_ACT_POP:
                if (!tb[TCA_MPLS_PROTO]) {
                        NL_SET_ERR_MSG_MOD(extack, "Protocol must be set for MPLS pop");
                        err = -EINVAL;
                        goto release_idr;
                }
                if (!eth_proto_is_802_3(nla_get_be16(tb[TCA_MPLS_PROTO]))) {
                        NL_SET_ERR_MSG_MOD(extack, "Invalid protocol type for MPLS pop");
                        err = -EINVAL;
                        goto release_idr;
                }
                if (tb[TCA_MPLS_LABEL] || tb[TCA_MPLS_TTL] || tb[TCA_MPLS_TC] ||
                    tb[TCA_MPLS_BOS]) {
                        NL_SET_ERR_MSG_MOD(extack, "Label, TTL, TC or BOS cannot be used with MPLS pop");
                        err = -EINVAL;
                        goto release_idr;
                }
                break;
        case TCA_MPLS_ACT_DEC_TTL:
                if (tb[TCA_MPLS_PROTO] || tb[TCA_MPLS_LABEL] ||
                    tb[TCA_MPLS_TTL] || tb[TCA_MPLS_TC] || tb[TCA_MPLS_BOS]) {
                        NL_SET_ERR_MSG_MOD(extack, "Label, TTL, TC, BOS or protocol cannot be used with MPLS dec_ttl");
                        err = -EINVAL;
                        goto release_idr;
                }
                break;
        case TCA_MPLS_ACT_PUSH:
        case TCA_MPLS_ACT_MAC_PUSH:
                if (!tb[TCA_MPLS_LABEL]) {
                        NL_SET_ERR_MSG_MOD(extack, "Label is required for MPLS push");
                        err = -EINVAL;
                        goto release_idr;
                }
                if (tb[TCA_MPLS_PROTO] &&
                    !eth_p_mpls(nla_get_be16(tb[TCA_MPLS_PROTO]))) {
                        NL_SET_ERR_MSG_MOD(extack, "Protocol must be an MPLS type for MPLS push");
                        err = -EPROTONOSUPPORT;
                        goto release_idr;
                }
                /* Push needs a TTL - if not specified, set a default value. */
                if (!tb[TCA_MPLS_TTL]) {
#if IS_ENABLED(CONFIG_MPLS)
                        mpls_ttl = net->mpls.default_ttl ?
                                   net->mpls.default_ttl : ACT_MPLS_TTL_DEFAULT;
#else
                        mpls_ttl = ACT_MPLS_TTL_DEFAULT;
#endif
                }
                break;
        case TCA_MPLS_ACT_MODIFY:
                if (tb[TCA_MPLS_PROTO]) {
                        NL_SET_ERR_MSG_MOD(extack, "Protocol cannot be used with MPLS modify");
                        err = -EINVAL;
                        goto release_idr;
                }
                break;
        default:
                NL_SET_ERR_MSG_MOD(extack, "Unknown MPLS action");
                err = -EINVAL;
                goto release_idr;
        }

        err = tcf_action_check_ctrlact(parm->action, tp, &goto_ch, extack);
        if (err < 0)
                goto release_idr;

        m = to_mpls(*a);

        p = kzalloc(sizeof(*p), GFP_KERNEL);
        if (!p) {
                err = -ENOMEM;
                goto put_chain;
        }

        p->tcfm_action = parm->m_action;
        p->tcfm_label = tb[TCA_MPLS_LABEL] ? nla_get_u32(tb[TCA_MPLS_LABEL]) :
                                             ACT_MPLS_LABEL_NOT_SET;
        p->tcfm_tc = tb[TCA_MPLS_TC] ? nla_get_u8(tb[TCA_MPLS_TC]) :
                                       ACT_MPLS_TC_NOT_SET;
        p->tcfm_ttl = tb[TCA_MPLS_TTL] ? nla_get_u8(tb[TCA_MPLS_TTL]) :
                                         mpls_ttl;
        p->tcfm_bos = tb[TCA_MPLS_BOS] ? nla_get_u8(tb[TCA_MPLS_BOS]) :
                                         ACT_MPLS_BOS_NOT_SET;
        p->tcfm_proto = tb[TCA_MPLS_PROTO] ? nla_get_be16(tb[TCA_MPLS_PROTO]) :
                                             htons(ETH_P_MPLS_UC);

        spin_lock_bh(&m->tcf_lock);
        goto_ch = tcf_action_set_ctrlact(*a, parm->action, goto_ch);
        p = rcu_replace_pointer(m->mpls_p, p, lockdep_is_held(&m->tcf_lock));
        spin_unlock_bh(&m->tcf_lock);

        if (goto_ch)
                tcf_chain_put_by_act(goto_ch);
        if (p)
                kfree_rcu(p, rcu);

        return ret;
put_chain:
        if (goto_ch)
                tcf_chain_put_by_act(goto_ch);
release_idr:
        tcf_idr_release(*a, bind);
        return err;
}

static void tcf_mpls_cleanup(struct tc_action *a)
{
        struct tcf_mpls *m = to_mpls(a);
        struct tcf_mpls_params *p;

        p = rcu_dereference_protected(m->mpls_p, 1);
        if (p)
                kfree_rcu(p, rcu);
}

static int tcf_mpls_dump(struct sk_buff *skb, struct tc_action *a,
                         int bind, int ref)
{
        unsigned char *b = skb_tail_pointer(skb);
        struct tcf_mpls *m = to_mpls(a);
        struct tcf_mpls_params *p;
        struct tc_mpls opt = {
                .index    = m->tcf_index,
                .refcnt   = refcount_read(&m->tcf_refcnt) - ref,
                .bindcnt  = atomic_read(&m->tcf_bindcnt) - bind,
        };
        struct tcf_t t;

        spin_lock_bh(&m->tcf_lock);
        opt.action = m->tcf_action;
        p = rcu_dereference_protected(m->mpls_p, lockdep_is_held(&m->tcf_lock));
        opt.m_action = p->tcfm_action;

        if (nla_put(skb, TCA_MPLS_PARMS, sizeof(opt), &opt))
                goto nla_put_failure;

        if (p->tcfm_label != ACT_MPLS_LABEL_NOT_SET &&
            nla_put_u32(skb, TCA_MPLS_LABEL, p->tcfm_label))
                goto nla_put_failure;

        if (p->tcfm_tc != ACT_MPLS_TC_NOT_SET &&
            nla_put_u8(skb, TCA_MPLS_TC, p->tcfm_tc))
                goto nla_put_failure;

        if (p->tcfm_ttl && nla_put_u8(skb, TCA_MPLS_TTL, p->tcfm_ttl))
                goto nla_put_failure;

        if (p->tcfm_bos != ACT_MPLS_BOS_NOT_SET &&
            nla_put_u8(skb, TCA_MPLS_BOS, p->tcfm_bos))
                goto nla_put_failure;

        if (nla_put_be16(skb, TCA_MPLS_PROTO, p->tcfm_proto))
                goto nla_put_failure;

        tcf_tm_dump(&t, &m->tcf_tm);

        if (nla_put_64bit(skb, TCA_MPLS_TM, sizeof(t), &t, TCA_MPLS_PAD))
                goto nla_put_failure;

        spin_unlock_bh(&m->tcf_lock);

        return skb->len;

nla_put_failure:
        spin_unlock_bh(&m->tcf_lock);
        nlmsg_trim(skb, b);
        return -EMSGSIZE;
}

static int tcf_mpls_walker(struct net *net, struct sk_buff *skb,
                           struct netlink_callback *cb, int type,
                           const struct tc_action_ops *ops,
                           struct netlink_ext_ack *extack)
{
        struct tc_action_net *tn = net_generic(net, mpls_net_id);

        return tcf_generic_walker(tn, skb, cb, type, ops, extack);
}

static int tcf_mpls_search(struct net *net, struct tc_action **a, u32 index)
{
        struct tc_action_net *tn = net_generic(net, mpls_net_id);

        return tcf_idr_search(tn, a, index);
}

static struct tc_action_ops act_mpls_ops = {
        .kind                =        "mpls",
        .id                =        TCA_ID_MPLS,
        .owner                =        THIS_MODULE,
        .act                =        tcf_mpls_act,
        .dump                =        tcf_mpls_dump,
        .init                =        tcf_mpls_init,
        .cleanup        =        tcf_mpls_cleanup,
        .walk                =        tcf_mpls_walker,
        .lookup                =        tcf_mpls_search,
        .size                =        sizeof(struct tcf_mpls),
};

static __net_init int mpls_init_net(struct net *net)
{
        struct tc_action_net *tn = net_generic(net, mpls_net_id);

        return tc_action_net_init(net, tn, &act_mpls_ops);
}

static void __net_exit mpls_exit_net(struct list_head *net_list)
{
        tc_action_net_exit(net_list, mpls_net_id);
}

static struct pernet_operations mpls_net_ops = {
        .init = mpls_init_net,
        .exit_batch = mpls_exit_net,
        .id   = &mpls_net_id,
        .size = sizeof(struct tc_action_net),
};

static int __init mpls_init_module(void)
{
        return tcf_register_action(&act_mpls_ops, &mpls_net_ops);
}

static void __exit mpls_cleanup_module(void)
{
        tcf_unregister_action(&act_mpls_ops, &mpls_net_ops);
}

module_init(mpls_init_module);
module_exit(mpls_cleanup_module);

MODULE_SOFTDEP("post: mpls_gso");
MODULE_AUTHOR("Netronome Systems <oss-drivers@netronome.com>");
MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("MPLS manipulation actions");











































































































































    5 










    5 






























    5 



































































































































    5 




























































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef INT_BLK_MQ_H
#define INT_BLK_MQ_H

#include "blk-stat.h"
#include "blk-mq-tag.h"

struct blk_mq_tag_set;

struct blk_mq_ctxs {
        struct kobject kobj;
        struct blk_mq_ctx __percpu        *queue_ctx;
};

/**
 * struct blk_mq_ctx - State for a software queue facing the submitting CPUs
 */
struct blk_mq_ctx {
        struct {
                spinlock_t                lock;
                struct list_head        rq_lists[HCTX_MAX_TYPES];
        } ____cacheline_aligned_in_smp;

        unsigned int                cpu;
        unsigned short                index_hw[HCTX_MAX_TYPES];
        struct blk_mq_hw_ctx         *hctxs[HCTX_MAX_TYPES];

        /* incremented at dispatch time */
        unsigned long                rq_dispatched[2];
        unsigned long                rq_merged;

        /* incremented at completion time */
        unsigned long                ____cacheline_aligned_in_smp rq_completed[2];

        struct request_queue        *queue;
        struct blk_mq_ctxs      *ctxs;
        struct kobject                kobj;
} ____cacheline_aligned_in_smp;

void blk_mq_exit_queue(struct request_queue *q);
int blk_mq_update_nr_requests(struct request_queue *q, unsigned int nr);
void blk_mq_wake_waiters(struct request_queue *q);
bool blk_mq_dispatch_rq_list(struct blk_mq_hw_ctx *hctx, struct list_head *,
                             unsigned int);
void blk_mq_add_to_requeue_list(struct request *rq, bool at_head,
                                bool kick_requeue_list);
void blk_mq_flush_busy_ctxs(struct blk_mq_hw_ctx *hctx, struct list_head *list);
struct request *blk_mq_dequeue_from_ctx(struct blk_mq_hw_ctx *hctx,
                                        struct blk_mq_ctx *start);
void blk_mq_put_rq_ref(struct request *rq);

/*
 * Internal helpers for allocating/freeing the request map
 */
void blk_mq_free_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags,
                     unsigned int hctx_idx);
void blk_mq_free_rq_map(struct blk_mq_tags *tags, unsigned int flags);
struct blk_mq_tags *blk_mq_alloc_rq_map(struct blk_mq_tag_set *set,
                                        unsigned int hctx_idx,
                                        unsigned int nr_tags,
                                        unsigned int reserved_tags,
                                        unsigned int flags);
int blk_mq_alloc_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags,
                     unsigned int hctx_idx, unsigned int depth);

/*
 * Internal helpers for request insertion into sw queues
 */
void __blk_mq_insert_request(struct blk_mq_hw_ctx *hctx, struct request *rq,
                                bool at_head);
void blk_mq_request_bypass_insert(struct request *rq, bool at_head,
                                  bool run_queue);
void blk_mq_insert_requests(struct blk_mq_hw_ctx *hctx, struct blk_mq_ctx *ctx,
                                struct list_head *list);

/* Used by blk_insert_cloned_request() to issue request directly */
blk_status_t blk_mq_request_issue_directly(struct request *rq, bool last);
void blk_mq_try_issue_list_directly(struct blk_mq_hw_ctx *hctx,
                                    struct list_head *list);

/*
 * CPU -> queue mappings
 */
extern int blk_mq_hw_queue_to_node(struct blk_mq_queue_map *qmap, unsigned int);

/*
 * blk_mq_map_queue_type() - map (hctx_type,cpu) to hardware queue
 * @q: request queue
 * @type: the hctx type index
 * @cpu: CPU
 */
static inline struct blk_mq_hw_ctx *blk_mq_map_queue_type(struct request_queue *q,
                                                          enum hctx_type type,
                                                          unsigned int cpu)
{
        return q->queue_hw_ctx[q->tag_set->map[type].mq_map[cpu]];
}

/*
 * blk_mq_map_queue() - map (cmd_flags,type) to hardware queue
 * @q: request queue
 * @flags: request command flags
 * @ctx: software queue cpu ctx
 */
static inline struct blk_mq_hw_ctx *blk_mq_map_queue(struct request_queue *q,
                                                     unsigned int flags,
                                                     struct blk_mq_ctx *ctx)
{
        enum hctx_type type = HCTX_TYPE_DEFAULT;

        /*
         * The caller ensure that if REQ_HIPRI, poll must be enabled.
         */
        if (flags & REQ_HIPRI)
                type = HCTX_TYPE_POLL;
        else if ((flags & REQ_OP_MASK) == REQ_OP_READ)
                type = HCTX_TYPE_READ;
        
        return ctx->hctxs[type];
}

/*
 * sysfs helpers
 */
extern void blk_mq_sysfs_init(struct request_queue *q);
extern void blk_mq_sysfs_deinit(struct request_queue *q);
extern int __blk_mq_register_dev(struct device *dev, struct request_queue *q);
extern int blk_mq_sysfs_register(struct request_queue *q);
extern void blk_mq_sysfs_unregister(struct request_queue *q);
extern void blk_mq_hctx_kobj_init(struct blk_mq_hw_ctx *hctx);

void blk_mq_cancel_work_sync(struct request_queue *q);

void blk_mq_release(struct request_queue *q);

static inline struct blk_mq_ctx *__blk_mq_get_ctx(struct request_queue *q,
                                           unsigned int cpu)
{
        return per_cpu_ptr(q->queue_ctx, cpu);
}

/*
 * This assumes per-cpu software queueing queues. They could be per-node
 * as well, for instance. For now this is hardcoded as-is. Note that we don't
 * care about preemption, since we know the ctx's are persistent. This does
 * mean that we can't rely on ctx always matching the currently running CPU.
 */
static inline struct blk_mq_ctx *blk_mq_get_ctx(struct request_queue *q)
{
        return __blk_mq_get_ctx(q, raw_smp_processor_id());
}

struct blk_mq_alloc_data {
        /* input parameter */
        struct request_queue *q;
        blk_mq_req_flags_t flags;
        unsigned int shallow_depth;
        unsigned int cmd_flags;

        /* input & output parameter */
        struct blk_mq_ctx *ctx;
        struct blk_mq_hw_ctx *hctx;
};

static inline bool blk_mq_is_sbitmap_shared(unsigned int flags)
{
        return flags & BLK_MQ_F_TAG_HCTX_SHARED;
}

static inline struct blk_mq_tags *blk_mq_tags_from_data(struct blk_mq_alloc_data *data)
{
        if (data->q->elevator)
                return data->hctx->sched_tags;

        return data->hctx->tags;
}

static inline bool blk_mq_hctx_stopped(struct blk_mq_hw_ctx *hctx)
{
        /* Fast path: hardware queue is not stopped most of the time. */
        if (likely(!test_bit(BLK_MQ_S_STOPPED, &hctx->state)))
                return false;

        /*
         * This barrier is used to order adding of dispatch list before and
         * the test of BLK_MQ_S_STOPPED below. Pairs with the memory barrier
         * in blk_mq_start_stopped_hw_queue() so that dispatch code could
         * either see BLK_MQ_S_STOPPED is cleared or dispatch list is not
         * empty to avoid missing dispatching requests.
         */
        smp_mb();

        return test_bit(BLK_MQ_S_STOPPED, &hctx->state);
}

static inline bool blk_mq_hw_queue_mapped(struct blk_mq_hw_ctx *hctx)
{
        return hctx->nr_ctx && hctx->tags;
}

unsigned int blk_mq_in_flight(struct request_queue *q,
                struct block_device *part);
void blk_mq_in_flight_rw(struct request_queue *q, struct block_device *part,
                unsigned int inflight[2]);

static inline void blk_mq_put_dispatch_budget(struct request_queue *q,
                                              int budget_token)
{
        if (q->mq_ops->put_budget)
                q->mq_ops->put_budget(q, budget_token);
}

static inline int blk_mq_get_dispatch_budget(struct request_queue *q)
{
        if (q->mq_ops->get_budget)
                return q->mq_ops->get_budget(q);
        return 0;
}

static inline void blk_mq_set_rq_budget_token(struct request *rq, int token)
{
        if (token < 0)
                return;

        if (rq->q->mq_ops->set_rq_budget_token)
                rq->q->mq_ops->set_rq_budget_token(rq, token);
}

static inline int blk_mq_get_rq_budget_token(struct request *rq)
{
        if (rq->q->mq_ops->get_rq_budget_token)
                return rq->q->mq_ops->get_rq_budget_token(rq);
        return -1;
}

static inline void __blk_mq_inc_active_requests(struct blk_mq_hw_ctx *hctx)
{
        if (blk_mq_is_sbitmap_shared(hctx->flags))
                atomic_inc(&hctx->queue->nr_active_requests_shared_sbitmap);
        else
                atomic_inc(&hctx->nr_active);
}

static inline void __blk_mq_dec_active_requests(struct blk_mq_hw_ctx *hctx)
{
        if (blk_mq_is_sbitmap_shared(hctx->flags))
                atomic_dec(&hctx->queue->nr_active_requests_shared_sbitmap);
        else
                atomic_dec(&hctx->nr_active);
}

static inline int __blk_mq_active_requests(struct blk_mq_hw_ctx *hctx)
{
        if (blk_mq_is_sbitmap_shared(hctx->flags))
                return atomic_read(&hctx->queue->nr_active_requests_shared_sbitmap);
        return atomic_read(&hctx->nr_active);
}
static inline void __blk_mq_put_driver_tag(struct blk_mq_hw_ctx *hctx,
                                           struct request *rq)
{
        blk_mq_put_tag(hctx->tags, rq->mq_ctx, rq->tag);
        rq->tag = BLK_MQ_NO_TAG;

        if (rq->rq_flags & RQF_MQ_INFLIGHT) {
                rq->rq_flags &= ~RQF_MQ_INFLIGHT;
                __blk_mq_dec_active_requests(hctx);
        }
}

static inline void blk_mq_put_driver_tag(struct request *rq)
{
        if (rq->tag == BLK_MQ_NO_TAG || rq->internal_tag == BLK_MQ_NO_TAG)
                return;

        __blk_mq_put_driver_tag(rq->mq_hctx, rq);
}

bool blk_mq_get_driver_tag(struct request *rq);

static inline void blk_mq_clear_mq_map(struct blk_mq_queue_map *qmap)
{
        int cpu;

        for_each_possible_cpu(cpu)
                qmap->mq_map[cpu] = 0;
}

/*
 * blk_mq_plug() - Get caller context plug
 * @q: request queue
 * @bio : the bio being submitted by the caller context
 *
 * Plugging, by design, may delay the insertion of BIOs into the elevator in
 * order to increase BIO merging opportunities. This however can cause BIO
 * insertion order to change from the order in which submit_bio() is being
 * executed in the case of multiple contexts concurrently issuing BIOs to a
 * device, even if these context are synchronized to tightly control BIO issuing
 * order. While this is not a problem with regular block devices, this ordering
 * change can cause write BIO failures with zoned block devices as these
 * require sequential write patterns to zones. Prevent this from happening by
 * ignoring the plug state of a BIO issuing context if the target request queue
 * is for a zoned block device and the BIO to plug is a write operation.
 *
 * Return current->plug if the bio can be plugged and NULL otherwise
 */
static inline struct blk_plug *blk_mq_plug(struct request_queue *q,
                                           struct bio *bio)
{
        /*
         * For regular block devices or read operations, use the context plug
         * which may be NULL if blk_start_plug() was not executed.
         */
        if (!blk_queue_is_zoned(q) || !op_is_write(bio_op(bio)))
                return current->plug;

        /* Zoned block device write operation case: do not plug the BIO */
        return NULL;
}

/* Free all requests on the list */
static inline void blk_mq_free_requests(struct list_head *list)
{
        while (!list_empty(list)) {
                struct request *rq = list_entry_rq(list->next);

                list_del_init(&rq->queuelist);
                blk_mq_free_request(rq);
        }
}

/*
 * For shared tag users, we track the number of currently active users
 * and attempt to provide a fair share of the tag depth for each of them.
 */
static inline bool hctx_may_queue(struct blk_mq_hw_ctx *hctx,
                                  struct sbitmap_queue *bt)
{
        unsigned int depth, users;

        if (!hctx || !(hctx->flags & BLK_MQ_F_TAG_QUEUE_SHARED))
                return true;

        /*
         * Don't try dividing an ant
         */
        if (bt->sb.depth == 1)
                return true;

        if (blk_mq_is_sbitmap_shared(hctx->flags)) {
                struct request_queue *q = hctx->queue;
                struct blk_mq_tag_set *set = q->tag_set;

                if (!test_bit(QUEUE_FLAG_HCTX_ACTIVE, &q->queue_flags))
                        return true;
                users = atomic_read(&set->active_queues_shared_sbitmap);
        } else {
                if (!test_bit(BLK_MQ_S_TAG_ACTIVE, &hctx->state))
                        return true;
                users = atomic_read(&hctx->tags->active_queues);
        }

        if (!users)
                return true;

        /*
         * Allow at least some tags
         */
        depth = max((bt->sb.depth + users - 1) / users, 4U);
        return __blk_mq_active_requests(hctx) < depth;
}


#endif












































































































   56 

   57 







   34 

    1 





















   46 














   29 
    1 
   28 






   72 























   72 







   64 





   64 

   39 






   46 






   45 

   38 
   38 















   85 



   64 
   84 
   64 











   20 
    1 










    8 

   27 













   46 





   23 

   24 











   46 














   46 












   41 










   33 
    3 

   34 
   34 
   34 


   34 






   36 





   38 







   42 








   19 


   68 


   68 
   34 


   68 



   34 





   34 


   34 






    3 
   32 





   35 




   21 






   15 







    5 





   27 






   36 







   33 



    4 


   22 












   72 
   34 
















  102 
   82 

















   65 








   86 

   68 











   40 

    8 











   68 









   86 

   49 

   65 

   74 
   32 


   34 
   34 


   68 


    4 








   13 






    9 










   10 







   10 

















    1 

    1 



































   68 




















   68 











   87 










   87 







   44 



   38 

   60 


   32 










   88 





    5 


   86 


   86 



   47 


   33 






   33 




   47 
   47 





    2 


   34 
   34 
   34 





    1 


   61 


    2 


   19 

   31 
























































   64 









    1 




   63 


   17 









   14 




    1 





    6 
    6 





    6 



















    3 













    2 



    1 
    2 


    3 
    1 









1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
/*
 * net/tipc/group.c: TIPC group messaging code
 *
 * Copyright (c) 2017, Ericsson AB
 * Copyright (c) 2020, Red Hat Inc
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. Neither the names of the copyright holders nor the names of its
 *    contributors may be used to endorse or promote products derived from
 *    this software without specific prior written permission.
 *
 * Alternatively, this software may be distributed under the terms of the
 * GNU General Public License ("GPL") version 2 as published by the Free
 * Software Foundation.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

#include "core.h"
#include "addr.h"
#include "group.h"
#include "bcast.h"
#include "topsrv.h"
#include "msg.h"
#include "socket.h"
#include "node.h"
#include "name_table.h"
#include "subscr.h"

#define ADV_UNIT (((MAX_MSG_SIZE + MAX_H_SIZE) / FLOWCTL_BLK_SZ) + 1)
#define ADV_IDLE ADV_UNIT
#define ADV_ACTIVE (ADV_UNIT * 12)

enum mbr_state {
        MBR_JOINING,
        MBR_PUBLISHED,
        MBR_JOINED,
        MBR_PENDING,
        MBR_ACTIVE,
        MBR_RECLAIMING,
        MBR_REMITTED,
        MBR_LEAVING
};

struct tipc_member {
        struct rb_node tree_node;
        struct list_head list;
        struct list_head small_win;
        struct sk_buff_head deferredq;
        struct tipc_group *group;
        u32 node;
        u32 port;
        u32 instance;
        enum mbr_state state;
        u16 advertised;
        u16 window;
        u16 bc_rcv_nxt;
        u16 bc_syncpt;
        u16 bc_acked;
};

struct tipc_group {
        struct rb_root members;
        struct list_head small_win;
        struct list_head pending;
        struct list_head active;
        struct tipc_nlist dests;
        struct net *net;
        int subid;
        u32 type;
        u32 instance;
        u32 scope;
        u32 portid;
        u16 member_cnt;
        u16 active_cnt;
        u16 max_active;
        u16 bc_snd_nxt;
        u16 bc_ackers;
        bool *open;
        bool loopback;
        bool events;
};

static void tipc_group_proto_xmit(struct tipc_group *grp, struct tipc_member *m,
                                  int mtyp, struct sk_buff_head *xmitq);

static void tipc_group_open(struct tipc_member *m, bool *wakeup)
{
        *wakeup = false;
        if (list_empty(&m->small_win))
                return;
        list_del_init(&m->small_win);
        *m->group->open = true;
        *wakeup = true;
}

static void tipc_group_decr_active(struct tipc_group *grp,
                                   struct tipc_member *m)
{
        if (m->state == MBR_ACTIVE || m->state == MBR_RECLAIMING ||
            m->state == MBR_REMITTED)
                grp->active_cnt--;
}

static int tipc_group_rcvbuf_limit(struct tipc_group *grp)
{
        int max_active, active_pool, idle_pool;
        int mcnt = grp->member_cnt + 1;

        /* Limit simultaneous reception from other members */
        max_active = min(mcnt / 8, 64);
        max_active = max(max_active, 16);
        grp->max_active = max_active;

        /* Reserve blocks for active and idle members */
        active_pool = max_active * ADV_ACTIVE;
        idle_pool = (mcnt - max_active) * ADV_IDLE;

        /* Scale to bytes, considering worst-case truesize/msgsize ratio */
        return (active_pool + idle_pool) * FLOWCTL_BLK_SZ * 4;
}

u16 tipc_group_bc_snd_nxt(struct tipc_group *grp)
{
        return grp->bc_snd_nxt;
}

static bool tipc_group_is_receiver(struct tipc_member *m)
{
        return m && m->state != MBR_JOINING && m->state != MBR_LEAVING;
}

static bool tipc_group_is_sender(struct tipc_member *m)
{
        return m && m->state != MBR_JOINING && m->state != MBR_PUBLISHED;
}

u32 tipc_group_exclude(struct tipc_group *grp)
{
        if (!grp->loopback)
                return grp->portid;
        return 0;
}

struct tipc_group *tipc_group_create(struct net *net, u32 portid,
                                     struct tipc_group_req *mreq,
                                     bool *group_is_open)
{
        u32 filter = TIPC_SUB_PORTS | TIPC_SUB_NO_STATUS;
        bool global = mreq->scope != TIPC_NODE_SCOPE;
        struct tipc_group *grp;
        u32 type = mreq->type;

        grp = kzalloc(sizeof(*grp), GFP_ATOMIC);
        if (!grp)
                return NULL;
        tipc_nlist_init(&grp->dests, tipc_own_addr(net));
        INIT_LIST_HEAD(&grp->small_win);
        INIT_LIST_HEAD(&grp->active);
        INIT_LIST_HEAD(&grp->pending);
        grp->members = RB_ROOT;
        grp->net = net;
        grp->portid = portid;
        grp->type = type;
        grp->instance = mreq->instance;
        grp->scope = mreq->scope;
        grp->loopback = mreq->flags & TIPC_GROUP_LOOPBACK;
        grp->events = mreq->flags & TIPC_GROUP_MEMBER_EVTS;
        grp->open = group_is_open;
        *grp->open = false;
        filter |= global ? TIPC_SUB_CLUSTER_SCOPE : TIPC_SUB_NODE_SCOPE;
        if (tipc_topsrv_kern_subscr(net, portid, type, 0, ~0,
                                    filter, &grp->subid))
                return grp;
        kfree(grp);
        return NULL;
}

void tipc_group_join(struct net *net, struct tipc_group *grp, int *sk_rcvbuf)
{
        struct rb_root *tree = &grp->members;
        struct tipc_member *m, *tmp;
        struct sk_buff_head xmitq;

        __skb_queue_head_init(&xmitq);
        rbtree_postorder_for_each_entry_safe(m, tmp, tree, tree_node) {
                tipc_group_proto_xmit(grp, m, GRP_JOIN_MSG, &xmitq);
                tipc_group_update_member(m, 0);
        }
        tipc_node_distr_xmit(net, &xmitq);
        *sk_rcvbuf = tipc_group_rcvbuf_limit(grp);
}

void tipc_group_delete(struct net *net, struct tipc_group *grp)
{
        struct rb_root *tree = &grp->members;
        struct tipc_member *m, *tmp;
        struct sk_buff_head xmitq;

        __skb_queue_head_init(&xmitq);

        rbtree_postorder_for_each_entry_safe(m, tmp, tree, tree_node) {
                tipc_group_proto_xmit(grp, m, GRP_LEAVE_MSG, &xmitq);
                __skb_queue_purge(&m->deferredq);
                list_del(&m->list);
                kfree(m);
        }
        tipc_node_distr_xmit(net, &xmitq);
        tipc_nlist_purge(&grp->dests);
        tipc_topsrv_kern_unsubscr(net, grp->subid);
        kfree(grp);
}

static struct tipc_member *tipc_group_find_member(struct tipc_group *grp,
                                                  u32 node, u32 port)
{
        struct rb_node *n = grp->members.rb_node;
        u64 nkey, key = (u64)node << 32 | port;
        struct tipc_member *m;

        while (n) {
                m = container_of(n, struct tipc_member, tree_node);
                nkey = (u64)m->node << 32 | m->port;
                if (key < nkey)
                        n = n->rb_left;
                else if (key > nkey)
                        n = n->rb_right;
                else
                        return m;
        }
        return NULL;
}

static struct tipc_member *tipc_group_find_dest(struct tipc_group *grp,
                                                u32 node, u32 port)
{
        struct tipc_member *m;

        m = tipc_group_find_member(grp, node, port);
        if (m && tipc_group_is_receiver(m))
                return m;
        return NULL;
}

static struct tipc_member *tipc_group_find_node(struct tipc_group *grp,
                                                u32 node)
{
        struct tipc_member *m;
        struct rb_node *n;

        for (n = rb_first(&grp->members); n; n = rb_next(n)) {
                m = container_of(n, struct tipc_member, tree_node);
                if (m->node == node)
                        return m;
        }
        return NULL;
}

static int tipc_group_add_to_tree(struct tipc_group *grp,
                                  struct tipc_member *m)
{
        u64 nkey, key = (u64)m->node << 32 | m->port;
        struct rb_node **n, *parent = NULL;
        struct tipc_member *tmp;

        n = &grp->members.rb_node;
        while (*n) {
                tmp = container_of(*n, struct tipc_member, tree_node);
                parent = *n;
                tmp = container_of(parent, struct tipc_member, tree_node);
                nkey = (u64)tmp->node << 32 | tmp->port;
                if (key < nkey)
                        n = &(*n)->rb_left;
                else if (key > nkey)
                        n = &(*n)->rb_right;
                else
                        return -EEXIST;
        }
        rb_link_node(&m->tree_node, parent, n);
        rb_insert_color(&m->tree_node, &grp->members);
        return 0;
}

static struct tipc_member *tipc_group_create_member(struct tipc_group *grp,
                                                    u32 node, u32 port,
                                                    u32 instance, int state)
{
        struct tipc_member *m;
        int ret;

        m = kzalloc(sizeof(*m), GFP_ATOMIC);
        if (!m)
                return NULL;
        INIT_LIST_HEAD(&m->list);
        INIT_LIST_HEAD(&m->small_win);
        __skb_queue_head_init(&m->deferredq);
        m->group = grp;
        m->node = node;
        m->port = port;
        m->instance = instance;
        m->bc_acked = grp->bc_snd_nxt - 1;
        ret = tipc_group_add_to_tree(grp, m);
        if (ret < 0) {
                kfree(m);
                return NULL;
        }
        grp->member_cnt++;
        tipc_nlist_add(&grp->dests, m->node);
        m->state = state;
        return m;
}

void tipc_group_add_member(struct tipc_group *grp, u32 node,
                           u32 port, u32 instance)
{
        tipc_group_create_member(grp, node, port, instance, MBR_PUBLISHED);
}

static void tipc_group_delete_member(struct tipc_group *grp,
                                     struct tipc_member *m)
{
        rb_erase(&m->tree_node, &grp->members);
        grp->member_cnt--;

        /* Check if we were waiting for replicast ack from this member */
        if (grp->bc_ackers && less(m->bc_acked, grp->bc_snd_nxt - 1))
                grp->bc_ackers--;

        list_del_init(&m->list);
        list_del_init(&m->small_win);
        tipc_group_decr_active(grp, m);

        /* If last member on a node, remove node from dest list */
        if (!tipc_group_find_node(grp, m->node))
                tipc_nlist_del(&grp->dests, m->node);

        kfree(m);
}

struct tipc_nlist *tipc_group_dests(struct tipc_group *grp)
{
        return &grp->dests;
}

void tipc_group_self(struct tipc_group *grp, struct tipc_service_range *seq,
                     int *scope)
{
        seq->type = grp->type;
        seq->lower = grp->instance;
        seq->upper = grp->instance;
        *scope = grp->scope;
}

void tipc_group_update_member(struct tipc_member *m, int len)
{
        struct tipc_group *grp = m->group;
        struct tipc_member *_m, *tmp;

        if (!tipc_group_is_receiver(m))
                return;

        m->window -= len;

        if (m->window >= ADV_IDLE)
                return;

        list_del_init(&m->small_win);

        /* Sort member into small_window members' list */
        list_for_each_entry_safe(_m, tmp, &grp->small_win, small_win) {
                if (_m->window > m->window)
                        break;
        }
        list_add_tail(&m->small_win, &_m->small_win);
}

void tipc_group_update_bc_members(struct tipc_group *grp, int len, bool ack)
{
        u16 prev = grp->bc_snd_nxt - 1;
        struct tipc_member *m;
        struct rb_node *n;
        u16 ackers = 0;

        for (n = rb_first(&grp->members); n; n = rb_next(n)) {
                m = container_of(n, struct tipc_member, tree_node);
                if (tipc_group_is_receiver(m)) {
                        tipc_group_update_member(m, len);
                        m->bc_acked = prev;
                        ackers++;
                }
        }

        /* Mark number of acknowledges to expect, if any */
        if (ack)
                grp->bc_ackers = ackers;
        grp->bc_snd_nxt++;
}

bool tipc_group_cong(struct tipc_group *grp, u32 dnode, u32 dport,
                     int len, struct tipc_member **mbr)
{
        struct sk_buff_head xmitq;
        struct tipc_member *m;
        int adv, state;

        m = tipc_group_find_dest(grp, dnode, dport);
        if (!tipc_group_is_receiver(m)) {
                *mbr = NULL;
                return false;
        }
        *mbr = m;

        if (m->window >= len)
                return false;

        *grp->open = false;

        /* If not fully advertised, do it now to prevent mutual blocking */
        adv = m->advertised;
        state = m->state;
        if (state == MBR_JOINED && adv == ADV_IDLE)
                return true;
        if (state == MBR_ACTIVE && adv == ADV_ACTIVE)
                return true;
        if (state == MBR_PENDING && adv == ADV_IDLE)
                return true;
        __skb_queue_head_init(&xmitq);
        tipc_group_proto_xmit(grp, m, GRP_ADV_MSG, &xmitq);
        tipc_node_distr_xmit(grp->net, &xmitq);
        return true;
}

bool tipc_group_bc_cong(struct tipc_group *grp, int len)
{
        struct tipc_member *m = NULL;

        /* If prev bcast was replicast, reject until all receivers have acked */
        if (grp->bc_ackers) {
                *grp->open = false;
                return true;
        }
        if (list_empty(&grp->small_win))
                return false;

        m = list_first_entry(&grp->small_win, struct tipc_member, small_win);
        if (m->window >= len)
                return false;

        return tipc_group_cong(grp, m->node, m->port, len, &m);
}

/* tipc_group_sort_msg() - sort msg into queue by bcast sequence number
 */
static void tipc_group_sort_msg(struct sk_buff *skb, struct sk_buff_head *defq)
{
        struct tipc_msg *_hdr, *hdr = buf_msg(skb);
        u16 bc_seqno = msg_grp_bc_seqno(hdr);
        struct sk_buff *_skb, *tmp;
        int mtyp = msg_type(hdr);

        /* Bcast/mcast may be bypassed by ucast or other bcast, - sort it in */
        if (mtyp == TIPC_GRP_BCAST_MSG || mtyp == TIPC_GRP_MCAST_MSG) {
                skb_queue_walk_safe(defq, _skb, tmp) {
                        _hdr = buf_msg(_skb);
                        if (!less(bc_seqno, msg_grp_bc_seqno(_hdr)))
                                continue;
                        __skb_queue_before(defq, _skb, skb);
                        return;
                }
                /* Bcast was not bypassed, - add to tail */
        }
        /* Unicasts are never bypassed, - always add to tail */
        __skb_queue_tail(defq, skb);
}

/* tipc_group_filter_msg() - determine if we should accept arriving message
 */
void tipc_group_filter_msg(struct tipc_group *grp, struct sk_buff_head *inputq,
                           struct sk_buff_head *xmitq)
{
        struct sk_buff *skb = __skb_dequeue(inputq);
        bool ack, deliver, update, leave = false;
        struct sk_buff_head *defq;
        struct tipc_member *m;
        struct tipc_msg *hdr;
        u32 node, port;
        int mtyp, blks;

        if (!skb)
                return;

        hdr = buf_msg(skb);
        node =  msg_orignode(hdr);
        port = msg_origport(hdr);

        if (!msg_in_group(hdr))
                goto drop;

        m = tipc_group_find_member(grp, node, port);
        if (!tipc_group_is_sender(m))
                goto drop;

        if (less(msg_grp_bc_seqno(hdr), m->bc_rcv_nxt))
                goto drop;

        TIPC_SKB_CB(skb)->orig_member = m->instance;
        defq = &m->deferredq;
        tipc_group_sort_msg(skb, defq);

        while ((skb = skb_peek(defq))) {
                hdr = buf_msg(skb);
                mtyp = msg_type(hdr);
                blks = msg_blocks(hdr);
                deliver = true;
                ack = false;
                update = false;

                if (more(msg_grp_bc_seqno(hdr), m->bc_rcv_nxt))
                        break;

                /* Decide what to do with message */
                switch (mtyp) {
                case TIPC_GRP_MCAST_MSG:
                        if (msg_nameinst(hdr) != grp->instance) {
                                update = true;
                                deliver = false;
                        }
                        fallthrough;
                case TIPC_GRP_BCAST_MSG:
                        m->bc_rcv_nxt++;
                        ack = msg_grp_bc_ack_req(hdr);
                        break;
                case TIPC_GRP_UCAST_MSG:
                        break;
                case TIPC_GRP_MEMBER_EVT:
                        if (m->state == MBR_LEAVING)
                                leave = true;
                        if (!grp->events)
                                deliver = false;
                        break;
                default:
                        break;
                }

                /* Execute decisions */
                __skb_dequeue(defq);
                if (deliver)
                        __skb_queue_tail(inputq, skb);
                else
                        kfree_skb(skb);

                if (ack)
                        tipc_group_proto_xmit(grp, m, GRP_ACK_MSG, xmitq);

                if (leave) {
                        __skb_queue_purge(defq);
                        tipc_group_delete_member(grp, m);
                        break;
                }
                if (!update)
                        continue;

                tipc_group_update_rcv_win(grp, blks, node, port, xmitq);
        }
        return;
drop:
        kfree_skb(skb);
}

void tipc_group_update_rcv_win(struct tipc_group *grp, int blks, u32 node,
                               u32 port, struct sk_buff_head *xmitq)
{
        struct list_head *active = &grp->active;
        int max_active = grp->max_active;
        int reclaim_limit = max_active * 3 / 4;
        int active_cnt = grp->active_cnt;
        struct tipc_member *m, *rm, *pm;

        m = tipc_group_find_member(grp, node, port);
        if (!m)
                return;

        m->advertised -= blks;

        switch (m->state) {
        case MBR_JOINED:
                /* First, decide if member can go active */
                if (active_cnt <= max_active) {
                        m->state = MBR_ACTIVE;
                        list_add_tail(&m->list, active);
                        grp->active_cnt++;
                        tipc_group_proto_xmit(grp, m, GRP_ADV_MSG, xmitq);
                } else {
                        m->state = MBR_PENDING;
                        list_add_tail(&m->list, &grp->pending);
                }

                if (active_cnt < reclaim_limit)
                        break;

                /* Reclaim from oldest active member, if possible */
                if (!list_empty(active)) {
                        rm = list_first_entry(active, struct tipc_member, list);
                        rm->state = MBR_RECLAIMING;
                        list_del_init(&rm->list);
                        tipc_group_proto_xmit(grp, rm, GRP_RECLAIM_MSG, xmitq);
                        break;
                }
                /* Nobody to reclaim from; - revert oldest pending to JOINED */
                pm = list_first_entry(&grp->pending, struct tipc_member, list);
                list_del_init(&pm->list);
                pm->state = MBR_JOINED;
                tipc_group_proto_xmit(grp, pm, GRP_ADV_MSG, xmitq);
                break;
        case MBR_ACTIVE:
                if (!list_is_last(&m->list, &grp->active))
                        list_move_tail(&m->list, &grp->active);
                if (m->advertised > (ADV_ACTIVE * 3 / 4))
                        break;
                tipc_group_proto_xmit(grp, m, GRP_ADV_MSG, xmitq);
                break;
        case MBR_REMITTED:
                if (m->advertised > ADV_IDLE)
                        break;
                m->state = MBR_JOINED;
                grp->active_cnt--;
                if (m->advertised < ADV_IDLE) {
                        pr_warn_ratelimited("Rcv unexpected msg after REMIT\n");
                        tipc_group_proto_xmit(grp, m, GRP_ADV_MSG, xmitq);
                }

                if (list_empty(&grp->pending))
                        return;

                /* Set oldest pending member to active and advertise */
                pm = list_first_entry(&grp->pending, struct tipc_member, list);
                pm->state = MBR_ACTIVE;
                list_move_tail(&pm->list, &grp->active);
                grp->active_cnt++;
                tipc_group_proto_xmit(grp, pm, GRP_ADV_MSG, xmitq);
                break;
        case MBR_RECLAIMING:
        case MBR_JOINING:
        case MBR_LEAVING:
        default:
                break;
        }
}

static void tipc_group_create_event(struct tipc_group *grp,
                                    struct tipc_member *m,
                                    u32 event, u16 seqno,
                                    struct sk_buff_head *inputq)
{        u32 dnode = tipc_own_addr(grp->net);
        struct tipc_event evt;
        struct sk_buff *skb;
        struct tipc_msg *hdr;

        memset(&evt, 0, sizeof(evt));
        evt.event = event;
        evt.found_lower = m->instance;
        evt.found_upper = m->instance;
        evt.port.ref = m->port;
        evt.port.node = m->node;
        evt.s.seq.type = grp->type;
        evt.s.seq.lower = m->instance;
        evt.s.seq.upper = m->instance;

        skb = tipc_msg_create(TIPC_CRITICAL_IMPORTANCE, TIPC_GRP_MEMBER_EVT,
                              GROUP_H_SIZE, sizeof(evt), dnode, m->node,
                              grp->portid, m->port, 0);
        if (!skb)
                return;

        hdr = buf_msg(skb);
        msg_set_nametype(hdr, grp->type);
        msg_set_grp_evt(hdr, event);
        msg_set_dest_droppable(hdr, true);
        msg_set_grp_bc_seqno(hdr, seqno);
        memcpy(msg_data(hdr), &evt, sizeof(evt));
        TIPC_SKB_CB(skb)->orig_member = m->instance;
        __skb_queue_tail(inputq, skb);
}

static void tipc_group_proto_xmit(struct tipc_group *grp, struct tipc_member *m,
                                  int mtyp, struct sk_buff_head *xmitq)
{
        struct tipc_msg *hdr;
        struct sk_buff *skb;
        int adv = 0;

        skb = tipc_msg_create(GROUP_PROTOCOL, mtyp, INT_H_SIZE, 0,
                              m->node, tipc_own_addr(grp->net),
                              m->port, grp->portid, 0);
        if (!skb)
                return;

        if (m->state == MBR_ACTIVE)
                adv = ADV_ACTIVE - m->advertised;
        else if (m->state == MBR_JOINED || m->state == MBR_PENDING)
                adv = ADV_IDLE - m->advertised;

        hdr = buf_msg(skb);

        if (mtyp == GRP_JOIN_MSG) {
                msg_set_grp_bc_syncpt(hdr, grp->bc_snd_nxt);
                msg_set_adv_win(hdr, adv);
                m->advertised += adv;
        } else if (mtyp == GRP_LEAVE_MSG) {
                msg_set_grp_bc_syncpt(hdr, grp->bc_snd_nxt);
        } else if (mtyp == GRP_ADV_MSG) {
                msg_set_adv_win(hdr, adv);
                m->advertised += adv;
        } else if (mtyp == GRP_ACK_MSG) {
                msg_set_grp_bc_acked(hdr, m->bc_rcv_nxt);
        } else if (mtyp == GRP_REMIT_MSG) {
                msg_set_grp_remitted(hdr, m->window);
        }
        msg_set_dest_droppable(hdr, true);
        __skb_queue_tail(xmitq, skb);
}

void tipc_group_proto_rcv(struct tipc_group *grp, bool *usr_wakeup,
                          struct tipc_msg *hdr, struct sk_buff_head *inputq,
                          struct sk_buff_head *xmitq)
{
        u32 node = msg_orignode(hdr);
        u32 port = msg_origport(hdr);
        struct tipc_member *m, *pm;
        u16 remitted, in_flight;

        if (!grp)
                return;

        if (grp->scope == TIPC_NODE_SCOPE && node != tipc_own_addr(grp->net))
                return;

        m = tipc_group_find_member(grp, node, port);

        switch (msg_type(hdr)) {
        case GRP_JOIN_MSG:
                if (!m)
                        m = tipc_group_create_member(grp, node, port,
                                                     0, MBR_JOINING);
                if (!m)
                        return;
                m->bc_syncpt = msg_grp_bc_syncpt(hdr);
                m->bc_rcv_nxt = m->bc_syncpt;
                m->window += msg_adv_win(hdr);

                /* Wait until PUBLISH event is received if necessary */
                if (m->state != MBR_PUBLISHED)
                        return;

                /* Member can be taken into service */
                m->state = MBR_JOINED;
                tipc_group_open(m, usr_wakeup);
                tipc_group_update_member(m, 0);
                tipc_group_proto_xmit(grp, m, GRP_ADV_MSG, xmitq);
                tipc_group_create_event(grp, m, TIPC_PUBLISHED,
                                        m->bc_syncpt, inputq);
                return;
        case GRP_LEAVE_MSG:
                if (!m)
                        return;
                m->bc_syncpt = msg_grp_bc_syncpt(hdr);
                list_del_init(&m->list);
                tipc_group_open(m, usr_wakeup);
                tipc_group_decr_active(grp, m);
                m->state = MBR_LEAVING;
                tipc_group_create_event(grp, m, TIPC_WITHDRAWN,
                                        m->bc_syncpt, inputq);
                return;
        case GRP_ADV_MSG:
                if (!m)
                        return;
                m->window += msg_adv_win(hdr);
                tipc_group_open(m, usr_wakeup);
                return;
        case GRP_ACK_MSG:
                if (!m)
                        return;
                m->bc_acked = msg_grp_bc_acked(hdr);
                if (--grp->bc_ackers)
                        return;
                list_del_init(&m->small_win);
                *m->group->open = true;
                *usr_wakeup = true;
                tipc_group_update_member(m, 0);
                return;
        case GRP_RECLAIM_MSG:
                if (!m)
                        return;
                tipc_group_proto_xmit(grp, m, GRP_REMIT_MSG, xmitq);
                m->window = ADV_IDLE;
                tipc_group_open(m, usr_wakeup);
                return;
        case GRP_REMIT_MSG:
                if (!m || m->state != MBR_RECLAIMING)
                        return;

                remitted = msg_grp_remitted(hdr);

                /* Messages preceding the REMIT still in receive queue */
                if (m->advertised > remitted) {
                        m->state = MBR_REMITTED;
                        in_flight = m->advertised - remitted;
                        m->advertised = ADV_IDLE + in_flight;
                        return;
                }
                /* This should never happen */
                if (m->advertised < remitted)
                        pr_warn_ratelimited("Unexpected REMIT msg\n");

                /* All messages preceding the REMIT have been read */
                m->state = MBR_JOINED;
                grp->active_cnt--;
                m->advertised = ADV_IDLE;

                /* Set oldest pending member to active and advertise */
                if (list_empty(&grp->pending))
                        return;
                pm = list_first_entry(&grp->pending, struct tipc_member, list);
                pm->state = MBR_ACTIVE;
                list_move_tail(&pm->list, &grp->active);
                grp->active_cnt++;
                if (pm->advertised <= (ADV_ACTIVE * 3 / 4))
                        tipc_group_proto_xmit(grp, pm, GRP_ADV_MSG, xmitq);
                return;
        default:
                pr_warn("Received unknown GROUP_PROTO message\n");
        }
}

/* tipc_group_member_evt() - receive and handle a member up/down event
 */
void tipc_group_member_evt(struct tipc_group *grp,
                           bool *usr_wakeup,
                           int *sk_rcvbuf,
                           struct tipc_msg *hdr,
                           struct sk_buff_head *inputq,
                           struct sk_buff_head *xmitq)
{
        struct tipc_event *evt = (void *)msg_data(hdr);
        u32 instance = evt->found_lower;
        u32 node = evt->port.node;
        u32 port = evt->port.ref;
        int event = evt->event;
        struct tipc_member *m;
        struct net *net;
        u32 self;

        if (!grp)
                return;

        net = grp->net;
        self = tipc_own_addr(net);
        if (!grp->loopback && node == self && port == grp->portid)
                return;

        m = tipc_group_find_member(grp, node, port);

        switch (event) {
        case TIPC_PUBLISHED:
                /* Send and wait for arrival of JOIN message if necessary */
                if (!m) {
                        m = tipc_group_create_member(grp, node, port, instance,
                                                     MBR_PUBLISHED);
                        if (!m)
                                break;
                        tipc_group_update_member(m, 0);
                        tipc_group_proto_xmit(grp, m, GRP_JOIN_MSG, xmitq);
                        break;
                }

                if (m->state != MBR_JOINING)
                        break;

                /* Member can be taken into service */
                m->instance = instance;
                m->state = MBR_JOINED;
                tipc_group_open(m, usr_wakeup);
                tipc_group_update_member(m, 0);
                tipc_group_proto_xmit(grp, m, GRP_JOIN_MSG, xmitq);
                tipc_group_create_event(grp, m, TIPC_PUBLISHED,
                                        m->bc_syncpt, inputq);
                break;
        case TIPC_WITHDRAWN:
                if (!m)
                        break;

                tipc_group_decr_active(grp, m);
                m->state = MBR_LEAVING;
                list_del_init(&m->list);
                tipc_group_open(m, usr_wakeup);

                /* Only send event if no LEAVE message can be expected */
                if (!tipc_node_is_up(net, node))
                        tipc_group_create_event(grp, m, TIPC_WITHDRAWN,
                                                m->bc_rcv_nxt, inputq);
                break;
        default:
                break;
        }
        *sk_rcvbuf = tipc_group_rcvbuf_limit(grp);
}

int tipc_group_fill_sock_diag(struct tipc_group *grp, struct sk_buff *skb)
{
        struct nlattr *group = nla_nest_start_noflag(skb, TIPC_NLA_SOCK_GROUP);

        if (!group)
                return -EMSGSIZE;

        if (nla_put_u32(skb, TIPC_NLA_SOCK_GROUP_ID,
                        grp->type) ||
            nla_put_u32(skb, TIPC_NLA_SOCK_GROUP_INSTANCE,
                        grp->instance) ||
            nla_put_u32(skb, TIPC_NLA_SOCK_GROUP_BC_SEND_NEXT,
                        grp->bc_snd_nxt))
                goto group_msg_cancel;

        if (grp->scope == TIPC_NODE_SCOPE)
                if (nla_put_flag(skb, TIPC_NLA_SOCK_GROUP_NODE_SCOPE))
                        goto group_msg_cancel;

        if (grp->scope == TIPC_CLUSTER_SCOPE)
                if (nla_put_flag(skb, TIPC_NLA_SOCK_GROUP_CLUSTER_SCOPE))
                        goto group_msg_cancel;

        if (*grp->open)
                if (nla_put_flag(skb, TIPC_NLA_SOCK_GROUP_OPEN))
                        goto group_msg_cancel;

        nla_nest_end(skb, group);
        return 0;

group_msg_cancel:
        nla_nest_cancel(skb, group);
        return -1;
}
























































































































































    9 

















    9 






















   55 


    2 


   53 


   53 
   53 








    9 
    9 
    9 

    9 
    9 































   86 





  226 




  235 

   90 
  205 




   13 
  196 



  206 
    8 

  198 




  204 














   92 
   92 










    6 



   85 



    1 







































  158 









  159 
  158 





















    4 
    3 

    3 















  159 







  159 
  148 



    1 
    8 
   13 

   75 
    3 

    2 
    1 
   92 

































  225 






  224 



















  225 
  226 

  206 

   17 







  226 









  225 


  226 






















  225 













































  407 







  339 



    1 






   45 


   23 














  219 






   42 




  169 
   21 

   19 





  197 
   11 














    3 





    1 
    2 











    2 












   39 





   39 











    1 








    2 





    1 




    2 




    2 
    6 
   11 

   16 




   28 

    1 


    2 







   29 
    1 


   30 








   29 
   28 
    1 



   25 
    1 

    2 


    2 





   24 
    2 











  312 



  312 







































































   98 





   98 














   97 



   96 





   97 



   96 
   96 















    1 















































































   98 



   98 
















































   11 

    9 




































































    1 



    1 




    1 


    2 


    2 



   13 











   13 

   10 











    3 
   13 



    6 
   11 

















    6 
    6 



   13 










    1 




  104 



    9 



    1 




   96 

   89 
   11 


  105 





  141 



   17 

   18 

  106 
























































































  137 
















  132 


  126 








    4 


    4 



  125 
  127 










   98 












  272 












    1 




    1 




  265 
    1 
    4 








    5 






  222 
    4 
   39 







  262 


























    5 























  258 
  257 



   39 


  219 


  257 


  202 
   25 
    3 





































    1 











  141 



   36 

  112 


  105 
   11 


  136 
   14 




  136 


  133 
    4 


  137 






   86 









  158 

   39 



  121 




  158 


   22 


  136 




















  508 
  509 


  501 


   43 


   21 



   31 



   43 


  480 

   13 















    7 


   35 









  508 



  506 























































































  241 
  238 





    7 












    6 






  242 


  241 





  239 



  242 









  241 


   15 








  242 










  241 

  180 
  118 
  243 






   16 










   42 



   42 






   42 
   42 


   36 










  167 
  167 













  167 



























   30 






















   61 



   61 
   56 
   30 












  311 









  312 




  294 



  167 


  166 

  166 













  166 






  166 
  165 





































































































  292 





   16 


  191 










  286 




  269 




  178 

  178 














  176 






































































  171 
   73 



  110 






  103 
    8 




  100 
  128 

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * INET                An implementation of the TCP/IP protocol suite for the LINUX
 *                operating system.  INET is implemented using the  BSD Socket
 *                interface as the means of communication with the user level.
 *
 *                IPv4 Forwarding Information Base: semantics.
 *
 * Authors:        Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
 */

#include <linux/uaccess.h>
#include <linux/bitops.h>
#include <linux/types.h>
#include <linux/kernel.h>
#include <linux/jiffies.h>
#include <linux/mm.h>
#include <linux/string.h>
#include <linux/socket.h>
#include <linux/sockios.h>
#include <linux/errno.h>
#include <linux/in.h>
#include <linux/inet.h>
#include <linux/inetdevice.h>
#include <linux/netdevice.h>
#include <linux/if_arp.h>
#include <linux/proc_fs.h>
#include <linux/skbuff.h>
#include <linux/init.h>
#include <linux/slab.h>
#include <linux/netlink.h>
#include <linux/hash.h>
#include <linux/nospec.h>

#include <net/arp.h>
#include <net/ip.h>
#include <net/protocol.h>
#include <net/route.h>
#include <net/tcp.h>
#include <net/sock.h>
#include <net/ip_fib.h>
#include <net/ip6_fib.h>
#include <net/nexthop.h>
#include <net/netlink.h>
#include <net/rtnh.h>
#include <net/lwtunnel.h>
#include <net/fib_notifier.h>
#include <net/addrconf.h>

#include "fib_lookup.h"

static DEFINE_SPINLOCK(fib_info_lock);
static struct hlist_head *fib_info_hash;
static struct hlist_head *fib_info_laddrhash;
static unsigned int fib_info_hash_size;
static unsigned int fib_info_cnt;

#define DEVINDEX_HASHBITS 8
#define DEVINDEX_HASHSIZE (1U << DEVINDEX_HASHBITS)
static struct hlist_head fib_info_devhash[DEVINDEX_HASHSIZE];

/* for_nexthops and change_nexthops only used when nexthop object
 * is not set in a fib_info. The logic within can reference fib_nh.
 */
#ifdef CONFIG_IP_ROUTE_MULTIPATH

#define for_nexthops(fi) {                                                \
        int nhsel; const struct fib_nh *nh;                                \
        for (nhsel = 0, nh = (fi)->fib_nh;                                \
             nhsel < fib_info_num_path((fi));                                \
             nh++, nhsel++)

#define change_nexthops(fi) {                                                \
        int nhsel; struct fib_nh *nexthop_nh;                                \
        for (nhsel = 0,        nexthop_nh = (struct fib_nh *)((fi)->fib_nh);        \
             nhsel < fib_info_num_path((fi));                                \
             nexthop_nh++, nhsel++)

#else /* CONFIG_IP_ROUTE_MULTIPATH */

/* Hope, that gcc will optimize it to get rid of dummy loop */

#define for_nexthops(fi) {                                                \
        int nhsel; const struct fib_nh *nh = (fi)->fib_nh;                \
        for (nhsel = 0; nhsel < 1; nhsel++)

#define change_nexthops(fi) {                                                \
        int nhsel;                                                        \
        struct fib_nh *nexthop_nh = (struct fib_nh *)((fi)->fib_nh);        \
        for (nhsel = 0; nhsel < 1; nhsel++)

#endif /* CONFIG_IP_ROUTE_MULTIPATH */

#define endfor_nexthops(fi) }


const struct fib_prop fib_props[RTN_MAX + 1] = {
        [RTN_UNSPEC] = {
                .error        = 0,
                .scope        = RT_SCOPE_NOWHERE,
        },
        [RTN_UNICAST] = {
                .error        = 0,
                .scope        = RT_SCOPE_UNIVERSE,
        },
        [RTN_LOCAL] = {
                .error        = 0,
                .scope        = RT_SCOPE_HOST,
        },
        [RTN_BROADCAST] = {
                .error        = 0,
                .scope        = RT_SCOPE_LINK,
        },
        [RTN_ANYCAST] = {
                .error        = 0,
                .scope        = RT_SCOPE_LINK,
        },
        [RTN_MULTICAST] = {
                .error        = 0,
                .scope        = RT_SCOPE_UNIVERSE,
        },
        [RTN_BLACKHOLE] = {
                .error        = -EINVAL,
                .scope        = RT_SCOPE_UNIVERSE,
        },
        [RTN_UNREACHABLE] = {
                .error        = -EHOSTUNREACH,
                .scope        = RT_SCOPE_UNIVERSE,
        },
        [RTN_PROHIBIT] = {
                .error        = -EACCES,
                .scope        = RT_SCOPE_UNIVERSE,
        },
        [RTN_THROW] = {
                .error        = -EAGAIN,
                .scope        = RT_SCOPE_UNIVERSE,
        },
        [RTN_NAT] = {
                .error        = -EINVAL,
                .scope        = RT_SCOPE_NOWHERE,
        },
        [RTN_XRESOLVE] = {
                .error        = -EINVAL,
                .scope        = RT_SCOPE_NOWHERE,
        },
};

static void rt_fibinfo_free(struct rtable __rcu **rtp)
{
        struct rtable *rt = rcu_dereference_protected(*rtp, 1);

        if (!rt)
                return;

        /* Not even needed : RCU_INIT_POINTER(*rtp, NULL);
         * because we waited an RCU grace period before calling
         * free_fib_info_rcu()
         */

        dst_dev_put(&rt->dst);
        dst_release_immediate(&rt->dst);
}

static void free_nh_exceptions(struct fib_nh_common *nhc)
{
        struct fnhe_hash_bucket *hash;
        int i;

        hash = rcu_dereference_protected(nhc->nhc_exceptions, 1);
        if (!hash)
                return;
        for (i = 0; i < FNHE_HASH_SIZE; i++) {
                struct fib_nh_exception *fnhe;

                fnhe = rcu_dereference_protected(hash[i].chain, 1);
                while (fnhe) {
                        struct fib_nh_exception *next;

                        next = rcu_dereference_protected(fnhe->fnhe_next, 1);

                        rt_fibinfo_free(&fnhe->fnhe_rth_input);
                        rt_fibinfo_free(&fnhe->fnhe_rth_output);

                        kfree(fnhe);

                        fnhe = next;
                }
        }
        kfree(hash);
}

static void rt_fibinfo_free_cpus(struct rtable __rcu * __percpu *rtp)
{
        int cpu;

        if (!rtp)
                return;

        for_each_possible_cpu(cpu) {
                struct rtable *rt;

                rt = rcu_dereference_protected(*per_cpu_ptr(rtp, cpu), 1);
                if (rt) {
                        dst_dev_put(&rt->dst);
                        dst_release_immediate(&rt->dst);
                }
        }
        free_percpu(rtp);
}

void fib_nh_common_release(struct fib_nh_common *nhc)
{
        dev_put(nhc->nhc_dev);
        lwtstate_put(nhc->nhc_lwtstate);
        rt_fibinfo_free_cpus(nhc->nhc_pcpu_rth_output);
        rt_fibinfo_free(&nhc->nhc_rth_input);
        free_nh_exceptions(nhc);
}
EXPORT_SYMBOL_GPL(fib_nh_common_release);

void fib_nh_release(struct net *net, struct fib_nh *fib_nh)
{
#ifdef CONFIG_IP_ROUTE_CLASSID
        if (fib_nh->nh_tclassid)
                atomic_dec(&net->ipv4.fib_num_tclassid_users);
#endif
        fib_nh_common_release(&fib_nh->nh_common);
}

/* Release a nexthop info record */
static void free_fib_info_rcu(struct rcu_head *head)
{
        struct fib_info *fi = container_of(head, struct fib_info, rcu);

        if (fi->nh) {
                nexthop_put(fi->nh);
        } else {
                change_nexthops(fi) {
                        fib_nh_release(fi->fib_net, nexthop_nh);
                } endfor_nexthops(fi);
        }

        ip_fib_metrics_put(fi->fib_metrics);

        kfree(fi);
}

void free_fib_info(struct fib_info *fi)
{
        if (fi->fib_dead == 0) {
                pr_warn("Freeing alive fib_info %p\n", fi);
                return;
        }

        call_rcu(&fi->rcu, free_fib_info_rcu);
}
EXPORT_SYMBOL_GPL(free_fib_info);

void fib_release_info(struct fib_info *fi)
{
        spin_lock_bh(&fib_info_lock);
        if (fi && refcount_dec_and_test(&fi->fib_treeref)) {
                hlist_del(&fi->fib_hash);

                /* Paired with READ_ONCE() in fib_create_info(). */
                WRITE_ONCE(fib_info_cnt, fib_info_cnt - 1);

                if (fi->fib_prefsrc)
                        hlist_del(&fi->fib_lhash);
                if (fi->nh) {
                        list_del(&fi->nh_list);
                } else {
                        change_nexthops(fi) {
                                if (!nexthop_nh->fib_nh_dev)
                                        continue;
                                hlist_del(&nexthop_nh->nh_hash);
                        } endfor_nexthops(fi)
                }
                /* Paired with READ_ONCE() from fib_table_lookup() */
                WRITE_ONCE(fi->fib_dead, 1);
                fib_info_put(fi);
        }
        spin_unlock_bh(&fib_info_lock);
}

static inline int nh_comp(struct fib_info *fi, struct fib_info *ofi)
{
        const struct fib_nh *onh;

        if (fi->nh || ofi->nh)
                return nexthop_cmp(fi->nh, ofi->nh) ? 0 : -1;

        if (ofi->fib_nhs == 0)
                return 0;

        for_nexthops(fi) {
                onh = fib_info_nh(ofi, nhsel);

                if (nh->fib_nh_oif != onh->fib_nh_oif ||
                    nh->fib_nh_gw_family != onh->fib_nh_gw_family ||
                    nh->fib_nh_scope != onh->fib_nh_scope ||
#ifdef CONFIG_IP_ROUTE_MULTIPATH
                    nh->fib_nh_weight != onh->fib_nh_weight ||
#endif
#ifdef CONFIG_IP_ROUTE_CLASSID
                    nh->nh_tclassid != onh->nh_tclassid ||
#endif
                    lwtunnel_cmp_encap(nh->fib_nh_lws, onh->fib_nh_lws) ||
                    ((nh->fib_nh_flags ^ onh->fib_nh_flags) & ~RTNH_COMPARE_MASK))
                        return -1;

                if (nh->fib_nh_gw_family == AF_INET &&
                    nh->fib_nh_gw4 != onh->fib_nh_gw4)
                        return -1;

                if (nh->fib_nh_gw_family == AF_INET6 &&
                    ipv6_addr_cmp(&nh->fib_nh_gw6, &onh->fib_nh_gw6))
                        return -1;
        } endfor_nexthops(fi);
        return 0;
}

static inline unsigned int fib_devindex_hashfn(unsigned int val)
{
        return hash_32(val, DEVINDEX_HASHBITS);
}

static struct hlist_head *
fib_info_devhash_bucket(const struct net_device *dev)
{
        u32 val = net_hash_mix(dev_net(dev)) ^ dev->ifindex;

        return &fib_info_devhash[fib_devindex_hashfn(val)];
}

static unsigned int fib_info_hashfn_1(int init_val, u8 protocol, u8 scope,
                                      u32 prefsrc, u32 priority)
{
        unsigned int val = init_val;

        val ^= (protocol << 8) | scope;
        val ^= prefsrc;
        val ^= priority;

        return val;
}

static unsigned int fib_info_hashfn_result(unsigned int val)
{
        unsigned int mask = (fib_info_hash_size - 1);

        return (val ^ (val >> 7) ^ (val >> 12)) & mask;
}

static inline unsigned int fib_info_hashfn(struct fib_info *fi)
{
        unsigned int val;

        val = fib_info_hashfn_1(fi->fib_nhs, fi->fib_protocol,
                                fi->fib_scope, (__force u32)fi->fib_prefsrc,
                                fi->fib_priority);

        if (fi->nh) {
                val ^= fib_devindex_hashfn(fi->nh->id);
        } else {
                for_nexthops(fi) {
                        val ^= fib_devindex_hashfn(nh->fib_nh_oif);
                } endfor_nexthops(fi)
        }

        return fib_info_hashfn_result(val);
}

/* no metrics, only nexthop id */
static struct fib_info *fib_find_info_nh(struct net *net,
                                         const struct fib_config *cfg)
{
        struct hlist_head *head;
        struct fib_info *fi;
        unsigned int hash;

        hash = fib_info_hashfn_1(fib_devindex_hashfn(cfg->fc_nh_id),
                                 cfg->fc_protocol, cfg->fc_scope,
                                 (__force u32)cfg->fc_prefsrc,
                                 cfg->fc_priority);
        hash = fib_info_hashfn_result(hash);
        head = &fib_info_hash[hash];

        hlist_for_each_entry(fi, head, fib_hash) {
                if (!net_eq(fi->fib_net, net))
                        continue;
                if (!fi->nh || fi->nh->id != cfg->fc_nh_id)
                        continue;
                if (cfg->fc_protocol == fi->fib_protocol &&
                    cfg->fc_scope == fi->fib_scope &&
                    cfg->fc_prefsrc == fi->fib_prefsrc &&
                    cfg->fc_priority == fi->fib_priority &&
                    cfg->fc_type == fi->fib_type &&
                    cfg->fc_table == fi->fib_tb_id &&
                    !((cfg->fc_flags ^ fi->fib_flags) & ~RTNH_COMPARE_MASK))
                        return fi;
        }

        return NULL;
}

static struct fib_info *fib_find_info(struct fib_info *nfi)
{
        struct hlist_head *head;
        struct fib_info *fi;
        unsigned int hash;

        hash = fib_info_hashfn(nfi);
        head = &fib_info_hash[hash];

        hlist_for_each_entry(fi, head, fib_hash) {
                if (!net_eq(fi->fib_net, nfi->fib_net))
                        continue;
                if (fi->fib_nhs != nfi->fib_nhs)
                        continue;
                if (nfi->fib_protocol == fi->fib_protocol &&
                    nfi->fib_scope == fi->fib_scope &&
                    nfi->fib_prefsrc == fi->fib_prefsrc &&
                    nfi->fib_priority == fi->fib_priority &&
                    nfi->fib_type == fi->fib_type &&
                    nfi->fib_tb_id == fi->fib_tb_id &&
                    memcmp(nfi->fib_metrics, fi->fib_metrics,
                           sizeof(u32) * RTAX_MAX) == 0 &&
                    !((nfi->fib_flags ^ fi->fib_flags) & ~RTNH_COMPARE_MASK) &&
                    nh_comp(fi, nfi) == 0)
                        return fi;
        }

        return NULL;
}

/* Check, that the gateway is already configured.
 * Used only by redirect accept routine.
 */
int ip_fib_check_default(__be32 gw, struct net_device *dev)
{
        struct hlist_head *head;
        struct fib_nh *nh;

        spin_lock(&fib_info_lock);

        head = fib_info_devhash_bucket(dev);

        hlist_for_each_entry(nh, head, nh_hash) {
                if (nh->fib_nh_dev == dev &&
                    nh->fib_nh_gw4 == gw &&
                    !(nh->fib_nh_flags & RTNH_F_DEAD)) {
                        spin_unlock(&fib_info_lock);
                        return 0;
                }
        }

        spin_unlock(&fib_info_lock);

        return -1;
}

size_t fib_nlmsg_size(struct fib_info *fi)
{
        size_t payload = NLMSG_ALIGN(sizeof(struct rtmsg))
                         + nla_total_size(4) /* RTA_TABLE */
                         + nla_total_size(4) /* RTA_DST */
                         + nla_total_size(4) /* RTA_PRIORITY */
                         + nla_total_size(4) /* RTA_PREFSRC */
                         + nla_total_size(TCP_CA_NAME_MAX); /* RTAX_CC_ALGO */
        unsigned int nhs = fib_info_num_path(fi);

        /* space for nested metrics */
        payload += nla_total_size((RTAX_MAX * nla_total_size(4)));

        if (fi->nh)
                payload += nla_total_size(4); /* RTA_NH_ID */

        if (nhs) {
                size_t nh_encapsize = 0;
                /* Also handles the special case nhs == 1 */

                /* each nexthop is packed in an attribute */
                size_t nhsize = nla_total_size(sizeof(struct rtnexthop));
                unsigned int i;

                /* may contain flow and gateway attribute */
                nhsize += 2 * nla_total_size(4);

                /* grab encap info */
                for (i = 0; i < fib_info_num_path(fi); i++) {
                        struct fib_nh_common *nhc = fib_info_nhc(fi, i);

                        if (nhc->nhc_lwtstate) {
                                /* RTA_ENCAP_TYPE */
                                nh_encapsize += lwtunnel_get_encap_size(
                                                nhc->nhc_lwtstate);
                                /* RTA_ENCAP */
                                nh_encapsize +=  nla_total_size(2);
                        }
                }

                /* all nexthops are packed in a nested attribute */
                payload += nla_total_size((nhs * nhsize) + nh_encapsize);

        }

        return payload;
}

void rtmsg_fib(int event, __be32 key, struct fib_alias *fa,
               int dst_len, u32 tb_id, const struct nl_info *info,
               unsigned int nlm_flags)
{
        struct fib_rt_info fri;
        struct sk_buff *skb;
        u32 seq = info->nlh ? info->nlh->nlmsg_seq : 0;
        int err = -ENOBUFS;

        skb = nlmsg_new(fib_nlmsg_size(fa->fa_info), GFP_KERNEL);
        if (!skb)
                goto errout;

        fri.fi = fa->fa_info;
        fri.tb_id = tb_id;
        fri.dst = key;
        fri.dst_len = dst_len;
        fri.tos = fa->fa_tos;
        fri.type = fa->fa_type;
        fri.offload = READ_ONCE(fa->offload);
        fri.trap = READ_ONCE(fa->trap);
        fri.offload_failed = READ_ONCE(fa->offload_failed);
        err = fib_dump_info(skb, info->portid, seq, event, &fri, nlm_flags);
        if (err < 0) {
                /* -EMSGSIZE implies BUG in fib_nlmsg_size() */
                WARN_ON(err == -EMSGSIZE);
                kfree_skb(skb);
                goto errout;
        }
        rtnl_notify(skb, info->nl_net, info->portid, RTNLGRP_IPV4_ROUTE,
                    info->nlh, GFP_KERNEL);
        return;
errout:
        if (err < 0)
                rtnl_set_sk_err(info->nl_net, RTNLGRP_IPV4_ROUTE, err);
}

static int fib_detect_death(struct fib_info *fi, int order,
                            struct fib_info **last_resort, int *last_idx,
                            int dflt)
{
        const struct fib_nh_common *nhc = fib_info_nhc(fi, 0);
        struct neighbour *n;
        int state = NUD_NONE;

        if (likely(nhc->nhc_gw_family == AF_INET))
                n = neigh_lookup(&arp_tbl, &nhc->nhc_gw.ipv4, nhc->nhc_dev);
        else if (nhc->nhc_gw_family == AF_INET6)
                n = neigh_lookup(ipv6_stub->nd_tbl, &nhc->nhc_gw.ipv6,
                                 nhc->nhc_dev);
        else
                n = NULL;

        if (n) {
                state = n->nud_state;
                neigh_release(n);
        } else {
                return 0;
        }
        if (state == NUD_REACHABLE)
                return 0;
        if ((state & NUD_VALID) && order != dflt)
                return 0;
        if ((state & NUD_VALID) ||
            (*last_idx < 0 && order > dflt && state != NUD_INCOMPLETE)) {
                *last_resort = fi;
                *last_idx = order;
        }
        return 1;
}

int fib_nh_common_init(struct net *net, struct fib_nh_common *nhc,
                       struct nlattr *encap, u16 encap_type,
                       void *cfg, gfp_t gfp_flags,
                       struct netlink_ext_ack *extack)
{
        int err;

        nhc->nhc_pcpu_rth_output = alloc_percpu_gfp(struct rtable __rcu *,
                                                    gfp_flags);
        if (!nhc->nhc_pcpu_rth_output)
                return -ENOMEM;

        if (encap) {
                struct lwtunnel_state *lwtstate;

                if (encap_type == LWTUNNEL_ENCAP_NONE) {
                        NL_SET_ERR_MSG(extack, "LWT encap type not specified");
                        err = -EINVAL;
                        goto lwt_failure;
                }
                err = lwtunnel_build_state(net, encap_type, encap,
                                           nhc->nhc_family, cfg, &lwtstate,
                                           extack);
                if (err)
                        goto lwt_failure;

                nhc->nhc_lwtstate = lwtstate_get(lwtstate);
        }

        return 0;

lwt_failure:
        rt_fibinfo_free_cpus(nhc->nhc_pcpu_rth_output);
        nhc->nhc_pcpu_rth_output = NULL;
        return err;
}
EXPORT_SYMBOL_GPL(fib_nh_common_init);

int fib_nh_init(struct net *net, struct fib_nh *nh,
                struct fib_config *cfg, int nh_weight,
                struct netlink_ext_ack *extack)
{
        int err;

        nh->fib_nh_family = AF_INET;

        err = fib_nh_common_init(net, &nh->nh_common, cfg->fc_encap,
                                 cfg->fc_encap_type, cfg, GFP_KERNEL, extack);
        if (err)
                return err;

        nh->fib_nh_oif = cfg->fc_oif;
        nh->fib_nh_gw_family = cfg->fc_gw_family;
        if (cfg->fc_gw_family == AF_INET)
                nh->fib_nh_gw4 = cfg->fc_gw4;
        else if (cfg->fc_gw_family == AF_INET6)
                nh->fib_nh_gw6 = cfg->fc_gw6;

        nh->fib_nh_flags = cfg->fc_flags;

#ifdef CONFIG_IP_ROUTE_CLASSID
        nh->nh_tclassid = cfg->fc_flow;
        if (nh->nh_tclassid)
                atomic_inc(&net->ipv4.fib_num_tclassid_users);
#endif
#ifdef CONFIG_IP_ROUTE_MULTIPATH
        nh->fib_nh_weight = nh_weight;
#endif
        return 0;
}

#ifdef CONFIG_IP_ROUTE_MULTIPATH

static int fib_count_nexthops(struct rtnexthop *rtnh, int remaining,
                              struct netlink_ext_ack *extack)
{
        int nhs = 0;

        while (rtnh_ok(rtnh, remaining)) {
                nhs++;
                rtnh = rtnh_next(rtnh, &remaining);
        }

        /* leftover implies invalid nexthop configuration, discard it */
        if (remaining > 0) {
                NL_SET_ERR_MSG(extack,
                               "Invalid nexthop configuration - extra data after nexthops");
                nhs = 0;
        }

        return nhs;
}

static int fib_gw_from_attr(__be32 *gw, struct nlattr *nla,
                            struct netlink_ext_ack *extack)
{
        if (nla_len(nla) < sizeof(*gw)) {
                NL_SET_ERR_MSG(extack, "Invalid IPv4 address in RTA_GATEWAY");
                return -EINVAL;
        }

        *gw = nla_get_in_addr(nla);

        return 0;
}

/* only called when fib_nh is integrated into fib_info */
static int fib_get_nhs(struct fib_info *fi, struct rtnexthop *rtnh,
                       int remaining, struct fib_config *cfg,
                       struct netlink_ext_ack *extack)
{
        struct net *net = fi->fib_net;
        struct fib_config fib_cfg;
        struct fib_nh *nh;
        int ret;

        change_nexthops(fi) {
                int attrlen;

                memset(&fib_cfg, 0, sizeof(fib_cfg));

                if (!rtnh_ok(rtnh, remaining)) {
                        NL_SET_ERR_MSG(extack,
                                       "Invalid nexthop configuration - extra data after nexthop");
                        return -EINVAL;
                }

                if (rtnh->rtnh_flags & (RTNH_F_DEAD | RTNH_F_LINKDOWN)) {
                        NL_SET_ERR_MSG(extack,
                                       "Invalid flags for nexthop - can not contain DEAD or LINKDOWN");
                        return -EINVAL;
                }

                fib_cfg.fc_flags = (cfg->fc_flags & ~0xFF) | rtnh->rtnh_flags;
                fib_cfg.fc_oif = rtnh->rtnh_ifindex;

                attrlen = rtnh_attrlen(rtnh);
                if (attrlen > 0) {
                        struct nlattr *nla, *nlav, *attrs = rtnh_attrs(rtnh);

                        nla = nla_find(attrs, attrlen, RTA_GATEWAY);
                        nlav = nla_find(attrs, attrlen, RTA_VIA);
                        if (nla && nlav) {
                                NL_SET_ERR_MSG(extack,
                                               "Nexthop configuration can not contain both GATEWAY and VIA");
                                return -EINVAL;
                        }
                        if (nla) {
                                ret = fib_gw_from_attr(&fib_cfg.fc_gw4, nla,
                                                       extack);
                                if (ret)
                                        goto errout;

                                if (fib_cfg.fc_gw4)
                                        fib_cfg.fc_gw_family = AF_INET;
                        } else if (nlav) {
                                ret = fib_gw_from_via(&fib_cfg, nlav, extack);
                                if (ret)
                                        goto errout;
                        }

                        nla = nla_find(attrs, attrlen, RTA_FLOW);
                        if (nla) {
                                if (nla_len(nla) < sizeof(u32)) {
                                        NL_SET_ERR_MSG(extack, "Invalid RTA_FLOW");
                                        return -EINVAL;
                                }
                                fib_cfg.fc_flow = nla_get_u32(nla);
                        }

                        fib_cfg.fc_encap = nla_find(attrs, attrlen, RTA_ENCAP);
                        /* RTA_ENCAP_TYPE length checked in
                         * lwtunnel_valid_encap_type_attr
                         */
                        nla = nla_find(attrs, attrlen, RTA_ENCAP_TYPE);
                        if (nla)
                                fib_cfg.fc_encap_type = nla_get_u16(nla);
                }

                ret = fib_nh_init(net, nexthop_nh, &fib_cfg,
                                  rtnh->rtnh_hops + 1, extack);
                if (ret)
                        goto errout;

                rtnh = rtnh_next(rtnh, &remaining);
        } endfor_nexthops(fi);

        ret = -EINVAL;
        nh = fib_info_nh(fi, 0);
        if (cfg->fc_oif && nh->fib_nh_oif != cfg->fc_oif) {
                NL_SET_ERR_MSG(extack,
                               "Nexthop device index does not match RTA_OIF");
                goto errout;
        }
        if (cfg->fc_gw_family) {
                if (cfg->fc_gw_family != nh->fib_nh_gw_family ||
                    (cfg->fc_gw_family == AF_INET &&
                     nh->fib_nh_gw4 != cfg->fc_gw4) ||
                    (cfg->fc_gw_family == AF_INET6 &&
                     ipv6_addr_cmp(&nh->fib_nh_gw6, &cfg->fc_gw6))) {
                        NL_SET_ERR_MSG(extack,
                                       "Nexthop gateway does not match RTA_GATEWAY or RTA_VIA");
                        goto errout;
                }
        }
#ifdef CONFIG_IP_ROUTE_CLASSID
        if (cfg->fc_flow && nh->nh_tclassid != cfg->fc_flow) {
                NL_SET_ERR_MSG(extack,
                               "Nexthop class id does not match RTA_FLOW");
                goto errout;
        }
#endif
        ret = 0;
errout:
        return ret;
}

/* only called when fib_nh is integrated into fib_info */
static void fib_rebalance(struct fib_info *fi)
{
        int total;
        int w;

        if (fib_info_num_path(fi) < 2)
                return;

        total = 0;
        for_nexthops(fi) {
                if (nh->fib_nh_flags & RTNH_F_DEAD)
                        continue;

                if (ip_ignore_linkdown(nh->fib_nh_dev) &&
                    nh->fib_nh_flags & RTNH_F_LINKDOWN)
                        continue;

                total += nh->fib_nh_weight;
        } endfor_nexthops(fi);

        w = 0;
        change_nexthops(fi) {
                int upper_bound;

                if (nexthop_nh->fib_nh_flags & RTNH_F_DEAD) {
                        upper_bound = -1;
                } else if (ip_ignore_linkdown(nexthop_nh->fib_nh_dev) &&
                           nexthop_nh->fib_nh_flags & RTNH_F_LINKDOWN) {
                        upper_bound = -1;
                } else {
                        w += nexthop_nh->fib_nh_weight;
                        upper_bound = DIV_ROUND_CLOSEST_ULL((u64)w << 31,
                                                            total) - 1;
                }

                atomic_set(&nexthop_nh->fib_nh_upper_bound, upper_bound);
        } endfor_nexthops(fi);
}
#else /* CONFIG_IP_ROUTE_MULTIPATH */

static int fib_get_nhs(struct fib_info *fi, struct rtnexthop *rtnh,
                       int remaining, struct fib_config *cfg,
                       struct netlink_ext_ack *extack)
{
        NL_SET_ERR_MSG(extack, "Multipath support not enabled in kernel");

        return -EINVAL;
}

#define fib_rebalance(fi) do { } while (0)

#endif /* CONFIG_IP_ROUTE_MULTIPATH */

static int fib_encap_match(struct net *net, u16 encap_type,
                           struct nlattr *encap,
                           const struct fib_nh *nh,
                           const struct fib_config *cfg,
                           struct netlink_ext_ack *extack)
{
        struct lwtunnel_state *lwtstate;
        int ret, result = 0;

        if (encap_type == LWTUNNEL_ENCAP_NONE)
                return 0;

        ret = lwtunnel_build_state(net, encap_type, encap, AF_INET,
                                   cfg, &lwtstate, extack);
        if (!ret) {
                result = lwtunnel_cmp_encap(lwtstate, nh->fib_nh_lws);
                lwtstate_free(lwtstate);
        }

        return result;
}

int fib_nh_match(struct net *net, struct fib_config *cfg, struct fib_info *fi,
                 struct netlink_ext_ack *extack)
{
#ifdef CONFIG_IP_ROUTE_MULTIPATH
        struct rtnexthop *rtnh;
        int remaining;
#endif

        if (cfg->fc_priority && cfg->fc_priority != fi->fib_priority)
                return 1;

        if (cfg->fc_nh_id) {
                if (fi->nh && cfg->fc_nh_id == fi->nh->id)
                        return 0;
                return 1;
        }

        if (fi->nh) {
                if (cfg->fc_oif || cfg->fc_gw_family || cfg->fc_mp)
                        return 1;
                return 0;
        }

        if (cfg->fc_oif || cfg->fc_gw_family) {
                struct fib_nh *nh;

                nh = fib_info_nh(fi, 0);
                if (cfg->fc_encap) {
                        if (fib_encap_match(net, cfg->fc_encap_type,
                                            cfg->fc_encap, nh, cfg, extack))
                                return 1;
                }
#ifdef CONFIG_IP_ROUTE_CLASSID
                if (cfg->fc_flow &&
                    cfg->fc_flow != nh->nh_tclassid)
                        return 1;
#endif
                if ((cfg->fc_oif && cfg->fc_oif != nh->fib_nh_oif) ||
                    (cfg->fc_gw_family &&
                     cfg->fc_gw_family != nh->fib_nh_gw_family))
                        return 1;

                if (cfg->fc_gw_family == AF_INET &&
                    cfg->fc_gw4 != nh->fib_nh_gw4)
                        return 1;

                if (cfg->fc_gw_family == AF_INET6 &&
                    ipv6_addr_cmp(&cfg->fc_gw6, &nh->fib_nh_gw6))
                        return 1;

                return 0;
        }

#ifdef CONFIG_IP_ROUTE_MULTIPATH
        if (!cfg->fc_mp)
                return 0;

        rtnh = cfg->fc_mp;
        remaining = cfg->fc_mp_len;

        for_nexthops(fi) {
                int attrlen;

                if (!rtnh_ok(rtnh, remaining))
                        return -EINVAL;

                if (rtnh->rtnh_ifindex && rtnh->rtnh_ifindex != nh->fib_nh_oif)
                        return 1;

                attrlen = rtnh_attrlen(rtnh);
                if (attrlen > 0) {
                        struct nlattr *nla, *nlav, *attrs = rtnh_attrs(rtnh);
                        int err;

                        nla = nla_find(attrs, attrlen, RTA_GATEWAY);
                        nlav = nla_find(attrs, attrlen, RTA_VIA);
                        if (nla && nlav) {
                                NL_SET_ERR_MSG(extack,
                                               "Nexthop configuration can not contain both GATEWAY and VIA");
                                return -EINVAL;
                        }

                        if (nla) {
                                __be32 gw;

                                err = fib_gw_from_attr(&gw, nla, extack);
                                if (err)
                                        return err;

                                if (nh->fib_nh_gw_family != AF_INET ||
                                    gw != nh->fib_nh_gw4)
                                        return 1;
                        } else if (nlav) {
                                struct fib_config cfg2;

                                err = fib_gw_from_via(&cfg2, nlav, extack);
                                if (err)
                                        return err;

                                switch (nh->fib_nh_gw_family) {
                                case AF_INET:
                                        if (cfg2.fc_gw_family != AF_INET ||
                                            cfg2.fc_gw4 != nh->fib_nh_gw4)
                                                return 1;
                                        break;
                                case AF_INET6:
                                        if (cfg2.fc_gw_family != AF_INET6 ||
                                            ipv6_addr_cmp(&cfg2.fc_gw6,
                                                          &nh->fib_nh_gw6))
                                                return 1;
                                        break;
                                }
                        }

#ifdef CONFIG_IP_ROUTE_CLASSID
                        nla = nla_find(attrs, attrlen, RTA_FLOW);
                        if (nla) {
                                if (nla_len(nla) < sizeof(u32)) {
                                        NL_SET_ERR_MSG(extack, "Invalid RTA_FLOW");
                                        return -EINVAL;
                                }
                                if (nla_get_u32(nla) != nh->nh_tclassid)
                                        return 1;
                        }
#endif
                }

                rtnh = rtnh_next(rtnh, &remaining);
        } endfor_nexthops(fi);
#endif
        return 0;
}

bool fib_metrics_match(struct fib_config *cfg, struct fib_info *fi)
{
        struct nlattr *nla;
        int remaining;

        if (!cfg->fc_mx)
                return true;

        nla_for_each_attr(nla, cfg->fc_mx, cfg->fc_mx_len, remaining) {
                int type = nla_type(nla);
                u32 fi_val, val;

                if (!type)
                        continue;
                if (type > RTAX_MAX)
                        return false;

                type = array_index_nospec(type, RTAX_MAX + 1);
                if (type == RTAX_CC_ALGO) {
                        char tmp[TCP_CA_NAME_MAX];
                        bool ecn_ca = false;

                        nla_strscpy(tmp, nla, sizeof(tmp));
                        val = tcp_ca_get_key_by_name(fi->fib_net, tmp, &ecn_ca);
                } else {
                        if (nla_len(nla) != sizeof(u32))
                                return false;
                        val = nla_get_u32(nla);
                }

                fi_val = fi->fib_metrics->metrics[type - 1];
                if (type == RTAX_FEATURES)
                        fi_val &= ~DST_FEATURE_ECN_CA;

                if (fi_val != val)
                        return false;
        }

        return true;
}

static int fib_check_nh_v6_gw(struct net *net, struct fib_nh *nh,
                              u32 table, struct netlink_ext_ack *extack)
{
        struct fib6_config cfg = {
                .fc_table = table,
                .fc_flags = nh->fib_nh_flags | RTF_GATEWAY,
                .fc_ifindex = nh->fib_nh_oif,
                .fc_gateway = nh->fib_nh_gw6,
        };
        struct fib6_nh fib6_nh = {};
        int err;

        err = ipv6_stub->fib6_nh_init(net, &fib6_nh, &cfg, GFP_KERNEL, extack);
        if (!err) {
                nh->fib_nh_dev = fib6_nh.fib_nh_dev;
                dev_hold(nh->fib_nh_dev);
                nh->fib_nh_oif = nh->fib_nh_dev->ifindex;
                nh->fib_nh_scope = RT_SCOPE_LINK;

                ipv6_stub->fib6_nh_release(&fib6_nh);
        }

        return err;
}

/*
 * Picture
 * -------
 *
 * Semantics of nexthop is very messy by historical reasons.
 * We have to take into account, that:
 * a) gateway can be actually local interface address,
 *    so that gatewayed route is direct.
 * b) gateway must be on-link address, possibly
 *    described not by an ifaddr, but also by a direct route.
 * c) If both gateway and interface are specified, they should not
 *    contradict.
 * d) If we use tunnel routes, gateway could be not on-link.
 *
 * Attempt to reconcile all of these (alas, self-contradictory) conditions
 * results in pretty ugly and hairy code with obscure logic.
 *
 * I chose to generalized it instead, so that the size
 * of code does not increase practically, but it becomes
 * much more general.
 * Every prefix is assigned a "scope" value: "host" is local address,
 * "link" is direct route,
 * [ ... "site" ... "interior" ... ]
 * and "universe" is true gateway route with global meaning.
 *
 * Every prefix refers to a set of "nexthop"s (gw, oif),
 * where gw must have narrower scope. This recursion stops
 * when gw has LOCAL scope or if "nexthop" is declared ONLINK,
 * which means that gw is forced to be on link.
 *
 * Code is still hairy, but now it is apparently logically
 * consistent and very flexible. F.e. as by-product it allows
 * to co-exists in peace independent exterior and interior
 * routing processes.
 *
 * Normally it looks as following.
 *
 * {universe prefix}  -> (gw, oif) [scope link]
 *                  |
 *                  |-> {link prefix} -> (gw, oif) [scope local]
 *                                        |
 *                                        |-> {local prefix} (terminal node)
 */
static int fib_check_nh_v4_gw(struct net *net, struct fib_nh *nh, u32 table,
                              u8 scope, struct netlink_ext_ack *extack)
{
        struct net_device *dev;
        struct fib_result res;
        int err = 0;

        if (nh->fib_nh_flags & RTNH_F_ONLINK) {
                unsigned int addr_type;

                if (scope >= RT_SCOPE_LINK) {
                        NL_SET_ERR_MSG(extack, "Nexthop has invalid scope");
                        return -EINVAL;
                }
                dev = __dev_get_by_index(net, nh->fib_nh_oif);
                if (!dev) {
                        NL_SET_ERR_MSG(extack, "Nexthop device required for onlink");
                        return -ENODEV;
                }
                if (!(dev->flags & IFF_UP)) {
                        NL_SET_ERR_MSG(extack, "Nexthop device is not up");
                        return -ENETDOWN;
                }
                addr_type = inet_addr_type_dev_table(net, dev, nh->fib_nh_gw4);
                if (addr_type != RTN_UNICAST) {
                        NL_SET_ERR_MSG(extack, "Nexthop has invalid gateway");
                        return -EINVAL;
                }
                if (!netif_carrier_ok(dev))
                        nh->fib_nh_flags |= RTNH_F_LINKDOWN;
                nh->fib_nh_dev = dev;
                dev_hold(dev);
                nh->fib_nh_scope = RT_SCOPE_LINK;
                return 0;
        }
        rcu_read_lock();
        {
                struct fib_table *tbl = NULL;
                struct flowi4 fl4 = {
                        .daddr = nh->fib_nh_gw4,
                        .flowi4_scope = scope + 1,
                        .flowi4_oif = nh->fib_nh_oif,
                        .flowi4_iif = LOOPBACK_IFINDEX,
                };

                /* It is not necessary, but requires a bit of thinking */
                if (fl4.flowi4_scope < RT_SCOPE_LINK)
                        fl4.flowi4_scope = RT_SCOPE_LINK;

                if (table && table != RT_TABLE_MAIN)
                        tbl = fib_get_table(net, table);

                if (tbl)
                        err = fib_table_lookup(tbl, &fl4, &res,
                                               FIB_LOOKUP_IGNORE_LINKSTATE |
                                               FIB_LOOKUP_NOREF);

                /* on error or if no table given do full lookup. This
                 * is needed for example when nexthops are in the local
                 * table rather than the given table
                 */
                if (!tbl || err) {
                        err = fib_lookup(net, &fl4, &res,
                                         FIB_LOOKUP_IGNORE_LINKSTATE);
                }

                if (err) {
                        NL_SET_ERR_MSG(extack, "Nexthop has invalid gateway");
                        goto out;
                }
        }

        err = -EINVAL;
        if (res.type != RTN_UNICAST && res.type != RTN_LOCAL) {
                NL_SET_ERR_MSG(extack, "Nexthop has invalid gateway");
                goto out;
        }
        nh->fib_nh_scope = res.scope;
        nh->fib_nh_oif = FIB_RES_OIF(res);
        nh->fib_nh_dev = dev = FIB_RES_DEV(res);
        if (!dev) {
                NL_SET_ERR_MSG(extack,
                               "No egress device for nexthop gateway");
                goto out;
        }
        dev_hold(dev);
        if (!netif_carrier_ok(dev))
                nh->fib_nh_flags |= RTNH_F_LINKDOWN;
        err = (dev->flags & IFF_UP) ? 0 : -ENETDOWN;
out:
        rcu_read_unlock();
        return err;
}

static int fib_check_nh_nongw(struct net *net, struct fib_nh *nh,
                              struct netlink_ext_ack *extack)
{
        struct in_device *in_dev;
        int err;

        if (nh->fib_nh_flags & (RTNH_F_PERVASIVE | RTNH_F_ONLINK)) {
                NL_SET_ERR_MSG(extack,
                               "Invalid flags for nexthop - PERVASIVE and ONLINK can not be set");
                return -EINVAL;
        }

        rcu_read_lock();

        err = -ENODEV;
        in_dev = inetdev_by_index(net, nh->fib_nh_oif);
        if (!in_dev)
                goto out;
        err = -ENETDOWN;
        if (!(in_dev->dev->flags & IFF_UP)) {
                NL_SET_ERR_MSG(extack, "Device for nexthop is not up");
                goto out;
        }

        nh->fib_nh_dev = in_dev->dev;
        dev_hold(nh->fib_nh_dev);
        nh->fib_nh_scope = RT_SCOPE_LINK;
        if (!netif_carrier_ok(nh->fib_nh_dev))
                nh->fib_nh_flags |= RTNH_F_LINKDOWN;
        err = 0;
out:
        rcu_read_unlock();
        return err;
}

int fib_check_nh(struct net *net, struct fib_nh *nh, u32 table, u8 scope,
                 struct netlink_ext_ack *extack)
{
        int err;

        if (nh->fib_nh_gw_family == AF_INET)
                err = fib_check_nh_v4_gw(net, nh, table, scope, extack);
        else if (nh->fib_nh_gw_family == AF_INET6)
                err = fib_check_nh_v6_gw(net, nh, table, extack);
        else
                err = fib_check_nh_nongw(net, nh, extack);

        return err;
}

static inline unsigned int fib_laddr_hashfn(__be32 val)
{
        unsigned int mask = (fib_info_hash_size - 1);

        return ((__force u32)val ^
                ((__force u32)val >> 7) ^
                ((__force u32)val >> 14)) & mask;
}

static struct hlist_head *fib_info_hash_alloc(int bytes)
{
        if (bytes <= PAGE_SIZE)
                return kzalloc(bytes, GFP_KERNEL);
        else
                return (struct hlist_head *)
                        __get_free_pages(GFP_KERNEL | __GFP_ZERO,
                                         get_order(bytes));
}

static void fib_info_hash_free(struct hlist_head *hash, int bytes)
{
        if (!hash)
                return;

        if (bytes <= PAGE_SIZE)
                kfree(hash);
        else
                free_pages((unsigned long) hash, get_order(bytes));
}

static void fib_info_hash_move(struct hlist_head *new_info_hash,
                               struct hlist_head *new_laddrhash,
                               unsigned int new_size)
{
        struct hlist_head *old_info_hash, *old_laddrhash;
        unsigned int old_size = fib_info_hash_size;
        unsigned int i, bytes;

        spin_lock_bh(&fib_info_lock);
        old_info_hash = fib_info_hash;
        old_laddrhash = fib_info_laddrhash;
        fib_info_hash_size = new_size;

        for (i = 0; i < old_size; i++) {
                struct hlist_head *head = &fib_info_hash[i];
                struct hlist_node *n;
                struct fib_info *fi;

                hlist_for_each_entry_safe(fi, n, head, fib_hash) {
                        struct hlist_head *dest;
                        unsigned int new_hash;

                        new_hash = fib_info_hashfn(fi);
                        dest = &new_info_hash[new_hash];
                        hlist_add_head(&fi->fib_hash, dest);
                }
        }
        fib_info_hash = new_info_hash;

        for (i = 0; i < old_size; i++) {
                struct hlist_head *lhead = &fib_info_laddrhash[i];
                struct hlist_node *n;
                struct fib_info *fi;

                hlist_for_each_entry_safe(fi, n, lhead, fib_lhash) {
                        struct hlist_head *ldest;
                        unsigned int new_hash;

                        new_hash = fib_laddr_hashfn(fi->fib_prefsrc);
                        ldest = &new_laddrhash[new_hash];
                        hlist_add_head(&fi->fib_lhash, ldest);
                }
        }
        fib_info_laddrhash = new_laddrhash;

        spin_unlock_bh(&fib_info_lock);

        bytes = old_size * sizeof(struct hlist_head *);
        fib_info_hash_free(old_info_hash, bytes);
        fib_info_hash_free(old_laddrhash, bytes);
}

__be32 fib_info_update_nhc_saddr(struct net *net, struct fib_nh_common *nhc,
                                 unsigned char scope)
{
        struct fib_nh *nh;
        __be32 saddr;

        if (nhc->nhc_family != AF_INET)
                return inet_select_addr(nhc->nhc_dev, 0, scope);

        nh = container_of(nhc, struct fib_nh, nh_common);
        saddr = inet_select_addr(nh->fib_nh_dev, nh->fib_nh_gw4, scope);

        WRITE_ONCE(nh->nh_saddr, saddr);
        WRITE_ONCE(nh->nh_saddr_genid, atomic_read(&net->ipv4.dev_addr_genid));

        return saddr;
}

__be32 fib_result_prefsrc(struct net *net, struct fib_result *res)
{
        struct fib_nh_common *nhc = res->nhc;

        if (res->fi->fib_prefsrc)
                return res->fi->fib_prefsrc;

        if (nhc->nhc_family == AF_INET) {
                struct fib_nh *nh;

                nh = container_of(nhc, struct fib_nh, nh_common);
                if (READ_ONCE(nh->nh_saddr_genid) ==
                    atomic_read(&net->ipv4.dev_addr_genid))
                        return READ_ONCE(nh->nh_saddr);
        }

        return fib_info_update_nhc_saddr(net, nhc, res->fi->fib_scope);
}

static bool fib_valid_prefsrc(struct fib_config *cfg, __be32 fib_prefsrc)
{
        if (cfg->fc_type != RTN_LOCAL || !cfg->fc_dst ||
            fib_prefsrc != cfg->fc_dst) {
                u32 tb_id = cfg->fc_table;
                int rc;

                if (tb_id == RT_TABLE_MAIN)
                        tb_id = RT_TABLE_LOCAL;

                rc = inet_addr_type_table(cfg->fc_nlinfo.nl_net,
                                          fib_prefsrc, tb_id);

                if (rc != RTN_LOCAL && tb_id != RT_TABLE_LOCAL) {
                        rc = inet_addr_type_table(cfg->fc_nlinfo.nl_net,
                                                  fib_prefsrc, RT_TABLE_LOCAL);
                }

                if (rc != RTN_LOCAL)
                        return false;
        }
        return true;
}

struct fib_info *fib_create_info(struct fib_config *cfg,
                                 struct netlink_ext_ack *extack)
{
        int err;
        struct fib_info *fi = NULL;
        struct nexthop *nh = NULL;
        struct fib_info *ofi;
        int nhs = 1;
        struct net *net = cfg->fc_nlinfo.nl_net;

        if (cfg->fc_type > RTN_MAX)
                goto err_inval;

        /* Fast check to catch the most weird cases */
        if (fib_props[cfg->fc_type].scope > cfg->fc_scope) {
                NL_SET_ERR_MSG(extack, "Invalid scope");
                goto err_inval;
        }

        if (cfg->fc_flags & (RTNH_F_DEAD | RTNH_F_LINKDOWN)) {
                NL_SET_ERR_MSG(extack,
                               "Invalid rtm_flags - can not contain DEAD or LINKDOWN");
                goto err_inval;
        }

        if (cfg->fc_nh_id) {
                if (!cfg->fc_mx) {
                        fi = fib_find_info_nh(net, cfg);
                        if (fi) {
                                refcount_inc(&fi->fib_treeref);
                                return fi;
                        }
                }

                nh = nexthop_find_by_id(net, cfg->fc_nh_id);
                if (!nh) {
                        NL_SET_ERR_MSG(extack, "Nexthop id does not exist");
                        goto err_inval;
                }
                nhs = 0;
        }

#ifdef CONFIG_IP_ROUTE_MULTIPATH
        if (cfg->fc_mp) {
                nhs = fib_count_nexthops(cfg->fc_mp, cfg->fc_mp_len, extack);
                if (nhs == 0)
                        goto err_inval;
        }
#endif

        err = -ENOBUFS;

        /* Paired with WRITE_ONCE() in fib_release_info() */
        if (READ_ONCE(fib_info_cnt) >= fib_info_hash_size) {
                unsigned int new_size = fib_info_hash_size << 1;
                struct hlist_head *new_info_hash;
                struct hlist_head *new_laddrhash;
                unsigned int bytes;

                if (!new_size)
                        new_size = 16;
                bytes = new_size * sizeof(struct hlist_head *);
                new_info_hash = fib_info_hash_alloc(bytes);
                new_laddrhash = fib_info_hash_alloc(bytes);
                if (!new_info_hash || !new_laddrhash) {
                        fib_info_hash_free(new_info_hash, bytes);
                        fib_info_hash_free(new_laddrhash, bytes);
                } else
                        fib_info_hash_move(new_info_hash, new_laddrhash, new_size);

                if (!fib_info_hash_size)
                        goto failure;
        }

        fi = kzalloc(struct_size(fi, fib_nh, nhs), GFP_KERNEL);
        if (!fi)
                goto failure;
        fi->fib_metrics = ip_fib_metrics_init(fi->fib_net, cfg->fc_mx,
                                              cfg->fc_mx_len, extack);
        if (IS_ERR(fi->fib_metrics)) {
                err = PTR_ERR(fi->fib_metrics);
                kfree(fi);
                return ERR_PTR(err);
        }

        fi->fib_net = net;
        fi->fib_protocol = cfg->fc_protocol;
        fi->fib_scope = cfg->fc_scope;
        fi->fib_flags = cfg->fc_flags;
        fi->fib_priority = cfg->fc_priority;
        fi->fib_prefsrc = cfg->fc_prefsrc;
        fi->fib_type = cfg->fc_type;
        fi->fib_tb_id = cfg->fc_table;

        fi->fib_nhs = nhs;
        if (nh) {
                if (!nexthop_get(nh)) {
                        NL_SET_ERR_MSG(extack, "Nexthop has been deleted");
                        err = -EINVAL;
                } else {
                        err = 0;
                        fi->nh = nh;
                }
        } else {
                change_nexthops(fi) {
                        nexthop_nh->nh_parent = fi;
                } endfor_nexthops(fi)

                if (cfg->fc_mp)
                        err = fib_get_nhs(fi, cfg->fc_mp, cfg->fc_mp_len, cfg,
                                          extack);
                else
                        err = fib_nh_init(net, fi->fib_nh, cfg, 1, extack);
        }

        if (err != 0)
                goto failure;

        if (fib_props[cfg->fc_type].error) {
                if (cfg->fc_gw_family || cfg->fc_oif || cfg->fc_mp) {
                        NL_SET_ERR_MSG(extack,
                                       "Gateway, device and multipath can not be specified for this route type");
                        goto err_inval;
                }
                goto link_it;
        } else {
                switch (cfg->fc_type) {
                case RTN_UNICAST:
                case RTN_LOCAL:
                case RTN_BROADCAST:
                case RTN_ANYCAST:
                case RTN_MULTICAST:
                        break;
                default:
                        NL_SET_ERR_MSG(extack, "Invalid route type");
                        goto err_inval;
                }
        }

        if (cfg->fc_scope > RT_SCOPE_HOST) {
                NL_SET_ERR_MSG(extack, "Invalid scope");
                goto err_inval;
        }

        if (fi->nh) {
                err = fib_check_nexthop(fi->nh, cfg->fc_scope, extack);
                if (err)
                        goto failure;
        } else if (cfg->fc_scope == RT_SCOPE_HOST) {
                struct fib_nh *nh = fi->fib_nh;

                /* Local address is added. */
                if (nhs != 1) {
                        NL_SET_ERR_MSG(extack,
                                       "Route with host scope can not have multiple nexthops");
                        goto err_inval;
                }
                if (nh->fib_nh_gw_family) {
                        NL_SET_ERR_MSG(extack,
                                       "Route with host scope can not have a gateway");
                        goto err_inval;
                }
                nh->fib_nh_scope = RT_SCOPE_NOWHERE;
                nh->fib_nh_dev = dev_get_by_index(net, nh->fib_nh_oif);
                err = -ENODEV;
                if (!nh->fib_nh_dev)
                        goto failure;
        } else {
                int linkdown = 0;

                change_nexthops(fi) {
                        err = fib_check_nh(cfg->fc_nlinfo.nl_net, nexthop_nh,
                                           cfg->fc_table, cfg->fc_scope,
                                           extack);
                        if (err != 0)
                                goto failure;
                        if (nexthop_nh->fib_nh_flags & RTNH_F_LINKDOWN)
                                linkdown++;
                } endfor_nexthops(fi)
                if (linkdown == fi->fib_nhs)
                        fi->fib_flags |= RTNH_F_LINKDOWN;
        }

        if (fi->fib_prefsrc && !fib_valid_prefsrc(cfg, fi->fib_prefsrc)) {
                NL_SET_ERR_MSG(extack, "Invalid prefsrc address");
                goto err_inval;
        }

        if (!fi->nh) {
                change_nexthops(fi) {
                        fib_info_update_nhc_saddr(net, &nexthop_nh->nh_common,
                                                  fi->fib_scope);
                        if (nexthop_nh->fib_nh_gw_family == AF_INET6)
                                fi->fib_nh_is_v6 = true;
                } endfor_nexthops(fi)

                fib_rebalance(fi);
        }

link_it:
        ofi = fib_find_info(fi);
        if (ofi) {
                /* fib_table_lookup() should not see @fi yet. */
                fi->fib_dead = 1;
                free_fib_info(fi);
                refcount_inc(&ofi->fib_treeref);
                return ofi;
        }

        refcount_set(&fi->fib_treeref, 1);
        refcount_set(&fi->fib_clntref, 1);
        spin_lock_bh(&fib_info_lock);
        fib_info_cnt++;
        hlist_add_head(&fi->fib_hash,
                       &fib_info_hash[fib_info_hashfn(fi)]);
        if (fi->fib_prefsrc) {
                struct hlist_head *head;

                head = &fib_info_laddrhash[fib_laddr_hashfn(fi->fib_prefsrc)];
                hlist_add_head(&fi->fib_lhash, head);
        }
        if (fi->nh) {
                list_add(&fi->nh_list, &nh->fi_list);
        } else {
                change_nexthops(fi) {
                        struct hlist_head *head;

                        if (!nexthop_nh->fib_nh_dev)
                                continue;
                        head = fib_info_devhash_bucket(nexthop_nh->fib_nh_dev);
                        hlist_add_head(&nexthop_nh->nh_hash, head);
                } endfor_nexthops(fi)
        }
        spin_unlock_bh(&fib_info_lock);
        return fi;

err_inval:
        err = -EINVAL;

failure:
        if (fi) {
                /* fib_table_lookup() should not see @fi yet. */
                fi->fib_dead = 1;
                free_fib_info(fi);
        }

        return ERR_PTR(err);
}

int fib_nexthop_info(struct sk_buff *skb, const struct fib_nh_common *nhc,
                     u8 rt_family, unsigned char *flags, bool skip_oif)
{
        if (nhc->nhc_flags & RTNH_F_DEAD)
                *flags |= RTNH_F_DEAD;

        if (nhc->nhc_flags & RTNH_F_LINKDOWN) {
                *flags |= RTNH_F_LINKDOWN;

                rcu_read_lock();
                switch (nhc->nhc_family) {
                case AF_INET:
                        if (ip_ignore_linkdown(nhc->nhc_dev))
                                *flags |= RTNH_F_DEAD;
                        break;
                case AF_INET6:
                        if (ip6_ignore_linkdown(nhc->nhc_dev))
                                *flags |= RTNH_F_DEAD;
                        break;
                }
                rcu_read_unlock();
        }

        switch (nhc->nhc_gw_family) {
        case AF_INET:
                if (nla_put_in_addr(skb, RTA_GATEWAY, nhc->nhc_gw.ipv4))
                        goto nla_put_failure;
                break;
        case AF_INET6:
                /* if gateway family does not match nexthop family
                 * gateway is encoded as RTA_VIA
                 */
                if (rt_family != nhc->nhc_gw_family) {
                        int alen = sizeof(struct in6_addr);
                        struct nlattr *nla;
                        struct rtvia *via;

                        nla = nla_reserve(skb, RTA_VIA, alen + 2);
                        if (!nla)
                                goto nla_put_failure;

                        via = nla_data(nla);
                        via->rtvia_family = AF_INET6;
                        memcpy(via->rtvia_addr, &nhc->nhc_gw.ipv6, alen);
                } else if (nla_put_in6_addr(skb, RTA_GATEWAY,
                                            &nhc->nhc_gw.ipv6) < 0) {
                        goto nla_put_failure;
                }
                break;
        }

        *flags |= (nhc->nhc_flags &
                   (RTNH_F_ONLINK | RTNH_F_OFFLOAD | RTNH_F_TRAP));

        if (!skip_oif && nhc->nhc_dev &&
            nla_put_u32(skb, RTA_OIF, nhc->nhc_dev->ifindex))
                goto nla_put_failure;

        if (nhc->nhc_lwtstate &&
            lwtunnel_fill_encap(skb, nhc->nhc_lwtstate,
                                RTA_ENCAP, RTA_ENCAP_TYPE) < 0)
                goto nla_put_failure;

        return 0;

nla_put_failure:
        return -EMSGSIZE;
}
EXPORT_SYMBOL_GPL(fib_nexthop_info);

#if IS_ENABLED(CONFIG_IP_ROUTE_MULTIPATH) || IS_ENABLED(CONFIG_IPV6)
int fib_add_nexthop(struct sk_buff *skb, const struct fib_nh_common *nhc,
                    int nh_weight, u8 rt_family, u32 nh_tclassid)
{
        const struct net_device *dev = nhc->nhc_dev;
        struct rtnexthop *rtnh;
        unsigned char flags = 0;

        rtnh = nla_reserve_nohdr(skb, sizeof(*rtnh));
        if (!rtnh)
                goto nla_put_failure;

        rtnh->rtnh_hops = nh_weight - 1;
        rtnh->rtnh_ifindex = dev ? dev->ifindex : 0;

        if (fib_nexthop_info(skb, nhc, rt_family, &flags, true) < 0)
                goto nla_put_failure;

        rtnh->rtnh_flags = flags;

        if (nh_tclassid && nla_put_u32(skb, RTA_FLOW, nh_tclassid))
                goto nla_put_failure;

        /* length of rtnetlink header + attributes */
        rtnh->rtnh_len = nlmsg_get_pos(skb) - (void *)rtnh;

        return 0;

nla_put_failure:
        return -EMSGSIZE;
}
EXPORT_SYMBOL_GPL(fib_add_nexthop);
#endif

#ifdef CONFIG_IP_ROUTE_MULTIPATH
static int fib_add_multipath(struct sk_buff *skb, struct fib_info *fi)
{
        struct nlattr *mp;

        mp = nla_nest_start_noflag(skb, RTA_MULTIPATH);
        if (!mp)
                goto nla_put_failure;

        if (unlikely(fi->nh)) {
                if (nexthop_mpath_fill_node(skb, fi->nh, AF_INET) < 0)
                        goto nla_put_failure;
                goto mp_end;
        }

        for_nexthops(fi) {
                u32 nh_tclassid = 0;
#ifdef CONFIG_IP_ROUTE_CLASSID
                nh_tclassid = nh->nh_tclassid;
#endif
                if (fib_add_nexthop(skb, &nh->nh_common, nh->fib_nh_weight,
                                    AF_INET, nh_tclassid) < 0)
                        goto nla_put_failure;
        } endfor_nexthops(fi);

mp_end:
        nla_nest_end(skb, mp);

        return 0;

nla_put_failure:
        return -EMSGSIZE;
}
#else
static int fib_add_multipath(struct sk_buff *skb, struct fib_info *fi)
{
        return 0;
}
#endif

int fib_dump_info(struct sk_buff *skb, u32 portid, u32 seq, int event,
                  const struct fib_rt_info *fri, unsigned int flags)
{
        unsigned int nhs = fib_info_num_path(fri->fi);
        struct fib_info *fi = fri->fi;
        u32 tb_id = fri->tb_id;
        struct nlmsghdr *nlh;
        struct rtmsg *rtm;

        nlh = nlmsg_put(skb, portid, seq, event, sizeof(*rtm), flags);
        if (!nlh)
                return -EMSGSIZE;

        rtm = nlmsg_data(nlh);
        rtm->rtm_family = AF_INET;
        rtm->rtm_dst_len = fri->dst_len;
        rtm->rtm_src_len = 0;
        rtm->rtm_tos = fri->tos;
        if (tb_id < 256)
                rtm->rtm_table = tb_id;
        else
                rtm->rtm_table = RT_TABLE_COMPAT;
        if (nla_put_u32(skb, RTA_TABLE, tb_id))
                goto nla_put_failure;
        rtm->rtm_type = fri->type;
        rtm->rtm_flags = fi->fib_flags;
        rtm->rtm_scope = fi->fib_scope;
        rtm->rtm_protocol = fi->fib_protocol;

        if (rtm->rtm_dst_len &&
            nla_put_in_addr(skb, RTA_DST, fri->dst))
                goto nla_put_failure;
        if (fi->fib_priority &&
            nla_put_u32(skb, RTA_PRIORITY, fi->fib_priority))
                goto nla_put_failure;
        if (rtnetlink_put_metrics(skb, fi->fib_metrics->metrics) < 0)
                goto nla_put_failure;

        if (fi->fib_prefsrc &&
            nla_put_in_addr(skb, RTA_PREFSRC, fi->fib_prefsrc))
                goto nla_put_failure;

        if (fi->nh) {
                if (nla_put_u32(skb, RTA_NH_ID, fi->nh->id))
                        goto nla_put_failure;
                if (nexthop_is_blackhole(fi->nh))
                        rtm->rtm_type = RTN_BLACKHOLE;
                if (!READ_ONCE(fi->fib_net->ipv4.sysctl_nexthop_compat_mode))
                        goto offload;
        }

        if (nhs == 1) {
                const struct fib_nh_common *nhc = fib_info_nhc(fi, 0);
                unsigned char flags = 0;

                if (fib_nexthop_info(skb, nhc, AF_INET, &flags, false) < 0)
                        goto nla_put_failure;

                rtm->rtm_flags = flags;
#ifdef CONFIG_IP_ROUTE_CLASSID
                if (nhc->nhc_family == AF_INET) {
                        struct fib_nh *nh;

                        nh = container_of(nhc, struct fib_nh, nh_common);
                        if (nh->nh_tclassid &&
                            nla_put_u32(skb, RTA_FLOW, nh->nh_tclassid))
                                goto nla_put_failure;
                }
#endif
        } else {
                if (fib_add_multipath(skb, fi) < 0)
                        goto nla_put_failure;
        }

offload:
        if (fri->offload)
                rtm->rtm_flags |= RTM_F_OFFLOAD;
        if (fri->trap)
                rtm->rtm_flags |= RTM_F_TRAP;
        if (fri->offload_failed)
                rtm->rtm_flags |= RTM_F_OFFLOAD_FAILED;

        nlmsg_end(skb, nlh);
        return 0;

nla_put_failure:
        nlmsg_cancel(skb, nlh);
        return -EMSGSIZE;
}

/*
 * Update FIB if:
 * - local address disappeared -> we must delete all the entries
 *   referring to it.
 * - device went down -> we must shutdown all nexthops going via it.
 */
int fib_sync_down_addr(struct net_device *dev, __be32 local)
{
        int ret = 0;
        unsigned int hash = fib_laddr_hashfn(local);
        struct hlist_head *head = &fib_info_laddrhash[hash];
        int tb_id = l3mdev_fib_table(dev) ? : RT_TABLE_MAIN;
        struct net *net = dev_net(dev);
        struct fib_info *fi;

        if (!fib_info_laddrhash || local == 0)
                return 0;

        hlist_for_each_entry(fi, head, fib_lhash) {
                if (!net_eq(fi->fib_net, net) ||
                    fi->fib_tb_id != tb_id)
                        continue;
                if (fi->fib_prefsrc == local) {
                        fi->fib_flags |= RTNH_F_DEAD;
                        fi->pfsrc_removed = true;
                        ret++;
                }
        }
        return ret;
}

static int call_fib_nh_notifiers(struct fib_nh *nh,
                                 enum fib_event_type event_type)
{
        bool ignore_link_down = ip_ignore_linkdown(nh->fib_nh_dev);
        struct fib_nh_notifier_info info = {
                .fib_nh = nh,
        };

        switch (event_type) {
        case FIB_EVENT_NH_ADD:
                if (nh->fib_nh_flags & RTNH_F_DEAD)
                        break;
                if (ignore_link_down && nh->fib_nh_flags & RTNH_F_LINKDOWN)
                        break;
                return call_fib4_notifiers(dev_net(nh->fib_nh_dev), event_type,
                                           &info.info);
        case FIB_EVENT_NH_DEL:
                if ((ignore_link_down && nh->fib_nh_flags & RTNH_F_LINKDOWN) ||
                    (nh->fib_nh_flags & RTNH_F_DEAD))
                        return call_fib4_notifiers(dev_net(nh->fib_nh_dev),
                                                   event_type, &info.info);
                break;
        default:
                break;
        }

        return NOTIFY_DONE;
}

/* Update the PMTU of exceptions when:
 * - the new MTU of the first hop becomes smaller than the PMTU
 * - the old MTU was the same as the PMTU, and it limited discovery of
 *   larger MTUs on the path. With that limit raised, we can now
 *   discover larger MTUs
 * A special case is locked exceptions, for which the PMTU is smaller
 * than the minimal accepted PMTU:
 * - if the new MTU is greater than the PMTU, don't make any change
 * - otherwise, unlock and set PMTU
 */
void fib_nhc_update_mtu(struct fib_nh_common *nhc, u32 new, u32 orig)
{
        struct fnhe_hash_bucket *bucket;
        int i;

        bucket = rcu_dereference_protected(nhc->nhc_exceptions, 1);
        if (!bucket)
                return;

        for (i = 0; i < FNHE_HASH_SIZE; i++) {
                struct fib_nh_exception *fnhe;

                for (fnhe = rcu_dereference_protected(bucket[i].chain, 1);
                     fnhe;
                     fnhe = rcu_dereference_protected(fnhe->fnhe_next, 1)) {
                        if (fnhe->fnhe_mtu_locked) {
                                if (new <= fnhe->fnhe_pmtu) {
                                        fnhe->fnhe_pmtu = new;
                                        fnhe->fnhe_mtu_locked = false;
                                }
                        } else if (new < fnhe->fnhe_pmtu ||
                                   orig == fnhe->fnhe_pmtu) {
                                fnhe->fnhe_pmtu = new;
                        }
                }
        }
}

void fib_sync_mtu(struct net_device *dev, u32 orig_mtu)
{
        struct hlist_head *head = fib_info_devhash_bucket(dev);
        struct fib_nh *nh;

        hlist_for_each_entry(nh, head, nh_hash) {
                if (nh->fib_nh_dev == dev)
                        fib_nhc_update_mtu(&nh->nh_common, dev->mtu, orig_mtu);
        }
}

/* Event              force Flags           Description
 * NETDEV_CHANGE      0     LINKDOWN        Carrier OFF, not for scope host
 * NETDEV_DOWN        0     LINKDOWN|DEAD   Link down, not for scope host
 * NETDEV_DOWN        1     LINKDOWN|DEAD   Last address removed
 * NETDEV_UNREGISTER  1     LINKDOWN|DEAD   Device removed
 *
 * only used when fib_nh is built into fib_info
 */
int fib_sync_down_dev(struct net_device *dev, unsigned long event, bool force)
{
        struct hlist_head *head = fib_info_devhash_bucket(dev);
        struct fib_info *prev_fi = NULL;
        int scope = RT_SCOPE_NOWHERE;
        struct fib_nh *nh;
        int ret = 0;

        if (force)
                scope = -1;

        hlist_for_each_entry(nh, head, nh_hash) {
                struct fib_info *fi = nh->nh_parent;
                int dead;

                BUG_ON(!fi->fib_nhs);
                if (nh->fib_nh_dev != dev || fi == prev_fi)
                        continue;
                prev_fi = fi;
                dead = 0;
                change_nexthops(fi) {
                        if (nexthop_nh->fib_nh_flags & RTNH_F_DEAD)
                                dead++;
                        else if (nexthop_nh->fib_nh_dev == dev &&
                                 nexthop_nh->fib_nh_scope != scope) {
                                switch (event) {
                                case NETDEV_DOWN:
                                case NETDEV_UNREGISTER:
                                        nexthop_nh->fib_nh_flags |= RTNH_F_DEAD;
                                        fallthrough;
                                case NETDEV_CHANGE:
                                        nexthop_nh->fib_nh_flags |= RTNH_F_LINKDOWN;
                                        break;
                                }
                                call_fib_nh_notifiers(nexthop_nh,
                                                      FIB_EVENT_NH_DEL);
                                dead++;
                        }
#ifdef CONFIG_IP_ROUTE_MULTIPATH
                        if (event == NETDEV_UNREGISTER &&
                            nexthop_nh->fib_nh_dev == dev) {
                                dead = fi->fib_nhs;
                                break;
                        }
#endif
                } endfor_nexthops(fi)
                if (dead == fi->fib_nhs) {
                        switch (event) {
                        case NETDEV_DOWN:
                        case NETDEV_UNREGISTER:
                                fi->fib_flags |= RTNH_F_DEAD;
                                fallthrough;
                        case NETDEV_CHANGE:
                                fi->fib_flags |= RTNH_F_LINKDOWN;
                                break;
                        }
                        ret++;
                }

                fib_rebalance(fi);
        }

        return ret;
}

/* Must be invoked inside of an RCU protected region.  */
static void fib_select_default(const struct flowi4 *flp, struct fib_result *res)
{
        struct fib_info *fi = NULL, *last_resort = NULL;
        struct hlist_head *fa_head = res->fa_head;
        struct fib_table *tb = res->table;
        u8 slen = 32 - res->prefixlen;
        int order = -1, last_idx = -1;
        struct fib_alias *fa, *fa1 = NULL;
        u32 last_prio = res->fi->fib_priority;
        u8 last_tos = 0;

        hlist_for_each_entry_rcu(fa, fa_head, fa_list) {
                struct fib_info *next_fi = fa->fa_info;
                struct fib_nh_common *nhc;

                if (fa->fa_slen != slen)
                        continue;
                if (fa->fa_tos && fa->fa_tos != flp->flowi4_tos)
                        continue;
                if (fa->tb_id != tb->tb_id)
                        continue;
                if (next_fi->fib_priority > last_prio &&
                    fa->fa_tos == last_tos) {
                        if (last_tos)
                                continue;
                        break;
                }
                if (next_fi->fib_flags & RTNH_F_DEAD)
                        continue;
                last_tos = fa->fa_tos;
                last_prio = next_fi->fib_priority;

                if (next_fi->fib_scope != res->scope ||
                    fa->fa_type != RTN_UNICAST)
                        continue;

                nhc = fib_info_nhc(next_fi, 0);
                if (!nhc->nhc_gw_family || nhc->nhc_scope != RT_SCOPE_LINK)
                        continue;

                fib_alias_accessed(fa);

                if (!fi) {
                        if (next_fi != res->fi)
                                break;
                        fa1 = fa;
                } else if (!fib_detect_death(fi, order, &last_resort,
                                             &last_idx, fa1->fa_default)) {
                        fib_result_assign(res, fi);
                        fa1->fa_default = order;
                        goto out;
                }
                fi = next_fi;
                order++;
        }

        if (order <= 0 || !fi) {
                if (fa1)
                        fa1->fa_default = -1;
                goto out;
        }

        if (!fib_detect_death(fi, order, &last_resort, &last_idx,
                              fa1->fa_default)) {
                fib_result_assign(res, fi);
                fa1->fa_default = order;
                goto out;
        }

        if (last_idx >= 0)
                fib_result_assign(res, last_resort);
        fa1->fa_default = last_idx;
out:
        return;
}

/*
 * Dead device goes up. We wake up dead nexthops.
 * It takes sense only on multipath routes.
 *
 * only used when fib_nh is built into fib_info
 */
int fib_sync_up(struct net_device *dev, unsigned char nh_flags)
{
        struct fib_info *prev_fi;
        struct hlist_head *head;
        struct fib_nh *nh;
        int ret;

        if (!(dev->flags & IFF_UP))
                return 0;

        if (nh_flags & RTNH_F_DEAD) {
                unsigned int flags = dev_get_flags(dev);

                if (flags & (IFF_RUNNING | IFF_LOWER_UP))
                        nh_flags |= RTNH_F_LINKDOWN;
        }

        prev_fi = NULL;
        head = fib_info_devhash_bucket(dev);
        ret = 0;

        hlist_for_each_entry(nh, head, nh_hash) {
                struct fib_info *fi = nh->nh_parent;
                int alive;

                BUG_ON(!fi->fib_nhs);
                if (nh->fib_nh_dev != dev || fi == prev_fi)
                        continue;

                prev_fi = fi;
                alive = 0;
                change_nexthops(fi) {
                        if (!(nexthop_nh->fib_nh_flags & nh_flags)) {
                                alive++;
                                continue;
                        }
                        if (!nexthop_nh->fib_nh_dev ||
                            !(nexthop_nh->fib_nh_dev->flags & IFF_UP))
                                continue;
                        if (nexthop_nh->fib_nh_dev != dev ||
                            !__in_dev_get_rtnl(dev))
                                continue;
                        alive++;
                        nexthop_nh->fib_nh_flags &= ~nh_flags;
                        call_fib_nh_notifiers(nexthop_nh, FIB_EVENT_NH_ADD);
                } endfor_nexthops(fi)

                if (alive > 0) {
                        fi->fib_flags &= ~nh_flags;
                        ret++;
                }

                fib_rebalance(fi);
        }

        return ret;
}

#ifdef CONFIG_IP_ROUTE_MULTIPATH
static bool fib_good_nh(const struct fib_nh *nh)
{
        int state = NUD_REACHABLE;

        if (nh->fib_nh_scope == RT_SCOPE_LINK) {
                struct neighbour *n;

                rcu_read_lock_bh();

                if (likely(nh->fib_nh_gw_family == AF_INET))
                        n = __ipv4_neigh_lookup_noref(nh->fib_nh_dev,
                                                   (__force u32)nh->fib_nh_gw4);
                else if (nh->fib_nh_gw_family == AF_INET6)
                        n = __ipv6_neigh_lookup_noref_stub(nh->fib_nh_dev,
                                                           &nh->fib_nh_gw6);
                else
                        n = NULL;
                if (n)
                        state = n->nud_state;

                rcu_read_unlock_bh();
        }

        return !!(state & NUD_VALID);
}

void fib_select_multipath(struct fib_result *res, int hash)
{
        struct fib_info *fi = res->fi;
        struct net *net = fi->fib_net;
        bool first = false;

        if (unlikely(res->fi->nh)) {
                nexthop_path_fib_result(res, hash);
                return;
        }

        change_nexthops(fi) {
                if (READ_ONCE(net->ipv4.sysctl_fib_multipath_use_neigh)) {
                        if (!fib_good_nh(nexthop_nh))
                                continue;
                        if (!first) {
                                res->nh_sel = nhsel;
                                res->nhc = &nexthop_nh->nh_common;
                                first = true;
                        }
                }

                if (hash > atomic_read(&nexthop_nh->fib_nh_upper_bound))
                        continue;

                res->nh_sel = nhsel;
                res->nhc = &nexthop_nh->nh_common;
                return;
        } endfor_nexthops(fi);
}
#endif

void fib_select_path(struct net *net, struct fib_result *res,
                     struct flowi4 *fl4, const struct sk_buff *skb)
{
        if (fl4->flowi4_oif)
                goto check_saddr;

#ifdef CONFIG_IP_ROUTE_MULTIPATH
        if (fib_info_num_path(res->fi) > 1) {
                int h = fib_multipath_hash(net, fl4, skb, NULL);

                fib_select_multipath(res, h);
        }
        else
#endif
        if (!res->prefixlen &&
            res->table->tb_num_default > 1 &&
            res->type == RTN_UNICAST)
                fib_select_default(fl4, res);

check_saddr:
        if (!fl4->saddr)
                fl4->saddr = fib_result_prefsrc(net, res);
}

























   17 




   66 




   21 











   47 
   38 








   14 

    7 










   62 
   48 










    8 
   22 

















   46 
    1 




   47 


   47 











   47 
   47 





   42 





   62 
   62 




  245 




  225 
   20 



  245 



1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
// SPDX-License-Identifier: GPL-2.0
#include <linux/module.h>
#include <linux/kernel.h>
#include <linux/string.h>
#include <linux/socket.h>
#include <linux/net.h>
#include <linux/fs.h>
#include <net/af_unix.h>
#include <net/scm.h>
#include <linux/init.h>
#include <linux/io_uring.h>

#include "scm.h"

unsigned int unix_tot_inflight;
EXPORT_SYMBOL(unix_tot_inflight);

LIST_HEAD(gc_inflight_list);
EXPORT_SYMBOL(gc_inflight_list);

DEFINE_SPINLOCK(unix_gc_lock);
EXPORT_SYMBOL(unix_gc_lock);

struct sock *unix_get_socket(struct file *filp)
{
        struct sock *u_sock = NULL;
        struct inode *inode = file_inode(filp);

        /* Socket ? */
        if (S_ISSOCK(inode->i_mode) && !(filp->f_mode & FMODE_PATH)) {
                struct socket *sock = SOCKET_I(inode);
                struct sock *s = sock->sk;

                /* PF_UNIX ? */
                if (s && sock->ops && sock->ops->family == PF_UNIX)
                        u_sock = s;
        }

        return u_sock;
}
EXPORT_SYMBOL(unix_get_socket);

/* Keep the number of times in flight count for the file
 * descriptor if it is for an AF_UNIX socket.
 */
void unix_inflight(struct user_struct *user, struct file *fp)
{
        struct sock *s = unix_get_socket(fp);

        spin_lock(&unix_gc_lock);

        if (s) {
                struct unix_sock *u = unix_sk(s);

                if (!u->inflight) {
                        BUG_ON(!list_empty(&u->link));
                        list_add_tail(&u->link, &gc_inflight_list);
                } else {
                        BUG_ON(list_empty(&u->link));
                }
                u->inflight++;
                /* Paired with READ_ONCE() in wait_for_unix_gc() */
                WRITE_ONCE(unix_tot_inflight, unix_tot_inflight + 1);
        }
        WRITE_ONCE(user->unix_inflight, user->unix_inflight + 1);
        spin_unlock(&unix_gc_lock);
}

void unix_notinflight(struct user_struct *user, struct file *fp)
{
        struct sock *s = unix_get_socket(fp);

        spin_lock(&unix_gc_lock);

        if (s) {
                struct unix_sock *u = unix_sk(s);

                BUG_ON(!u->inflight);
                BUG_ON(list_empty(&u->link));

                u->inflight--;
                if (!u->inflight)
                        list_del_init(&u->link);
                /* Paired with READ_ONCE() in wait_for_unix_gc() */
                WRITE_ONCE(unix_tot_inflight, unix_tot_inflight - 1);
        }
        WRITE_ONCE(user->unix_inflight, user->unix_inflight - 1);
        spin_unlock(&unix_gc_lock);
}

/*
 * The "user->unix_inflight" variable is protected by the garbage
 * collection lock, and we just read it locklessly here. If you go
 * over the limit, there might be a tiny race in actually noticing
 * it across threads. Tough.
 */
static inline bool too_many_unix_fds(struct task_struct *p)
{
        struct user_struct *user = current_user();

        if (unlikely(READ_ONCE(user->unix_inflight) > task_rlimit(p, RLIMIT_NOFILE)))
                return !capable(CAP_SYS_RESOURCE) && !capable(CAP_SYS_ADMIN);
        return false;
}

int unix_attach_fds(struct scm_cookie *scm, struct sk_buff *skb)
{
        int i;

        if (too_many_unix_fds(current))
                return -ETOOMANYREFS;

        /*
         * Need to duplicate file references for the sake of garbage
         * collection.  Otherwise a socket in the fps might become a
         * candidate for GC while the skb is not yet queued.
         */
        UNIXCB(skb).fp = scm_fp_dup(scm->fp);
        if (!UNIXCB(skb).fp)
                return -ENOMEM;

        for (i = scm->fp->count - 1; i >= 0; i--)
                unix_inflight(scm->fp->user, scm->fp->fp[i]);
        return 0;
}
EXPORT_SYMBOL(unix_attach_fds);

void unix_detach_fds(struct scm_cookie *scm, struct sk_buff *skb)
{
        int i;

        scm->fp = UNIXCB(skb).fp;
        UNIXCB(skb).fp = NULL;

        for (i = scm->fp->count-1; i >= 0; i--)
                unix_notinflight(scm->fp->user, scm->fp->fp[i]);
}
EXPORT_SYMBOL(unix_detach_fds);

void unix_destruct_scm(struct sk_buff *skb)
{
        struct scm_cookie scm;

        memset(&scm, 0, sizeof(scm));
        scm.pid  = UNIXCB(skb).pid;
        if (UNIXCB(skb).fp)
                unix_detach_fds(&scm, skb);

        /* Alas, it calls VFS */
        /* So fscking what? fput() had been SMP-safe since the last Summer */
        scm_destroy(&scm);
        sock_wfree(skb);
}
EXPORT_SYMBOL(unix_destruct_scm);

























































    6 

    6 

    6 





    1 



    5 



    5 






    6 











































































































































    5 











    5 

















































































   14 



   15 




   15 
   15 







   13 



   13 




   20 







   20 

    5 



   20 

   20 
   20 
    5 

    1 

    1 












   15 


   15 




   15 

   15 










    6 
    6 






    6 







1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
// SPDX-License-Identifier: GPL-2.0
/*
 * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
 */

#include "queueing.h"
#include "timers.h"
#include "device.h"
#include "peer.h"
#include "socket.h"
#include "messages.h"
#include "cookie.h"

#include <linux/uio.h>
#include <linux/inetdevice.h>
#include <linux/socket.h>
#include <net/ip_tunnels.h>
#include <net/udp.h>
#include <net/sock.h>

static void wg_packet_send_handshake_initiation(struct wg_peer *peer)
{
        struct message_handshake_initiation packet;

        if (!wg_birthdate_has_expired(atomic64_read(&peer->last_sent_handshake),
                                      REKEY_TIMEOUT))
                return; /* This function is rate limited. */

        atomic64_set(&peer->last_sent_handshake, ktime_get_coarse_boottime_ns());
        net_dbg_ratelimited("%s: Sending handshake initiation to peer %llu (%pISpfsc)\n",
                            peer->device->dev->name, peer->internal_id,
                            &peer->endpoint.addr);

        if (wg_noise_handshake_create_initiation(&packet, &peer->handshake)) {
                wg_cookie_add_mac_to_packet(&packet, sizeof(packet), peer);
                wg_timers_any_authenticated_packet_traversal(peer);
                wg_timers_any_authenticated_packet_sent(peer);
                atomic64_set(&peer->last_sent_handshake,
                             ktime_get_coarse_boottime_ns());
                wg_socket_send_buffer_to_peer(peer, &packet, sizeof(packet),
                                              HANDSHAKE_DSCP);
                wg_timers_handshake_initiated(peer);
        }
}

void wg_packet_handshake_send_worker(struct work_struct *work)
{
        struct wg_peer *peer = container_of(work, struct wg_peer,
                                            transmit_handshake_work);

        wg_packet_send_handshake_initiation(peer);
        wg_peer_put(peer);
}

void wg_packet_send_queued_handshake_initiation(struct wg_peer *peer,
                                                bool is_retry)
{
        if (!is_retry)
                peer->timer_handshake_attempts = 0;

        rcu_read_lock_bh();
        /* We check last_sent_handshake here in addition to the actual function
         * we're queueing up, so that we don't queue things if not strictly
         * necessary:
         */
        if (!wg_birthdate_has_expired(atomic64_read(&peer->last_sent_handshake),
                                      REKEY_TIMEOUT) ||
                        unlikely(READ_ONCE(peer->is_dead)))
                goto out;

        wg_peer_get(peer);
        /* Queues up calling packet_send_queued_handshakes(peer), where we do a
         * peer_put(peer) after:
         */
        if (!queue_work(peer->device->handshake_send_wq,
                        &peer->transmit_handshake_work))
                /* If the work was already queued, we want to drop the
                 * extra reference:
                 */
                wg_peer_put(peer);
out:
        rcu_read_unlock_bh();
}

void wg_packet_send_handshake_response(struct wg_peer *peer)
{
        struct message_handshake_response packet;

        atomic64_set(&peer->last_sent_handshake, ktime_get_coarse_boottime_ns());
        net_dbg_ratelimited("%s: Sending handshake response to peer %llu (%pISpfsc)\n",
                            peer->device->dev->name, peer->internal_id,
                            &peer->endpoint.addr);

        if (wg_noise_handshake_create_response(&packet, &peer->handshake)) {
                wg_cookie_add_mac_to_packet(&packet, sizeof(packet), peer);
                if (wg_noise_handshake_begin_session(&peer->handshake,
                                                     &peer->keypairs)) {
                        wg_timers_session_derived(peer);
                        wg_timers_any_authenticated_packet_traversal(peer);
                        wg_timers_any_authenticated_packet_sent(peer);
                        atomic64_set(&peer->last_sent_handshake,
                                     ktime_get_coarse_boottime_ns());
                        wg_socket_send_buffer_to_peer(peer, &packet,
                                                      sizeof(packet),
                                                      HANDSHAKE_DSCP);
                }
        }
}

void wg_packet_send_handshake_cookie(struct wg_device *wg,
                                     struct sk_buff *initiating_skb,
                                     __le32 sender_index)
{
        struct message_handshake_cookie packet;

        net_dbg_skb_ratelimited("%s: Sending cookie response for denied handshake message for %pISpfsc\n",
                                wg->dev->name, initiating_skb);
        wg_cookie_message_create(&packet, initiating_skb, sender_index,
                                 &wg->cookie_checker);
        wg_socket_send_buffer_as_reply_to_skb(wg, initiating_skb, &packet,
                                              sizeof(packet));
}

static void keep_key_fresh(struct wg_peer *peer)
{
        struct noise_keypair *keypair;
        bool send;

        rcu_read_lock_bh();
        keypair = rcu_dereference_bh(peer->keypairs.current_keypair);
        send = keypair && READ_ONCE(keypair->sending.is_valid) &&
               (atomic64_read(&keypair->sending_counter) > REKEY_AFTER_MESSAGES ||
                (keypair->i_am_the_initiator &&
                 wg_birthdate_has_expired(keypair->sending.birthdate, REKEY_AFTER_TIME)));
        rcu_read_unlock_bh();

        if (unlikely(send))
                wg_packet_send_queued_handshake_initiation(peer, false);
}

static unsigned int calculate_skb_padding(struct sk_buff *skb)
{
        unsigned int padded_size, last_unit = skb->len;

        if (unlikely(!PACKET_CB(skb)->mtu))
                return ALIGN(last_unit, MESSAGE_PADDING_MULTIPLE) - last_unit;

        /* We do this modulo business with the MTU, just in case the networking
         * layer gives us a packet that's bigger than the MTU. In that case, we
         * wouldn't want the final subtraction to overflow in the case of the
         * padded_size being clamped. Fortunately, that's very rarely the case,
         * so we optimize for that not happening.
         */
        if (unlikely(last_unit > PACKET_CB(skb)->mtu))
                last_unit %= PACKET_CB(skb)->mtu;

        padded_size = min(PACKET_CB(skb)->mtu,
                          ALIGN(last_unit, MESSAGE_PADDING_MULTIPLE));
        return padded_size - last_unit;
}

static bool encrypt_packet(struct sk_buff *skb, struct noise_keypair *keypair)
{
        unsigned int padding_len, plaintext_len, trailer_len;
        struct scatterlist sg[MAX_SKB_FRAGS + 8];
        struct message_data *header;
        struct sk_buff *trailer;
        int num_frags;

        /* Force hash calculation before encryption so that flow analysis is
         * consistent over the inner packet.
         */
        skb_get_hash(skb);

        /* Calculate lengths. */
        padding_len = calculate_skb_padding(skb);
        trailer_len = padding_len + noise_encrypted_len(0);
        plaintext_len = skb->len + padding_len;

        /* Expand data section to have room for padding and auth tag. */
        num_frags = skb_cow_data(skb, trailer_len, &trailer);
        if (unlikely(num_frags < 0 || num_frags > ARRAY_SIZE(sg)))
                return false;

        /* Set the padding to zeros, and make sure it and the auth tag are part
         * of the skb.
         */
        memset(skb_tail_pointer(trailer), 0, padding_len);

        /* Expand head section to have room for our header and the network
         * stack's headers.
         */
        if (unlikely(skb_cow_head(skb, DATA_PACKET_HEAD_ROOM) < 0))
                return false;

        /* Finalize checksum calculation for the inner packet, if required. */
        if (unlikely(skb->ip_summed == CHECKSUM_PARTIAL &&
                     skb_checksum_help(skb)))
                return false;

        /* Only after checksumming can we safely add on the padding at the end
         * and the header.
         */
        skb_set_inner_network_header(skb, 0);
        header = (struct message_data *)skb_push(skb, sizeof(*header));
        header->header.type = cpu_to_le32(MESSAGE_DATA);
        header->key_idx = keypair->remote_index;
        header->counter = cpu_to_le64(PACKET_CB(skb)->nonce);
        pskb_put(skb, trailer, trailer_len);

        /* Now we can encrypt the scattergather segments */
        sg_init_table(sg, num_frags);
        if (skb_to_sgvec(skb, sg, sizeof(struct message_data),
                         noise_encrypted_len(plaintext_len)) <= 0)
                return false;
        return chacha20poly1305_encrypt_sg_inplace(sg, plaintext_len, NULL, 0,
                                                   PACKET_CB(skb)->nonce,
                                                   keypair->sending.key);
}

void wg_packet_send_keepalive(struct wg_peer *peer)
{
        struct sk_buff *skb;

        if (skb_queue_empty_lockless(&peer->staged_packet_queue)) {
                skb = alloc_skb(DATA_PACKET_HEAD_ROOM + MESSAGE_MINIMUM_LENGTH,
                                GFP_ATOMIC);
                if (unlikely(!skb))
                        return;
                skb_reserve(skb, DATA_PACKET_HEAD_ROOM);
                skb->dev = peer->device->dev;
                PACKET_CB(skb)->mtu = skb->dev->mtu;
                skb_queue_tail(&peer->staged_packet_queue, skb);
                net_dbg_ratelimited("%s: Sending keepalive packet to peer %llu (%pISpfsc)\n",
                                    peer->device->dev->name, peer->internal_id,
                                    &peer->endpoint.addr);
        }

        wg_packet_send_staged_packets(peer);
}

static void wg_packet_create_data_done(struct wg_peer *peer, struct sk_buff *first)
{
        struct sk_buff *skb, *next;
        bool is_keepalive, data_sent = false;

        wg_timers_any_authenticated_packet_traversal(peer);
        wg_timers_any_authenticated_packet_sent(peer);
        skb_list_walk_safe(first, skb, next) {
                is_keepalive = skb->len == message_data_len(0);
                if (likely(!wg_socket_send_skb_to_peer(peer, skb,
                                PACKET_CB(skb)->ds) && !is_keepalive))
                        data_sent = true;
        }

        if (likely(data_sent))
                wg_timers_data_sent(peer);

        keep_key_fresh(peer);
}

void wg_packet_tx_worker(struct work_struct *work)
{
        struct wg_peer *peer = container_of(work, struct wg_peer, transmit_packet_work);
        struct noise_keypair *keypair;
        enum packet_state state;
        struct sk_buff *first;

        while ((first = wg_prev_queue_peek(&peer->tx_queue)) != NULL &&
               (state = atomic_read_acquire(&PACKET_CB(first)->state)) !=
                       PACKET_STATE_UNCRYPTED) {
                wg_prev_queue_drop_peeked(&peer->tx_queue);
                keypair = PACKET_CB(first)->keypair;

                if (likely(state == PACKET_STATE_CRYPTED))
                        wg_packet_create_data_done(peer, first);
                else
                        kfree_skb_list(first);

                wg_noise_keypair_put(keypair, false);
                wg_peer_put(peer);
                if (need_resched())
                        cond_resched();
        }
}

void wg_packet_encrypt_worker(struct work_struct *work)
{
        struct crypt_queue *queue = container_of(work, struct multicore_worker,
                                                 work)->ptr;
        struct sk_buff *first, *skb, *next;

        while ((first = ptr_ring_consume_bh(&queue->ring)) != NULL) {
                enum packet_state state = PACKET_STATE_CRYPTED;

                skb_list_walk_safe(first, skb, next) {
                        if (likely(encrypt_packet(skb,
                                        PACKET_CB(first)->keypair))) {
                                wg_reset_packet(skb, true);
                        } else {
                                state = PACKET_STATE_DEAD;
                                break;
                        }
                }
                wg_queue_enqueue_per_peer_tx(first, state);
                if (need_resched())
                        cond_resched();
        }
}

static void wg_packet_create_data(struct wg_peer *peer, struct sk_buff *first)
{
        struct wg_device *wg = peer->device;
        int ret = -EINVAL;

        rcu_read_lock_bh();
        if (unlikely(READ_ONCE(peer->is_dead)))
                goto err;

        ret = wg_queue_enqueue_per_device_and_peer(&wg->encrypt_queue, &peer->tx_queue, first,
                                                   wg->packet_crypt_wq);
        if (unlikely(ret == -EPIPE))
                wg_queue_enqueue_per_peer_tx(first, PACKET_STATE_DEAD);
err:
        rcu_read_unlock_bh();
        if (likely(!ret || ret == -EPIPE))
                return;
        wg_noise_keypair_put(PACKET_CB(first)->keypair, false);
        wg_peer_put(peer);
        kfree_skb_list(first);
}

void wg_packet_purge_staged_packets(struct wg_peer *peer)
{
        spin_lock_bh(&peer->staged_packet_queue.lock);
        DEV_STATS_ADD(peer->device->dev, tx_dropped,
                      peer->staged_packet_queue.qlen);
        __skb_queue_purge(&peer->staged_packet_queue);
        spin_unlock_bh(&peer->staged_packet_queue.lock);
}

void wg_packet_send_staged_packets(struct wg_peer *peer)
{
        struct noise_keypair *keypair;
        struct sk_buff_head packets;
        struct sk_buff *skb;

        /* Steal the current queue into our local one. */
        __skb_queue_head_init(&packets);
        spin_lock_bh(&peer->staged_packet_queue.lock);
        skb_queue_splice_init(&peer->staged_packet_queue, &packets);
        spin_unlock_bh(&peer->staged_packet_queue.lock);
        if (unlikely(skb_queue_empty(&packets)))
                return;

        /* First we make sure we have a valid reference to a valid key. */
        rcu_read_lock_bh();
        keypair = wg_noise_keypair_get(
                rcu_dereference_bh(peer->keypairs.current_keypair));
        rcu_read_unlock_bh();
        if (unlikely(!keypair))
                goto out_nokey;
        if (unlikely(!READ_ONCE(keypair->sending.is_valid)))
                goto out_nokey;
        if (unlikely(wg_birthdate_has_expired(keypair->sending.birthdate,
                                              REJECT_AFTER_TIME)))
                goto out_invalid;

        /* After we know we have a somewhat valid key, we now try to assign
         * nonces to all of the packets in the queue. If we can't assign nonces
         * for all of them, we just consider it a failure and wait for the next
         * handshake.
         */
        skb_queue_walk(&packets, skb) {
                /* 0 for no outer TOS: no leak. TODO: at some later point, we
                 * might consider using flowi->tos as outer instead.
                 */
                PACKET_CB(skb)->ds = ip_tunnel_ecn_encap(0, ip_hdr(skb), skb);
                PACKET_CB(skb)->nonce =
                                atomic64_inc_return(&keypair->sending_counter) - 1;
                if (unlikely(PACKET_CB(skb)->nonce >= REJECT_AFTER_MESSAGES))
                        goto out_invalid;
        }

        packets.prev->next = NULL;
        wg_peer_get(keypair->entry.peer);
        PACKET_CB(packets.next)->keypair = keypair;
        wg_packet_create_data(peer, packets.next);
        return;

out_invalid:
        WRITE_ONCE(keypair->sending.is_valid, false);
out_nokey:
        wg_noise_keypair_put(keypair, false);

        /* We orphan the packets if we're waiting on a handshake, so that they
         * don't block a socket's pool.
         */
        skb_queue_walk(&packets, skb)
                skb_orphan(skb);
        /* Then we put them back on the top of the queue. We're not too
         * concerned about accidentally getting things a little out of order if
         * packets are being added really fast, because this queue is for before
         * packets can even be sent and it's small anyway.
         */
        spin_lock_bh(&peer->staged_packet_queue.lock);
        skb_queue_splice(&packets, &peer->staged_packet_queue);
        spin_unlock_bh(&peer->staged_packet_queue.lock);

        /* If we're exiting because there's something wrong with the key, it
         * means we should initiate a new handshake.
         */
        wg_packet_send_queued_handshake_initiation(peer, false);
}






















































































   33 




   33 



    4 
   31 











    1 








   26 














































































































































































































































































































    1 



    1 



























   20 
   20 

















   20 










   17 

































































   20 



   17 
    4 






    3 



    1 



    2 





   27 








    2 

    1 

    1 

    1 



    1 


    1 



   20 




   18 

   18 




    8 

















    8 



    8 


















    1 


    6 





    3 

    3 





    4 

    1 


    1 




    1 

    1 




   36 
   36 


































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
// SPDX-License-Identifier: GPL-2.0-only
/* bpf/cpumap.c
 *
 * Copyright (c) 2017 Jesper Dangaard Brouer, Red Hat Inc.
 */

/* The 'cpumap' is primarily used as a backend map for XDP BPF helper
 * call bpf_redirect_map() and XDP_REDIRECT action, like 'devmap'.
 *
 * Unlike devmap which redirects XDP frames out another NIC device,
 * this map type redirects raw XDP frames to another CPU.  The remote
 * CPU will do SKB-allocation and call the normal network stack.
 *
 * This is a scalability and isolation mechanism, that allow
 * separating the early driver network XDP layer, from the rest of the
 * netstack, and assigning dedicated CPUs for this stage.  This
 * basically allows for 10G wirespeed pre-filtering via bpf.
 */
#include <linux/bitops.h>
#include <linux/bpf.h>
#include <linux/filter.h>
#include <linux/ptr_ring.h>
#include <net/xdp.h>

#include <linux/sched.h>
#include <linux/workqueue.h>
#include <linux/kthread.h>
#include <linux/capability.h>
#include <linux/completion.h>
#include <trace/events/xdp.h>

#include <linux/netdevice.h>   /* netif_receive_skb_list */
#include <linux/etherdevice.h> /* eth_type_trans */

/* General idea: XDP packets getting XDP redirected to another CPU,
 * will maximum be stored/queued for one driver ->poll() call.  It is
 * guaranteed that queueing the frame and the flush operation happen on
 * same CPU.  Thus, cpu_map_flush operation can deduct via this_cpu_ptr()
 * which queue in bpf_cpu_map_entry contains packets.
 */

#define CPU_MAP_BULK_SIZE 8  /* 8 == one cacheline on 64-bit archs */
struct bpf_cpu_map_entry;
struct bpf_cpu_map;

struct xdp_bulk_queue {
        void *q[CPU_MAP_BULK_SIZE];
        struct list_head flush_node;
        struct bpf_cpu_map_entry *obj;
        unsigned int count;
};

/* Struct for every remote "destination" CPU in map */
struct bpf_cpu_map_entry {
        u32 cpu;    /* kthread CPU and map index */
        int map_id; /* Back reference to map */

        /* XDP can run multiple RX-ring queues, need __percpu enqueue store */
        struct xdp_bulk_queue __percpu *bulkq;

        struct bpf_cpu_map *cmap;

        /* Queue with potential multi-producers, and single-consumer kthread */
        struct ptr_ring *queue;
        struct task_struct *kthread;

        struct bpf_cpumap_val value;
        struct bpf_prog *prog;

        atomic_t refcnt; /* Control when this struct can be free'ed */
        struct rcu_head rcu;

        struct work_struct kthread_stop_wq;
        struct completion kthread_running;
};

struct bpf_cpu_map {
        struct bpf_map map;
        /* Below members specific for map type */
        struct bpf_cpu_map_entry __rcu **cpu_map;
};

static DEFINE_PER_CPU(struct list_head, cpu_map_flush_list);

static struct bpf_map *cpu_map_alloc(union bpf_attr *attr)
{
        u32 value_size = attr->value_size;
        struct bpf_cpu_map *cmap;
        int err = -ENOMEM;

        if (!bpf_capable())
                return ERR_PTR(-EPERM);

        /* check sanity of attributes */
        if (attr->max_entries == 0 || attr->key_size != 4 ||
            (value_size != offsetofend(struct bpf_cpumap_val, qsize) &&
             value_size != offsetofend(struct bpf_cpumap_val, bpf_prog.fd)) ||
            attr->map_flags & ~BPF_F_NUMA_NODE)
                return ERR_PTR(-EINVAL);

        cmap = kzalloc(sizeof(*cmap), GFP_USER | __GFP_ACCOUNT);
        if (!cmap)
                return ERR_PTR(-ENOMEM);

        bpf_map_init_from_attr(&cmap->map, attr);

        /* Pre-limit array size based on NR_CPUS, not final CPU check */
        if (cmap->map.max_entries > NR_CPUS) {
                err = -E2BIG;
                goto free_cmap;
        }

        /* Alloc array for possible remote "destination" CPUs */
        cmap->cpu_map = bpf_map_area_alloc(cmap->map.max_entries *
                                           sizeof(struct bpf_cpu_map_entry *),
                                           cmap->map.numa_node);
        if (!cmap->cpu_map)
                goto free_cmap;

        return &cmap->map;
free_cmap:
        kfree(cmap);
        return ERR_PTR(err);
}

static void get_cpu_map_entry(struct bpf_cpu_map_entry *rcpu)
{
        atomic_inc(&rcpu->refcnt);
}

static void __cpu_map_ring_cleanup(struct ptr_ring *ring)
{
        /* The tear-down procedure should have made sure that queue is
         * empty.  See __cpu_map_entry_replace() and work-queue
         * invoked cpu_map_kthread_stop(). Catch any broken behaviour
         * gracefully and warn once.
         */
        void *ptr;

        while ((ptr = ptr_ring_consume(ring))) {
                WARN_ON_ONCE(1);
                if (unlikely(__ptr_test_bit(0, &ptr))) {
                        __ptr_clear_bit(0, &ptr);
                        kfree_skb(ptr);
                        continue;
                }
                xdp_return_frame(ptr);
        }
}

static void put_cpu_map_entry(struct bpf_cpu_map_entry *rcpu)
{
        if (atomic_dec_and_test(&rcpu->refcnt)) {
                if (rcpu->prog)
                        bpf_prog_put(rcpu->prog);
                /* The queue should be empty at this point */
                __cpu_map_ring_cleanup(rcpu->queue);
                ptr_ring_cleanup(rcpu->queue, NULL);
                kfree(rcpu->queue);
                kfree(rcpu);
        }
}

/* called from workqueue, to workaround syscall using preempt_disable */
static void cpu_map_kthread_stop(struct work_struct *work)
{
        struct bpf_cpu_map_entry *rcpu;

        rcpu = container_of(work, struct bpf_cpu_map_entry, kthread_stop_wq);

        /* Wait for flush in __cpu_map_entry_free(), via full RCU barrier,
         * as it waits until all in-flight call_rcu() callbacks complete.
         */
        rcu_barrier();

        /* kthread_stop will wake_up_process and wait for it to complete */
        kthread_stop(rcpu->kthread);
}

static void cpu_map_bpf_prog_run_skb(struct bpf_cpu_map_entry *rcpu,
                                     struct list_head *listp,
                                     struct xdp_cpumap_stats *stats)
{
        struct sk_buff *skb, *tmp;
        struct xdp_buff xdp;
        u32 act;
        int err;

        list_for_each_entry_safe(skb, tmp, listp, list) {
                act = bpf_prog_run_generic_xdp(skb, &xdp, rcpu->prog);
                switch (act) {
                case XDP_PASS:
                        break;
                case XDP_REDIRECT:
                        skb_list_del_init(skb);
                        err = xdp_do_generic_redirect(skb->dev, skb, &xdp,
                                                      rcpu->prog);
                        if (unlikely(err)) {
                                kfree_skb(skb);
                                stats->drop++;
                        } else {
                                stats->redirect++;
                        }
                        return;
                default:
                        bpf_warn_invalid_xdp_action(act);
                        fallthrough;
                case XDP_ABORTED:
                        trace_xdp_exception(skb->dev, rcpu->prog, act);
                        fallthrough;
                case XDP_DROP:
                        skb_list_del_init(skb);
                        kfree_skb(skb);
                        stats->drop++;
                        return;
                }
        }
}

static int cpu_map_bpf_prog_run_xdp(struct bpf_cpu_map_entry *rcpu,
                                    void **frames, int n,
                                    struct xdp_cpumap_stats *stats)
{
        struct xdp_rxq_info rxq = {};
        struct xdp_buff xdp;
        int i, nframes = 0;

        xdp_set_return_frame_no_direct();
        xdp.rxq = &rxq;

        for (i = 0; i < n; i++) {
                struct xdp_frame *xdpf = frames[i];
                u32 act;
                int err;

                rxq.dev = xdpf->dev_rx;
                rxq.mem = xdpf->mem;
                /* TODO: report queue_index to xdp_rxq_info */

                xdp_convert_frame_to_buff(xdpf, &xdp);

                act = bpf_prog_run_xdp(rcpu->prog, &xdp);
                switch (act) {
                case XDP_PASS:
                        err = xdp_update_frame_from_buff(&xdp, xdpf);
                        if (err < 0) {
                                xdp_return_frame(xdpf);
                                stats->drop++;
                        } else {
                                frames[nframes++] = xdpf;
                                stats->pass++;
                        }
                        break;
                case XDP_REDIRECT:
                        err = xdp_do_redirect(xdpf->dev_rx, &xdp,
                                              rcpu->prog);
                        if (unlikely(err)) {
                                xdp_return_frame(xdpf);
                                stats->drop++;
                        } else {
                                stats->redirect++;
                        }
                        break;
                default:
                        bpf_warn_invalid_xdp_action(act);
                        fallthrough;
                case XDP_DROP:
                        xdp_return_frame(xdpf);
                        stats->drop++;
                        break;
                }
        }

        xdp_clear_return_frame_no_direct();

        return nframes;
}

#define CPUMAP_BATCH 8

static int cpu_map_bpf_prog_run(struct bpf_cpu_map_entry *rcpu, void **frames,
                                int xdp_n, struct xdp_cpumap_stats *stats,
                                struct list_head *list)
{
        int nframes;

        if (!rcpu->prog)
                return xdp_n;

        rcu_read_lock_bh();

        nframes = cpu_map_bpf_prog_run_xdp(rcpu, frames, xdp_n, stats);

        if (stats->redirect)
                xdp_do_flush();

        if (unlikely(!list_empty(list)))
                cpu_map_bpf_prog_run_skb(rcpu, list, stats);

        rcu_read_unlock_bh(); /* resched point, may call do_softirq() */

        return nframes;
}

static int cpu_map_kthread_run(void *data)
{
        struct bpf_cpu_map_entry *rcpu = data;
        unsigned long last_qs = jiffies;

        complete(&rcpu->kthread_running);
        set_current_state(TASK_INTERRUPTIBLE);

        /* When kthread gives stop order, then rcpu have been disconnected
         * from map, thus no new packets can enter. Remaining in-flight
         * per CPU stored packets are flushed to this queue.  Wait honoring
         * kthread_stop signal until queue is empty.
         */
        while (!kthread_should_stop() || !__ptr_ring_empty(rcpu->queue)) {
                struct xdp_cpumap_stats stats = {}; /* zero stats */
                unsigned int kmem_alloc_drops = 0, sched = 0;
                gfp_t gfp = __GFP_ZERO | GFP_ATOMIC;
                int i, n, m, nframes, xdp_n;
                void *frames[CPUMAP_BATCH];
                void *skbs[CPUMAP_BATCH];
                LIST_HEAD(list);

                /* Release CPU reschedule checks */
                if (__ptr_ring_empty(rcpu->queue)) {
                        set_current_state(TASK_INTERRUPTIBLE);
                        /* Recheck to avoid lost wake-up */
                        if (__ptr_ring_empty(rcpu->queue)) {
                                schedule();
                                sched = 1;
                                last_qs = jiffies;
                        } else {
                                __set_current_state(TASK_RUNNING);
                        }
                } else {
                        rcu_softirq_qs_periodic(last_qs);
                        sched = cond_resched();
                }

                /*
                 * The bpf_cpu_map_entry is single consumer, with this
                 * kthread CPU pinned. Lockless access to ptr_ring
                 * consume side valid as no-resize allowed of queue.
                 */
                n = __ptr_ring_consume_batched(rcpu->queue, frames,
                                               CPUMAP_BATCH);
                for (i = 0, xdp_n = 0; i < n; i++) {
                        void *f = frames[i];
                        struct page *page;

                        if (unlikely(__ptr_test_bit(0, &f))) {
                                struct sk_buff *skb = f;

                                __ptr_clear_bit(0, &skb);
                                list_add_tail(&skb->list, &list);
                                continue;
                        }

                        frames[xdp_n++] = f;
                        page = virt_to_page(f);

                        /* Bring struct page memory area to curr CPU. Read by
                         * build_skb_around via page_is_pfmemalloc(), and when
                         * freed written by page_frag_free call.
                         */
                        prefetchw(page);
                }

                /* Support running another XDP prog on this CPU */
                nframes = cpu_map_bpf_prog_run(rcpu, frames, xdp_n, &stats, &list);
                if (nframes) {
                        m = kmem_cache_alloc_bulk(skbuff_head_cache, gfp, nframes, skbs);
                        if (unlikely(m == 0)) {
                                for (i = 0; i < nframes; i++)
                                        skbs[i] = NULL; /* effect: xdp_return_frame */
                                kmem_alloc_drops += nframes;
                        }
                }

                local_bh_disable();
                for (i = 0; i < nframes; i++) {
                        struct xdp_frame *xdpf = frames[i];
                        struct sk_buff *skb = skbs[i];

                        skb = __xdp_build_skb_from_frame(xdpf, skb,
                                                         xdpf->dev_rx);
                        if (!skb) {
                                xdp_return_frame(xdpf);
                                continue;
                        }

                        list_add_tail(&skb->list, &list);
                }
                netif_receive_skb_list(&list);

                /* Feedback loop via tracepoint */
                trace_xdp_cpumap_kthread(rcpu->map_id, n, kmem_alloc_drops,
                                         sched, &stats);

                local_bh_enable(); /* resched point, may call do_softirq() */
        }
        __set_current_state(TASK_RUNNING);

        put_cpu_map_entry(rcpu);
        return 0;
}

static int __cpu_map_load_bpf_program(struct bpf_cpu_map_entry *rcpu, int fd)
{
        struct bpf_prog *prog;

        prog = bpf_prog_get_type(fd, BPF_PROG_TYPE_XDP);
        if (IS_ERR(prog))
                return PTR_ERR(prog);

        if (prog->expected_attach_type != BPF_XDP_CPUMAP) {
                bpf_prog_put(prog);
                return -EINVAL;
        }

        rcpu->value.bpf_prog.id = prog->aux->id;
        rcpu->prog = prog;

        return 0;
}

static struct bpf_cpu_map_entry *
__cpu_map_entry_alloc(struct bpf_map *map, struct bpf_cpumap_val *value,
                      u32 cpu)
{
        int numa, err, i, fd = value->bpf_prog.fd;
        gfp_t gfp = GFP_KERNEL | __GFP_NOWARN;
        struct bpf_cpu_map_entry *rcpu;
        struct xdp_bulk_queue *bq;

        /* Have map->numa_node, but choose node of redirect target CPU */
        numa = cpu_to_node(cpu);

        rcpu = bpf_map_kmalloc_node(map, sizeof(*rcpu), gfp | __GFP_ZERO, numa);
        if (!rcpu)
                return NULL;

        /* Alloc percpu bulkq */
        rcpu->bulkq = bpf_map_alloc_percpu(map, sizeof(*rcpu->bulkq),
                                           sizeof(void *), gfp);
        if (!rcpu->bulkq)
                goto free_rcu;

        for_each_possible_cpu(i) {
                bq = per_cpu_ptr(rcpu->bulkq, i);
                bq->obj = rcpu;
        }

        /* Alloc queue */
        rcpu->queue = bpf_map_kmalloc_node(map, sizeof(*rcpu->queue), gfp,
                                           numa);
        if (!rcpu->queue)
                goto free_bulkq;

        err = ptr_ring_init(rcpu->queue, value->qsize, gfp);
        if (err)
                goto free_queue;

        rcpu->cpu    = cpu;
        rcpu->map_id = map->id;
        rcpu->value.qsize  = value->qsize;

        if (fd > 0 && __cpu_map_load_bpf_program(rcpu, fd))
                goto free_ptr_ring;

        /* Setup kthread */
        init_completion(&rcpu->kthread_running);
        rcpu->kthread = kthread_create_on_node(cpu_map_kthread_run, rcpu, numa,
                                               "cpumap/%d/map:%d", cpu,
                                               map->id);
        if (IS_ERR(rcpu->kthread))
                goto free_prog;

        get_cpu_map_entry(rcpu); /* 1-refcnt for being in cmap->cpu_map[] */
        get_cpu_map_entry(rcpu); /* 1-refcnt for kthread */

        /* Make sure kthread runs on a single CPU */
        kthread_bind(rcpu->kthread, cpu);
        wake_up_process(rcpu->kthread);

        /* Make sure kthread has been running, so kthread_stop() will not
         * stop the kthread prematurely and all pending frames or skbs
         * will be handled by the kthread before kthread_stop() returns.
         */
        wait_for_completion(&rcpu->kthread_running);

        return rcpu;

free_prog:
        if (rcpu->prog)
                bpf_prog_put(rcpu->prog);
free_ptr_ring:
        ptr_ring_cleanup(rcpu->queue, NULL);
free_queue:
        kfree(rcpu->queue);
free_bulkq:
        free_percpu(rcpu->bulkq);
free_rcu:
        kfree(rcpu);
        return NULL;
}

static void __cpu_map_entry_free(struct rcu_head *rcu)
{
        struct bpf_cpu_map_entry *rcpu;

        /* This cpu_map_entry have been disconnected from map and one
         * RCU grace-period have elapsed.  Thus, XDP cannot queue any
         * new packets and cannot change/set flush_needed that can
         * find this entry.
         */
        rcpu = container_of(rcu, struct bpf_cpu_map_entry, rcu);

        free_percpu(rcpu->bulkq);
        /* Cannot kthread_stop() here, last put free rcpu resources */
        put_cpu_map_entry(rcpu);
}

/* After xchg pointer to bpf_cpu_map_entry, use the call_rcu() to
 * ensure any driver rcu critical sections have completed, but this
 * does not guarantee a flush has happened yet. Because driver side
 * rcu_read_lock/unlock only protects the running XDP program.  The
 * atomic xchg and NULL-ptr check in __cpu_map_flush() makes sure a
 * pending flush op doesn't fail.
 *
 * The bpf_cpu_map_entry is still used by the kthread, and there can
 * still be pending packets (in queue and percpu bulkq).  A refcnt
 * makes sure to last user (kthread_stop vs. call_rcu) free memory
 * resources.
 *
 * The rcu callback __cpu_map_entry_free flush remaining packets in
 * percpu bulkq to queue.  Due to caller map_delete_elem() disable
 * preemption, cannot call kthread_stop() to make sure queue is empty.
 * Instead a work_queue is started for stopping kthread,
 * cpu_map_kthread_stop, which waits for an RCU grace period before
 * stopping kthread, emptying the queue.
 */
static void __cpu_map_entry_replace(struct bpf_cpu_map *cmap,
                                    u32 key_cpu, struct bpf_cpu_map_entry *rcpu)
{
        struct bpf_cpu_map_entry *old_rcpu;

        old_rcpu = unrcu_pointer(xchg(&cmap->cpu_map[key_cpu], RCU_INITIALIZER(rcpu)));
        if (old_rcpu) {
                call_rcu(&old_rcpu->rcu, __cpu_map_entry_free);
                INIT_WORK(&old_rcpu->kthread_stop_wq, cpu_map_kthread_stop);
                schedule_work(&old_rcpu->kthread_stop_wq);
        }
}

static int cpu_map_delete_elem(struct bpf_map *map, void *key)
{
        struct bpf_cpu_map *cmap = container_of(map, struct bpf_cpu_map, map);
        u32 key_cpu = *(u32 *)key;

        if (key_cpu >= map->max_entries)
                return -EINVAL;

        /* notice caller map_delete_elem() use preempt_disable() */
        __cpu_map_entry_replace(cmap, key_cpu, NULL);
        return 0;
}

static int cpu_map_update_elem(struct bpf_map *map, void *key, void *value,
                               u64 map_flags)
{
        struct bpf_cpu_map *cmap = container_of(map, struct bpf_cpu_map, map);
        struct bpf_cpumap_val cpumap_value = {};
        struct bpf_cpu_map_entry *rcpu;
        /* Array index key correspond to CPU number */
        u32 key_cpu = *(u32 *)key;

        memcpy(&cpumap_value, value, map->value_size);

        if (unlikely(map_flags > BPF_EXIST))
                return -EINVAL;
        if (unlikely(key_cpu >= cmap->map.max_entries))
                return -E2BIG;
        if (unlikely(map_flags == BPF_NOEXIST))
                return -EEXIST;
        if (unlikely(cpumap_value.qsize > 16384)) /* sanity limit on qsize */
                return -EOVERFLOW;

        /* Make sure CPU is a valid possible cpu */
        if (key_cpu >= nr_cpumask_bits || !cpu_possible(key_cpu))
                return -ENODEV;

        if (cpumap_value.qsize == 0) {
                rcpu = NULL; /* Same as deleting */
        } else {
                /* Updating qsize cause re-allocation of bpf_cpu_map_entry */
                rcpu = __cpu_map_entry_alloc(map, &cpumap_value, key_cpu);
                if (!rcpu)
                        return -ENOMEM;
                rcpu->cmap = cmap;
        }
        rcu_read_lock();
        __cpu_map_entry_replace(cmap, key_cpu, rcpu);
        rcu_read_unlock();
        return 0;
}

static void cpu_map_free(struct bpf_map *map)
{
        struct bpf_cpu_map *cmap = container_of(map, struct bpf_cpu_map, map);
        u32 i;

        /* At this point bpf_prog->aux->refcnt == 0 and this map->refcnt == 0,
         * so the bpf programs (can be more than one that used this map) were
         * disconnected from events. Wait for outstanding critical sections in
         * these programs to complete. The rcu critical section only guarantees
         * no further "XDP/bpf-side" reads against bpf_cpu_map->cpu_map.
         * It does __not__ ensure pending flush operations (if any) are
         * complete.
         */

        synchronize_rcu();

        /* For cpu_map the remote CPUs can still be using the entries
         * (struct bpf_cpu_map_entry).
         */
        for (i = 0; i < cmap->map.max_entries; i++) {
                struct bpf_cpu_map_entry *rcpu;

                rcpu = rcu_dereference_raw(cmap->cpu_map[i]);
                if (!rcpu)
                        continue;

                /* bq flush and cleanup happens after RCU grace-period */
                __cpu_map_entry_replace(cmap, i, NULL); /* call_rcu */
        }
        bpf_map_area_free(cmap->cpu_map);
        kfree(cmap);
}

/* Elements are kept alive by RCU; either by rcu_read_lock() (from syscall) or
 * by local_bh_disable() (from XDP calls inside NAPI). The
 * rcu_read_lock_bh_held() below makes lockdep accept both.
 */
static void *__cpu_map_lookup_elem(struct bpf_map *map, u32 key)
{
        struct bpf_cpu_map *cmap = container_of(map, struct bpf_cpu_map, map);
        struct bpf_cpu_map_entry *rcpu;

        if (key >= map->max_entries)
                return NULL;

        rcpu = rcu_dereference_check(cmap->cpu_map[key],
                                     rcu_read_lock_bh_held());
        return rcpu;
}

static void *cpu_map_lookup_elem(struct bpf_map *map, void *key)
{
        struct bpf_cpu_map_entry *rcpu =
                __cpu_map_lookup_elem(map, *(u32 *)key);

        return rcpu ? &rcpu->value : NULL;
}

static int cpu_map_get_next_key(struct bpf_map *map, void *key, void *next_key)
{
        struct bpf_cpu_map *cmap = container_of(map, struct bpf_cpu_map, map);
        u32 index = key ? *(u32 *)key : U32_MAX;
        u32 *next = next_key;

        if (index >= cmap->map.max_entries) {
                *next = 0;
                return 0;
        }

        if (index == cmap->map.max_entries - 1)
                return -ENOENT;
        *next = index + 1;
        return 0;
}

static int cpu_map_redirect(struct bpf_map *map, u32 ifindex, u64 flags)
{
        return __bpf_xdp_redirect_map(map, ifindex, flags, 0,
                                      __cpu_map_lookup_elem);
}

static int cpu_map_btf_id;
const struct bpf_map_ops cpu_map_ops = {
        .map_meta_equal                = bpf_map_meta_equal,
        .map_alloc                = cpu_map_alloc,
        .map_free                = cpu_map_free,
        .map_delete_elem        = cpu_map_delete_elem,
        .map_update_elem        = cpu_map_update_elem,
        .map_lookup_elem        = cpu_map_lookup_elem,
        .map_get_next_key        = cpu_map_get_next_key,
        .map_check_btf                = map_check_no_btf,
        .map_btf_name                = "bpf_cpu_map",
        .map_btf_id                = &cpu_map_btf_id,
        .map_redirect                = cpu_map_redirect,
};

static void bq_flush_to_queue(struct xdp_bulk_queue *bq)
{
        struct bpf_cpu_map_entry *rcpu = bq->obj;
        unsigned int processed = 0, drops = 0;
        const int to_cpu = rcpu->cpu;
        struct ptr_ring *q;
        int i;

        if (unlikely(!bq->count))
                return;

        q = rcpu->queue;
        spin_lock(&q->producer_lock);

        for (i = 0; i < bq->count; i++) {
                struct xdp_frame *xdpf = bq->q[i];
                int err;

                err = __ptr_ring_produce(q, xdpf);
                if (err) {
                        drops++;
                        xdp_return_frame_rx_napi(xdpf);
                }
                processed++;
        }
        bq->count = 0;
        spin_unlock(&q->producer_lock);

        __list_del_clearprev(&bq->flush_node);

        /* Feedback loop via tracepoints */
        trace_xdp_cpumap_enqueue(rcpu->map_id, processed, drops, to_cpu);
}

/* Runs under RCU-read-side, plus in softirq under NAPI protection.
 * Thus, safe percpu variable access.
 */
static void bq_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_frame *xdpf)
{
        struct list_head *flush_list = this_cpu_ptr(&cpu_map_flush_list);
        struct xdp_bulk_queue *bq = this_cpu_ptr(rcpu->bulkq);

        if (unlikely(bq->count == CPU_MAP_BULK_SIZE))
                bq_flush_to_queue(bq);

        /* Notice, xdp_buff/page MUST be queued here, long enough for
         * driver to code invoking us to finished, due to driver
         * (e.g. ixgbe) recycle tricks based on page-refcnt.
         *
         * Thus, incoming xdp_frame is always queued here (else we race
         * with another CPU on page-refcnt and remaining driver code).
         * Queue time is very short, as driver will invoke flush
         * operation, when completing napi->poll call.
         */
        bq->q[bq->count++] = xdpf;

        if (!bq->flush_node.prev)
                list_add(&bq->flush_node, flush_list);
}

int cpu_map_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_frame *xdpf,
                    struct net_device *dev_rx)
{
        /* Info needed when constructing SKB on remote CPU */
        xdpf->dev_rx = dev_rx;

        bq_enqueue(rcpu, xdpf);
        return 0;
}

int cpu_map_generic_redirect(struct bpf_cpu_map_entry *rcpu,
                             struct sk_buff *skb)
{
        int ret;

        __skb_pull(skb, skb->mac_len);
        skb_set_redirected(skb, false);
        __ptr_set_bit(0, &skb);

        ret = ptr_ring_produce(rcpu->queue, skb);
        if (ret < 0)
                goto trace;

        wake_up_process(rcpu->kthread);
trace:
        trace_xdp_cpumap_enqueue(rcpu->map_id, !ret, !!ret, rcpu->cpu);
        return ret;
}

void __cpu_map_flush(void)
{
        struct list_head *flush_list = this_cpu_ptr(&cpu_map_flush_list);
        struct xdp_bulk_queue *bq, *tmp;

        list_for_each_entry_safe(bq, tmp, flush_list, flush_node) {
                bq_flush_to_queue(bq);

                /* If already running, costs spin_lock_irqsave + smb_mb */
                wake_up_process(bq->obj->kthread);
        }
}

static int __init cpu_map_init(void)
{
        int cpu;

        for_each_possible_cpu(cpu)
                INIT_LIST_HEAD(&per_cpu(cpu_map_flush_list, cpu));
        return 0;
}

subsys_initcall(cpu_map_init);


















































































































































































































































































































































































































































































   49 















   49 


















































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
/*
 *  Mapping of UID/GIDs to name and vice versa.
 *
 *  Copyright (c) 2002, 2003 The Regents of the University of
 *  Michigan.  All rights reserved.
 *
 *  Marius Aamodt Eriksen <marius@umich.edu>
 *
 *  Redistribution and use in source and binary forms, with or without
 *  modification, are permitted provided that the following conditions
 *  are met:
 *
 *  1. Redistributions of source code must retain the above copyright
 *     notice, this list of conditions and the following disclaimer.
 *  2. Redistributions in binary form must reproduce the above copyright
 *     notice, this list of conditions and the following disclaimer in the
 *     documentation and/or other materials provided with the distribution.
 *  3. Neither the name of the University nor the names of its
 *     contributors may be used to endorse or promote products derived
 *     from this software without specific prior written permission.
 *
 *  THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED
 *  WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
 *  MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 *  DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
 *  FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 *  CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 *  SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
 *  BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
 *  LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
 *  NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 *  SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#include <linux/module.h>
#include <linux/seq_file.h>
#include <linux/sched.h>
#include <linux/slab.h>
#include <linux/sunrpc/svc_xprt.h>
#include <net/net_namespace.h>
#include "idmap.h"
#include "nfsd.h"
#include "netns.h"
#include "vfs.h"

/*
 * Turn off idmapping when using AUTH_SYS.
 */
static bool nfs4_disable_idmapping = true;
module_param(nfs4_disable_idmapping, bool, 0644);
MODULE_PARM_DESC(nfs4_disable_idmapping,
                "Turn off server's NFSv4 idmapping when using 'sec=sys'");

/*
 * Cache entry
 */

/*
 * XXX we know that IDMAP_NAMESZ < PAGE_SIZE, but it's ugly to rely on
 * that.
 */

struct ent {
        struct cache_head h;
        int               type;                       /* User / Group */
        u32               id;
        char              name[IDMAP_NAMESZ];
        char              authname[IDMAP_NAMESZ];
        struct rcu_head          rcu_head;
};

/* Common entry handling */

#define ENT_HASHBITS          8
#define ENT_HASHMAX           (1 << ENT_HASHBITS)

static void
ent_init(struct cache_head *cnew, struct cache_head *citm)
{
        struct ent *new = container_of(cnew, struct ent, h);
        struct ent *itm = container_of(citm, struct ent, h);

        new->id = itm->id;
        new->type = itm->type;

        strscpy(new->name, itm->name, sizeof(new->name));
        strscpy(new->authname, itm->authname, sizeof(new->authname));
}

static void
ent_put(struct kref *ref)
{
        struct ent *map = container_of(ref, struct ent, h.ref);
        kfree_rcu(map, rcu_head);
}

static struct cache_head *
ent_alloc(void)
{
        struct ent *e = kmalloc(sizeof(*e), GFP_KERNEL);
        if (e)
                return &e->h;
        else
                return NULL;
}

/*
 * ID -> Name cache
 */

static uint32_t
idtoname_hash(struct ent *ent)
{
        uint32_t hash;

        hash = hash_str(ent->authname, ENT_HASHBITS);
        hash = hash_long(hash ^ ent->id, ENT_HASHBITS);

        /* Flip LSB for user/group */
        if (ent->type == IDMAP_TYPE_GROUP)
                hash ^= 1;

        return hash;
}

static int
idtoname_upcall(struct cache_detail *cd, struct cache_head *h)
{
        return sunrpc_cache_pipe_upcall_timeout(cd, h);
}

static void
idtoname_request(struct cache_detail *cd, struct cache_head *ch, char **bpp,
    int *blen)
{
         struct ent *ent = container_of(ch, struct ent, h);
        char idstr[11];

        qword_add(bpp, blen, ent->authname);
        snprintf(idstr, sizeof(idstr), "%u", ent->id);
        qword_add(bpp, blen, ent->type == IDMAP_TYPE_GROUP ? "group" : "user");
        qword_add(bpp, blen, idstr);

        (*bpp)[-1] = '\n';
}

static int
idtoname_match(struct cache_head *ca, struct cache_head *cb)
{
        struct ent *a = container_of(ca, struct ent, h);
        struct ent *b = container_of(cb, struct ent, h);

        return (a->id == b->id && a->type == b->type &&
            strcmp(a->authname, b->authname) == 0);
}

static int
idtoname_show(struct seq_file *m, struct cache_detail *cd, struct cache_head *h)
{
        struct ent *ent;

        if (h == NULL) {
                seq_puts(m, "#domain type id [name]\n");
                return 0;
        }
        ent = container_of(h, struct ent, h);
        seq_printf(m, "%s %s %u", ent->authname,
                        ent->type == IDMAP_TYPE_GROUP ? "group" : "user",
                        ent->id);
        if (test_bit(CACHE_VALID, &h->flags))
                seq_printf(m, " %s", ent->name);
        seq_putc(m, '\n');
        return 0;
}

static void
warn_no_idmapd(struct cache_detail *detail, int has_died)
{
        printk("nfsd: nfsv4 idmapping failing: has idmapd %s?\n",
                        has_died ? "died" : "not been started");
}


static int         idtoname_parse(struct cache_detail *, char *, int);
static struct ent *idtoname_lookup(struct cache_detail *, struct ent *);
static struct ent *idtoname_update(struct cache_detail *, struct ent *,
                                   struct ent *);

static const struct cache_detail idtoname_cache_template = {
        .owner                = THIS_MODULE,
        .hash_size        = ENT_HASHMAX,
        .name                = "nfs4.idtoname",
        .cache_put        = ent_put,
        .cache_upcall        = idtoname_upcall,
        .cache_request        = idtoname_request,
        .cache_parse        = idtoname_parse,
        .cache_show        = idtoname_show,
        .warn_no_listener = warn_no_idmapd,
        .match                = idtoname_match,
        .init                = ent_init,
        .update                = ent_init,
        .alloc                = ent_alloc,
};

static int
idtoname_parse(struct cache_detail *cd, char *buf, int buflen)
{
        struct ent ent, *res;
        char *buf1, *bp;
        int len;
        int error = -EINVAL;

        if (buf[buflen - 1] != '\n')
                return (-EINVAL);
        buf[buflen - 1]= '\0';

        buf1 = kmalloc(PAGE_SIZE, GFP_KERNEL);
        if (buf1 == NULL)
                return (-ENOMEM);

        memset(&ent, 0, sizeof(ent));

        /* Authentication name */
        len = qword_get(&buf, buf1, PAGE_SIZE);
        if (len <= 0 || len >= IDMAP_NAMESZ)
                goto out;
        memcpy(ent.authname, buf1, sizeof(ent.authname));

        /* Type */
        if (qword_get(&buf, buf1, PAGE_SIZE) <= 0)
                goto out;
        ent.type = strcmp(buf1, "user") == 0 ?
                IDMAP_TYPE_USER : IDMAP_TYPE_GROUP;

        /* ID */
        if (qword_get(&buf, buf1, PAGE_SIZE) <= 0)
                goto out;
        ent.id = simple_strtoul(buf1, &bp, 10);
        if (bp == buf1)
                goto out;

        /* expiry */
        ent.h.expiry_time = get_expiry(&buf);
        if (ent.h.expiry_time == 0)
                goto out;

        error = -ENOMEM;
        res = idtoname_lookup(cd, &ent);
        if (!res)
                goto out;

        /* Name */
        error = -EINVAL;
        len = qword_get(&buf, buf1, PAGE_SIZE);
        if (len < 0 || len >= IDMAP_NAMESZ)
                goto out;
        if (len == 0)
                set_bit(CACHE_NEGATIVE, &ent.h.flags);
        else
                memcpy(ent.name, buf1, sizeof(ent.name));
        error = -ENOMEM;
        res = idtoname_update(cd, &ent, res);
        if (res == NULL)
                goto out;

        cache_put(&res->h, cd);
        error = 0;
out:
        kfree(buf1);
        return error;
}

static struct ent *
idtoname_lookup(struct cache_detail *cd, struct ent *item)
{
        struct cache_head *ch = sunrpc_cache_lookup_rcu(cd, &item->h,
                                                        idtoname_hash(item));
        if (ch)
                return container_of(ch, struct ent, h);
        else
                return NULL;
}

static struct ent *
idtoname_update(struct cache_detail *cd, struct ent *new, struct ent *old)
{
        struct cache_head *ch = sunrpc_cache_update(cd, &new->h, &old->h,
                                                    idtoname_hash(new));
        if (ch)
                return container_of(ch, struct ent, h);
        else
                return NULL;
}


/*
 * Name -> ID cache
 */

static inline int
nametoid_hash(struct ent *ent)
{
        return hash_str(ent->name, ENT_HASHBITS);
}

static int
nametoid_upcall(struct cache_detail *cd, struct cache_head *h)
{
        return sunrpc_cache_pipe_upcall_timeout(cd, h);
}

static void
nametoid_request(struct cache_detail *cd, struct cache_head *ch, char **bpp,
    int *blen)
{
         struct ent *ent = container_of(ch, struct ent, h);

        qword_add(bpp, blen, ent->authname);
        qword_add(bpp, blen, ent->type == IDMAP_TYPE_GROUP ? "group" : "user");
        qword_add(bpp, blen, ent->name);

        (*bpp)[-1] = '\n';
}

static int
nametoid_match(struct cache_head *ca, struct cache_head *cb)
{
        struct ent *a = container_of(ca, struct ent, h);
        struct ent *b = container_of(cb, struct ent, h);

        return (a->type == b->type && strcmp(a->name, b->name) == 0 &&
            strcmp(a->authname, b->authname) == 0);
}

static int
nametoid_show(struct seq_file *m, struct cache_detail *cd, struct cache_head *h)
{
        struct ent *ent;

        if (h == NULL) {
                seq_puts(m, "#domain type name [id]\n");
                return 0;
        }
        ent = container_of(h, struct ent, h);
        seq_printf(m, "%s %s %s", ent->authname,
                        ent->type == IDMAP_TYPE_GROUP ? "group" : "user",
                        ent->name);
        if (test_bit(CACHE_VALID, &h->flags))
                seq_printf(m, " %u", ent->id);
        seq_putc(m, '\n');
        return 0;
}

static struct ent *nametoid_lookup(struct cache_detail *, struct ent *);
static struct ent *nametoid_update(struct cache_detail *, struct ent *,
                                   struct ent *);
static int         nametoid_parse(struct cache_detail *, char *, int);

static const struct cache_detail nametoid_cache_template = {
        .owner                = THIS_MODULE,
        .hash_size        = ENT_HASHMAX,
        .name                = "nfs4.nametoid",
        .cache_put        = ent_put,
        .cache_upcall        = nametoid_upcall,
        .cache_request        = nametoid_request,
        .cache_parse        = nametoid_parse,
        .cache_show        = nametoid_show,
        .warn_no_listener = warn_no_idmapd,
        .match                = nametoid_match,
        .init                = ent_init,
        .update                = ent_init,
        .alloc                = ent_alloc,
};

static int
nametoid_parse(struct cache_detail *cd, char *buf, int buflen)
{
        struct ent ent, *res;
        char *buf1;
        int len, error = -EINVAL;

        if (buf[buflen - 1] != '\n')
                return (-EINVAL);
        buf[buflen - 1]= '\0';

        buf1 = kmalloc(PAGE_SIZE, GFP_KERNEL);
        if (buf1 == NULL)
                return (-ENOMEM);

        memset(&ent, 0, sizeof(ent));

        /* Authentication name */
        len = qword_get(&buf, buf1, PAGE_SIZE);
        if (len <= 0 || len >= IDMAP_NAMESZ)
                goto out;
        memcpy(ent.authname, buf1, sizeof(ent.authname));

        /* Type */
        if (qword_get(&buf, buf1, PAGE_SIZE) <= 0)
                goto out;
        ent.type = strcmp(buf1, "user") == 0 ?
                IDMAP_TYPE_USER : IDMAP_TYPE_GROUP;

        /* Name */
        len = qword_get(&buf, buf1, PAGE_SIZE);
        if (len <= 0 || len >= IDMAP_NAMESZ)
                goto out;
        memcpy(ent.name, buf1, sizeof(ent.name));

        /* expiry */
        ent.h.expiry_time = get_expiry(&buf);
        if (ent.h.expiry_time == 0)
                goto out;

        /* ID */
        error = get_int(&buf, &ent.id);
        if (error == -EINVAL)
                goto out;
        if (error == -ENOENT)
                set_bit(CACHE_NEGATIVE, &ent.h.flags);

        error = -ENOMEM;
        res = nametoid_lookup(cd, &ent);
        if (res == NULL)
                goto out;
        res = nametoid_update(cd, &ent, res);
        if (res == NULL)
                goto out;

        cache_put(&res->h, cd);
        error = 0;
out:
        kfree(buf1);
        return (error);
}


static struct ent *
nametoid_lookup(struct cache_detail *cd, struct ent *item)
{
        struct cache_head *ch = sunrpc_cache_lookup_rcu(cd, &item->h,
                                                        nametoid_hash(item));
        if (ch)
                return container_of(ch, struct ent, h);
        else
                return NULL;
}

static struct ent *
nametoid_update(struct cache_detail *cd, struct ent *new, struct ent *old)
{
        struct cache_head *ch = sunrpc_cache_update(cd, &new->h, &old->h,
                                                    nametoid_hash(new));
        if (ch)
                return container_of(ch, struct ent, h);
        else
                return NULL;
}

/*
 * Exported API
 */

int
nfsd_idmap_init(struct net *net)
{
        int rv;
        struct nfsd_net *nn = net_generic(net, nfsd_net_id);

        nn->idtoname_cache = cache_create_net(&idtoname_cache_template, net);
        if (IS_ERR(nn->idtoname_cache))
                return PTR_ERR(nn->idtoname_cache);
        rv = cache_register_net(nn->idtoname_cache, net);
        if (rv)
                goto destroy_idtoname_cache;
        nn->nametoid_cache = cache_create_net(&nametoid_cache_template, net);
        if (IS_ERR(nn->nametoid_cache)) {
                rv = PTR_ERR(nn->nametoid_cache);
                goto unregister_idtoname_cache;
        }
        rv = cache_register_net(nn->nametoid_cache, net);
        if (rv)
                goto destroy_nametoid_cache;
        return 0;

destroy_nametoid_cache:
        cache_destroy_net(nn->nametoid_cache, net);
unregister_idtoname_cache:
        cache_unregister_net(nn->idtoname_cache, net);
destroy_idtoname_cache:
        cache_destroy_net(nn->idtoname_cache, net);
        return rv;
}

void
nfsd_idmap_shutdown(struct net *net)
{
        struct nfsd_net *nn = net_generic(net, nfsd_net_id);

        cache_unregister_net(nn->idtoname_cache, net);
        cache_unregister_net(nn->nametoid_cache, net);
        cache_destroy_net(nn->idtoname_cache, net);
        cache_destroy_net(nn->nametoid_cache, net);
}

static int
idmap_lookup(struct svc_rqst *rqstp,
                struct ent *(*lookup_fn)(struct cache_detail *, struct ent *),
                struct ent *key, struct cache_detail *detail, struct ent **item)
{
        int ret;

        *item = lookup_fn(detail, key);
        if (!*item)
                return -ENOMEM;
 retry:
        ret = cache_check(detail, &(*item)->h, &rqstp->rq_chandle);

        if (ret == -ETIMEDOUT) {
                struct ent *prev_item = *item;
                *item = lookup_fn(detail, key);
                if (*item != prev_item)
                        goto retry;
                cache_put(&(*item)->h, detail);
        }
        return ret;
}

static char *
rqst_authname(struct svc_rqst *rqstp)
{
        struct auth_domain *clp;

        clp = rqstp->rq_gssclient ? rqstp->rq_gssclient : rqstp->rq_client;
        return clp->name;
}

static __be32
idmap_name_to_id(struct svc_rqst *rqstp, int type, const char *name, u32 namelen,
                u32 *id)
{
        struct ent *item, key = {
                .type = type,
        };
        int ret;
        struct nfsd_net *nn = net_generic(SVC_NET(rqstp), nfsd_net_id);

        if (namelen + 1 > sizeof(key.name))
                return nfserr_badowner;
        memcpy(key.name, name, namelen);
        key.name[namelen] = '\0';
        strscpy(key.authname, rqst_authname(rqstp), sizeof(key.authname));
        ret = idmap_lookup(rqstp, nametoid_lookup, &key, nn->nametoid_cache, &item);
        if (ret == -ENOENT)
                return nfserr_badowner;
        if (ret)
                return nfserrno(ret);
        *id = item->id;
        cache_put(&item->h, nn->nametoid_cache);
        return 0;
}

static __be32 encode_ascii_id(struct xdr_stream *xdr, u32 id)
{
        char buf[11];
        int len;
        __be32 *p;

        len = sprintf(buf, "%u", id);
        p = xdr_reserve_space(xdr, len + 4);
        if (!p)
                return nfserr_resource;
        p = xdr_encode_opaque(p, buf, len);
        return 0;
}

static __be32 idmap_id_to_name(struct xdr_stream *xdr,
                               struct svc_rqst *rqstp, int type, u32 id)
{
        struct ent *item, key = {
                .id = id,
                .type = type,
        };
        __be32 status = nfs_ok;
        __be32 *p;
        int ret;
        struct nfsd_net *nn = net_generic(SVC_NET(rqstp), nfsd_net_id);

        strscpy(key.authname, rqst_authname(rqstp), sizeof(key.authname));
        ret = idmap_lookup(rqstp, idtoname_lookup, &key, nn->idtoname_cache, &item);
        if (ret == -ENOENT)
                return encode_ascii_id(xdr, id);
        if (ret)
                return nfserrno(ret);
        ret = strlen(item->name);
        WARN_ON_ONCE(ret > IDMAP_NAMESZ);

        p = xdr_reserve_space(xdr, ret + 4);
        if (unlikely(!p)) {
                status = nfserr_resource;
                goto out_put;
        }
        xdr_encode_opaque(p, item->name, ret);
out_put:
        cache_put(&item->h, nn->idtoname_cache);
        return status;
}

static bool
numeric_name_to_id(struct svc_rqst *rqstp, int type, const char *name, u32 namelen, u32 *id)
{
        int ret;
        char buf[11];

        if (namelen + 1 > sizeof(buf))
                /* too long to represent a 32-bit id: */
                return false;
        /* Just to make sure it's null-terminated: */
        memcpy(buf, name, namelen);
        buf[namelen] = '\0';
        ret = kstrtouint(buf, 10, id);
        return ret == 0;
}

static __be32
do_name_to_id(struct svc_rqst *rqstp, int type, const char *name, u32 namelen, u32 *id)
{
        if (nfs4_disable_idmapping && rqstp->rq_cred.cr_flavor < RPC_AUTH_GSS)
                if (numeric_name_to_id(rqstp, type, name, namelen, id))
                        return 0;
                /*
                 * otherwise, fall through and try idmapping, for
                 * backwards compatibility with clients sending names:
                 */
        return idmap_name_to_id(rqstp, type, name, namelen, id);
}

static __be32 encode_name_from_id(struct xdr_stream *xdr,
                                  struct svc_rqst *rqstp, int type, u32 id)
{
        if (nfs4_disable_idmapping && rqstp->rq_cred.cr_flavor < RPC_AUTH_GSS)
                return encode_ascii_id(xdr, id);
        return idmap_id_to_name(xdr, rqstp, type, id);
}

__be32
nfsd_map_name_to_uid(struct svc_rqst *rqstp, const char *name, size_t namelen,
                kuid_t *uid)
{
        __be32 status;
        u32 id = -1;

        if (name == NULL || namelen == 0)
                return nfserr_inval;

        status = do_name_to_id(rqstp, IDMAP_TYPE_USER, name, namelen, &id);
        *uid = make_kuid(nfsd_user_namespace(rqstp), id);
        if (!uid_valid(*uid))
                status = nfserr_badowner;
        return status;
}

__be32
nfsd_map_name_to_gid(struct svc_rqst *rqstp, const char *name, size_t namelen,
                kgid_t *gid)
{
        __be32 status;
        u32 id = -1;

        if (name == NULL || namelen == 0)
                return nfserr_inval;

        status = do_name_to_id(rqstp, IDMAP_TYPE_GROUP, name, namelen, &id);
        *gid = make_kgid(nfsd_user_namespace(rqstp), id);
        if (!gid_valid(*gid))
                status = nfserr_badowner;
        return status;
}

__be32 nfsd4_encode_user(struct xdr_stream *xdr, struct svc_rqst *rqstp,
                         kuid_t uid)
{
        u32 id = from_kuid_munged(nfsd_user_namespace(rqstp), uid);
        return encode_name_from_id(xdr, rqstp, IDMAP_TYPE_USER, id);
}

__be32 nfsd4_encode_group(struct xdr_stream *xdr, struct svc_rqst *rqstp,
                          kgid_t gid)
{
        u32 id = from_kgid_munged(nfsd_user_namespace(rqstp), gid);
        return encode_name_from_id(xdr, rqstp, IDMAP_TYPE_GROUP, id);
}




















































































































































































































































































































   49 


   49 






























1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
// SPDX-License-Identifier: GPL-2.0-only
/*
 * net/sched/act_sample.c - Packet sampling tc action
 * Copyright (c) 2017 Yotam Gigi <yotamg@mellanox.com>
 */

#include <linux/types.h>
#include <linux/kernel.h>
#include <linux/string.h>
#include <linux/errno.h>
#include <linux/skbuff.h>
#include <linux/rtnetlink.h>
#include <linux/module.h>
#include <linux/init.h>
#include <linux/gfp.h>
#include <net/net_namespace.h>
#include <net/netlink.h>
#include <net/pkt_sched.h>
#include <linux/tc_act/tc_sample.h>
#include <net/tc_act/tc_sample.h>
#include <net/psample.h>
#include <net/pkt_cls.h>

#include <linux/if_arp.h>

static unsigned int sample_net_id;
static struct tc_action_ops act_sample_ops;

static const struct nla_policy sample_policy[TCA_SAMPLE_MAX + 1] = {
        [TCA_SAMPLE_PARMS]                = { .len = sizeof(struct tc_sample) },
        [TCA_SAMPLE_RATE]                = { .type = NLA_U32 },
        [TCA_SAMPLE_TRUNC_SIZE]                = { .type = NLA_U32 },
        [TCA_SAMPLE_PSAMPLE_GROUP]        = { .type = NLA_U32 },
};

static int tcf_sample_init(struct net *net, struct nlattr *nla,
                           struct nlattr *est, struct tc_action **a,
                           struct tcf_proto *tp,
                           u32 flags, struct netlink_ext_ack *extack)
{
        struct tc_action_net *tn = net_generic(net, sample_net_id);
        bool bind = flags & TCA_ACT_FLAGS_BIND;
        struct nlattr *tb[TCA_SAMPLE_MAX + 1];
        struct psample_group *psample_group;
        u32 psample_group_num, rate, index;
        struct tcf_chain *goto_ch = NULL;
        struct tc_sample *parm;
        struct tcf_sample *s;
        bool exists = false;
        int ret, err;

        if (!nla)
                return -EINVAL;
        ret = nla_parse_nested_deprecated(tb, TCA_SAMPLE_MAX, nla,
                                          sample_policy, NULL);
        if (ret < 0)
                return ret;

        if (!tb[TCA_SAMPLE_PARMS])
                return -EINVAL;

        parm = nla_data(tb[TCA_SAMPLE_PARMS]);
        index = parm->index;
        err = tcf_idr_check_alloc(tn, &index, a, bind);
        if (err < 0)
                return err;
        exists = err;
        if (exists && bind)
                return 0;

        if (!exists) {
                ret = tcf_idr_create(tn, index, est, a,
                                     &act_sample_ops, bind, true, flags);
                if (ret) {
                        tcf_idr_cleanup(tn, index);
                        return ret;
                }
                ret = ACT_P_CREATED;
        } else if (!(flags & TCA_ACT_FLAGS_REPLACE)) {
                tcf_idr_release(*a, bind);
                return -EEXIST;
        }

        if (!tb[TCA_SAMPLE_RATE] || !tb[TCA_SAMPLE_PSAMPLE_GROUP]) {
                NL_SET_ERR_MSG(extack, "sample rate and group are required");
                err = -EINVAL;
                goto release_idr;
        }

        err = tcf_action_check_ctrlact(parm->action, tp, &goto_ch, extack);
        if (err < 0)
                goto release_idr;

        rate = nla_get_u32(tb[TCA_SAMPLE_RATE]);
        if (!rate) {
                NL_SET_ERR_MSG(extack, "invalid sample rate");
                err = -EINVAL;
                goto put_chain;
        }
        psample_group_num = nla_get_u32(tb[TCA_SAMPLE_PSAMPLE_GROUP]);
        psample_group = psample_group_get(net, psample_group_num);
        if (!psample_group) {
                err = -ENOMEM;
                goto put_chain;
        }

        s = to_sample(*a);

        spin_lock_bh(&s->tcf_lock);
        goto_ch = tcf_action_set_ctrlact(*a, parm->action, goto_ch);
        s->rate = rate;
        s->psample_group_num = psample_group_num;
        psample_group = rcu_replace_pointer(s->psample_group, psample_group,
                                            lockdep_is_held(&s->tcf_lock));

        if (tb[TCA_SAMPLE_TRUNC_SIZE]) {
                s->truncate = true;
                s->trunc_size = nla_get_u32(tb[TCA_SAMPLE_TRUNC_SIZE]);
        }
        spin_unlock_bh(&s->tcf_lock);

        if (psample_group)
                psample_group_put(psample_group);
        if (goto_ch)
                tcf_chain_put_by_act(goto_ch);

        return ret;
put_chain:
        if (goto_ch)
                tcf_chain_put_by_act(goto_ch);
release_idr:
        tcf_idr_release(*a, bind);
        return err;
}

static void tcf_sample_cleanup(struct tc_action *a)
{
        struct tcf_sample *s = to_sample(a);
        struct psample_group *psample_group;

        /* last reference to action, no need to lock */
        psample_group = rcu_dereference_protected(s->psample_group, 1);
        RCU_INIT_POINTER(s->psample_group, NULL);
        if (psample_group)
                psample_group_put(psample_group);
}

static bool tcf_sample_dev_ok_push(struct net_device *dev)
{
        switch (dev->type) {
        case ARPHRD_TUNNEL:
        case ARPHRD_TUNNEL6:
        case ARPHRD_SIT:
        case ARPHRD_IPGRE:
        case ARPHRD_IP6GRE:
        case ARPHRD_VOID:
        case ARPHRD_NONE:
                return false;
        default:
                return true;
        }
}

static int tcf_sample_act(struct sk_buff *skb, const struct tc_action *a,
                          struct tcf_result *res)
{
        struct tcf_sample *s = to_sample(a);
        struct psample_group *psample_group;
        struct psample_metadata md = {};
        int retval;

        tcf_lastuse_update(&s->tcf_tm);
        bstats_cpu_update(this_cpu_ptr(s->common.cpu_bstats), skb);
        retval = READ_ONCE(s->tcf_action);

        psample_group = rcu_dereference_bh(s->psample_group);

        /* randomly sample packets according to rate */
        if (psample_group && (prandom_u32() % s->rate == 0)) {
                if (!skb_at_tc_ingress(skb)) {
                        md.in_ifindex = skb->skb_iif;
                        md.out_ifindex = skb->dev->ifindex;
                } else {
                        md.in_ifindex = skb->dev->ifindex;
                }

                /* on ingress, the mac header gets popped, so push it back */
                if (skb_at_tc_ingress(skb) && tcf_sample_dev_ok_push(skb->dev))
                        skb_push(skb, skb->mac_len);

                md.trunc_size = s->truncate ? s->trunc_size : skb->len;
                psample_sample_packet(psample_group, skb, s->rate, &md);

                if (skb_at_tc_ingress(skb) && tcf_sample_dev_ok_push(skb->dev))
                        skb_pull(skb, skb->mac_len);
        }

        return retval;
}

static void tcf_sample_stats_update(struct tc_action *a, u64 bytes, u64 packets,
                                    u64 drops, u64 lastuse, bool hw)
{
        struct tcf_sample *s = to_sample(a);
        struct tcf_t *tm = &s->tcf_tm;

        tcf_action_update_stats(a, bytes, packets, drops, hw);
        tm->lastuse = max_t(u64, tm->lastuse, lastuse);
}

static int tcf_sample_dump(struct sk_buff *skb, struct tc_action *a,
                           int bind, int ref)
{
        unsigned char *b = skb_tail_pointer(skb);
        struct tcf_sample *s = to_sample(a);
        struct tc_sample opt = {
                .index      = s->tcf_index,
                .refcnt     = refcount_read(&s->tcf_refcnt) - ref,
                .bindcnt    = atomic_read(&s->tcf_bindcnt) - bind,
        };
        struct tcf_t t;

        spin_lock_bh(&s->tcf_lock);
        opt.action = s->tcf_action;
        if (nla_put(skb, TCA_SAMPLE_PARMS, sizeof(opt), &opt))
                goto nla_put_failure;

        tcf_tm_dump(&t, &s->tcf_tm);
        if (nla_put_64bit(skb, TCA_SAMPLE_TM, sizeof(t), &t, TCA_SAMPLE_PAD))
                goto nla_put_failure;

        if (nla_put_u32(skb, TCA_SAMPLE_RATE, s->rate))
                goto nla_put_failure;

        if (s->truncate)
                if (nla_put_u32(skb, TCA_SAMPLE_TRUNC_SIZE, s->trunc_size))
                        goto nla_put_failure;

        if (nla_put_u32(skb, TCA_SAMPLE_PSAMPLE_GROUP, s->psample_group_num))
                goto nla_put_failure;
        spin_unlock_bh(&s->tcf_lock);

        return skb->len;

nla_put_failure:
        spin_unlock_bh(&s->tcf_lock);
        nlmsg_trim(skb, b);
        return -1;
}

static int tcf_sample_walker(struct net *net, struct sk_buff *skb,
                             struct netlink_callback *cb, int type,
                             const struct tc_action_ops *ops,
                             struct netlink_ext_ack *extack)
{
        struct tc_action_net *tn = net_generic(net, sample_net_id);

        return tcf_generic_walker(tn, skb, cb, type, ops, extack);
}

static int tcf_sample_search(struct net *net, struct tc_action **a, u32 index)
{
        struct tc_action_net *tn = net_generic(net, sample_net_id);

        return tcf_idr_search(tn, a, index);
}

static void tcf_psample_group_put(void *priv)
{
        struct psample_group *group = priv;

        psample_group_put(group);
}

static struct psample_group *
tcf_sample_get_group(const struct tc_action *a,
                     tc_action_priv_destructor *destructor)
{
        struct tcf_sample *s = to_sample(a);
        struct psample_group *group;

        group = rcu_dereference_protected(s->psample_group,
                                          lockdep_is_held(&s->tcf_lock));
        if (group) {
                psample_group_take(group);
                *destructor = tcf_psample_group_put;
        }

        return group;
}

static struct tc_action_ops act_sample_ops = {
        .kind          = "sample",
        .id          = TCA_ID_SAMPLE,
        .owner          = THIS_MODULE,
        .act          = tcf_sample_act,
        .stats_update = tcf_sample_stats_update,
        .dump          = tcf_sample_dump,
        .init          = tcf_sample_init,
        .cleanup  = tcf_sample_cleanup,
        .walk          = tcf_sample_walker,
        .lookup          = tcf_sample_search,
        .get_psample_group = tcf_sample_get_group,
        .size          = sizeof(struct tcf_sample),
};

static __net_init int sample_init_net(struct net *net)
{
        struct tc_action_net *tn = net_generic(net, sample_net_id);

        return tc_action_net_init(net, tn, &act_sample_ops);
}

static void __net_exit sample_exit_net(struct list_head *net_list)
{
        tc_action_net_exit(net_list, sample_net_id);
}

static struct pernet_operations sample_net_ops = {
        .init = sample_init_net,
        .exit_batch = sample_exit_net,
        .id   = &sample_net_id,
        .size = sizeof(struct tc_action_net),
};

static int __init sample_init_module(void)
{
        return tcf_register_action(&act_sample_ops, &sample_net_ops);
}

static void __exit sample_cleanup_module(void)
{
        tcf_unregister_action(&act_sample_ops, &sample_net_ops);
}

module_init(sample_init_module);
module_exit(sample_cleanup_module);

MODULE_AUTHOR("Yotam Gigi <yotam.gi@gmail.com>");
MODULE_DESCRIPTION("Packet sampling action");
MODULE_LICENSE("GPL v2");
































































































    7 






















    7 



































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
// SPDX-License-Identifier: GPL-2.0
/*
 *  linux/drivers/base/map.c
 *
 * (C) Copyright Al Viro 2002,2003
 *
 * NOTE: data structure needs to be changed.  It works, but for large dev_t
 * it will be too slow.  It is isolated, though, so these changes will be
 * local to that file.
 */

#include <linux/module.h>
#include <linux/slab.h>
#include <linux/mutex.h>
#include <linux/kdev_t.h>
#include <linux/kobject.h>
#include <linux/kobj_map.h>

struct kobj_map {
        struct probe {
                struct probe *next;
                dev_t dev;
                unsigned long range;
                struct module *owner;
                kobj_probe_t *get;
                int (*lock)(dev_t, void *);
                void *data;
        } *probes[255];
        struct mutex *lock;
};

int kobj_map(struct kobj_map *domain, dev_t dev, unsigned long range,
             struct module *module, kobj_probe_t *probe,
             int (*lock)(dev_t, void *), void *data)
{
        unsigned int n = MAJOR(dev + range - 1) - MAJOR(dev) + 1;
        unsigned int index = MAJOR(dev);
        unsigned int i;
        struct probe *p;

        if (n > 255)
                n = 255;

        p = kmalloc_array(n, sizeof(struct probe), GFP_KERNEL);
        if (p == NULL)
                return -ENOMEM;

        for (i = 0; i < n; i++, p++) {
                p->owner = module;
                p->get = probe;
                p->lock = lock;
                p->dev = dev;
                p->range = range;
                p->data = data;
        }
        mutex_lock(domain->lock);
        for (i = 0, p -= n; i < n; i++, p++, index++) {
                struct probe **s = &domain->probes[index % 255];
                while (*s && (*s)->range < range)
                        s = &(*s)->next;
                p->next = *s;
                *s = p;
        }
        mutex_unlock(domain->lock);
        return 0;
}

void kobj_unmap(struct kobj_map *domain, dev_t dev, unsigned long range)
{
        unsigned int n = MAJOR(dev + range - 1) - MAJOR(dev) + 1;
        unsigned int index = MAJOR(dev);
        unsigned int i;
        struct probe *found = NULL;

        if (n > 255)
                n = 255;

        mutex_lock(domain->lock);
        for (i = 0; i < n; i++, index++) {
                struct probe **s;
                for (s = &domain->probes[index % 255]; *s; s = &(*s)->next) {
                        struct probe *p = *s;
                        if (p->dev == dev && p->range == range) {
                                *s = p->next;
                                if (!found)
                                        found = p;
                                break;
                        }
                }
        }
        mutex_unlock(domain->lock);
        kfree(found);
}

struct kobject *kobj_lookup(struct kobj_map *domain, dev_t dev, int *index)
{
        struct kobject *kobj;
        struct probe *p;
        unsigned long best = ~0UL;

retry:
        mutex_lock(domain->lock);
        for (p = domain->probes[MAJOR(dev) % 255]; p; p = p->next) {
                struct kobject *(*probe)(dev_t, int *, void *);
                struct module *owner;
                void *data;

                if (p->dev > dev || p->dev + p->range - 1 < dev)
                        continue;
                if (p->range - 1 >= best)
                        break;
                if (!try_module_get(p->owner))
                        continue;
                owner = p->owner;
                data = p->data;
                probe = p->get;
                best = p->range - 1;
                *index = dev - p->dev;
                if (p->lock && p->lock(dev, data) < 0) {
                        module_put(owner);
                        continue;
                }
                mutex_unlock(domain->lock);
                kobj = probe(dev, index, data);
                /* Currently ->owner protects _only_ ->probe() itself. */
                module_put(owner);
                if (kobj)
                        return kobj;
                goto retry;
        }
        mutex_unlock(domain->lock);
        return NULL;
}

struct kobj_map *kobj_map_init(kobj_probe_t *base_probe, struct mutex *lock)
{
        struct kobj_map *p = kmalloc(sizeof(struct kobj_map), GFP_KERNEL);
        struct probe *base = kzalloc(sizeof(*base), GFP_KERNEL);
        int i;

        if ((p == NULL) || (base == NULL)) {
                kfree(p);
                kfree(base);
                return NULL;
        }

        base->dev = 1;
        base->range = ~0;
        base->get = base_probe;
        for (i = 0; i < 255; i++)
                p->probes[i] = base;
        p->lock = lock;
        return p;
}
































   49 








   52 





   51 








    6 

































































   33 

    2 



















1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _IPV6_FRAG_H
#define _IPV6_FRAG_H
#include <linux/kernel.h>
#include <net/addrconf.h>
#include <net/ipv6.h>
#include <net/inet_frag.h>

enum ip6_defrag_users {
        IP6_DEFRAG_LOCAL_DELIVER,
        IP6_DEFRAG_CONNTRACK_IN,
        __IP6_DEFRAG_CONNTRACK_IN        = IP6_DEFRAG_CONNTRACK_IN + USHRT_MAX,
        IP6_DEFRAG_CONNTRACK_OUT,
        __IP6_DEFRAG_CONNTRACK_OUT        = IP6_DEFRAG_CONNTRACK_OUT + USHRT_MAX,
        IP6_DEFRAG_CONNTRACK_BRIDGE_IN,
        __IP6_DEFRAG_CONNTRACK_BRIDGE_IN = IP6_DEFRAG_CONNTRACK_BRIDGE_IN + USHRT_MAX,
};

/*
 *        Equivalent of ipv4 struct ip
 */
struct frag_queue {
        struct inet_frag_queue        q;

        int                        iif;
        __u16                        nhoffset;
        u8                        ecn;
};

#if IS_ENABLED(CONFIG_IPV6)
static inline void ip6frag_init(struct inet_frag_queue *q, const void *a)
{
        struct frag_queue *fq = container_of(q, struct frag_queue, q);
        const struct frag_v6_compare_key *key = a;

        q->key.v6 = *key;
        fq->ecn = 0;
}

static inline u32 ip6frag_key_hashfn(const void *data, u32 len, u32 seed)
{
        return jhash2(data,
                      sizeof(struct frag_v6_compare_key) / sizeof(u32), seed);
}

static inline u32 ip6frag_obj_hashfn(const void *data, u32 len, u32 seed)
{
        const struct inet_frag_queue *fq = data;

        return jhash2((const u32 *)&fq->key.v6,
                      sizeof(struct frag_v6_compare_key) / sizeof(u32), seed);
}

static inline int
ip6frag_obj_cmpfn(struct rhashtable_compare_arg *arg, const void *ptr)
{
        const struct frag_v6_compare_key *key = arg->key;
        const struct inet_frag_queue *fq = ptr;

        return !!memcmp(&fq->key, key, sizeof(*key));
}

static inline void
ip6frag_expire_frag_queue(struct net *net, struct frag_queue *fq)
{
        struct net_device *dev = NULL;
        struct sk_buff *head;

        rcu_read_lock();
        /* Paired with the WRITE_ONCE() in fqdir_pre_exit(). */
        if (READ_ONCE(fq->q.fqdir->dead))
                goto out_rcu_unlock;
        spin_lock(&fq->q.lock);

        if (fq->q.flags & INET_FRAG_COMPLETE)
                goto out;

        inet_frag_kill(&fq->q);

        dev = dev_get_by_index_rcu(net, fq->iif);
        if (!dev)
                goto out;

        __IP6_INC_STATS(net, __in6_dev_get(dev), IPSTATS_MIB_REASMFAILS);
        __IP6_INC_STATS(net, __in6_dev_get(dev), IPSTATS_MIB_REASMTIMEOUT);

        /* Don't send error if the first segment did not arrive. */
        if (!(fq->q.flags & INET_FRAG_FIRST_IN))
                goto out;

        /* sk_buff::dev and sk_buff::rbnode are unionized. So we
         * pull the head out of the tree in order to be able to
         * deal with head->dev.
         */
        head = inet_frag_pull_head(&fq->q);
        if (!head)
                goto out;

        head->dev = dev;
        spin_unlock(&fq->q.lock);

        icmpv6_send(head, ICMPV6_TIME_EXCEED, ICMPV6_EXC_FRAGTIME, 0);
        kfree_skb(head);
        goto out_rcu_unlock;

out:
        spin_unlock(&fq->q.lock);
out_rcu_unlock:
        rcu_read_unlock();
        inet_frag_put(&fq->q);
}

/* Check if the upper layer header is truncated in the first fragment. */
static inline bool
ipv6frag_thdr_truncated(struct sk_buff *skb, int start, u8 *nexthdrp)
{
        u8 nexthdr = *nexthdrp;
        __be16 frag_off;
        int offset;

        offset = ipv6_skip_exthdr(skb, start, &nexthdr, &frag_off);
        if (offset < 0 || (frag_off & htons(IP6_OFFSET)))
                return false;
        switch (nexthdr) {
        case NEXTHDR_TCP:
                offset += sizeof(struct tcphdr);
                break;
        case NEXTHDR_UDP:
                offset += sizeof(struct udphdr);
                break;
        case NEXTHDR_ICMP:
                offset += sizeof(struct icmp6hdr);
                break;
        default:
                offset += 1;
        }
        if (offset > skb->len)
                return true;
        return false;
}

#endif
#endif



























































































































































































































































































































































  142 






























































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
/* SPDX-License-Identifier: GPL-2.0 */
#undef TRACE_SYSTEM
#define TRACE_SYSTEM xdp

#if !defined(_TRACE_XDP_H) || defined(TRACE_HEADER_MULTI_READ)
#define _TRACE_XDP_H

#include <linux/netdevice.h>
#include <linux/filter.h>
#include <linux/tracepoint.h>
#include <linux/bpf.h>

#define __XDP_ACT_MAP(FN)        \
        FN(ABORTED)                \
        FN(DROP)                \
        FN(PASS)                \
        FN(TX)                        \
        FN(REDIRECT)

#define __XDP_ACT_TP_FN(x)        \
        TRACE_DEFINE_ENUM(XDP_##x);
#define __XDP_ACT_SYM_FN(x)        \
        { XDP_##x, #x },
#define __XDP_ACT_SYM_TAB        \
        __XDP_ACT_MAP(__XDP_ACT_SYM_FN) { -1, NULL }
__XDP_ACT_MAP(__XDP_ACT_TP_FN)

TRACE_EVENT(xdp_exception,

        TP_PROTO(const struct net_device *dev,
                 const struct bpf_prog *xdp, u32 act),

        TP_ARGS(dev, xdp, act),

        TP_STRUCT__entry(
                __field(int, prog_id)
                __field(u32, act)
                __field(int, ifindex)
        ),

        TP_fast_assign(
                __entry->prog_id        = xdp->aux->id;
                __entry->act                = act;
                __entry->ifindex        = dev->ifindex;
        ),

        TP_printk("prog_id=%d action=%s ifindex=%d",
                  __entry->prog_id,
                  __print_symbolic(__entry->act, __XDP_ACT_SYM_TAB),
                  __entry->ifindex)
);

TRACE_EVENT(xdp_bulk_tx,

        TP_PROTO(const struct net_device *dev,
                 int sent, int drops, int err),

        TP_ARGS(dev, sent, drops, err),

        TP_STRUCT__entry(
                __field(int, ifindex)
                __field(u32, act)
                __field(int, drops)
                __field(int, sent)
                __field(int, err)
        ),

        TP_fast_assign(
                __entry->ifindex        = dev->ifindex;
                __entry->act                = XDP_TX;
                __entry->drops                = drops;
                __entry->sent                = sent;
                __entry->err                = err;
        ),

        TP_printk("ifindex=%d action=%s sent=%d drops=%d err=%d",
                  __entry->ifindex,
                  __print_symbolic(__entry->act, __XDP_ACT_SYM_TAB),
                  __entry->sent, __entry->drops, __entry->err)
);

#ifndef __DEVMAP_OBJ_TYPE
#define __DEVMAP_OBJ_TYPE
struct _bpf_dtab_netdev {
        struct net_device *dev;
};
#endif /* __DEVMAP_OBJ_TYPE */

DECLARE_EVENT_CLASS(xdp_redirect_template,

        TP_PROTO(const struct net_device *dev,
                 const struct bpf_prog *xdp,
                 const void *tgt, int err,
                 enum bpf_map_type map_type,
                 u32 map_id, u32 index),

        TP_ARGS(dev, xdp, tgt, err, map_type, map_id, index),

        TP_STRUCT__entry(
                __field(int, prog_id)
                __field(u32, act)
                __field(int, ifindex)
                __field(int, err)
                __field(int, to_ifindex)
                __field(u32, map_id)
                __field(int, map_index)
        ),

        TP_fast_assign(
                u32 ifindex = 0, map_index = index;

                if (map_type == BPF_MAP_TYPE_DEVMAP || map_type == BPF_MAP_TYPE_DEVMAP_HASH) {
                        /* Just leave to_ifindex to 0 if do broadcast redirect,
                         * as tgt will be NULL.
                         */
                        if (tgt)
                                ifindex = ((struct _bpf_dtab_netdev *)tgt)->dev->ifindex;
                } else if (map_type == BPF_MAP_TYPE_UNSPEC && map_id == INT_MAX) {
                        ifindex = index;
                        map_index = 0;
                }

                __entry->prog_id        = xdp->aux->id;
                __entry->act                = XDP_REDIRECT;
                __entry->ifindex        = dev->ifindex;
                __entry->err                = err;
                __entry->to_ifindex        = ifindex;
                __entry->map_id                = map_id;
                __entry->map_index        = map_index;
        ),

        TP_printk("prog_id=%d action=%s ifindex=%d to_ifindex=%d err=%d"
                  " map_id=%d map_index=%d",
                  __entry->prog_id,
                  __print_symbolic(__entry->act, __XDP_ACT_SYM_TAB),
                  __entry->ifindex, __entry->to_ifindex,
                  __entry->err, __entry->map_id, __entry->map_index)
);

DEFINE_EVENT(xdp_redirect_template, xdp_redirect,
        TP_PROTO(const struct net_device *dev,
                 const struct bpf_prog *xdp,
                 const void *tgt, int err,
                 enum bpf_map_type map_type,
                 u32 map_id, u32 index),
        TP_ARGS(dev, xdp, tgt, err, map_type, map_id, index)
);

DEFINE_EVENT(xdp_redirect_template, xdp_redirect_err,
        TP_PROTO(const struct net_device *dev,
                 const struct bpf_prog *xdp,
                 const void *tgt, int err,
                 enum bpf_map_type map_type,
                 u32 map_id, u32 index),
        TP_ARGS(dev, xdp, tgt, err, map_type, map_id, index)
);

#define _trace_xdp_redirect(dev, xdp, to)                                                \
         trace_xdp_redirect(dev, xdp, NULL, 0, BPF_MAP_TYPE_UNSPEC, INT_MAX, to)

#define _trace_xdp_redirect_err(dev, xdp, to, err)                                        \
         trace_xdp_redirect_err(dev, xdp, NULL, err, BPF_MAP_TYPE_UNSPEC, INT_MAX, to)

#define _trace_xdp_redirect_map(dev, xdp, to, map_type, map_id, index) \
         trace_xdp_redirect(dev, xdp, to, 0, map_type, map_id, index)

#define _trace_xdp_redirect_map_err(dev, xdp, to, map_type, map_id, index, err) \
         trace_xdp_redirect_err(dev, xdp, to, err, map_type, map_id, index)

/* not used anymore, but kept around so as not to break old programs */
DEFINE_EVENT(xdp_redirect_template, xdp_redirect_map,
        TP_PROTO(const struct net_device *dev,
                 const struct bpf_prog *xdp,
                 const void *tgt, int err,
                 enum bpf_map_type map_type,
                 u32 map_id, u32 index),
        TP_ARGS(dev, xdp, tgt, err, map_type, map_id, index)
);

DEFINE_EVENT(xdp_redirect_template, xdp_redirect_map_err,
        TP_PROTO(const struct net_device *dev,
                 const struct bpf_prog *xdp,
                 const void *tgt, int err,
                 enum bpf_map_type map_type,
                 u32 map_id, u32 index),
        TP_ARGS(dev, xdp, tgt, err, map_type, map_id, index)
);

TRACE_EVENT(xdp_cpumap_kthread,

        TP_PROTO(int map_id, unsigned int processed,  unsigned int drops,
                 int sched, struct xdp_cpumap_stats *xdp_stats),

        TP_ARGS(map_id, processed, drops, sched, xdp_stats),

        TP_STRUCT__entry(
                __field(int, map_id)
                __field(u32, act)
                __field(int, cpu)
                __field(unsigned int, drops)
                __field(unsigned int, processed)
                __field(int, sched)
                __field(unsigned int, xdp_pass)
                __field(unsigned int, xdp_drop)
                __field(unsigned int, xdp_redirect)
        ),

        TP_fast_assign(
                __entry->map_id                = map_id;
                __entry->act                = XDP_REDIRECT;
                __entry->cpu                = smp_processor_id();
                __entry->drops                = drops;
                __entry->processed        = processed;
                __entry->sched        = sched;
                __entry->xdp_pass        = xdp_stats->pass;
                __entry->xdp_drop        = xdp_stats->drop;
                __entry->xdp_redirect        = xdp_stats->redirect;
        ),

        TP_printk("kthread"
                  " cpu=%d map_id=%d action=%s"
                  " processed=%u drops=%u"
                  " sched=%d"
                  " xdp_pass=%u xdp_drop=%u xdp_redirect=%u",
                  __entry->cpu, __entry->map_id,
                  __print_symbolic(__entry->act, __XDP_ACT_SYM_TAB),
                  __entry->processed, __entry->drops,
                  __entry->sched,
                  __entry->xdp_pass, __entry->xdp_drop, __entry->xdp_redirect)
);

TRACE_EVENT(xdp_cpumap_enqueue,

        TP_PROTO(int map_id, unsigned int processed,  unsigned int drops,
                 int to_cpu),

        TP_ARGS(map_id, processed, drops, to_cpu),

        TP_STRUCT__entry(
                __field(int, map_id)
                __field(u32, act)
                __field(int, cpu)
                __field(unsigned int, drops)
                __field(unsigned int, processed)
                __field(int, to_cpu)
        ),

        TP_fast_assign(
                __entry->map_id                = map_id;
                __entry->act                = XDP_REDIRECT;
                __entry->cpu                = smp_processor_id();
                __entry->drops                = drops;
                __entry->processed        = processed;
                __entry->to_cpu                = to_cpu;
        ),

        TP_printk("enqueue"
                  " cpu=%d map_id=%d action=%s"
                  " processed=%u drops=%u"
                  " to_cpu=%d",
                  __entry->cpu, __entry->map_id,
                  __print_symbolic(__entry->act, __XDP_ACT_SYM_TAB),
                  __entry->processed, __entry->drops,
                  __entry->to_cpu)
);

TRACE_EVENT(xdp_devmap_xmit,

        TP_PROTO(const struct net_device *from_dev,
                 const struct net_device *to_dev,
                 int sent, int drops, int err),

        TP_ARGS(from_dev, to_dev, sent, drops, err),

        TP_STRUCT__entry(
                __field(int, from_ifindex)
                __field(u32, act)
                __field(int, to_ifindex)
                __field(int, drops)
                __field(int, sent)
                __field(int, err)
        ),

        TP_fast_assign(
                __entry->from_ifindex        = from_dev->ifindex;
                __entry->act                = XDP_REDIRECT;
                __entry->to_ifindex        = to_dev->ifindex;
                __entry->drops                = drops;
                __entry->sent                = sent;
                __entry->err                = err;
        ),

        TP_printk("ndo_xdp_xmit"
                  " from_ifindex=%d to_ifindex=%d action=%s"
                  " sent=%d drops=%d"
                  " err=%d",
                  __entry->from_ifindex, __entry->to_ifindex,
                  __print_symbolic(__entry->act, __XDP_ACT_SYM_TAB),
                  __entry->sent, __entry->drops,
                  __entry->err)
);

/* Expect users already include <net/xdp.h>, but not xdp_priv.h */
#include <net/xdp_priv.h>

#define __MEM_TYPE_MAP(FN)        \
        FN(PAGE_SHARED)                \
        FN(PAGE_ORDER0)                \
        FN(PAGE_POOL)                \
        FN(XSK_BUFF_POOL)

#define __MEM_TYPE_TP_FN(x)        \
        TRACE_DEFINE_ENUM(MEM_TYPE_##x);
#define __MEM_TYPE_SYM_FN(x)        \
        { MEM_TYPE_##x, #x },
#define __MEM_TYPE_SYM_TAB        \
        __MEM_TYPE_MAP(__MEM_TYPE_SYM_FN) { -1, 0 }
__MEM_TYPE_MAP(__MEM_TYPE_TP_FN)

TRACE_EVENT(mem_disconnect,

        TP_PROTO(const struct xdp_mem_allocator *xa),

        TP_ARGS(xa),

        TP_STRUCT__entry(
                __field(const struct xdp_mem_allocator *,        xa)
                __field(u32,                mem_id)
                __field(u32,                mem_type)
                __field(const void *,        allocator)
        ),

        TP_fast_assign(
                __entry->xa                = xa;
                __entry->mem_id                = xa->mem.id;
                __entry->mem_type        = xa->mem.type;
                __entry->allocator        = xa->allocator;
        ),

        TP_printk("mem_id=%d mem_type=%s allocator=%p",
                  __entry->mem_id,
                  __print_symbolic(__entry->mem_type, __MEM_TYPE_SYM_TAB),
                  __entry->allocator
        )
);

TRACE_EVENT(mem_connect,

        TP_PROTO(const struct xdp_mem_allocator *xa,
                 const struct xdp_rxq_info *rxq),

        TP_ARGS(xa, rxq),

        TP_STRUCT__entry(
                __field(const struct xdp_mem_allocator *,        xa)
                __field(u32,                mem_id)
                __field(u32,                mem_type)
                __field(const void *,        allocator)
                __field(const struct xdp_rxq_info *,                rxq)
                __field(int,                ifindex)
        ),

        TP_fast_assign(
                __entry->xa                = xa;
                __entry->mem_id                = xa->mem.id;
                __entry->mem_type        = xa->mem.type;
                __entry->allocator        = xa->allocator;
                __entry->rxq                = rxq;
                __entry->ifindex        = rxq->dev->ifindex;
        ),

        TP_printk("mem_id=%d mem_type=%s allocator=%p"
                  " ifindex=%d",
                  __entry->mem_id,
                  __print_symbolic(__entry->mem_type, __MEM_TYPE_SYM_TAB),
                  __entry->allocator,
                  __entry->ifindex
        )
);

TRACE_EVENT(mem_return_failed,

        TP_PROTO(const struct xdp_mem_info *mem,
                 const struct page *page),

        TP_ARGS(mem, page),

        TP_STRUCT__entry(
                __field(const struct page *,        page)
                __field(u32,                mem_id)
                __field(u32,                mem_type)
        ),

        TP_fast_assign(
                __entry->page                = page;
                __entry->mem_id                = mem->id;
                __entry->mem_type        = mem->type;
        ),

        TP_printk("mem_id=%d mem_type=%s page=%p",
                  __entry->mem_id,
                  __print_symbolic(__entry->mem_type, __MEM_TYPE_SYM_TAB),
                  __entry->page
        )
);

#endif /* _TRACE_XDP_H */

#include <trace/define_trace.h>





























   12 



   12 

































   23 






   24 
   24 






















   24 


































   24 



   24 







   12 

   12 

   12 







   25 

















   25 



    2 



   24 






   24 
























    2 














































































































































   23 








    2 







    2 

    1 




    1 



    1 



   19 




























   25 


   25 
















































   24 








   24 













   24 
























































































































    4 


    4 













    3 


    3 

















    3 
























   12 
   12 






   12 
























































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
// SPDX-License-Identifier: GPL-2.0-only
/*
 * File: socket.c
 *
 * Phonet sockets
 *
 * Copyright (C) 2008 Nokia Corporation.
 *
 * Authors: Sakari Ailus <sakari.ailus@nokia.com>
 *          Rémi Denis-Courmont
 */

#include <linux/gfp.h>
#include <linux/kernel.h>
#include <linux/net.h>
#include <linux/poll.h>
#include <linux/sched/signal.h>

#include <net/sock.h>
#include <net/tcp_states.h>

#include <linux/phonet.h>
#include <linux/export.h>
#include <net/phonet/phonet.h>
#include <net/phonet/pep.h>
#include <net/phonet/pn_dev.h>

static int pn_socket_release(struct socket *sock)
{
        struct sock *sk = sock->sk;

        if (sk) {
                sock->sk = NULL;
                sk->sk_prot->close(sk, 0);
        }
        return 0;
}

#define PN_HASHSIZE        16
#define PN_HASHMASK        (PN_HASHSIZE-1)


static struct  {
        struct hlist_head hlist[PN_HASHSIZE];
        struct mutex lock;
} pnsocks;

void __init pn_sock_init(void)
{
        unsigned int i;

        for (i = 0; i < PN_HASHSIZE; i++)
                INIT_HLIST_HEAD(pnsocks.hlist + i);
        mutex_init(&pnsocks.lock);
}

static struct hlist_head *pn_hash_list(u16 obj)
{
        return pnsocks.hlist + (obj & PN_HASHMASK);
}

/*
 * Find address based on socket address, match only certain fields.
 * Also grab sock if it was found. Remember to sock_put it later.
 */
struct sock *pn_find_sock_by_sa(struct net *net, const struct sockaddr_pn *spn)
{
        struct sock *sknode;
        struct sock *rval = NULL;
        u16 obj = pn_sockaddr_get_object(spn);
        u8 res = spn->spn_resource;
        struct hlist_head *hlist = pn_hash_list(obj);

        rcu_read_lock();
        sk_for_each_rcu(sknode, hlist) {
                struct pn_sock *pn = pn_sk(sknode);
                BUG_ON(!pn->sobject); /* unbound socket */

                if (!net_eq(sock_net(sknode), net))
                        continue;
                if (pn_port(obj)) {
                        /* Look up socket by port */
                        if (pn_port(pn->sobject) != pn_port(obj))
                                continue;
                } else {
                        /* If port is zero, look up by resource */
                        if (pn->resource != res)
                                continue;
                }
                if (pn_addr(pn->sobject) &&
                    pn_addr(pn->sobject) != pn_addr(obj))
                        continue;

                rval = sknode;
                sock_hold(sknode);
                break;
        }
        rcu_read_unlock();

        return rval;
}

/* Deliver a broadcast packet (only in bottom-half) */
void pn_deliver_sock_broadcast(struct net *net, struct sk_buff *skb)
{
        struct hlist_head *hlist = pnsocks.hlist;
        unsigned int h;

        rcu_read_lock();
        for (h = 0; h < PN_HASHSIZE; h++) {
                struct sock *sknode;

                sk_for_each(sknode, hlist) {
                        struct sk_buff *clone;

                        if (!net_eq(sock_net(sknode), net))
                                continue;
                        if (!sock_flag(sknode, SOCK_BROADCAST))
                                continue;

                        clone = skb_clone(skb, GFP_ATOMIC);
                        if (clone) {
                                sock_hold(sknode);
                                sk_receive_skb(sknode, clone, 0);
                        }
                }
                hlist++;
        }
        rcu_read_unlock();
}

int pn_sock_hash(struct sock *sk)
{
        struct hlist_head *hlist = pn_hash_list(pn_sk(sk)->sobject);

        mutex_lock(&pnsocks.lock);
        sk_add_node_rcu(sk, hlist);
        mutex_unlock(&pnsocks.lock);

        return 0;
}
EXPORT_SYMBOL(pn_sock_hash);

void pn_sock_unhash(struct sock *sk)
{
        mutex_lock(&pnsocks.lock);
        sk_del_node_init_rcu(sk);
        mutex_unlock(&pnsocks.lock);
        pn_sock_unbind_all_res(sk);
        synchronize_rcu();
}
EXPORT_SYMBOL(pn_sock_unhash);

static DEFINE_MUTEX(port_mutex);

static int pn_socket_bind(struct socket *sock, struct sockaddr *addr, int len)
{
        struct sock *sk = sock->sk;
        struct pn_sock *pn = pn_sk(sk);
        struct sockaddr_pn *spn = (struct sockaddr_pn *)addr;
        int err;
        u16 handle;
        u8 saddr;

        if (sk->sk_prot->bind)
                return sk->sk_prot->bind(sk, addr, len);

        if (len < sizeof(struct sockaddr_pn))
                return -EINVAL;
        if (spn->spn_family != AF_PHONET)
                return -EAFNOSUPPORT;

        handle = pn_sockaddr_get_object((struct sockaddr_pn *)addr);
        saddr = pn_addr(handle);
        if (saddr && phonet_address_lookup(sock_net(sk), saddr))
                return -EADDRNOTAVAIL;

        lock_sock(sk);
        if (sk->sk_state != TCP_CLOSE || pn_port(pn->sobject)) {
                err = -EINVAL; /* attempt to rebind */
                goto out;
        }
        WARN_ON(sk_hashed(sk));
        mutex_lock(&port_mutex);
        err = sk->sk_prot->get_port(sk, pn_port(handle));
        if (err)
                goto out_port;

        /* get_port() sets the port, bind() sets the address if applicable */
        pn->sobject = pn_object(saddr, pn_port(pn->sobject));
        pn->resource = spn->spn_resource;

        /* Enable RX on the socket */
        err = sk->sk_prot->hash(sk);
out_port:
        mutex_unlock(&port_mutex);
out:
        release_sock(sk);
        return err;
}

static int pn_socket_autobind(struct socket *sock)
{
        struct sockaddr_pn sa;
        int err;

        memset(&sa, 0, sizeof(sa));
        sa.spn_family = AF_PHONET;
        err = pn_socket_bind(sock, (struct sockaddr *)&sa,
                                sizeof(struct sockaddr_pn));
        if (err != -EINVAL)
                return err;
        BUG_ON(!pn_port(pn_sk(sock->sk)->sobject));
        return 0; /* socket was already bound */
}

static int pn_socket_connect(struct socket *sock, struct sockaddr *addr,
                int len, int flags)
{
        struct sock *sk = sock->sk;
        struct pn_sock *pn = pn_sk(sk);
        struct sockaddr_pn *spn = (struct sockaddr_pn *)addr;
        struct task_struct *tsk = current;
        long timeo = sock_rcvtimeo(sk, flags & O_NONBLOCK);
        int err;

        if (pn_socket_autobind(sock))
                return -ENOBUFS;
        if (len < sizeof(struct sockaddr_pn))
                return -EINVAL;
        if (spn->spn_family != AF_PHONET)
                return -EAFNOSUPPORT;

        lock_sock(sk);

        switch (sock->state) {
        case SS_UNCONNECTED:
                if (sk->sk_state != TCP_CLOSE) {
                        err = -EISCONN;
                        goto out;
                }
                break;
        case SS_CONNECTING:
                err = -EALREADY;
                goto out;
        default:
                err = -EISCONN;
                goto out;
        }

        pn->dobject = pn_sockaddr_get_object(spn);
        pn->resource = pn_sockaddr_get_resource(spn);
        sock->state = SS_CONNECTING;

        err = sk->sk_prot->connect(sk, addr, len);
        if (err) {
                sock->state = SS_UNCONNECTED;
                pn->dobject = 0;
                goto out;
        }

        while (sk->sk_state == TCP_SYN_SENT) {
                DEFINE_WAIT(wait);

                if (!timeo) {
                        err = -EINPROGRESS;
                        goto out;
                }
                if (signal_pending(tsk)) {
                        err = sock_intr_errno(timeo);
                        goto out;
                }

                prepare_to_wait_exclusive(sk_sleep(sk), &wait,
                                                TASK_INTERRUPTIBLE);
                release_sock(sk);
                timeo = schedule_timeout(timeo);
                lock_sock(sk);
                finish_wait(sk_sleep(sk), &wait);
        }

        if ((1 << sk->sk_state) & (TCPF_SYN_RECV|TCPF_ESTABLISHED))
                err = 0;
        else if (sk->sk_state == TCP_CLOSE_WAIT)
                err = -ECONNRESET;
        else
                err = -ECONNREFUSED;
        sock->state = err ? SS_UNCONNECTED : SS_CONNECTED;
out:
        release_sock(sk);
        return err;
}

static int pn_socket_accept(struct socket *sock, struct socket *newsock,
                            int flags, bool kern)
{
        struct sock *sk = sock->sk;
        struct sock *newsk;
        int err;

        if (unlikely(sk->sk_state != TCP_LISTEN))
                return -EINVAL;

        newsk = sk->sk_prot->accept(sk, flags, &err, kern);
        if (!newsk)
                return err;

        lock_sock(newsk);
        sock_graft(newsk, newsock);
        newsock->state = SS_CONNECTED;
        release_sock(newsk);
        return 0;
}

static int pn_socket_getname(struct socket *sock, struct sockaddr *addr,
                                int peer)
{
        struct sock *sk = sock->sk;
        struct pn_sock *pn = pn_sk(sk);

        memset(addr, 0, sizeof(struct sockaddr_pn));
        addr->sa_family = AF_PHONET;
        if (!peer) /* Race with bind() here is userland's problem. */
                pn_sockaddr_set_object((struct sockaddr_pn *)addr,
                                        pn->sobject);

        return sizeof(struct sockaddr_pn);
}

static __poll_t pn_socket_poll(struct file *file, struct socket *sock,
                                        poll_table *wait)
{
        struct sock *sk = sock->sk;
        struct pep_sock *pn = pep_sk(sk);
        __poll_t mask = 0;

        poll_wait(file, sk_sleep(sk), wait);

        if (sk->sk_state == TCP_CLOSE)
                return EPOLLERR;
        if (!skb_queue_empty_lockless(&sk->sk_receive_queue))
                mask |= EPOLLIN | EPOLLRDNORM;
        if (!skb_queue_empty_lockless(&pn->ctrlreq_queue))
                mask |= EPOLLPRI;
        if (!mask && sk->sk_state == TCP_CLOSE_WAIT)
                return EPOLLHUP;

        if (sk->sk_state == TCP_ESTABLISHED &&
                refcount_read(&sk->sk_wmem_alloc) < sk->sk_sndbuf &&
                atomic_read(&pn->tx_credits))
                mask |= EPOLLOUT | EPOLLWRNORM | EPOLLWRBAND;

        return mask;
}

static int pn_socket_ioctl(struct socket *sock, unsigned int cmd,
                                unsigned long arg)
{
        struct sock *sk = sock->sk;
        struct pn_sock *pn = pn_sk(sk);

        if (cmd == SIOCPNGETOBJECT) {
                struct net_device *dev;
                u16 handle;
                u8 saddr;

                if (get_user(handle, (__u16 __user *)arg))
                        return -EFAULT;

                lock_sock(sk);
                if (sk->sk_bound_dev_if)
                        dev = dev_get_by_index(sock_net(sk),
                                                sk->sk_bound_dev_if);
                else
                        dev = phonet_device_get(sock_net(sk));
                if (dev && (dev->flags & IFF_UP))
                        saddr = phonet_address_get(dev, pn_addr(handle));
                else
                        saddr = PN_NO_ADDR;
                release_sock(sk);

                dev_put(dev);
                if (saddr == PN_NO_ADDR)
                        return -EHOSTUNREACH;

                handle = pn_object(saddr, pn_port(pn->sobject));
                return put_user(handle, (__u16 __user *)arg);
        }

        return sk->sk_prot->ioctl(sk, cmd, arg);
}

static int pn_socket_listen(struct socket *sock, int backlog)
{
        struct sock *sk = sock->sk;
        int err = 0;

        if (pn_socket_autobind(sock))
                return -ENOBUFS;

        lock_sock(sk);
        if (sock->state != SS_UNCONNECTED) {
                err = -EINVAL;
                goto out;
        }

        if (sk->sk_state != TCP_LISTEN) {
                sk->sk_state = TCP_LISTEN;
                sk->sk_ack_backlog = 0;
        }
        sk->sk_max_ack_backlog = backlog;
out:
        release_sock(sk);
        return err;
}

static int pn_socket_sendmsg(struct socket *sock, struct msghdr *m,
                             size_t total_len)
{
        struct sock *sk = sock->sk;

        if (pn_socket_autobind(sock))
                return -EAGAIN;

        return sk->sk_prot->sendmsg(sk, m, total_len);
}

const struct proto_ops phonet_dgram_ops = {
        .family                = AF_PHONET,
        .owner                = THIS_MODULE,
        .release        = pn_socket_release,
        .bind                = pn_socket_bind,
        .connect        = sock_no_connect,
        .socketpair        = sock_no_socketpair,
        .accept                = sock_no_accept,
        .getname        = pn_socket_getname,
        .poll                = datagram_poll,
        .ioctl                = pn_socket_ioctl,
        .listen                = sock_no_listen,
        .shutdown        = sock_no_shutdown,
        .sendmsg        = pn_socket_sendmsg,
        .recvmsg        = sock_common_recvmsg,
        .mmap                = sock_no_mmap,
        .sendpage        = sock_no_sendpage,
};

const struct proto_ops phonet_stream_ops = {
        .family                = AF_PHONET,
        .owner                = THIS_MODULE,
        .release        = pn_socket_release,
        .bind                = pn_socket_bind,
        .connect        = pn_socket_connect,
        .socketpair        = sock_no_socketpair,
        .accept                = pn_socket_accept,
        .getname        = pn_socket_getname,
        .poll                = pn_socket_poll,
        .ioctl                = pn_socket_ioctl,
        .listen                = pn_socket_listen,
        .shutdown        = sock_no_shutdown,
        .setsockopt        = sock_common_setsockopt,
        .getsockopt        = sock_common_getsockopt,
        .sendmsg        = pn_socket_sendmsg,
        .recvmsg        = sock_common_recvmsg,
        .mmap                = sock_no_mmap,
        .sendpage        = sock_no_sendpage,
};
EXPORT_SYMBOL(phonet_stream_ops);

/* allocate port for a socket */
int pn_sock_get_port(struct sock *sk, unsigned short sport)
{
        static int port_cur;
        struct net *net = sock_net(sk);
        struct pn_sock *pn = pn_sk(sk);
        struct sockaddr_pn try_sa;
        struct sock *tmpsk;

        memset(&try_sa, 0, sizeof(struct sockaddr_pn));
        try_sa.spn_family = AF_PHONET;
        WARN_ON(!mutex_is_locked(&port_mutex));
        if (!sport) {
                /* search free port */
                int port, pmin, pmax;

                phonet_get_local_port_range(&pmin, &pmax);
                for (port = pmin; port <= pmax; port++) {
                        port_cur++;
                        if (port_cur < pmin || port_cur > pmax)
                                port_cur = pmin;

                        pn_sockaddr_set_port(&try_sa, port_cur);
                        tmpsk = pn_find_sock_by_sa(net, &try_sa);
                        if (tmpsk == NULL) {
                                sport = port_cur;
                                goto found;
                        } else
                                sock_put(tmpsk);
                }
        } else {
                /* try to find specific port */
                pn_sockaddr_set_port(&try_sa, sport);
                tmpsk = pn_find_sock_by_sa(net, &try_sa);
                if (tmpsk == NULL)
                        /* No sock there! We can use that port... */
                        goto found;
                else
                        sock_put(tmpsk);
        }
        /* the port must be in use already */
        return -EADDRINUSE;

found:
        pn->sobject = pn_object(pn_addr(pn->sobject), sport);
        return 0;
}
EXPORT_SYMBOL(pn_sock_get_port);

#ifdef CONFIG_PROC_FS
static struct sock *pn_sock_get_idx(struct seq_file *seq, loff_t pos)
{
        struct net *net = seq_file_net(seq);
        struct hlist_head *hlist = pnsocks.hlist;
        struct sock *sknode;
        unsigned int h;

        for (h = 0; h < PN_HASHSIZE; h++) {
                sk_for_each_rcu(sknode, hlist) {
                        if (!net_eq(net, sock_net(sknode)))
                                continue;
                        if (!pos)
                                return sknode;
                        pos--;
                }
                hlist++;
        }
        return NULL;
}

static struct sock *pn_sock_get_next(struct seq_file *seq, struct sock *sk)
{
        struct net *net = seq_file_net(seq);

        do
                sk = sk_next(sk);
        while (sk && !net_eq(net, sock_net(sk)));

        return sk;
}

static void *pn_sock_seq_start(struct seq_file *seq, loff_t *pos)
        __acquires(rcu)
{
        rcu_read_lock();
        return *pos ? pn_sock_get_idx(seq, *pos - 1) : SEQ_START_TOKEN;
}

static void *pn_sock_seq_next(struct seq_file *seq, void *v, loff_t *pos)
{
        struct sock *sk;

        if (v == SEQ_START_TOKEN)
                sk = pn_sock_get_idx(seq, 0);
        else
                sk = pn_sock_get_next(seq, v);
        (*pos)++;
        return sk;
}

static void pn_sock_seq_stop(struct seq_file *seq, void *v)
        __releases(rcu)
{
        rcu_read_unlock();
}

static int pn_sock_seq_show(struct seq_file *seq, void *v)
{
        seq_setwidth(seq, 127);
        if (v == SEQ_START_TOKEN)
                seq_puts(seq, "pt  loc  rem rs st tx_queue rx_queue "
                        "  uid inode ref pointer drops");
        else {
                struct sock *sk = v;
                struct pn_sock *pn = pn_sk(sk);

                seq_printf(seq, "%2d %04X:%04X:%02X %02X %08X:%08X %5d %lu "
                        "%d %pK %u",
                        sk->sk_protocol, pn->sobject, pn->dobject,
                        pn->resource, sk->sk_state,
                        sk_wmem_alloc_get(sk), sk_rmem_alloc_get(sk),
                        from_kuid_munged(seq_user_ns(seq), sock_i_uid(sk)),
                        sock_i_ino(sk),
                        refcount_read(&sk->sk_refcnt), sk,
                        atomic_read(&sk->sk_drops));
        }
        seq_pad(seq, '\n');
        return 0;
}

const struct seq_operations pn_sock_seq_ops = {
        .start = pn_sock_seq_start,
        .next = pn_sock_seq_next,
        .stop = pn_sock_seq_stop,
        .show = pn_sock_seq_show,
};
#endif

static struct  {
        struct sock *sk[256];
} pnres;

/*
 * Find and hold socket based on resource.
 */
struct sock *pn_find_sock_by_res(struct net *net, u8 res)
{
        struct sock *sk;

        if (!net_eq(net, &init_net))
                return NULL;

        rcu_read_lock();
        sk = rcu_dereference(pnres.sk[res]);
        if (sk)
                sock_hold(sk);
        rcu_read_unlock();
        return sk;
}

static DEFINE_MUTEX(resource_mutex);

int pn_sock_bind_res(struct sock *sk, u8 res)
{
        int ret = -EADDRINUSE;

        if (!net_eq(sock_net(sk), &init_net))
                return -ENOIOCTLCMD;
        if (!capable(CAP_SYS_ADMIN))
                return -EPERM;
        if (pn_socket_autobind(sk->sk_socket))
                return -EAGAIN;

        mutex_lock(&resource_mutex);
        if (pnres.sk[res] == NULL) {
                sock_hold(sk);
                rcu_assign_pointer(pnres.sk[res], sk);
                ret = 0;
        }
        mutex_unlock(&resource_mutex);
        return ret;
}

int pn_sock_unbind_res(struct sock *sk, u8 res)
{
        int ret = -ENOENT;

        if (!capable(CAP_SYS_ADMIN))
                return -EPERM;

        mutex_lock(&resource_mutex);
        if (pnres.sk[res] == sk) {
                RCU_INIT_POINTER(pnres.sk[res], NULL);
                ret = 0;
        }
        mutex_unlock(&resource_mutex);

        if (ret == 0) {
                synchronize_rcu();
                sock_put(sk);
        }
        return ret;
}

void pn_sock_unbind_all_res(struct sock *sk)
{
        unsigned int res, match = 0;

        mutex_lock(&resource_mutex);
        for (res = 0; res < 256; res++) {
                if (pnres.sk[res] == sk) {
                        RCU_INIT_POINTER(pnres.sk[res], NULL);
                        match++;
                }
        }
        mutex_unlock(&resource_mutex);

        while (match > 0) {
                __sock_put(sk);
                match--;
        }
        /* Caller is responsible for RCU sync before final sock_put() */
}

#ifdef CONFIG_PROC_FS
static struct sock **pn_res_get_idx(struct seq_file *seq, loff_t pos)
{
        struct net *net = seq_file_net(seq);
        unsigned int i;

        if (!net_eq(net, &init_net))
                return NULL;

        for (i = 0; i < 256; i++) {
                if (pnres.sk[i] == NULL)
                        continue;
                if (!pos)
                        return pnres.sk + i;
                pos--;
        }
        return NULL;
}

static struct sock **pn_res_get_next(struct seq_file *seq, struct sock **sk)
{
        struct net *net = seq_file_net(seq);
        unsigned int i;

        BUG_ON(!net_eq(net, &init_net));

        for (i = (sk - pnres.sk) + 1; i < 256; i++)
                if (pnres.sk[i])
                        return pnres.sk + i;
        return NULL;
}

static void *pn_res_seq_start(struct seq_file *seq, loff_t *pos)
        __acquires(resource_mutex)
{
        mutex_lock(&resource_mutex);
        return *pos ? pn_res_get_idx(seq, *pos - 1) : SEQ_START_TOKEN;
}

static void *pn_res_seq_next(struct seq_file *seq, void *v, loff_t *pos)
{
        struct sock **sk;

        if (v == SEQ_START_TOKEN)
                sk = pn_res_get_idx(seq, 0);
        else
                sk = pn_res_get_next(seq, v);
        (*pos)++;
        return sk;
}

static void pn_res_seq_stop(struct seq_file *seq, void *v)
        __releases(resource_mutex)
{
        mutex_unlock(&resource_mutex);
}

static int pn_res_seq_show(struct seq_file *seq, void *v)
{
        seq_setwidth(seq, 63);
        if (v == SEQ_START_TOKEN)
                seq_puts(seq, "rs   uid inode");
        else {
                struct sock **psk = v;
                struct sock *sk = *psk;

                seq_printf(seq, "%02X %5u %lu",
                           (int) (psk - pnres.sk),
                           from_kuid_munged(seq_user_ns(seq), sock_i_uid(sk)),
                           sock_i_ino(sk));
        }
        seq_pad(seq, '\n');
        return 0;
}

const struct seq_operations pn_res_seq_ops = {
        .start = pn_res_seq_start,
        .next = pn_res_seq_next,
        .stop = pn_res_seq_stop,
        .show = pn_res_seq_show,
};
#endif




























  485 





  485 









  483 
  408 




  485 


    1 




































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
// SPDX-License-Identifier: GPL-2.0-only
/*
 * fs/kernfs/symlink.c - kernfs symlink implementation
 *
 * Copyright (c) 2001-3 Patrick Mochel
 * Copyright (c) 2007 SUSE Linux Products GmbH
 * Copyright (c) 2007, 2013 Tejun Heo <tj@kernel.org>
 */

#include <linux/fs.h>
#include <linux/gfp.h>
#include <linux/namei.h>

#include "kernfs-internal.h"

/**
 * kernfs_create_link - create a symlink
 * @parent: directory to create the symlink in
 * @name: name of the symlink
 * @target: target node for the symlink to point to
 *
 * Returns the created node on success, ERR_PTR() value on error.
 * Ownership of the link matches ownership of the target.
 */
struct kernfs_node *kernfs_create_link(struct kernfs_node *parent,
                                       const char *name,
                                       struct kernfs_node *target)
{
        struct kernfs_node *kn;
        int error;
        kuid_t uid = GLOBAL_ROOT_UID;
        kgid_t gid = GLOBAL_ROOT_GID;

        if (target->iattr) {
                uid = target->iattr->ia_uid;
                gid = target->iattr->ia_gid;
        }

        kn = kernfs_new_node(parent, name, S_IFLNK|S_IRWXUGO, uid, gid,
                             KERNFS_LINK);
        if (!kn)
                return ERR_PTR(-ENOMEM);

        if (kernfs_ns_enabled(parent))
                kn->ns = target->ns;
        kn->symlink.target_kn = target;
        kernfs_get(target);        /* ref owned by symlink */

        error = kernfs_add_one(kn);
        if (!error)
                return kn;

        kernfs_put(kn);
        return ERR_PTR(error);
}

static int kernfs_get_target_path(struct kernfs_node *parent,
                                  struct kernfs_node *target, char *path)
{
        struct kernfs_node *base, *kn;
        char *s = path;
        int len = 0;

        /* go up to the root, stop at the base */
        base = parent;
        while (base->parent) {
                kn = target->parent;
                while (kn->parent && base != kn)
                        kn = kn->parent;

                if (base == kn)
                        break;

                if ((s - path) + 3 >= PATH_MAX)
                        return -ENAMETOOLONG;

                strcpy(s, "../");
                s += 3;
                base = base->parent;
        }

        /* determine end of target string for reverse fillup */
        kn = target;
        while (kn->parent && kn != base) {
                len += strlen(kn->name) + 1;
                kn = kn->parent;
        }

        /* check limits */
        if (len < 2)
                return -EINVAL;
        len--;
        if ((s - path) + len >= PATH_MAX)
                return -ENAMETOOLONG;

        /* reverse fillup of target string from target to base */
        kn = target;
        while (kn->parent && kn != base) {
                int slen = strlen(kn->name);

                len -= slen;
                memcpy(s + len, kn->name, slen);
                if (len)
                        s[--len] = '/';

                kn = kn->parent;
        }

        return 0;
}

static int kernfs_getlink(struct inode *inode, char *path)
{
        struct kernfs_node *kn = inode->i_private;
        struct kernfs_node *parent = kn->parent;
        struct kernfs_node *target = kn->symlink.target_kn;
        int error;

        down_read(&kernfs_rwsem);
        error = kernfs_get_target_path(parent, target, path);
        up_read(&kernfs_rwsem);

        return error;
}

static const char *kernfs_iop_get_link(struct dentry *dentry,
                                       struct inode *inode,
                                       struct delayed_call *done)
{
        char *body;
        int error;

        if (!dentry)
                return ERR_PTR(-ECHILD);
        body = kzalloc(PAGE_SIZE, GFP_KERNEL);
        if (!body)
                return ERR_PTR(-ENOMEM);
        error = kernfs_getlink(inode, body);
        if (unlikely(error < 0)) {
                kfree(body);
                return ERR_PTR(error);
        }
        set_delayed_call(done, kfree_link, body);
        return body;
}

const struct inode_operations kernfs_symlink_iops = {
        .listxattr        = kernfs_iop_listxattr,
        .get_link        = kernfs_iop_get_link,
        .setattr        = kernfs_iop_setattr,
        .getattr        = kernfs_iop_getattr,
        .permission        = kernfs_iop_permission,
};






















































































































































































































































































































































































































































































































































































































































































































































































































































































  318 





   25 

  315 




























1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
/*
 * Copyright (c) 2005 Voltaire Inc.  All rights reserved.
 * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved.
 * Copyright (c) 1999-2005, Mellanox Technologies, Inc. All rights reserved.
 * Copyright (c) 2005 Intel Corporation.  All rights reserved.
 *
 * This software is available to you under a choice of one of two
 * licenses.  You may choose to be licensed under the terms of the GNU
 * General Public License (GPL) Version 2, available from the file
 * COPYING in the main directory of this source tree, or the
 * OpenIB.org BSD license below:
 *
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
 *
 *      - Redistributions of source code must retain the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer.
 *
 *      - Redistributions in binary form must reproduce the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer in the documentation and/or other materials
 *        provided with the distribution.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

#include <linux/mutex.h>
#include <linux/inetdevice.h>
#include <linux/slab.h>
#include <linux/workqueue.h>
#include <linux/module.h>
#include <net/arp.h>
#include <net/neighbour.h>
#include <net/route.h>
#include <net/netevent.h>
#include <net/ipv6_stubs.h>
#include <net/ip6_route.h>
#include <rdma/ib_addr.h>
#include <rdma/ib_cache.h>
#include <rdma/ib_sa.h>
#include <rdma/ib.h>
#include <rdma/rdma_netlink.h>
#include <net/netlink.h>

#include "core_priv.h"

struct addr_req {
        struct list_head list;
        struct sockaddr_storage src_addr;
        struct sockaddr_storage dst_addr;
        struct rdma_dev_addr *addr;
        void *context;
        void (*callback)(int status, struct sockaddr *src_addr,
                         struct rdma_dev_addr *addr, void *context);
        unsigned long timeout;
        struct delayed_work work;
        bool resolve_by_gid_attr;        /* Consider gid attr in resolve phase */
        int status;
        u32 seq;
};

static atomic_t ib_nl_addr_request_seq = ATOMIC_INIT(0);

static DEFINE_SPINLOCK(lock);
static LIST_HEAD(req_list);
static struct workqueue_struct *addr_wq;

static const struct nla_policy ib_nl_addr_policy[LS_NLA_TYPE_MAX] = {
        [LS_NLA_TYPE_DGID] = {.type = NLA_BINARY,
                .len = sizeof(struct rdma_nla_ls_gid),
                .validation_type = NLA_VALIDATE_MIN,
                .min = sizeof(struct rdma_nla_ls_gid)},
};

static inline bool ib_nl_is_good_ip_resp(const struct nlmsghdr *nlh)
{
        struct nlattr *tb[LS_NLA_TYPE_MAX] = {};
        int ret;

        if (nlh->nlmsg_flags & RDMA_NL_LS_F_ERR)
                return false;

        ret = nla_parse_deprecated(tb, LS_NLA_TYPE_MAX - 1, nlmsg_data(nlh),
                                   nlmsg_len(nlh), ib_nl_addr_policy, NULL);
        if (ret)
                return false;

        return true;
}

static void ib_nl_process_good_ip_rsep(const struct nlmsghdr *nlh)
{
        const struct nlattr *head, *curr;
        union ib_gid gid;
        struct addr_req *req;
        int len, rem;
        int found = 0;

        head = (const struct nlattr *)nlmsg_data(nlh);
        len = nlmsg_len(nlh);

        nla_for_each_attr(curr, head, len, rem) {
                if (curr->nla_type == LS_NLA_TYPE_DGID)
                        memcpy(&gid, nla_data(curr), nla_len(curr));
        }

        spin_lock_bh(&lock);
        list_for_each_entry(req, &req_list, list) {
                if (nlh->nlmsg_seq != req->seq)
                        continue;
                /* We set the DGID part, the rest was set earlier */
                rdma_addr_set_dgid(req->addr, &gid);
                req->status = 0;
                found = 1;
                break;
        }
        spin_unlock_bh(&lock);

        if (!found)
                pr_info("Couldn't find request waiting for DGID: %pI6\n",
                        &gid);
}

int ib_nl_handle_ip_res_resp(struct sk_buff *skb,
                             struct nlmsghdr *nlh,
                             struct netlink_ext_ack *extack)
{
        if ((nlh->nlmsg_flags & NLM_F_REQUEST) ||
            !(NETLINK_CB(skb).sk))
                return -EPERM;

        if (ib_nl_is_good_ip_resp(nlh))
                ib_nl_process_good_ip_rsep(nlh);

        return 0;
}

static int ib_nl_ip_send_msg(struct rdma_dev_addr *dev_addr,
                             const void *daddr,
                             u32 seq, u16 family)
{
        struct sk_buff *skb = NULL;
        struct nlmsghdr *nlh;
        struct rdma_ls_ip_resolve_header *header;
        void *data;
        size_t size;
        int attrtype;
        int len;

        if (family == AF_INET) {
                size = sizeof(struct in_addr);
                attrtype = RDMA_NLA_F_MANDATORY | LS_NLA_TYPE_IPV4;
        } else {
                size = sizeof(struct in6_addr);
                attrtype = RDMA_NLA_F_MANDATORY | LS_NLA_TYPE_IPV6;
        }

        len = nla_total_size(sizeof(size));
        len += NLMSG_ALIGN(sizeof(*header));

        skb = nlmsg_new(len, GFP_KERNEL);
        if (!skb)
                return -ENOMEM;

        data = ibnl_put_msg(skb, &nlh, seq, 0, RDMA_NL_LS,
                            RDMA_NL_LS_OP_IP_RESOLVE, NLM_F_REQUEST);
        if (!data) {
                nlmsg_free(skb);
                return -ENODATA;
        }

        /* Construct the family header first */
        header = skb_put(skb, NLMSG_ALIGN(sizeof(*header)));
        header->ifindex = dev_addr->bound_dev_if;
        nla_put(skb, attrtype, size, daddr);

        /* Repair the nlmsg header length */
        nlmsg_end(skb, nlh);
        rdma_nl_multicast(&init_net, skb, RDMA_NL_GROUP_LS, GFP_KERNEL);

        /* Make the request retry, so when we get the response from userspace
         * we will have something.
         */
        return -ENODATA;
}

int rdma_addr_size(const struct sockaddr *addr)
{
        switch (addr->sa_family) {
        case AF_INET:
                return sizeof(struct sockaddr_in);
        case AF_INET6:
                return sizeof(struct sockaddr_in6);
        case AF_IB:
                return sizeof(struct sockaddr_ib);
        default:
                return 0;
        }
}
EXPORT_SYMBOL(rdma_addr_size);

int rdma_addr_size_in6(struct sockaddr_in6 *addr)
{
        int ret = rdma_addr_size((struct sockaddr *) addr);

        return ret <= sizeof(*addr) ? ret : 0;
}
EXPORT_SYMBOL(rdma_addr_size_in6);

int rdma_addr_size_kss(struct __kernel_sockaddr_storage *addr)
{
        int ret = rdma_addr_size((struct sockaddr *) addr);

        return ret <= sizeof(*addr) ? ret : 0;
}
EXPORT_SYMBOL(rdma_addr_size_kss);

/**
 * rdma_copy_src_l2_addr - Copy netdevice source addresses
 * @dev_addr:        Destination address pointer where to copy the addresses
 * @dev:        Netdevice whose source addresses to copy
 *
 * rdma_copy_src_l2_addr() copies source addresses from the specified netdevice.
 * This includes unicast address, broadcast address, device type and
 * interface index.
 */
void rdma_copy_src_l2_addr(struct rdma_dev_addr *dev_addr,
                           const struct net_device *dev)
{
        dev_addr->dev_type = dev->type;
        memcpy(dev_addr->src_dev_addr, dev->dev_addr, MAX_ADDR_LEN);
        memcpy(dev_addr->broadcast, dev->broadcast, MAX_ADDR_LEN);
        dev_addr->bound_dev_if = dev->ifindex;
}
EXPORT_SYMBOL(rdma_copy_src_l2_addr);

static struct net_device *
rdma_find_ndev_for_src_ip_rcu(struct net *net, const struct sockaddr *src_in)
{
        struct net_device *dev = NULL;
        int ret = -EADDRNOTAVAIL;

        switch (src_in->sa_family) {
        case AF_INET:
                dev = __ip_dev_find(net,
                                    ((const struct sockaddr_in *)src_in)->sin_addr.s_addr,
                                    false);
                if (dev)
                        ret = 0;
                break;
#if IS_ENABLED(CONFIG_IPV6)
        case AF_INET6:
                for_each_netdev_rcu(net, dev) {
                        if (ipv6_chk_addr(net,
                                          &((const struct sockaddr_in6 *)src_in)->sin6_addr,
                                          dev, 1)) {
                                ret = 0;
                                break;
                        }
                }
                break;
#endif
        }
        return ret ? ERR_PTR(ret) : dev;
}

int rdma_translate_ip(const struct sockaddr *addr,
                      struct rdma_dev_addr *dev_addr)
{
        struct net_device *dev;

        if (dev_addr->bound_dev_if) {
                dev = dev_get_by_index(dev_addr->net, dev_addr->bound_dev_if);
                if (!dev)
                        return -ENODEV;
                rdma_copy_src_l2_addr(dev_addr, dev);
                dev_put(dev);
                return 0;
        }

        rcu_read_lock();
        dev = rdma_find_ndev_for_src_ip_rcu(dev_addr->net, addr);
        if (!IS_ERR(dev))
                rdma_copy_src_l2_addr(dev_addr, dev);
        rcu_read_unlock();
        return PTR_ERR_OR_ZERO(dev);
}
EXPORT_SYMBOL(rdma_translate_ip);

static void set_timeout(struct addr_req *req, unsigned long time)
{
        unsigned long delay;

        delay = time - jiffies;
        if ((long)delay < 0)
                delay = 0;

        mod_delayed_work(addr_wq, &req->work, delay);
}

static void queue_req(struct addr_req *req)
{
        spin_lock_bh(&lock);
        list_add_tail(&req->list, &req_list);
        set_timeout(req, req->timeout);
        spin_unlock_bh(&lock);
}

static int ib_nl_fetch_ha(struct rdma_dev_addr *dev_addr,
                          const void *daddr, u32 seq, u16 family)
{
        if (!rdma_nl_chk_listeners(RDMA_NL_GROUP_LS))
                return -EADDRNOTAVAIL;

        return ib_nl_ip_send_msg(dev_addr, daddr, seq, family);
}

static int dst_fetch_ha(const struct dst_entry *dst,
                        struct rdma_dev_addr *dev_addr,
                        const void *daddr)
{
        struct neighbour *n;
        int ret = 0;

        n = dst_neigh_lookup(dst, daddr);
        if (!n)
                return -ENODATA;

        if (!(n->nud_state & NUD_VALID)) {
                neigh_event_send(n, NULL);
                ret = -ENODATA;
        } else {
                neigh_ha_snapshot(dev_addr->dst_dev_addr, n, dst->dev);
        }

        neigh_release(n);

        return ret;
}

static bool has_gateway(const struct dst_entry *dst, sa_family_t family)
{
        struct rtable *rt;
        struct rt6_info *rt6;

        if (family == AF_INET) {
                rt = container_of(dst, struct rtable, dst);
                return rt->rt_uses_gateway;
        }

        rt6 = container_of(dst, struct rt6_info, dst);
        return rt6->rt6i_flags & RTF_GATEWAY;
}

static int fetch_ha(const struct dst_entry *dst, struct rdma_dev_addr *dev_addr,
                    const struct sockaddr *dst_in, u32 seq)
{
        const struct sockaddr_in *dst_in4 =
                (const struct sockaddr_in *)dst_in;
        const struct sockaddr_in6 *dst_in6 =
                (const struct sockaddr_in6 *)dst_in;
        const void *daddr = (dst_in->sa_family == AF_INET) ?
                (const void *)&dst_in4->sin_addr.s_addr :
                (const void *)&dst_in6->sin6_addr;
        sa_family_t family = dst_in->sa_family;

        might_sleep();

        /* If we have a gateway in IB mode then it must be an IB network */
        if (has_gateway(dst, family) && dev_addr->network == RDMA_NETWORK_IB)
                return ib_nl_fetch_ha(dev_addr, daddr, seq, family);
        else
                return dst_fetch_ha(dst, dev_addr, daddr);
}

static int addr4_resolve(struct sockaddr *src_sock,
                         const struct sockaddr *dst_sock,
                         struct rdma_dev_addr *addr,
                         struct rtable **prt)
{
        struct sockaddr_in *src_in = (struct sockaddr_in *)src_sock;
        const struct sockaddr_in *dst_in =
                        (const struct sockaddr_in *)dst_sock;

        __be32 src_ip = src_in->sin_addr.s_addr;
        __be32 dst_ip = dst_in->sin_addr.s_addr;
        struct rtable *rt;
        struct flowi4 fl4;
        int ret;

        memset(&fl4, 0, sizeof(fl4));
        fl4.daddr = dst_ip;
        fl4.saddr = src_ip;
        fl4.flowi4_oif = addr->bound_dev_if;
        rt = ip_route_output_key(addr->net, &fl4);
        ret = PTR_ERR_OR_ZERO(rt);
        if (ret)
                return ret;

        src_in->sin_addr.s_addr = fl4.saddr;

        addr->hoplimit = ip4_dst_hoplimit(&rt->dst);

        *prt = rt;
        return 0;
}

#if IS_ENABLED(CONFIG_IPV6)
static int addr6_resolve(struct sockaddr *src_sock,
                         const struct sockaddr *dst_sock,
                         struct rdma_dev_addr *addr,
                         struct dst_entry **pdst)
{
        struct sockaddr_in6 *src_in = (struct sockaddr_in6 *)src_sock;
        const struct sockaddr_in6 *dst_in =
                                (const struct sockaddr_in6 *)dst_sock;
        struct flowi6 fl6;
        struct dst_entry *dst;

        memset(&fl6, 0, sizeof fl6);
        fl6.daddr = dst_in->sin6_addr;
        fl6.saddr = src_in->sin6_addr;
        fl6.flowi6_oif = addr->bound_dev_if;

        dst = ipv6_stub->ipv6_dst_lookup_flow(addr->net, NULL, &fl6, NULL);
        if (IS_ERR(dst))
                return PTR_ERR(dst);

        if (ipv6_addr_any(&src_in->sin6_addr))
                src_in->sin6_addr = fl6.saddr;

        addr->hoplimit = ip6_dst_hoplimit(dst);

        *pdst = dst;
        return 0;
}
#else
static int addr6_resolve(struct sockaddr *src_sock,
                         const struct sockaddr *dst_sock,
                         struct rdma_dev_addr *addr,
                         struct dst_entry **pdst)
{
        return -EADDRNOTAVAIL;
}
#endif

static int addr_resolve_neigh(const struct dst_entry *dst,
                              const struct sockaddr *dst_in,
                              struct rdma_dev_addr *addr,
                              unsigned int ndev_flags,
                              u32 seq)
{
        int ret = 0;

        if (ndev_flags & IFF_LOOPBACK) {
                memcpy(addr->dst_dev_addr, addr->src_dev_addr, MAX_ADDR_LEN);
        } else {
                if (!(ndev_flags & IFF_NOARP)) {
                        /* If the device doesn't do ARP internally */
                        ret = fetch_ha(dst, addr, dst_in, seq);
                }
        }
        return ret;
}

static int copy_src_l2_addr(struct rdma_dev_addr *dev_addr,
                            const struct sockaddr *dst_in,
                            const struct dst_entry *dst,
                            const struct net_device *ndev)
{
        int ret = 0;

        if (dst->dev->flags & IFF_LOOPBACK)
                ret = rdma_translate_ip(dst_in, dev_addr);
        else
                rdma_copy_src_l2_addr(dev_addr, dst->dev);

        /*
         * If there's a gateway and type of device not ARPHRD_INFINIBAND,
         * we're definitely in RoCE v2 (as RoCE v1 isn't routable) set the
         * network type accordingly.
         */
        if (has_gateway(dst, dst_in->sa_family) &&
            ndev->type != ARPHRD_INFINIBAND)
                dev_addr->network = dst_in->sa_family == AF_INET ?
                                                RDMA_NETWORK_IPV4 :
                                                RDMA_NETWORK_IPV6;
        else
                dev_addr->network = RDMA_NETWORK_IB;

        return ret;
}

static int rdma_set_src_addr_rcu(struct rdma_dev_addr *dev_addr,
                                 unsigned int *ndev_flags,
                                 const struct sockaddr *dst_in,
                                 const struct dst_entry *dst)
{
        struct net_device *ndev = READ_ONCE(dst->dev);

        *ndev_flags = ndev->flags;
        /* A physical device must be the RDMA device to use */
        if (ndev->flags & IFF_LOOPBACK) {
                /*
                 * RDMA (IB/RoCE, iWarp) doesn't run on lo interface or
                 * loopback IP address. So if route is resolved to loopback
                 * interface, translate that to a real ndev based on non
                 * loopback IP address.
                 */
                ndev = rdma_find_ndev_for_src_ip_rcu(dev_net(ndev), dst_in);
                if (IS_ERR(ndev))
                        return -ENODEV;
        }

        return copy_src_l2_addr(dev_addr, dst_in, dst, ndev);
}

static int set_addr_netns_by_gid_rcu(struct rdma_dev_addr *addr)
{
        struct net_device *ndev;

        ndev = rdma_read_gid_attr_ndev_rcu(addr->sgid_attr);
        if (IS_ERR(ndev))
                return PTR_ERR(ndev);

        /*
         * Since we are holding the rcu, reading net and ifindex
         * are safe without any additional reference; because
         * change_net_namespace() in net/core/dev.c does rcu sync
         * after it changes the state to IFF_DOWN and before
         * updating netdev fields {net, ifindex}.
         */
        addr->net = dev_net(ndev);
        addr->bound_dev_if = ndev->ifindex;
        return 0;
}

static void rdma_addr_set_net_defaults(struct rdma_dev_addr *addr)
{
        addr->net = &init_net;
        addr->bound_dev_if = 0;
}

static int addr_resolve(struct sockaddr *src_in,
                        const struct sockaddr *dst_in,
                        struct rdma_dev_addr *addr,
                        bool resolve_neigh,
                        bool resolve_by_gid_attr,
                        u32 seq)
{
        struct dst_entry *dst = NULL;
        unsigned int ndev_flags = 0;
        struct rtable *rt = NULL;
        int ret;

        if (!addr->net) {
                pr_warn_ratelimited("%s: missing namespace\n", __func__);
                return -EINVAL;
        }

        rcu_read_lock();
        if (resolve_by_gid_attr) {
                if (!addr->sgid_attr) {
                        rcu_read_unlock();
                        pr_warn_ratelimited("%s: missing gid_attr\n", __func__);
                        return -EINVAL;
                }
                /*
                 * If the request is for a specific gid attribute of the
                 * rdma_dev_addr, derive net from the netdevice of the
                 * GID attribute.
                 */
                ret = set_addr_netns_by_gid_rcu(addr);
                if (ret) {
                        rcu_read_unlock();
                        return ret;
                }
        }
        if (src_in->sa_family == AF_INET) {
                ret = addr4_resolve(src_in, dst_in, addr, &rt);
                dst = &rt->dst;
        } else {
                ret = addr6_resolve(src_in, dst_in, addr, &dst);
        }
        if (ret) {
                rcu_read_unlock();
                goto done;
        }
        ret = rdma_set_src_addr_rcu(addr, &ndev_flags, dst_in, dst);
        rcu_read_unlock();

        /*
         * Resolve neighbor destination address if requested and
         * only if src addr translation didn't fail.
         */
        if (!ret && resolve_neigh)
                ret = addr_resolve_neigh(dst, dst_in, addr, ndev_flags, seq);

        if (src_in->sa_family == AF_INET)
                ip_rt_put(rt);
        else
                dst_release(dst);
done:
        /*
         * Clear the addr net to go back to its original state, only if it was
         * derived from GID attribute in this context.
         */
        if (resolve_by_gid_attr)
                rdma_addr_set_net_defaults(addr);
        return ret;
}

static void process_one_req(struct work_struct *_work)
{
        struct addr_req *req;
        struct sockaddr *src_in, *dst_in;

        req = container_of(_work, struct addr_req, work.work);

        if (req->status == -ENODATA) {
                src_in = (struct sockaddr *)&req->src_addr;
                dst_in = (struct sockaddr *)&req->dst_addr;
                req->status = addr_resolve(src_in, dst_in, req->addr,
                                           true, req->resolve_by_gid_attr,
                                           req->seq);
                if (req->status && time_after_eq(jiffies, req->timeout)) {
                        req->status = -ETIMEDOUT;
                } else if (req->status == -ENODATA) {
                        /* requeue the work for retrying again */
                        spin_lock_bh(&lock);
                        if (!list_empty(&req->list))
                                set_timeout(req, req->timeout);
                        spin_unlock_bh(&lock);
                        return;
                }
        }

        req->callback(req->status, (struct sockaddr *)&req->src_addr,
                req->addr, req->context);
        req->callback = NULL;

        spin_lock_bh(&lock);
        /*
         * Although the work will normally have been canceled by the workqueue,
         * it can still be requeued as long as it is on the req_list.
         */
        cancel_delayed_work(&req->work);
        if (!list_empty(&req->list)) {
                list_del_init(&req->list);
                kfree(req);
        }
        spin_unlock_bh(&lock);
}

int rdma_resolve_ip(struct sockaddr *src_addr, const struct sockaddr *dst_addr,
                    struct rdma_dev_addr *addr, unsigned long timeout_ms,
                    void (*callback)(int status, struct sockaddr *src_addr,
                                     struct rdma_dev_addr *addr, void *context),
                    bool resolve_by_gid_attr, void *context)
{
        struct sockaddr *src_in, *dst_in;
        struct addr_req *req;
        int ret = 0;

        req = kzalloc(sizeof *req, GFP_KERNEL);
        if (!req)
                return -ENOMEM;

        src_in = (struct sockaddr *) &req->src_addr;
        dst_in = (struct sockaddr *) &req->dst_addr;

        if (src_addr) {
                if (src_addr->sa_family != dst_addr->sa_family) {
                        ret = -EINVAL;
                        goto err;
                }

                memcpy(src_in, src_addr, rdma_addr_size(src_addr));
        } else {
                src_in->sa_family = dst_addr->sa_family;
        }

        memcpy(dst_in, dst_addr, rdma_addr_size(dst_addr));
        req->addr = addr;
        req->callback = callback;
        req->context = context;
        req->resolve_by_gid_attr = resolve_by_gid_attr;
        INIT_DELAYED_WORK(&req->work, process_one_req);
        req->seq = (u32)atomic_inc_return(&ib_nl_addr_request_seq);

        req->status = addr_resolve(src_in, dst_in, addr, true,
                                   req->resolve_by_gid_attr, req->seq);
        switch (req->status) {
        case 0:
                req->timeout = jiffies;
                queue_req(req);
                break;
        case -ENODATA:
                req->timeout = msecs_to_jiffies(timeout_ms) + jiffies;
                queue_req(req);
                break;
        default:
                ret = req->status;
                goto err;
        }
        return ret;
err:
        kfree(req);
        return ret;
}
EXPORT_SYMBOL(rdma_resolve_ip);

int roce_resolve_route_from_path(struct sa_path_rec *rec,
                                 const struct ib_gid_attr *attr)
{
        union {
                struct sockaddr     _sockaddr;
                struct sockaddr_in  _sockaddr_in;
                struct sockaddr_in6 _sockaddr_in6;
        } sgid, dgid;
        struct rdma_dev_addr dev_addr = {};
        int ret;

        might_sleep();

        if (rec->roce.route_resolved)
                return 0;

        rdma_gid2ip((struct sockaddr *)&sgid, &rec->sgid);
        rdma_gid2ip((struct sockaddr *)&dgid, &rec->dgid);

        if (sgid._sockaddr.sa_family != dgid._sockaddr.sa_family)
                return -EINVAL;

        if (!attr || !attr->ndev)
                return -EINVAL;

        dev_addr.net = &init_net;
        dev_addr.sgid_attr = attr;

        ret = addr_resolve((struct sockaddr *)&sgid, (struct sockaddr *)&dgid,
                           &dev_addr, false, true, 0);
        if (ret)
                return ret;

        if ((dev_addr.network == RDMA_NETWORK_IPV4 ||
             dev_addr.network == RDMA_NETWORK_IPV6) &&
            rec->rec_type != SA_PATH_REC_TYPE_ROCE_V2)
                return -EINVAL;

        rec->roce.route_resolved = true;
        return 0;
}

/**
 * rdma_addr_cancel - Cancel resolve ip request
 * @addr:        Pointer to address structure given previously
 *                during rdma_resolve_ip().
 * rdma_addr_cancel() is synchronous function which cancels any pending
 * request if there is any.
 */
void rdma_addr_cancel(struct rdma_dev_addr *addr)
{
        struct addr_req *req, *temp_req;
        struct addr_req *found = NULL;

        spin_lock_bh(&lock);
        list_for_each_entry_safe(req, temp_req, &req_list, list) {
                if (req->addr == addr) {
                        /*
                         * Removing from the list means we take ownership of
                         * the req
                         */
                        list_del_init(&req->list);
                        found = req;
                        break;
                }
        }
        spin_unlock_bh(&lock);

        if (!found)
                return;

        /*
         * sync canceling the work after removing it from the req_list
         * guarentees no work is running and none will be started.
         */
        cancel_delayed_work_sync(&found->work);
        kfree(found);
}
EXPORT_SYMBOL(rdma_addr_cancel);

struct resolve_cb_context {
        struct completion comp;
        int status;
};

static void resolve_cb(int status, struct sockaddr *src_addr,
             struct rdma_dev_addr *addr, void *context)
{
        ((struct resolve_cb_context *)context)->status = status;
        complete(&((struct resolve_cb_context *)context)->comp);
}

int rdma_addr_find_l2_eth_by_grh(const union ib_gid *sgid,
                                 const union ib_gid *dgid,
                                 u8 *dmac, const struct ib_gid_attr *sgid_attr,
                                 int *hoplimit)
{
        struct rdma_dev_addr dev_addr;
        struct resolve_cb_context ctx;
        union {
                struct sockaddr_in  _sockaddr_in;
                struct sockaddr_in6 _sockaddr_in6;
        } sgid_addr, dgid_addr;
        int ret;

        rdma_gid2ip((struct sockaddr *)&sgid_addr, sgid);
        rdma_gid2ip((struct sockaddr *)&dgid_addr, dgid);

        memset(&dev_addr, 0, sizeof(dev_addr));
        dev_addr.net = &init_net;
        dev_addr.sgid_attr = sgid_attr;

        init_completion(&ctx.comp);
        ret = rdma_resolve_ip((struct sockaddr *)&sgid_addr,
                              (struct sockaddr *)&dgid_addr, &dev_addr, 1000,
                              resolve_cb, true, &ctx);
        if (ret)
                return ret;

        wait_for_completion(&ctx.comp);

        ret = ctx.status;
        if (ret)
                return ret;

        memcpy(dmac, dev_addr.dst_dev_addr, ETH_ALEN);
        *hoplimit = dev_addr.hoplimit;
        return 0;
}

static int netevent_callback(struct notifier_block *self, unsigned long event,
        void *ctx)
{
        struct addr_req *req;

        if (event == NETEVENT_NEIGH_UPDATE) {
                struct neighbour *neigh = ctx;

                if (neigh->nud_state & NUD_VALID) {
                        spin_lock_bh(&lock);
                        list_for_each_entry(req, &req_list, list)
                                set_timeout(req, jiffies);
                        spin_unlock_bh(&lock);
                }
        }
        return 0;
}

static struct notifier_block nb = {
        .notifier_call = netevent_callback
};

int addr_init(void)
{
        addr_wq = alloc_ordered_workqueue("ib_addr", 0);
        if (!addr_wq)
                return -ENOMEM;

        register_netevent_notifier(&nb);

        return 0;
}

void addr_cleanup(void)
{
        unregister_netevent_notifier(&nb);
        destroy_workqueue(addr_wq);
        WARN_ON(!list_empty(&req_list));
}

































































































































































































































































































































































































































































































   49 



































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
// SPDX-License-Identifier: GPL-2.0-only
/*
 * Crypto user configuration API.
 *
 * Copyright (C) 2011 secunet Security Networks AG
 * Copyright (C) 2011 Steffen Klassert <steffen.klassert@secunet.com>
 */

#include <linux/module.h>
#include <linux/crypto.h>
#include <linux/cryptouser.h>
#include <linux/sched.h>
#include <linux/security.h>
#include <net/netlink.h>
#include <net/net_namespace.h>
#include <net/sock.h>
#include <crypto/internal/skcipher.h>
#include <crypto/internal/rng.h>
#include <crypto/akcipher.h>
#include <crypto/kpp.h>
#include <crypto/internal/cryptouser.h>

#include "internal.h"

#define null_terminated(x)        (strnlen(x, sizeof(x)) < sizeof(x))

static DEFINE_MUTEX(crypto_cfg_mutex);

struct crypto_dump_info {
        struct sk_buff *in_skb;
        struct sk_buff *out_skb;
        u32 nlmsg_seq;
        u16 nlmsg_flags;
};

struct crypto_alg *crypto_alg_match(struct crypto_user_alg *p, int exact)
{
        struct crypto_alg *q, *alg = NULL;

        down_read(&crypto_alg_sem);

        list_for_each_entry(q, &crypto_alg_list, cra_list) {
                int match = 0;

                if (crypto_is_larval(q))
                        continue;

                if ((q->cra_flags ^ p->cru_type) & p->cru_mask)
                        continue;

                if (strlen(p->cru_driver_name))
                        match = !strcmp(q->cra_driver_name,
                                        p->cru_driver_name);
                else if (!exact)
                        match = !strcmp(q->cra_name, p->cru_name);

                if (!match)
                        continue;

                if (unlikely(!crypto_mod_get(q)))
                        continue;

                alg = q;
                break;
        }

        up_read(&crypto_alg_sem);

        return alg;
}

static int crypto_report_cipher(struct sk_buff *skb, struct crypto_alg *alg)
{
        struct crypto_report_cipher rcipher;

        memset(&rcipher, 0, sizeof(rcipher));

        strscpy(rcipher.type, "cipher", sizeof(rcipher.type));

        rcipher.blocksize = alg->cra_blocksize;
        rcipher.min_keysize = alg->cra_cipher.cia_min_keysize;
        rcipher.max_keysize = alg->cra_cipher.cia_max_keysize;

        return nla_put(skb, CRYPTOCFGA_REPORT_CIPHER,
                       sizeof(rcipher), &rcipher);
}

static int crypto_report_comp(struct sk_buff *skb, struct crypto_alg *alg)
{
        struct crypto_report_comp rcomp;

        memset(&rcomp, 0, sizeof(rcomp));

        strscpy(rcomp.type, "compression", sizeof(rcomp.type));

        return nla_put(skb, CRYPTOCFGA_REPORT_COMPRESS, sizeof(rcomp), &rcomp);
}

static int crypto_report_one(struct crypto_alg *alg,
                             struct crypto_user_alg *ualg, struct sk_buff *skb)
{
        memset(ualg, 0, sizeof(*ualg));

        strscpy(ualg->cru_name, alg->cra_name, sizeof(ualg->cru_name));
        strscpy(ualg->cru_driver_name, alg->cra_driver_name,
                sizeof(ualg->cru_driver_name));
        strscpy(ualg->cru_module_name, module_name(alg->cra_module),
                sizeof(ualg->cru_module_name));

        ualg->cru_type = 0;
        ualg->cru_mask = 0;
        ualg->cru_flags = alg->cra_flags;
        ualg->cru_refcnt = refcount_read(&alg->cra_refcnt);

        if (nla_put_u32(skb, CRYPTOCFGA_PRIORITY_VAL, alg->cra_priority))
                goto nla_put_failure;
        if (alg->cra_flags & CRYPTO_ALG_LARVAL) {
                struct crypto_report_larval rl;

                memset(&rl, 0, sizeof(rl));
                strscpy(rl.type, "larval", sizeof(rl.type));
                if (nla_put(skb, CRYPTOCFGA_REPORT_LARVAL, sizeof(rl), &rl))
                        goto nla_put_failure;
                goto out;
        }

        if (alg->cra_type && alg->cra_type->report) {
                if (alg->cra_type->report(skb, alg))
                        goto nla_put_failure;

                goto out;
        }

        switch (alg->cra_flags & (CRYPTO_ALG_TYPE_MASK | CRYPTO_ALG_LARVAL)) {
        case CRYPTO_ALG_TYPE_CIPHER:
                if (crypto_report_cipher(skb, alg))
                        goto nla_put_failure;

                break;
        case CRYPTO_ALG_TYPE_COMPRESS:
                if (crypto_report_comp(skb, alg))
                        goto nla_put_failure;

                break;
        }

out:
        return 0;

nla_put_failure:
        return -EMSGSIZE;
}

static int crypto_report_alg(struct crypto_alg *alg,
                             struct crypto_dump_info *info)
{
        struct sk_buff *in_skb = info->in_skb;
        struct sk_buff *skb = info->out_skb;
        struct nlmsghdr *nlh;
        struct crypto_user_alg *ualg;
        int err = 0;

        nlh = nlmsg_put(skb, NETLINK_CB(in_skb).portid, info->nlmsg_seq,
                        CRYPTO_MSG_GETALG, sizeof(*ualg), info->nlmsg_flags);
        if (!nlh) {
                err = -EMSGSIZE;
                goto out;
        }

        ualg = nlmsg_data(nlh);

        err = crypto_report_one(alg, ualg, skb);
        if (err) {
                nlmsg_cancel(skb, nlh);
                goto out;
        }

        nlmsg_end(skb, nlh);

out:
        return err;
}

static int crypto_report(struct sk_buff *in_skb, struct nlmsghdr *in_nlh,
                         struct nlattr **attrs)
{
        struct net *net = sock_net(in_skb->sk);
        struct crypto_user_alg *p = nlmsg_data(in_nlh);
        struct crypto_alg *alg;
        struct sk_buff *skb;
        struct crypto_dump_info info;
        int err;

        if (!null_terminated(p->cru_name) || !null_terminated(p->cru_driver_name))
                return -EINVAL;

        alg = crypto_alg_match(p, 0);
        if (!alg)
                return -ENOENT;

        err = -ENOMEM;
        skb = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
        if (!skb)
                goto drop_alg;

        info.in_skb = in_skb;
        info.out_skb = skb;
        info.nlmsg_seq = in_nlh->nlmsg_seq;
        info.nlmsg_flags = 0;

        err = crypto_report_alg(alg, &info);

drop_alg:
        crypto_mod_put(alg);

        if (err) {
                kfree_skb(skb);
                return err;
        }

        return nlmsg_unicast(net->crypto_nlsk, skb, NETLINK_CB(in_skb).portid);
}

static int crypto_dump_report(struct sk_buff *skb, struct netlink_callback *cb)
{
        const size_t start_pos = cb->args[0];
        size_t pos = 0;
        struct crypto_dump_info info;
        struct crypto_alg *alg;
        int res;

        info.in_skb = cb->skb;
        info.out_skb = skb;
        info.nlmsg_seq = cb->nlh->nlmsg_seq;
        info.nlmsg_flags = NLM_F_MULTI;

        down_read(&crypto_alg_sem);
        list_for_each_entry(alg, &crypto_alg_list, cra_list) {
                if (pos >= start_pos) {
                        res = crypto_report_alg(alg, &info);
                        if (res == -EMSGSIZE)
                                break;
                        if (res)
                                goto out;
                }
                pos++;
        }
        cb->args[0] = pos;
        res = skb->len;
out:
        up_read(&crypto_alg_sem);
        return res;
}

static int crypto_dump_report_done(struct netlink_callback *cb)
{
        return 0;
}

static int crypto_update_alg(struct sk_buff *skb, struct nlmsghdr *nlh,
                             struct nlattr **attrs)
{
        struct crypto_alg *alg;
        struct crypto_user_alg *p = nlmsg_data(nlh);
        struct nlattr *priority = attrs[CRYPTOCFGA_PRIORITY_VAL];
        LIST_HEAD(list);

        if (!netlink_capable(skb, CAP_NET_ADMIN))
                return -EPERM;

        if (!null_terminated(p->cru_name) || !null_terminated(p->cru_driver_name))
                return -EINVAL;

        if (priority && !strlen(p->cru_driver_name))
                return -EINVAL;

        alg = crypto_alg_match(p, 1);
        if (!alg)
                return -ENOENT;

        down_write(&crypto_alg_sem);

        crypto_remove_spawns(alg, &list, NULL);

        if (priority)
                alg->cra_priority = nla_get_u32(priority);

        up_write(&crypto_alg_sem);

        crypto_mod_put(alg);
        crypto_remove_final(&list);

        return 0;
}

static int crypto_del_alg(struct sk_buff *skb, struct nlmsghdr *nlh,
                          struct nlattr **attrs)
{
        struct crypto_alg *alg;
        struct crypto_user_alg *p = nlmsg_data(nlh);
        int err;

        if (!netlink_capable(skb, CAP_NET_ADMIN))
                return -EPERM;

        if (!null_terminated(p->cru_name) || !null_terminated(p->cru_driver_name))
                return -EINVAL;

        alg = crypto_alg_match(p, 1);
        if (!alg)
                return -ENOENT;

        /* We can not unregister core algorithms such as aes-generic.
         * We would loose the reference in the crypto_alg_list to this algorithm
         * if we try to unregister. Unregistering such an algorithm without
         * removing the module is not possible, so we restrict to crypto
         * instances that are build from templates. */
        err = -EINVAL;
        if (!(alg->cra_flags & CRYPTO_ALG_INSTANCE))
                goto drop_alg;

        err = -EBUSY;
        if (refcount_read(&alg->cra_refcnt) > 2)
                goto drop_alg;

        crypto_unregister_instance((struct crypto_instance *)alg);
        err = 0;

drop_alg:
        crypto_mod_put(alg);
        return err;
}

static int crypto_add_alg(struct sk_buff *skb, struct nlmsghdr *nlh,
                          struct nlattr **attrs)
{
        int exact = 0;
        const char *name;
        struct crypto_alg *alg;
        struct crypto_user_alg *p = nlmsg_data(nlh);
        struct nlattr *priority = attrs[CRYPTOCFGA_PRIORITY_VAL];

        if (!netlink_capable(skb, CAP_NET_ADMIN))
                return -EPERM;

        if (!null_terminated(p->cru_name) || !null_terminated(p->cru_driver_name))
                return -EINVAL;

        if (strlen(p->cru_driver_name))
                exact = 1;

        if (priority && !exact)
                return -EINVAL;

        alg = crypto_alg_match(p, exact);
        if (alg) {
                crypto_mod_put(alg);
                return -EEXIST;
        }

        if (strlen(p->cru_driver_name))
                name = p->cru_driver_name;
        else
                name = p->cru_name;

        alg = crypto_alg_mod_lookup(name, p->cru_type, p->cru_mask);
        if (IS_ERR(alg))
                return PTR_ERR(alg);

        down_write(&crypto_alg_sem);

        if (priority)
                alg->cra_priority = nla_get_u32(priority);

        up_write(&crypto_alg_sem);

        crypto_mod_put(alg);

        return 0;
}

static int crypto_del_rng(struct sk_buff *skb, struct nlmsghdr *nlh,
                          struct nlattr **attrs)
{
        if (!netlink_capable(skb, CAP_NET_ADMIN))
                return -EPERM;
        return crypto_del_default_rng();
}

#define MSGSIZE(type) sizeof(struct type)

static const int crypto_msg_min[CRYPTO_NR_MSGTYPES] = {
        [CRYPTO_MSG_NEWALG        - CRYPTO_MSG_BASE] = MSGSIZE(crypto_user_alg),
        [CRYPTO_MSG_DELALG        - CRYPTO_MSG_BASE] = MSGSIZE(crypto_user_alg),
        [CRYPTO_MSG_UPDATEALG        - CRYPTO_MSG_BASE] = MSGSIZE(crypto_user_alg),
        [CRYPTO_MSG_GETALG        - CRYPTO_MSG_BASE] = MSGSIZE(crypto_user_alg),
        [CRYPTO_MSG_DELRNG        - CRYPTO_MSG_BASE] = 0,
        [CRYPTO_MSG_GETSTAT        - CRYPTO_MSG_BASE] = MSGSIZE(crypto_user_alg),
};

static const struct nla_policy crypto_policy[CRYPTOCFGA_MAX+1] = {
        [CRYPTOCFGA_PRIORITY_VAL]   = { .type = NLA_U32},
};

#undef MSGSIZE

static const struct crypto_link {
        int (*doit)(struct sk_buff *, struct nlmsghdr *, struct nlattr **);
        int (*dump)(struct sk_buff *, struct netlink_callback *);
        int (*done)(struct netlink_callback *);
} crypto_dispatch[CRYPTO_NR_MSGTYPES] = {
        [CRYPTO_MSG_NEWALG        - CRYPTO_MSG_BASE] = { .doit = crypto_add_alg},
        [CRYPTO_MSG_DELALG        - CRYPTO_MSG_BASE] = { .doit = crypto_del_alg},
        [CRYPTO_MSG_UPDATEALG        - CRYPTO_MSG_BASE] = { .doit = crypto_update_alg},
        [CRYPTO_MSG_GETALG        - CRYPTO_MSG_BASE] = { .doit = crypto_report,
                                                       .dump = crypto_dump_report,
                                                       .done = crypto_dump_report_done},
        [CRYPTO_MSG_DELRNG        - CRYPTO_MSG_BASE] = { .doit = crypto_del_rng },
        [CRYPTO_MSG_GETSTAT        - CRYPTO_MSG_BASE] = { .doit = crypto_reportstat},
};

static int crypto_user_rcv_msg(struct sk_buff *skb, struct nlmsghdr *nlh,
                               struct netlink_ext_ack *extack)
{
        struct net *net = sock_net(skb->sk);
        struct nlattr *attrs[CRYPTOCFGA_MAX+1];
        const struct crypto_link *link;
        int type, err;

        type = nlh->nlmsg_type;
        if (type > CRYPTO_MSG_MAX)
                return -EINVAL;

        type -= CRYPTO_MSG_BASE;
        link = &crypto_dispatch[type];

        if ((type == (CRYPTO_MSG_GETALG - CRYPTO_MSG_BASE) &&
            (nlh->nlmsg_flags & NLM_F_DUMP))) {
                struct crypto_alg *alg;
                unsigned long dump_alloc = 0;

                if (link->dump == NULL)
                        return -EINVAL;

                down_read(&crypto_alg_sem);
                list_for_each_entry(alg, &crypto_alg_list, cra_list)
                        dump_alloc += CRYPTO_REPORT_MAXSIZE;
                up_read(&crypto_alg_sem);

                {
                        struct netlink_dump_control c = {
                                .dump = link->dump,
                                .done = link->done,
                                .min_dump_alloc = min(dump_alloc, 65535UL),
                        };
                        err = netlink_dump_start(net->crypto_nlsk, skb, nlh, &c);
                }

                return err;
        }

        err = nlmsg_parse_deprecated(nlh, crypto_msg_min[type], attrs,
                                     CRYPTOCFGA_MAX, crypto_policy, extack);
        if (err < 0)
                return err;

        if (link->doit == NULL)
                return -EINVAL;

        return link->doit(skb, nlh, attrs);
}

static void crypto_netlink_rcv(struct sk_buff *skb)
{
        mutex_lock(&crypto_cfg_mutex);
        netlink_rcv_skb(skb, &crypto_user_rcv_msg);
        mutex_unlock(&crypto_cfg_mutex);
}

static int __net_init crypto_netlink_init(struct net *net)
{
        struct netlink_kernel_cfg cfg = {
                .input        = crypto_netlink_rcv,
        };

        net->crypto_nlsk = netlink_kernel_create(net, NETLINK_CRYPTO, &cfg);
        return net->crypto_nlsk == NULL ? -ENOMEM : 0;
}

static void __net_exit crypto_netlink_exit(struct net *net)
{
        netlink_kernel_release(net->crypto_nlsk);
        net->crypto_nlsk = NULL;
}

static struct pernet_operations crypto_netlink_net_ops = {
        .init = crypto_netlink_init,
        .exit = crypto_netlink_exit,
};

static int __init crypto_user_init(void)
{
        return register_pernet_subsys(&crypto_netlink_net_ops);
}

static void __exit crypto_user_exit(void)
{
        unregister_pernet_subsys(&crypto_netlink_net_ops);
}

module_init(crypto_user_init);
module_exit(crypto_user_exit);
MODULE_LICENSE("GPL");
MODULE_AUTHOR("Steffen Klassert <steffen.klassert@secunet.com>");
MODULE_DESCRIPTION("Crypto userspace configuration API");
MODULE_ALIAS("net-pf-16-proto-21");





























































































































































































































  301 




  300 
  300 

  301 




  299 

  297 
  298 
  299 

  298 






























































































































































































































































































































































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
// SPDX-License-Identifier: GPL-2.0-only
/*
 * Common framework for low-level network console, dump, and debugger code
 *
 * Sep 8 2003  Matt Mackall <mpm@selenic.com>
 *
 * based on the netconsole code from:
 *
 * Copyright (C) 2001  Ingo Molnar <mingo@redhat.com>
 * Copyright (C) 2002  Red Hat, Inc.
 */

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#include <linux/moduleparam.h>
#include <linux/kernel.h>
#include <linux/netdevice.h>
#include <linux/etherdevice.h>
#include <linux/string.h>
#include <linux/if_arp.h>
#include <linux/inetdevice.h>
#include <linux/inet.h>
#include <linux/interrupt.h>
#include <linux/netpoll.h>
#include <linux/sched.h>
#include <linux/delay.h>
#include <linux/rcupdate.h>
#include <linux/workqueue.h>
#include <linux/slab.h>
#include <linux/export.h>
#include <linux/if_vlan.h>
#include <net/tcp.h>
#include <net/udp.h>
#include <net/addrconf.h>
#include <net/ndisc.h>
#include <net/ip6_checksum.h>
#include <asm/unaligned.h>
#include <trace/events/napi.h>
#include <linux/kconfig.h>

/*
 * We maintain a small pool of fully-sized skbs, to make sure the
 * message gets out even in extreme OOM situations.
 */

#define MAX_UDP_CHUNK 1460
#define MAX_SKBS 32

static struct sk_buff_head skb_pool;

DEFINE_STATIC_SRCU(netpoll_srcu);

#define USEC_PER_POLL        50

#define MAX_SKB_SIZE                                                        \
        (sizeof(struct ethhdr) +                                        \
         sizeof(struct iphdr) +                                                \
         sizeof(struct udphdr) +                                        \
         MAX_UDP_CHUNK)

static void zap_completion_queue(void);

static unsigned int carrier_timeout = 4;
module_param(carrier_timeout, uint, 0644);

#define np_info(np, fmt, ...)                                \
        pr_info("%s: " fmt, np->name, ##__VA_ARGS__)
#define np_err(np, fmt, ...)                                \
        pr_err("%s: " fmt, np->name, ##__VA_ARGS__)
#define np_notice(np, fmt, ...)                                \
        pr_notice("%s: " fmt, np->name, ##__VA_ARGS__)

static netdev_tx_t netpoll_start_xmit(struct sk_buff *skb,
                                      struct net_device *dev,
                                      struct netdev_queue *txq)
{
        netdev_tx_t status = NETDEV_TX_OK;
        netdev_features_t features;

        features = netif_skb_features(skb);

        if (skb_vlan_tag_present(skb) &&
            !vlan_hw_offload_capable(features, skb->vlan_proto)) {
                skb = __vlan_hwaccel_push_inside(skb);
                if (unlikely(!skb)) {
                        /* This is actually a packet drop, but we
                         * don't want the code that calls this
                         * function to try and operate on a NULL skb.
                         */
                        goto out;
                }
        }

        status = netdev_start_xmit(skb, dev, txq, false);

out:
        return status;
}

static void queue_process(struct work_struct *work)
{
        struct netpoll_info *npinfo =
                container_of(work, struct netpoll_info, tx_work.work);
        struct sk_buff *skb;
        unsigned long flags;

        while ((skb = skb_dequeue(&npinfo->txq))) {
                struct net_device *dev = skb->dev;
                struct netdev_queue *txq;
                unsigned int q_index;

                if (!netif_device_present(dev) || !netif_running(dev)) {
                        kfree_skb(skb);
                        continue;
                }

                local_irq_save(flags);
                /* check if skb->queue_mapping is still valid */
                q_index = skb_get_queue_mapping(skb);
                if (unlikely(q_index >= dev->real_num_tx_queues)) {
                        q_index = q_index % dev->real_num_tx_queues;
                        skb_set_queue_mapping(skb, q_index);
                }
                txq = netdev_get_tx_queue(dev, q_index);
                HARD_TX_LOCK(dev, txq, smp_processor_id());
                if (netif_xmit_frozen_or_stopped(txq) ||
                    !dev_xmit_complete(netpoll_start_xmit(skb, dev, txq))) {
                        skb_queue_head(&npinfo->txq, skb);
                        HARD_TX_UNLOCK(dev, txq);
                        local_irq_restore(flags);

                        schedule_delayed_work(&npinfo->tx_work, HZ/10);
                        return;
                }
                HARD_TX_UNLOCK(dev, txq);
                local_irq_restore(flags);
        }
}

static int netif_local_xmit_active(struct net_device *dev)
{
        int i;

        for (i = 0; i < dev->num_tx_queues; i++) {
                struct netdev_queue *txq = netdev_get_tx_queue(dev, i);

                if (READ_ONCE(txq->xmit_lock_owner) == smp_processor_id())
                        return 1;
        }

        return 0;
}

static void poll_one_napi(struct napi_struct *napi)
{
        int work;

        /* If we set this bit but see that it has already been set,
         * that indicates that napi has been disabled and we need
         * to abort this operation
         */
        if (test_and_set_bit(NAPI_STATE_NPSVC, &napi->state))
                return;

        /* We explicilty pass the polling call a budget of 0 to
         * indicate that we are clearing the Tx path only.
         */
        work = napi->poll(napi, 0);
        WARN_ONCE(work, "%pS exceeded budget in poll\n", napi->poll);
        trace_napi_poll(napi, work, 0);

        clear_bit(NAPI_STATE_NPSVC, &napi->state);
}

static void poll_napi(struct net_device *dev)
{
        struct napi_struct *napi;
        int cpu = smp_processor_id();

        list_for_each_entry_rcu(napi, &dev->napi_list, dev_list) {
                if (cmpxchg(&napi->poll_owner, -1, cpu) == -1) {
                        poll_one_napi(napi);
                        smp_store_release(&napi->poll_owner, -1);
                }
        }
}

void netpoll_poll_dev(struct net_device *dev)
{
        struct netpoll_info *ni = rcu_dereference_bh(dev->npinfo);
        const struct net_device_ops *ops;

        /* Don't do any rx activity if the dev_lock mutex is held
         * the dev_open/close paths use this to block netpoll activity
         * while changing device state
         */
        if (!ni || down_trylock(&ni->dev_lock))
                return;

        /* Some drivers will take the same locks in poll and xmit,
         * we can't poll if local CPU is already in xmit.
         */
        if (!netif_running(dev) || netif_local_xmit_active(dev)) {
                up(&ni->dev_lock);
                return;
        }

        ops = dev->netdev_ops;
        if (ops->ndo_poll_controller)
                ops->ndo_poll_controller(dev);

        poll_napi(dev);

        up(&ni->dev_lock);

        zap_completion_queue();
}
EXPORT_SYMBOL(netpoll_poll_dev);

void netpoll_poll_disable(struct net_device *dev)
{
        struct netpoll_info *ni;
        int idx;
        might_sleep();
        idx = srcu_read_lock(&netpoll_srcu);
        ni = srcu_dereference(dev->npinfo, &netpoll_srcu);
        if (ni)
                down(&ni->dev_lock);
        srcu_read_unlock(&netpoll_srcu, idx);
}
EXPORT_SYMBOL(netpoll_poll_disable);

void netpoll_poll_enable(struct net_device *dev)
{
        struct netpoll_info *ni;
        rcu_read_lock();
        ni = rcu_dereference(dev->npinfo);
        if (ni)
                up(&ni->dev_lock);
        rcu_read_unlock();
}
EXPORT_SYMBOL(netpoll_poll_enable);

static void refill_skbs(void)
{
        struct sk_buff *skb;
        unsigned long flags;

        spin_lock_irqsave(&skb_pool.lock, flags);
        while (skb_pool.qlen < MAX_SKBS) {
                skb = alloc_skb(MAX_SKB_SIZE, GFP_ATOMIC);
                if (!skb)
                        break;

                __skb_queue_tail(&skb_pool, skb);
        }
        spin_unlock_irqrestore(&skb_pool.lock, flags);
}

static void zap_completion_queue(void)
{
        unsigned long flags;
        struct softnet_data *sd = &get_cpu_var(softnet_data);

        if (sd->completion_queue) {
                struct sk_buff *clist;

                local_irq_save(flags);
                clist = sd->completion_queue;
                sd->completion_queue = NULL;
                local_irq_restore(flags);

                while (clist != NULL) {
                        struct sk_buff *skb = clist;
                        clist = clist->next;
                        if (!skb_irq_freeable(skb)) {
                                refcount_set(&skb->users, 1);
                                dev_kfree_skb_any(skb); /* put this one back */
                        } else {
                                __kfree_skb(skb);
                        }
                }
        }

        put_cpu_var(softnet_data);
}

static struct sk_buff *find_skb(struct netpoll *np, int len, int reserve)
{
        int count = 0;
        struct sk_buff *skb;

        zap_completion_queue();
        refill_skbs();
repeat:

        skb = alloc_skb(len, GFP_ATOMIC);
        if (!skb)
                skb = skb_dequeue(&skb_pool);

        if (!skb) {
                if (++count < 10) {
                        netpoll_poll_dev(np->dev);
                        goto repeat;
                }
                return NULL;
        }

        refcount_set(&skb->users, 1);
        skb_reserve(skb, reserve);
        return skb;
}

static int netpoll_owner_active(struct net_device *dev)
{
        struct napi_struct *napi;

        list_for_each_entry_rcu(napi, &dev->napi_list, dev_list) {
                if (READ_ONCE(napi->poll_owner) == smp_processor_id())
                        return 1;
        }
        return 0;
}

/* call with IRQ disabled */
static netdev_tx_t __netpoll_send_skb(struct netpoll *np, struct sk_buff *skb)
{
        netdev_tx_t status = NETDEV_TX_BUSY;
        struct net_device *dev;
        unsigned long tries;
        /* It is up to the caller to keep npinfo alive. */
        struct netpoll_info *npinfo;

        lockdep_assert_irqs_disabled();

        dev = np->dev;
        npinfo = rcu_dereference_bh(dev->npinfo);

        if (!npinfo || !netif_running(dev) || !netif_device_present(dev)) {
                dev_kfree_skb_irq(skb);
                return NET_XMIT_DROP;
        }

        /* don't get messages out of order, and no recursion */
        if (skb_queue_len(&npinfo->txq) == 0 && !netpoll_owner_active(dev)) {
                struct netdev_queue *txq;

                txq = netdev_core_pick_tx(dev, skb, NULL);

                /* try until next clock tick */
                for (tries = jiffies_to_usecs(1)/USEC_PER_POLL;
                     tries > 0; --tries) {
                        if (HARD_TX_TRYLOCK(dev, txq)) {
                                if (!netif_xmit_stopped(txq))
                                        status = netpoll_start_xmit(skb, dev, txq);

                                HARD_TX_UNLOCK(dev, txq);

                                if (dev_xmit_complete(status))
                                        break;

                        }

                        /* tickle device maybe there is some cleanup */
                        netpoll_poll_dev(np->dev);

                        udelay(USEC_PER_POLL);
                }

                WARN_ONCE(!irqs_disabled(),
                        "netpoll_send_skb_on_dev(): %s enabled interrupts in poll (%pS)\n",
                        dev->name, dev->netdev_ops->ndo_start_xmit);

        }

        if (!dev_xmit_complete(status)) {
                skb_queue_tail(&npinfo->txq, skb);
                schedule_delayed_work(&npinfo->tx_work,0);
        }
        return NETDEV_TX_OK;
}

netdev_tx_t netpoll_send_skb(struct netpoll *np, struct sk_buff *skb)
{
        unsigned long flags;
        netdev_tx_t ret;

        if (unlikely(!np)) {
                dev_kfree_skb_irq(skb);
                ret = NET_XMIT_DROP;
        } else {
                local_irq_save(flags);
                ret = __netpoll_send_skb(np, skb);
                local_irq_restore(flags);
        }
        return ret;
}
EXPORT_SYMBOL(netpoll_send_skb);

void netpoll_send_udp(struct netpoll *np, const char *msg, int len)
{
        int total_len, ip_len, udp_len;
        struct sk_buff *skb;
        struct udphdr *udph;
        struct iphdr *iph;
        struct ethhdr *eth;
        static atomic_t ip_ident;
        struct ipv6hdr *ip6h;

        if (!IS_ENABLED(CONFIG_PREEMPT_RT))
                WARN_ON_ONCE(!irqs_disabled());

        udp_len = len + sizeof(*udph);
        if (np->ipv6)
                ip_len = udp_len + sizeof(*ip6h);
        else
                ip_len = udp_len + sizeof(*iph);

        total_len = ip_len + LL_RESERVED_SPACE(np->dev);

        skb = find_skb(np, total_len + np->dev->needed_tailroom,
                       total_len - len);
        if (!skb)
                return;

        skb_copy_to_linear_data(skb, msg, len);
        skb_put(skb, len);

        skb_push(skb, sizeof(*udph));
        skb_reset_transport_header(skb);
        udph = udp_hdr(skb);
        udph->source = htons(np->local_port);
        udph->dest = htons(np->remote_port);
        udph->len = htons(udp_len);

        if (np->ipv6) {
                udph->check = 0;
                udph->check = csum_ipv6_magic(&np->local_ip.in6,
                                              &np->remote_ip.in6,
                                              udp_len, IPPROTO_UDP,
                                              csum_partial(udph, udp_len, 0));
                if (udph->check == 0)
                        udph->check = CSUM_MANGLED_0;

                skb_push(skb, sizeof(*ip6h));
                skb_reset_network_header(skb);
                ip6h = ipv6_hdr(skb);

                /* ip6h->version = 6; ip6h->priority = 0; */
                *(unsigned char *)ip6h = 0x60;
                ip6h->flow_lbl[0] = 0;
                ip6h->flow_lbl[1] = 0;
                ip6h->flow_lbl[2] = 0;

                ip6h->payload_len = htons(sizeof(struct udphdr) + len);
                ip6h->nexthdr = IPPROTO_UDP;
                ip6h->hop_limit = 32;
                ip6h->saddr = np->local_ip.in6;
                ip6h->daddr = np->remote_ip.in6;

                eth = skb_push(skb, ETH_HLEN);
                skb_reset_mac_header(skb);
                skb->protocol = eth->h_proto = htons(ETH_P_IPV6);
        } else {
                udph->check = 0;
                udph->check = csum_tcpudp_magic(np->local_ip.ip,
                                                np->remote_ip.ip,
                                                udp_len, IPPROTO_UDP,
                                                csum_partial(udph, udp_len, 0));
                if (udph->check == 0)
                        udph->check = CSUM_MANGLED_0;

                skb_push(skb, sizeof(*iph));
                skb_reset_network_header(skb);
                iph = ip_hdr(skb);

                /* iph->version = 4; iph->ihl = 5; */
                *(unsigned char *)iph = 0x45;
                iph->tos      = 0;
                put_unaligned(htons(ip_len), &(iph->tot_len));
                iph->id       = htons(atomic_inc_return(&ip_ident));
                iph->frag_off = 0;
                iph->ttl      = 64;
                iph->protocol = IPPROTO_UDP;
                iph->check    = 0;
                put_unaligned(np->local_ip.ip, &(iph->saddr));
                put_unaligned(np->remote_ip.ip, &(iph->daddr));
                iph->check    = ip_fast_csum((unsigned char *)iph, iph->ihl);

                eth = skb_push(skb, ETH_HLEN);
                skb_reset_mac_header(skb);
                skb->protocol = eth->h_proto = htons(ETH_P_IP);
        }

        ether_addr_copy(eth->h_source, np->dev->dev_addr);
        ether_addr_copy(eth->h_dest, np->remote_mac);

        skb->dev = np->dev;

        netpoll_send_skb(np, skb);
}
EXPORT_SYMBOL(netpoll_send_udp);

void netpoll_print_options(struct netpoll *np)
{
        np_info(np, "local port %d\n", np->local_port);
        if (np->ipv6)
                np_info(np, "local IPv6 address %pI6c\n", &np->local_ip.in6);
        else
                np_info(np, "local IPv4 address %pI4\n", &np->local_ip.ip);
        np_info(np, "interface '%s'\n", np->dev_name);
        np_info(np, "remote port %d\n", np->remote_port);
        if (np->ipv6)
                np_info(np, "remote IPv6 address %pI6c\n", &np->remote_ip.in6);
        else
                np_info(np, "remote IPv4 address %pI4\n", &np->remote_ip.ip);
        np_info(np, "remote ethernet address %pM\n", np->remote_mac);
}
EXPORT_SYMBOL(netpoll_print_options);

static int netpoll_parse_ip_addr(const char *str, union inet_addr *addr)
{
        const char *end;

        if (!strchr(str, ':') &&
            in4_pton(str, -1, (void *)addr, -1, &end) > 0) {
                if (!*end)
                        return 0;
        }
        if (in6_pton(str, -1, addr->in6.s6_addr, -1, &end) > 0) {
#if IS_ENABLED(CONFIG_IPV6)
                if (!*end)
                        return 1;
#else
                return -1;
#endif
        }
        return -1;
}

int netpoll_parse_options(struct netpoll *np, char *opt)
{
        char *cur=opt, *delim;
        int ipv6;
        bool ipversion_set = false;

        if (*cur != '@') {
                if ((delim = strchr(cur, '@')) == NULL)
                        goto parse_failed;
                *delim = 0;
                if (kstrtou16(cur, 10, &np->local_port))
                        goto parse_failed;
                cur = delim;
        }
        cur++;

        if (*cur != '/') {
                ipversion_set = true;
                if ((delim = strchr(cur, '/')) == NULL)
                        goto parse_failed;
                *delim = 0;
                ipv6 = netpoll_parse_ip_addr(cur, &np->local_ip);
                if (ipv6 < 0)
                        goto parse_failed;
                else
                        np->ipv6 = (bool)ipv6;
                cur = delim;
        }
        cur++;

        if (*cur != ',') {
                /* parse out dev name */
                if ((delim = strchr(cur, ',')) == NULL)
                        goto parse_failed;
                *delim = 0;
                strscpy(np->dev_name, cur, sizeof(np->dev_name));
                cur = delim;
        }
        cur++;

        if (*cur != '@') {
                /* dst port */
                if ((delim = strchr(cur, '@')) == NULL)
                        goto parse_failed;
                *delim = 0;
                if (*cur == ' ' || *cur == '\t')
                        np_info(np, "warning: whitespace is not allowed\n");
                if (kstrtou16(cur, 10, &np->remote_port))
                        goto parse_failed;
                cur = delim;
        }
        cur++;

        /* dst ip */
        if ((delim = strchr(cur, '/')) == NULL)
                goto parse_failed;
        *delim = 0;
        ipv6 = netpoll_parse_ip_addr(cur, &np->remote_ip);
        if (ipv6 < 0)
                goto parse_failed;
        else if (ipversion_set && np->ipv6 != (bool)ipv6)
                goto parse_failed;
        else
                np->ipv6 = (bool)ipv6;
        cur = delim + 1;

        if (*cur != 0) {
                /* MAC address */
                if (!mac_pton(cur, np->remote_mac))
                        goto parse_failed;
        }

        netpoll_print_options(np);

        return 0;

 parse_failed:
        np_info(np, "couldn't parse config at '%s'!\n", cur);
        return -1;
}
EXPORT_SYMBOL(netpoll_parse_options);

int __netpoll_setup(struct netpoll *np, struct net_device *ndev)
{
        struct netpoll_info *npinfo;
        const struct net_device_ops *ops;
        int err;

        np->dev = ndev;
        strscpy(np->dev_name, ndev->name, IFNAMSIZ);

        if (ndev->priv_flags & IFF_DISABLE_NETPOLL) {
                np_err(np, "%s doesn't support polling, aborting\n",
                       np->dev_name);
                err = -ENOTSUPP;
                goto out;
        }

        if (!rcu_access_pointer(ndev->npinfo)) {
                npinfo = kmalloc(sizeof(*npinfo), GFP_KERNEL);
                if (!npinfo) {
                        err = -ENOMEM;
                        goto out;
                }

                sema_init(&npinfo->dev_lock, 1);
                skb_queue_head_init(&npinfo->txq);
                INIT_DELAYED_WORK(&npinfo->tx_work, queue_process);

                refcount_set(&npinfo->refcnt, 1);

                ops = np->dev->netdev_ops;
                if (ops->ndo_netpoll_setup) {
                        err = ops->ndo_netpoll_setup(ndev, npinfo);
                        if (err)
                                goto free_npinfo;
                }
        } else {
                npinfo = rtnl_dereference(ndev->npinfo);
                refcount_inc(&npinfo->refcnt);
        }

        npinfo->netpoll = np;

        /* last thing to do is link it to the net device structure */
        rcu_assign_pointer(ndev->npinfo, npinfo);

        return 0;

free_npinfo:
        kfree(npinfo);
out:
        return err;
}
EXPORT_SYMBOL_GPL(__netpoll_setup);

int netpoll_setup(struct netpoll *np)
{
        struct net_device *ndev = NULL;
        struct in_device *in_dev;
        int err;

        rtnl_lock();
        if (np->dev_name[0]) {
                struct net *net = current->nsproxy->net_ns;
                ndev = __dev_get_by_name(net, np->dev_name);
        }
        if (!ndev) {
                np_err(np, "%s doesn't exist, aborting\n", np->dev_name);
                err = -ENODEV;
                goto unlock;
        }
        dev_hold(ndev);

        if (netdev_master_upper_dev_get(ndev)) {
                np_err(np, "%s is a slave device, aborting\n", np->dev_name);
                err = -EBUSY;
                goto put;
        }

        if (!netif_running(ndev)) {
                unsigned long atmost, atleast;

                np_info(np, "device %s not up yet, forcing it\n", np->dev_name);

                err = dev_open(ndev, NULL);

                if (err) {
                        np_err(np, "failed to open %s\n", ndev->name);
                        goto put;
                }

                rtnl_unlock();
                atleast = jiffies + HZ/10;
                atmost = jiffies + carrier_timeout * HZ;
                while (!netif_carrier_ok(ndev)) {
                        if (time_after(jiffies, atmost)) {
                                np_notice(np, "timeout waiting for carrier\n");
                                break;
                        }
                        msleep(1);
                }

                /* If carrier appears to come up instantly, we don't
                 * trust it and pause so that we don't pump all our
                 * queued console messages into the bitbucket.
                 */

                if (time_before(jiffies, atleast)) {
                        np_notice(np, "carrier detect appears untrustworthy, waiting 4 seconds\n");
                        msleep(4000);
                }
                rtnl_lock();
        }

        if (!np->local_ip.ip) {
                if (!np->ipv6) {
                        const struct in_ifaddr *ifa;

                        in_dev = __in_dev_get_rtnl(ndev);
                        if (!in_dev)
                                goto put_noaddr;

                        ifa = rtnl_dereference(in_dev->ifa_list);
                        if (!ifa) {
put_noaddr:
                                np_err(np, "no IP address for %s, aborting\n",
                                       np->dev_name);
                                err = -EDESTADDRREQ;
                                goto put;
                        }

                        np->local_ip.ip = ifa->ifa_local;
                        np_info(np, "local IP %pI4\n", &np->local_ip.ip);
                } else {
#if IS_ENABLED(CONFIG_IPV6)
                        struct inet6_dev *idev;

                        err = -EDESTADDRREQ;
                        idev = __in6_dev_get(ndev);
                        if (idev) {
                                struct inet6_ifaddr *ifp;

                                read_lock_bh(&idev->lock);
                                list_for_each_entry(ifp, &idev->addr_list, if_list) {
                                        if (!!(ipv6_addr_type(&ifp->addr) & IPV6_ADDR_LINKLOCAL) !=
                                            !!(ipv6_addr_type(&np->remote_ip.in6) & IPV6_ADDR_LINKLOCAL))
                                                continue;
                                        np->local_ip.in6 = ifp->addr;
                                        err = 0;
                                        break;
                                }
                                read_unlock_bh(&idev->lock);
                        }
                        if (err) {
                                np_err(np, "no IPv6 address for %s, aborting\n",
                                       np->dev_name);
                                goto put;
                        } else
                                np_info(np, "local IPv6 %pI6c\n", &np->local_ip.in6);
#else
                        np_err(np, "IPv6 is not supported %s, aborting\n",
                               np->dev_name);
                        err = -EINVAL;
                        goto put;
#endif
                }
        }

        /* fill up the skb queue */
        refill_skbs();

        err = __netpoll_setup(np, ndev);
        if (err)
                goto put;

        rtnl_unlock();
        return 0;

put:
        dev_put(ndev);
unlock:
        rtnl_unlock();
        return err;
}
EXPORT_SYMBOL(netpoll_setup);

static int __init netpoll_init(void)
{
        skb_queue_head_init(&skb_pool);
        return 0;
}
core_initcall(netpoll_init);

static void rcu_cleanup_netpoll_info(struct rcu_head *rcu_head)
{
        struct netpoll_info *npinfo =
                        container_of(rcu_head, struct netpoll_info, rcu);

        skb_queue_purge(&npinfo->txq);

        /* we can't call cancel_delayed_work_sync here, as we are in softirq */
        cancel_delayed_work(&npinfo->tx_work);

        /* clean after last, unfinished work */
        __skb_queue_purge(&npinfo->txq);
        /* now cancel it again */
        cancel_delayed_work(&npinfo->tx_work);
        kfree(npinfo);
}

void __netpoll_cleanup(struct netpoll *np)
{
        struct netpoll_info *npinfo;

        npinfo = rtnl_dereference(np->dev->npinfo);
        if (!npinfo)
                return;

        synchronize_srcu(&netpoll_srcu);

        if (refcount_dec_and_test(&npinfo->refcnt)) {
                const struct net_device_ops *ops;

                ops = np->dev->netdev_ops;
                if (ops->ndo_netpoll_cleanup)
                        ops->ndo_netpoll_cleanup(np->dev);

                RCU_INIT_POINTER(np->dev->npinfo, NULL);
                call_rcu(&npinfo->rcu, rcu_cleanup_netpoll_info);
        } else
                RCU_INIT_POINTER(np->dev->npinfo, NULL);
}
EXPORT_SYMBOL_GPL(__netpoll_cleanup);

void __netpoll_free(struct netpoll *np)
{
        ASSERT_RTNL();

        /* Wait for transmitting packets to finish before freeing. */
        synchronize_rcu();
        __netpoll_cleanup(np);
        kfree(np);
}
EXPORT_SYMBOL_GPL(__netpoll_free);

void netpoll_cleanup(struct netpoll *np)
{
        rtnl_lock();
        if (!np->dev)
                goto out;
        __netpoll_cleanup(np);
        dev_put(np->dev);
        np->dev = NULL;
out:
        rtnl_unlock();
}
EXPORT_SYMBOL(netpoll_cleanup);



































   28 









   27 


   28 
   28 


   28 





   26 
   28 














   20 



   20 
   20 
   20 

   20 


















   20 

   20 









   14 
   14 

   14 













   14 

   14 













    2 



    2 



    2 













































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * lib/plist.c
 *
 * Descending-priority-sorted double-linked list
 *
 * (C) 2002-2003 Intel Corp
 * Inaky Perez-Gonzalez <inaky.perez-gonzalez@intel.com>.
 *
 * 2001-2005 (c) MontaVista Software, Inc.
 * Daniel Walker <dwalker@mvista.com>
 *
 * (C) 2005 Thomas Gleixner <tglx@linutronix.de>
 *
 * Simplifications of the original code by
 * Oleg Nesterov <oleg@tv-sign.ru>
 *
 * Based on simple lists (include/linux/list.h).
 *
 * This file contains the add / del functions which are considered to
 * be too large to inline. See include/linux/plist.h for further
 * information.
 */

#include <linux/bug.h>
#include <linux/plist.h>

#ifdef CONFIG_DEBUG_PLIST

static struct plist_head test_head;

static void plist_check_prev_next(struct list_head *t, struct list_head *p,
                                  struct list_head *n)
{
        WARN(n->prev != p || p->next != n,
                        "top: %p, n: %p, p: %p\n"
                        "prev: %p, n: %p, p: %p\n"
                        "next: %p, n: %p, p: %p\n",
                         t, t->next, t->prev,
                        p, p->next, p->prev,
                        n, n->next, n->prev);
}

static void plist_check_list(struct list_head *top)
{
        struct list_head *prev = top, *next = top->next;

        plist_check_prev_next(top, prev, next);
        while (next != top) {
                prev = next;
                next = prev->next;
                plist_check_prev_next(top, prev, next);
        }
}

static void plist_check_head(struct plist_head *head)
{
        if (!plist_head_empty(head))
                plist_check_list(&plist_first(head)->prio_list);
        plist_check_list(&head->node_list);
}

#else
# define plist_check_head(h)        do { } while (0)
#endif

/**
 * plist_add - add @node to @head
 *
 * @node:        &struct plist_node pointer
 * @head:        &struct plist_head pointer
 */
void plist_add(struct plist_node *node, struct plist_head *head)
{
        struct plist_node *first, *iter, *prev = NULL;
        struct list_head *node_next = &head->node_list;

        plist_check_head(head);
        WARN_ON(!plist_node_empty(node));
        WARN_ON(!list_empty(&node->prio_list));

        if (plist_head_empty(head))
                goto ins_node;

        first = iter = plist_first(head);

        do {
                if (node->prio < iter->prio) {
                        node_next = &iter->node_list;
                        break;
                }

                prev = iter;
                iter = list_entry(iter->prio_list.next,
                                struct plist_node, prio_list);
        } while (iter != first);

        if (!prev || prev->prio != node->prio)
                list_add_tail(&node->prio_list, &iter->prio_list);
ins_node:
        list_add_tail(&node->node_list, node_next);

        plist_check_head(head);
}

/**
 * plist_del - Remove a @node from plist.
 *
 * @node:        &struct plist_node pointer - entry to be removed
 * @head:        &struct plist_head pointer - list head
 */
void plist_del(struct plist_node *node, struct plist_head *head)
{
        plist_check_head(head);

        if (!list_empty(&node->prio_list)) {
                if (node->node_list.next != &head->node_list) {
                        struct plist_node *next;

                        next = list_entry(node->node_list.next,
                                        struct plist_node, node_list);

                        /* add the next plist_node into prio_list */
                        if (list_empty(&next->prio_list))
                                list_add(&next->prio_list, &node->prio_list);
                }
                list_del_init(&node->prio_list);
        }

        list_del_init(&node->node_list);

        plist_check_head(head);
}

/**
 * plist_requeue - Requeue @node at end of same-prio entries.
 *
 * This is essentially an optimized plist_del() followed by
 * plist_add().  It moves an entry already in the plist to
 * after any other same-priority entries.
 *
 * @node:        &struct plist_node pointer - entry to be moved
 * @head:        &struct plist_head pointer - list head
 */
void plist_requeue(struct plist_node *node, struct plist_head *head)
{
        struct plist_node *iter;
        struct list_head *node_next = &head->node_list;

        plist_check_head(head);
        BUG_ON(plist_head_empty(head));
        BUG_ON(plist_node_empty(node));

        if (node == plist_last(head))
                return;

        iter = plist_next(node);

        if (node->prio != iter->prio)
                return;

        plist_del(node, head);

        plist_for_each_continue(iter, head) {
                if (node->prio != iter->prio) {
                        node_next = &iter->node_list;
                        break;
                }
        }
        list_add_tail(&node->node_list, node_next);

        plist_check_head(head);
}

#ifdef CONFIG_DEBUG_PLIST
#include <linux/sched.h>
#include <linux/sched/clock.h>
#include <linux/module.h>
#include <linux/init.h>

static struct plist_node __initdata test_node[241];

static void __init plist_test_check(int nr_expect)
{
        struct plist_node *first, *prio_pos, *node_pos;

        if (plist_head_empty(&test_head)) {
                BUG_ON(nr_expect != 0);
                return;
        }

        prio_pos = first = plist_first(&test_head);
        plist_for_each(node_pos, &test_head) {
                if (nr_expect-- < 0)
                        break;
                if (node_pos == first)
                        continue;
                if (node_pos->prio == prio_pos->prio) {
                        BUG_ON(!list_empty(&node_pos->prio_list));
                        continue;
                }

                BUG_ON(prio_pos->prio > node_pos->prio);
                BUG_ON(prio_pos->prio_list.next != &node_pos->prio_list);
                prio_pos = node_pos;
        }

        BUG_ON(nr_expect != 0);
        BUG_ON(prio_pos->prio_list.next != &first->prio_list);
}

static void __init plist_test_requeue(struct plist_node *node)
{
        plist_requeue(node, &test_head);

        if (node != plist_last(&test_head))
                BUG_ON(node->prio == plist_next(node)->prio);
}

static int  __init plist_test(void)
{
        int nr_expect = 0, i, loop;
        unsigned int r = local_clock();

        printk(KERN_DEBUG "start plist test\n");
        plist_head_init(&test_head);
        for (i = 0; i < ARRAY_SIZE(test_node); i++)
                plist_node_init(test_node + i, 0);

        for (loop = 0; loop < 1000; loop++) {
                r = r * 193939 % 47629;
                i = r % ARRAY_SIZE(test_node);
                if (plist_node_empty(test_node + i)) {
                        r = r * 193939 % 47629;
                        test_node[i].prio = r % 99;
                        plist_add(test_node + i, &test_head);
                        nr_expect++;
                } else {
                        plist_del(test_node + i, &test_head);
                        nr_expect--;
                }
                plist_test_check(nr_expect);
                if (!plist_node_empty(test_node + i)) {
                        plist_test_requeue(test_node + i);
                        plist_test_check(nr_expect);
                }
        }

        for (i = 0; i < ARRAY_SIZE(test_node); i++) {
                if (plist_node_empty(test_node + i))
                        continue;
                plist_del(test_node + i, &test_head);
                nr_expect--;
                plist_test_check(nr_expect);
        }

        printk(KERN_DEBUG "end plist test\n");
        return 0;
}

module_init(plist_test);

#endif























   30 


   29 
   30 
    6 

   30 



   29 

    3 


   27 

   27 






















   30 










   30 










   30 







    1 




    1 































   30 













































   29 



    7 




   30 


   30 











   30 



   30 





    4 



   23 



    2 







   30 




   30 







    1 

   29 










































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 *        Forwarding decision
 *        Linux ethernet bridge
 *
 *        Authors:
 *        Lennert Buytenhek                <buytenh@gnu.org>
 */

#include <linux/err.h>
#include <linux/slab.h>
#include <linux/kernel.h>
#include <linux/netdevice.h>
#include <linux/netpoll.h>
#include <linux/skbuff.h>
#include <linux/if_vlan.h>
#include <linux/netfilter_bridge.h>
#include "br_private.h"

/* Don't forward packets to originating port or forwarding disabled */
static inline int should_deliver(const struct net_bridge_port *p,
                                 const struct sk_buff *skb)
{
        struct net_bridge_vlan_group *vg;

        vg = nbp_vlan_group_rcu(p);
        return ((p->flags & BR_HAIRPIN_MODE) || skb->dev != p->dev) &&
                p->state == BR_STATE_FORWARDING && br_allowed_egress(vg, skb) &&
                nbp_switchdev_allowed_egress(p, skb) &&
                !br_skb_isolated(p, skb);
}

int br_dev_queue_push_xmit(struct net *net, struct sock *sk, struct sk_buff *skb)
{
        skb_push(skb, ETH_HLEN);
        if (!is_skb_forwardable(skb->dev, skb))
                goto drop;

        br_drop_fake_rtable(skb);

        if (skb->ip_summed == CHECKSUM_PARTIAL &&
            eth_type_vlan(skb->protocol)) {
                int depth;

                if (!vlan_get_protocol_and_depth(skb, skb->protocol, &depth))
                        goto drop;

                skb_set_network_header(skb, depth);
        }

        br_switchdev_frame_set_offload_fwd_mark(skb);

        dev_queue_xmit(skb);

        return 0;

drop:
        kfree_skb(skb);
        return 0;
}
EXPORT_SYMBOL_GPL(br_dev_queue_push_xmit);

int br_forward_finish(struct net *net, struct sock *sk, struct sk_buff *skb)
{
        skb->tstamp = 0;
        return NF_HOOK(NFPROTO_BRIDGE, NF_BR_POST_ROUTING,
                       net, sk, skb, NULL, skb->dev,
                       br_dev_queue_push_xmit);

}
EXPORT_SYMBOL_GPL(br_forward_finish);

static void __br_forward(const struct net_bridge_port *to,
                         struct sk_buff *skb, bool local_orig)
{
        struct net_bridge_vlan_group *vg;
        struct net_device *indev;
        struct net *net;
        int br_hook;

        /* Mark the skb for forwarding offload early so that br_handle_vlan()
         * can know whether to pop the VLAN header on egress or keep it.
         */
        nbp_switchdev_frame_mark_tx_fwd_offload(to, skb);

        vg = nbp_vlan_group_rcu(to);
        skb = br_handle_vlan(to->br, to, vg, skb);
        if (!skb)
                return;

        indev = skb->dev;
        skb->dev = to->dev;
        if (!local_orig) {
                if (skb_warn_if_lro(skb)) {
                        kfree_skb(skb);
                        return;
                }
                br_hook = NF_BR_FORWARD;
                skb_forward_csum(skb);
                net = dev_net(indev);
        } else {
                if (unlikely(netpoll_tx_running(to->br->dev))) {
                        skb_push(skb, ETH_HLEN);
                        if (!is_skb_forwardable(skb->dev, skb))
                                kfree_skb(skb);
                        else
                                br_netpoll_send_skb(to, skb);
                        return;
                }
                br_hook = NF_BR_LOCAL_OUT;
                net = dev_net(skb->dev);
                indev = NULL;
        }

        NF_HOOK(NFPROTO_BRIDGE, br_hook,
                net, NULL, skb, indev, skb->dev,
                br_forward_finish);
}

static int deliver_clone(const struct net_bridge_port *prev,
                         struct sk_buff *skb, bool local_orig)
{
        struct net_device *dev = BR_INPUT_SKB_CB(skb)->brdev;

        skb = skb_clone(skb, GFP_ATOMIC);
        if (!skb) {
                DEV_STATS_INC(dev, tx_dropped);
                return -ENOMEM;
        }

        __br_forward(prev, skb, local_orig);
        return 0;
}

/**
 * br_forward - forward a packet to a specific port
 * @to: destination port
 * @skb: packet being forwarded
 * @local_rcv: packet will be received locally after forwarding
 * @local_orig: packet is locally originated
 *
 * Should be called with rcu_read_lock.
 */
void br_forward(const struct net_bridge_port *to,
                struct sk_buff *skb, bool local_rcv, bool local_orig)
{
        if (unlikely(!to))
                goto out;

        /* redirect to backup link if the destination port is down */
        if (rcu_access_pointer(to->backup_port) && !netif_carrier_ok(to->dev)) {
                struct net_bridge_port *backup_port;

                backup_port = rcu_dereference(to->backup_port);
                if (unlikely(!backup_port))
                        goto out;
                to = backup_port;
        }

        if (should_deliver(to, skb)) {
                if (local_rcv)
                        deliver_clone(to, skb, local_orig);
                else
                        __br_forward(to, skb, local_orig);
                return;
        }

out:
        if (!local_rcv)
                kfree_skb(skb);
}
EXPORT_SYMBOL_GPL(br_forward);

static struct net_bridge_port *maybe_deliver(
        struct net_bridge_port *prev, struct net_bridge_port *p,
        struct sk_buff *skb, bool local_orig)
{
        u8 igmp_type = br_multicast_igmp_type(skb);
        int err;

        if (!should_deliver(p, skb))
                return prev;

        nbp_switchdev_frame_mark_tx_fwd_to_hwdom(p, skb);

        if (!prev)
                goto out;

        err = deliver_clone(prev, skb, local_orig);
        if (err)
                return ERR_PTR(err);
out:
        br_multicast_count(p->br, p, skb, igmp_type, BR_MCAST_DIR_TX);

        return p;
}

/* called under rcu_read_lock */
void br_flood(struct net_bridge *br, struct sk_buff *skb,
              enum br_pkt_type pkt_type, bool local_rcv, bool local_orig)
{
        struct net_bridge_port *prev = NULL;
        struct net_bridge_port *p;

        list_for_each_entry_rcu(p, &br->port_list, list) {
                /* Do not flood unicast traffic to ports that turn it off, nor
                 * other traffic if flood off, except for traffic we originate
                 */
                switch (pkt_type) {
                case BR_PKT_UNICAST:
                        if (!(p->flags & BR_FLOOD))
                                continue;
                        break;
                case BR_PKT_MULTICAST:
                        if (!(p->flags & BR_MCAST_FLOOD) && skb->dev != br->dev)
                                continue;
                        break;
                case BR_PKT_BROADCAST:
                        if (!(p->flags & BR_BCAST_FLOOD) && skb->dev != br->dev)
                                continue;
                        break;
                }

                /* Do not flood to ports that enable proxy ARP */
                if (p->flags & BR_PROXYARP)
                        continue;
                if ((p->flags & (BR_PROXYARP_WIFI | BR_NEIGH_SUPPRESS)) &&
                    BR_INPUT_SKB_CB(skb)->proxyarp_replied)
                        continue;

                prev = maybe_deliver(prev, p, skb, local_orig);
                if (IS_ERR(prev))
                        goto out;
        }

        if (!prev)
                goto out;

        if (local_rcv)
                deliver_clone(prev, skb, local_orig);
        else
                __br_forward(prev, skb, local_orig);
        return;

out:
        if (!local_rcv)
                kfree_skb(skb);
}

#ifdef CONFIG_BRIDGE_IGMP_SNOOPING
static void maybe_deliver_addr(struct net_bridge_port *p, struct sk_buff *skb,
                               const unsigned char *addr, bool local_orig)
{
        struct net_device *dev = BR_INPUT_SKB_CB(skb)->brdev;
        const unsigned char *src = eth_hdr(skb)->h_source;
        struct sk_buff *nskb;

        if (!should_deliver(p, skb))
                return;

        /* Even with hairpin, no soliloquies - prevent breaking IPv6 DAD */
        if (skb->dev == p->dev && ether_addr_equal(src, addr))
                return;

        __skb_push(skb, ETH_HLEN);
        nskb = pskb_copy(skb, GFP_ATOMIC);
        __skb_pull(skb, ETH_HLEN);
        if (!nskb) {
                DEV_STATS_INC(dev, tx_dropped);
                return;
        }

        skb = nskb;
        __skb_pull(skb, ETH_HLEN);
        if (!is_broadcast_ether_addr(addr))
                memcpy(eth_hdr(skb)->h_dest, addr, ETH_ALEN);

        __br_forward(p, skb, local_orig);
}

/* called with rcu_read_lock */
void br_multicast_flood(struct net_bridge_mdb_entry *mdst,
                        struct sk_buff *skb,
                        struct net_bridge_mcast *brmctx,
                        bool local_rcv, bool local_orig)
{
        struct net_bridge_port *prev = NULL;
        struct net_bridge_port_group *p;
        bool allow_mode_include = true;
        struct hlist_node *rp;

        rp = br_multicast_get_first_rport_node(brmctx, skb);

        if (mdst) {
                p = rcu_dereference(mdst->ports);
                if (br_multicast_should_handle_mode(brmctx, mdst->addr.proto) &&
                    br_multicast_is_star_g(&mdst->addr))
                        allow_mode_include = false;
        } else {
                p = NULL;
        }

        while (p || rp) {
                struct net_bridge_port *port, *lport, *rport;

                lport = p ? p->key.port : NULL;
                rport = br_multicast_rport_from_node_skb(rp, skb);

                if ((unsigned long)lport > (unsigned long)rport) {
                        port = lport;

                        if (port->flags & BR_MULTICAST_TO_UNICAST) {
                                maybe_deliver_addr(lport, skb, p->eth_addr,
                                                   local_orig);
                                goto delivered;
                        }
                        if ((!allow_mode_include &&
                             p->filter_mode == MCAST_INCLUDE) ||
                            (p->flags & MDB_PG_FLAGS_BLOCKED))
                                goto delivered;
                } else {
                        port = rport;
                }

                prev = maybe_deliver(prev, port, skb, local_orig);
                if (IS_ERR(prev))
                        goto out;
delivered:
                if ((unsigned long)lport >= (unsigned long)port)
                        p = rcu_dereference(p->next);
                if ((unsigned long)rport >= (unsigned long)port)
                        rp = rcu_dereference(hlist_next_rcu(rp));
        }

        if (!prev)
                goto out;

        if (local_rcv)
                deliver_clone(prev, skb, local_orig);
        else
                __br_forward(prev, skb, local_orig);
        return;

out:
        if (!local_rcv)
                kfree_skb(skb);
}
#endif




































   15 








   15 










   84 








   99 









   99 








    7 


   26 












   34 










  259 





  261 
  261 

  262 


  259 







  253 











  254 





  254 
  253 





  253 




  253 




1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
// SPDX-License-Identifier: GPL-2.0-only
/*
 * IPv6 library code, needed by static components when full IPv6 support is
 * not configured or static.  These functions are needed by GSO/GRO implementation.
 */
#include <linux/export.h>
#include <net/ip.h>
#include <net/ipv6.h>
#include <net/ip6_fib.h>
#include <net/addrconf.h>
#include <net/secure_seq.h>
#include <linux/netfilter.h>

static u32 __ipv6_select_ident(struct net *net,
                               const struct in6_addr *dst,
                               const struct in6_addr *src)
{
        u32 id;

        do {
                id = prandom_u32();
        } while (!id);

        return id;
}

/* This function exists only for tap drivers that must support broken
 * clients requesting UFO without specifying an IPv6 fragment ID.
 *
 * This is similar to ipv6_select_ident() but we use an independent hash
 * seed to limit information leakage.
 *
 * The network header must be set before calling this.
 */
__be32 ipv6_proxy_select_ident(struct net *net, struct sk_buff *skb)
{
        struct in6_addr buf[2];
        struct in6_addr *addrs;
        u32 id;

        addrs = skb_header_pointer(skb,
                                   skb_network_offset(skb) +
                                   offsetof(struct ipv6hdr, saddr),
                                   sizeof(buf), buf);
        if (!addrs)
                return 0;

        id = __ipv6_select_ident(net, &addrs[1], &addrs[0]);
        return htonl(id);
}
EXPORT_SYMBOL_GPL(ipv6_proxy_select_ident);

__be32 ipv6_select_ident(struct net *net,
                         const struct in6_addr *daddr,
                         const struct in6_addr *saddr)
{
        u32 id;

        id = __ipv6_select_ident(net, daddr, saddr);
        return htonl(id);
}
EXPORT_SYMBOL(ipv6_select_ident);

int ip6_find_1stfragopt(struct sk_buff *skb, u8 **nexthdr)
{
        unsigned int offset = sizeof(struct ipv6hdr);
        unsigned int packet_len = skb_tail_pointer(skb) -
                skb_network_header(skb);
        int found_rhdr = 0;
        *nexthdr = &ipv6_hdr(skb)->nexthdr;

        while (offset <= packet_len) {
                struct ipv6_opt_hdr *exthdr;

                switch (**nexthdr) {

                case NEXTHDR_HOP:
                        break;
                case NEXTHDR_ROUTING:
                        found_rhdr = 1;
                        break;
                case NEXTHDR_DEST:
#if IS_ENABLED(CONFIG_IPV6_MIP6)
                        if (ipv6_find_tlv(skb, offset, IPV6_TLV_HAO) >= 0)
                                break;
#endif
                        if (found_rhdr)
                                return offset;
                        break;
                default:
                        return offset;
                }

                if (offset + sizeof(struct ipv6_opt_hdr) > packet_len)
                        return -EINVAL;

                exthdr = (struct ipv6_opt_hdr *)(skb_network_header(skb) +
                                                 offset);
                offset += ipv6_optlen(exthdr);
                if (offset > IPV6_MAXPLEN)
                        return -EINVAL;
                *nexthdr = &exthdr->nexthdr;
        }

        return -EINVAL;
}
EXPORT_SYMBOL(ip6_find_1stfragopt);

#if IS_ENABLED(CONFIG_IPV6)
int ip6_dst_hoplimit(struct dst_entry *dst)
{
        int hoplimit = dst_metric_raw(dst, RTAX_HOPLIMIT);
        if (hoplimit == 0) {
                struct net_device *dev = dst->dev;
                struct inet6_dev *idev;

                rcu_read_lock();
                idev = __in6_dev_get(dev);
                if (idev)
                        hoplimit = idev->cnf.hop_limit;
                else
                        hoplimit = dev_net(dev)->ipv6.devconf_all->hop_limit;
                rcu_read_unlock();
        }
        return hoplimit;
}
EXPORT_SYMBOL(ip6_dst_hoplimit);
#endif

int __ip6_local_out(struct net *net, struct sock *sk, struct sk_buff *skb)
{
        int len;

        len = skb->len - sizeof(struct ipv6hdr);
        if (len > IPV6_MAXPLEN)
                len = 0;
        ipv6_hdr(skb)->payload_len = htons(len);
        IP6CB(skb)->nhoff = offsetof(struct ipv6hdr, nexthdr);

        /* if egress device is enslaved to an L3 master device pass the
         * skb to its handler for processing
         */
        skb = l3mdev_ip6_out(sk, skb);
        if (unlikely(!skb))
                return 0;

        skb->protocol = htons(ETH_P_IPV6);

        return nf_hook(NFPROTO_IPV6, NF_INET_LOCAL_OUT,
                       net, sk, skb, NULL, skb_dst(skb)->dev,
                       dst_output);
}
EXPORT_SYMBOL_GPL(__ip6_local_out);

int ip6_local_out(struct net *net, struct sock *sk, struct sk_buff *skb)
{
        int err;

        err = __ip6_local_out(net, sk, skb);
        if (likely(err == 1))
                err = dst_output(net, sk, skb);

        return err;
}
EXPORT_SYMBOL_GPL(ip6_local_out);





















































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































   43 























































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
// SPDX-License-Identifier: GPL-2.0
/*
 * linux/ipc/msg.c
 * Copyright (C) 1992 Krishna Balasubramanian
 *
 * Removed all the remaining kerneld mess
 * Catch the -EFAULT stuff properly
 * Use GFP_KERNEL for messages as in 1.2
 * Fixed up the unchecked user space derefs
 * Copyright (C) 1998 Alan Cox & Andi Kleen
 *
 * /proc/sysvipc/msg support (c) 1999 Dragos Acostachioaie <dragos@iname.com>
 *
 * mostly rewritten, threaded and wake-one semantics added
 * MSGMAX limit removed, sysctl's added
 * (c) 1999 Manfred Spraul <manfred@colorfullife.com>
 *
 * support for audit of ipc object properties and permission changes
 * Dustin Kirkland <dustin.kirkland@us.ibm.com>
 *
 * namespaces support
 * OpenVZ, SWsoft Inc.
 * Pavel Emelianov <xemul@openvz.org>
 */

#include <linux/capability.h>
#include <linux/msg.h>
#include <linux/spinlock.h>
#include <linux/init.h>
#include <linux/mm.h>
#include <linux/proc_fs.h>
#include <linux/list.h>
#include <linux/security.h>
#include <linux/sched/wake_q.h>
#include <linux/syscalls.h>
#include <linux/audit.h>
#include <linux/seq_file.h>
#include <linux/rwsem.h>
#include <linux/nsproxy.h>
#include <linux/ipc_namespace.h>
#include <linux/rhashtable.h>

#include <asm/current.h>
#include <linux/uaccess.h>
#include "util.h"

/* one msq_queue structure for each present queue on the system */
struct msg_queue {
        struct kern_ipc_perm q_perm;
        time64_t q_stime;                /* last msgsnd time */
        time64_t q_rtime;                /* last msgrcv time */
        time64_t q_ctime;                /* last change time */
        unsigned long q_cbytes;                /* current number of bytes on queue */
        unsigned long q_qnum;                /* number of messages in queue */
        unsigned long q_qbytes;                /* max number of bytes on queue */
        struct pid *q_lspid;                /* pid of last msgsnd */
        struct pid *q_lrpid;                /* last receive pid */

        struct list_head q_messages;
        struct list_head q_receivers;
        struct list_head q_senders;
} __randomize_layout;

/*
 * MSG_BARRIER Locking:
 *
 * Similar to the optimization used in ipc/mqueue.c, one syscall return path
 * does not acquire any locks when it sees that a message exists in
 * msg_receiver.r_msg. Therefore r_msg is set using smp_store_release()
 * and accessed using READ_ONCE()+smp_acquire__after_ctrl_dep(). In addition,
 * wake_q_add_safe() is used. See ipc/mqueue.c for more details
 */

/* one msg_receiver structure for each sleeping receiver */
struct msg_receiver {
        struct list_head        r_list;
        struct task_struct        *r_tsk;

        int                        r_mode;
        long                        r_msgtype;
        long                        r_maxsize;

        struct msg_msg                *r_msg;
};

/* one msg_sender for each sleeping sender */
struct msg_sender {
        struct list_head        list;
        struct task_struct        *tsk;
        size_t                  msgsz;
};

#define SEARCH_ANY                1
#define SEARCH_EQUAL                2
#define SEARCH_NOTEQUAL                3
#define SEARCH_LESSEQUAL        4
#define SEARCH_NUMBER                5

#define msg_ids(ns)        ((ns)->ids[IPC_MSG_IDS])

static inline struct msg_queue *msq_obtain_object(struct ipc_namespace *ns, int id)
{
        struct kern_ipc_perm *ipcp = ipc_obtain_object_idr(&msg_ids(ns), id);

        if (IS_ERR(ipcp))
                return ERR_CAST(ipcp);

        return container_of(ipcp, struct msg_queue, q_perm);
}

static inline struct msg_queue *msq_obtain_object_check(struct ipc_namespace *ns,
                                                        int id)
{
        struct kern_ipc_perm *ipcp = ipc_obtain_object_check(&msg_ids(ns), id);

        if (IS_ERR(ipcp))
                return ERR_CAST(ipcp);

        return container_of(ipcp, struct msg_queue, q_perm);
}

static inline void msg_rmid(struct ipc_namespace *ns, struct msg_queue *s)
{
        ipc_rmid(&msg_ids(ns), &s->q_perm);
}

static void msg_rcu_free(struct rcu_head *head)
{
        struct kern_ipc_perm *p = container_of(head, struct kern_ipc_perm, rcu);
        struct msg_queue *msq = container_of(p, struct msg_queue, q_perm);

        security_msg_queue_free(&msq->q_perm);
        kfree(msq);
}

/**
 * newque - Create a new msg queue
 * @ns: namespace
 * @params: ptr to the structure that contains the key and msgflg
 *
 * Called with msg_ids.rwsem held (writer)
 */
static int newque(struct ipc_namespace *ns, struct ipc_params *params)
{
        struct msg_queue *msq;
        int retval;
        key_t key = params->key;
        int msgflg = params->flg;

        msq = kmalloc(sizeof(*msq), GFP_KERNEL_ACCOUNT);
        if (unlikely(!msq))
                return -ENOMEM;

        msq->q_perm.mode = msgflg & S_IRWXUGO;
        msq->q_perm.key = key;

        msq->q_perm.security = NULL;
        retval = security_msg_queue_alloc(&msq->q_perm);
        if (retval) {
                kfree(msq);
                return retval;
        }

        msq->q_stime = msq->q_rtime = 0;
        msq->q_ctime = ktime_get_real_seconds();
        msq->q_cbytes = msq->q_qnum = 0;
        msq->q_qbytes = ns->msg_ctlmnb;
        msq->q_lspid = msq->q_lrpid = NULL;
        INIT_LIST_HEAD(&msq->q_messages);
        INIT_LIST_HEAD(&msq->q_receivers);
        INIT_LIST_HEAD(&msq->q_senders);

        /* ipc_addid() locks msq upon success. */
        retval = ipc_addid(&msg_ids(ns), &msq->q_perm, ns->msg_ctlmni);
        if (retval < 0) {
                ipc_rcu_putref(&msq->q_perm, msg_rcu_free);
                return retval;
        }

        ipc_unlock_object(&msq->q_perm);
        rcu_read_unlock();

        return msq->q_perm.id;
}

static inline bool msg_fits_inqueue(struct msg_queue *msq, size_t msgsz)
{
        return msgsz + msq->q_cbytes <= msq->q_qbytes &&
                1 + msq->q_qnum <= msq->q_qbytes;
}

static inline void ss_add(struct msg_queue *msq,
                          struct msg_sender *mss, size_t msgsz)
{
        mss->tsk = current;
        mss->msgsz = msgsz;
        /*
         * No memory barrier required: we did ipc_lock_object(),
         * and the waker obtains that lock before calling wake_q_add().
         */
        __set_current_state(TASK_INTERRUPTIBLE);
        list_add_tail(&mss->list, &msq->q_senders);
}

static inline void ss_del(struct msg_sender *mss)
{
        if (mss->list.next)
                list_del(&mss->list);
}

static void ss_wakeup(struct msg_queue *msq,
                      struct wake_q_head *wake_q, bool kill)
{
        struct msg_sender *mss, *t;
        struct task_struct *stop_tsk = NULL;
        struct list_head *h = &msq->q_senders;

        list_for_each_entry_safe(mss, t, h, list) {
                if (kill)
                        mss->list.next = NULL;

                /*
                 * Stop at the first task we don't wakeup,
                 * we've already iterated the original
                 * sender queue.
                 */
                else if (stop_tsk == mss->tsk)
                        break;
                /*
                 * We are not in an EIDRM scenario here, therefore
                 * verify that we really need to wakeup the task.
                 * To maintain current semantics and wakeup order,
                 * move the sender to the tail on behalf of the
                 * blocked task.
                 */
                else if (!msg_fits_inqueue(msq, mss->msgsz)) {
                        if (!stop_tsk)
                                stop_tsk = mss->tsk;

                        list_move_tail(&mss->list, &msq->q_senders);
                        continue;
                }

                wake_q_add(wake_q, mss->tsk);
        }
}

static void expunge_all(struct msg_queue *msq, int res,
                        struct wake_q_head *wake_q)
{
        struct msg_receiver *msr, *t;

        list_for_each_entry_safe(msr, t, &msq->q_receivers, r_list) {
                struct task_struct *r_tsk;

                r_tsk = get_task_struct(msr->r_tsk);

                /* see MSG_BARRIER for purpose/pairing */
                smp_store_release(&msr->r_msg, ERR_PTR(res));
                wake_q_add_safe(wake_q, r_tsk);
        }
}

/*
 * freeque() wakes up waiters on the sender and receiver waiting queue,
 * removes the message queue from message queue ID IDR, and cleans up all the
 * messages associated with this queue.
 *
 * msg_ids.rwsem (writer) and the spinlock for this message queue are held
 * before freeque() is called. msg_ids.rwsem remains locked on exit.
 */
static void freeque(struct ipc_namespace *ns, struct kern_ipc_perm *ipcp)
        __releases(RCU)
        __releases(&msq->q_perm)
{
        struct msg_msg *msg, *t;
        struct msg_queue *msq = container_of(ipcp, struct msg_queue, q_perm);
        DEFINE_WAKE_Q(wake_q);

        expunge_all(msq, -EIDRM, &wake_q);
        ss_wakeup(msq, &wake_q, true);
        msg_rmid(ns, msq);
        ipc_unlock_object(&msq->q_perm);
        wake_up_q(&wake_q);
        rcu_read_unlock();

        list_for_each_entry_safe(msg, t, &msq->q_messages, m_list) {
                atomic_dec(&ns->msg_hdrs);
                free_msg(msg);
        }
        atomic_sub(msq->q_cbytes, &ns->msg_bytes);
        ipc_update_pid(&msq->q_lspid, NULL);
        ipc_update_pid(&msq->q_lrpid, NULL);
        ipc_rcu_putref(&msq->q_perm, msg_rcu_free);
}

long ksys_msgget(key_t key, int msgflg)
{
        struct ipc_namespace *ns;
        static const struct ipc_ops msg_ops = {
                .getnew = newque,
                .associate = security_msg_queue_associate,
        };
        struct ipc_params msg_params;

        ns = current->nsproxy->ipc_ns;

        msg_params.key = key;
        msg_params.flg = msgflg;

        return ipcget(ns, &msg_ids(ns), &msg_ops, &msg_params);
}

SYSCALL_DEFINE2(msgget, key_t, key, int, msgflg)
{
        return ksys_msgget(key, msgflg);
}

static inline unsigned long
copy_msqid_to_user(void __user *buf, struct msqid64_ds *in, int version)
{
        switch (version) {
        case IPC_64:
                return copy_to_user(buf, in, sizeof(*in));
        case IPC_OLD:
        {
                struct msqid_ds out;

                memset(&out, 0, sizeof(out));

                ipc64_perm_to_ipc_perm(&in->msg_perm, &out.msg_perm);

                out.msg_stime                = in->msg_stime;
                out.msg_rtime                = in->msg_rtime;
                out.msg_ctime                = in->msg_ctime;

                if (in->msg_cbytes > USHRT_MAX)
                        out.msg_cbytes        = USHRT_MAX;
                else
                        out.msg_cbytes        = in->msg_cbytes;
                out.msg_lcbytes                = in->msg_cbytes;

                if (in->msg_qnum > USHRT_MAX)
                        out.msg_qnum        = USHRT_MAX;
                else
                        out.msg_qnum        = in->msg_qnum;

                if (in->msg_qbytes > USHRT_MAX)
                        out.msg_qbytes        = USHRT_MAX;
                else
                        out.msg_qbytes        = in->msg_qbytes;
                out.msg_lqbytes                = in->msg_qbytes;

                out.msg_lspid                = in->msg_lspid;
                out.msg_lrpid                = in->msg_lrpid;

                return copy_to_user(buf, &out, sizeof(out));
        }
        default:
                return -EINVAL;
        }
}

static inline unsigned long
copy_msqid_from_user(struct msqid64_ds *out, void __user *buf, int version)
{
        switch (version) {
        case IPC_64:
                if (copy_from_user(out, buf, sizeof(*out)))
                        return -EFAULT;
                return 0;
        case IPC_OLD:
        {
                struct msqid_ds tbuf_old;

                if (copy_from_user(&tbuf_old, buf, sizeof(tbuf_old)))
                        return -EFAULT;

                out->msg_perm.uid        = tbuf_old.msg_perm.uid;
                out->msg_perm.gid        = tbuf_old.msg_perm.gid;
                out->msg_perm.mode        = tbuf_old.msg_perm.mode;

                if (tbuf_old.msg_qbytes == 0)
                        out->msg_qbytes        = tbuf_old.msg_lqbytes;
                else
                        out->msg_qbytes        = tbuf_old.msg_qbytes;

                return 0;
        }
        default:
                return -EINVAL;
        }
}

/*
 * This function handles some msgctl commands which require the rwsem
 * to be held in write mode.
 * NOTE: no locks must be held, the rwsem is taken inside this function.
 */
static int msgctl_down(struct ipc_namespace *ns, int msqid, int cmd,
                        struct ipc64_perm *perm, int msg_qbytes)
{
        struct kern_ipc_perm *ipcp;
        struct msg_queue *msq;
        int err;

        down_write(&msg_ids(ns).rwsem);
        rcu_read_lock();

        ipcp = ipcctl_obtain_check(ns, &msg_ids(ns), msqid, cmd,
                                      perm, msg_qbytes);
        if (IS_ERR(ipcp)) {
                err = PTR_ERR(ipcp);
                goto out_unlock1;
        }

        msq = container_of(ipcp, struct msg_queue, q_perm);

        err = security_msg_queue_msgctl(&msq->q_perm, cmd);
        if (err)
                goto out_unlock1;

        switch (cmd) {
        case IPC_RMID:
                ipc_lock_object(&msq->q_perm);
                /* freeque unlocks the ipc object and rcu */
                freeque(ns, ipcp);
                goto out_up;
        case IPC_SET:
        {
                DEFINE_WAKE_Q(wake_q);

                if (msg_qbytes > ns->msg_ctlmnb &&
                    !capable(CAP_SYS_RESOURCE)) {
                        err = -EPERM;
                        goto out_unlock1;
                }

                ipc_lock_object(&msq->q_perm);
                err = ipc_update_perm(perm, ipcp);
                if (err)
                        goto out_unlock0;

                msq->q_qbytes = msg_qbytes;

                msq->q_ctime = ktime_get_real_seconds();
                /*
                 * Sleeping receivers might be excluded by
                 * stricter permissions.
                 */
                expunge_all(msq, -EAGAIN, &wake_q);
                /*
                 * Sleeping senders might be able to send
                 * due to a larger queue size.
                 */
                ss_wakeup(msq, &wake_q, false);
                ipc_unlock_object(&msq->q_perm);
                wake_up_q(&wake_q);

                goto out_unlock1;
        }
        default:
                err = -EINVAL;
                goto out_unlock1;
        }

out_unlock0:
        ipc_unlock_object(&msq->q_perm);
out_unlock1:
        rcu_read_unlock();
out_up:
        up_write(&msg_ids(ns).rwsem);
        return err;
}

static int msgctl_info(struct ipc_namespace *ns, int msqid,
                         int cmd, struct msginfo *msginfo)
{
        int err;
        int max_idx;

        /*
         * We must not return kernel stack data.
         * due to padding, it's not enough
         * to set all member fields.
         */
        err = security_msg_queue_msgctl(NULL, cmd);
        if (err)
                return err;

        memset(msginfo, 0, sizeof(*msginfo));
        msginfo->msgmni = ns->msg_ctlmni;
        msginfo->msgmax = ns->msg_ctlmax;
        msginfo->msgmnb = ns->msg_ctlmnb;
        msginfo->msgssz = MSGSSZ;
        msginfo->msgseg = MSGSEG;
        down_read(&msg_ids(ns).rwsem);
        if (cmd == MSG_INFO) {
                msginfo->msgpool = msg_ids(ns).in_use;
                msginfo->msgmap = atomic_read(&ns->msg_hdrs);
                msginfo->msgtql = atomic_read(&ns->msg_bytes);
        } else {
                msginfo->msgmap = MSGMAP;
                msginfo->msgpool = MSGPOOL;
                msginfo->msgtql = MSGTQL;
        }
        max_idx = ipc_get_maxidx(&msg_ids(ns));
        up_read(&msg_ids(ns).rwsem);
        return (max_idx < 0) ? 0 : max_idx;
}

static int msgctl_stat(struct ipc_namespace *ns, int msqid,
                         int cmd, struct msqid64_ds *p)
{
        struct msg_queue *msq;
        int err;

        memset(p, 0, sizeof(*p));

        rcu_read_lock();
        if (cmd == MSG_STAT || cmd == MSG_STAT_ANY) {
                msq = msq_obtain_object(ns, msqid);
                if (IS_ERR(msq)) {
                        err = PTR_ERR(msq);
                        goto out_unlock;
                }
        } else { /* IPC_STAT */
                msq = msq_obtain_object_check(ns, msqid);
                if (IS_ERR(msq)) {
                        err = PTR_ERR(msq);
                        goto out_unlock;
                }
        }

        /* see comment for SHM_STAT_ANY */
        if (cmd == MSG_STAT_ANY)
                audit_ipc_obj(&msq->q_perm);
        else {
                err = -EACCES;
                if (ipcperms(ns, &msq->q_perm, S_IRUGO))
                        goto out_unlock;
        }

        err = security_msg_queue_msgctl(&msq->q_perm, cmd);
        if (err)
                goto out_unlock;

        ipc_lock_object(&msq->q_perm);

        if (!ipc_valid_object(&msq->q_perm)) {
                ipc_unlock_object(&msq->q_perm);
                err = -EIDRM;
                goto out_unlock;
        }

        kernel_to_ipc64_perm(&msq->q_perm, &p->msg_perm);
        p->msg_stime  = msq->q_stime;
        p->msg_rtime  = msq->q_rtime;
        p->msg_ctime  = msq->q_ctime;
#ifndef CONFIG_64BIT
        p->msg_stime_high = msq->q_stime >> 32;
        p->msg_rtime_high = msq->q_rtime >> 32;
        p->msg_ctime_high = msq->q_ctime >> 32;
#endif
        p->msg_cbytes = msq->q_cbytes;
        p->msg_qnum   = msq->q_qnum;
        p->msg_qbytes = msq->q_qbytes;
        p->msg_lspid  = pid_vnr(msq->q_lspid);
        p->msg_lrpid  = pid_vnr(msq->q_lrpid);

        if (cmd == IPC_STAT) {
                /*
                 * As defined in SUS:
                 * Return 0 on success
                 */
                err = 0;
        } else {
                /*
                 * MSG_STAT and MSG_STAT_ANY (both Linux specific)
                 * Return the full id, including the sequence number
                 */
                err = msq->q_perm.id;
        }

        ipc_unlock_object(&msq->q_perm);
out_unlock:
        rcu_read_unlock();
        return err;
}

static long ksys_msgctl(int msqid, int cmd, struct msqid_ds __user *buf, int version)
{
        struct ipc_namespace *ns;
        struct msqid64_ds msqid64;
        int err;

        if (msqid < 0 || cmd < 0)
                return -EINVAL;

        ns = current->nsproxy->ipc_ns;

        switch (cmd) {
        case IPC_INFO:
        case MSG_INFO: {
                struct msginfo msginfo;
                err = msgctl_info(ns, msqid, cmd, &msginfo);
                if (err < 0)
                        return err;
                if (copy_to_user(buf, &msginfo, sizeof(struct msginfo)))
                        err = -EFAULT;
                return err;
        }
        case MSG_STAT:        /* msqid is an index rather than a msg queue id */
        case MSG_STAT_ANY:
        case IPC_STAT:
                err = msgctl_stat(ns, msqid, cmd, &msqid64);
                if (err < 0)
                        return err;
                if (copy_msqid_to_user(buf, &msqid64, version))
                        err = -EFAULT;
                return err;
        case IPC_SET:
                if (copy_msqid_from_user(&msqid64, buf, version))
                        return -EFAULT;
                return msgctl_down(ns, msqid, cmd, &msqid64.msg_perm,
                                   msqid64.msg_qbytes);
        case IPC_RMID:
                return msgctl_down(ns, msqid, cmd, NULL, 0);
        default:
                return  -EINVAL;
        }
}

SYSCALL_DEFINE3(msgctl, int, msqid, int, cmd, struct msqid_ds __user *, buf)
{
        return ksys_msgctl(msqid, cmd, buf, IPC_64);
}

#ifdef CONFIG_ARCH_WANT_IPC_PARSE_VERSION
long ksys_old_msgctl(int msqid, int cmd, struct msqid_ds __user *buf)
{
        int version = ipc_parse_version(&cmd);

        return ksys_msgctl(msqid, cmd, buf, version);
}

SYSCALL_DEFINE3(old_msgctl, int, msqid, int, cmd, struct msqid_ds __user *, buf)
{
        return ksys_old_msgctl(msqid, cmd, buf);
}
#endif

#ifdef CONFIG_COMPAT

struct compat_msqid_ds {
        struct compat_ipc_perm msg_perm;
        compat_uptr_t msg_first;
        compat_uptr_t msg_last;
        old_time32_t msg_stime;
        old_time32_t msg_rtime;
        old_time32_t msg_ctime;
        compat_ulong_t msg_lcbytes;
        compat_ulong_t msg_lqbytes;
        unsigned short msg_cbytes;
        unsigned short msg_qnum;
        unsigned short msg_qbytes;
        compat_ipc_pid_t msg_lspid;
        compat_ipc_pid_t msg_lrpid;
};

static int copy_compat_msqid_from_user(struct msqid64_ds *out, void __user *buf,
                                        int version)
{
        memset(out, 0, sizeof(*out));
        if (version == IPC_64) {
                struct compat_msqid64_ds __user *p = buf;
                if (get_compat_ipc64_perm(&out->msg_perm, &p->msg_perm))
                        return -EFAULT;
                if (get_user(out->msg_qbytes, &p->msg_qbytes))
                        return -EFAULT;
        } else {
                struct compat_msqid_ds __user *p = buf;
                if (get_compat_ipc_perm(&out->msg_perm, &p->msg_perm))
                        return -EFAULT;
                if (get_user(out->msg_qbytes, &p->msg_qbytes))
                        return -EFAULT;
        }
        return 0;
}

static int copy_compat_msqid_to_user(void __user *buf, struct msqid64_ds *in,
                                        int version)
{
        if (version == IPC_64) {
                struct compat_msqid64_ds v;
                memset(&v, 0, sizeof(v));
                to_compat_ipc64_perm(&v.msg_perm, &in->msg_perm);
                v.msg_stime         = lower_32_bits(in->msg_stime);
                v.msg_stime_high = upper_32_bits(in->msg_stime);
                v.msg_rtime         = lower_32_bits(in->msg_rtime);
                v.msg_rtime_high = upper_32_bits(in->msg_rtime);
                v.msg_ctime         = lower_32_bits(in->msg_ctime);
                v.msg_ctime_high = upper_32_bits(in->msg_ctime);
                v.msg_cbytes = in->msg_cbytes;
                v.msg_qnum = in->msg_qnum;
                v.msg_qbytes = in->msg_qbytes;
                v.msg_lspid = in->msg_lspid;
                v.msg_lrpid = in->msg_lrpid;
                return copy_to_user(buf, &v, sizeof(v));
        } else {
                struct compat_msqid_ds v;
                memset(&v, 0, sizeof(v));
                to_compat_ipc_perm(&v.msg_perm, &in->msg_perm);
                v.msg_stime = in->msg_stime;
                v.msg_rtime = in->msg_rtime;
                v.msg_ctime = in->msg_ctime;
                v.msg_cbytes = in->msg_cbytes;
                v.msg_qnum = in->msg_qnum;
                v.msg_qbytes = in->msg_qbytes;
                v.msg_lspid = in->msg_lspid;
                v.msg_lrpid = in->msg_lrpid;
                return copy_to_user(buf, &v, sizeof(v));
        }
}

static long compat_ksys_msgctl(int msqid, int cmd, void __user *uptr, int version)
{
        struct ipc_namespace *ns;
        int err;
        struct msqid64_ds msqid64;

        ns = current->nsproxy->ipc_ns;

        if (msqid < 0 || cmd < 0)
                return -EINVAL;

        switch (cmd & (~IPC_64)) {
        case IPC_INFO:
        case MSG_INFO: {
                struct msginfo msginfo;
                err = msgctl_info(ns, msqid, cmd, &msginfo);
                if (err < 0)
                        return err;
                if (copy_to_user(uptr, &msginfo, sizeof(struct msginfo)))
                        err = -EFAULT;
                return err;
        }
        case IPC_STAT:
        case MSG_STAT:
        case MSG_STAT_ANY:
                err = msgctl_stat(ns, msqid, cmd, &msqid64);
                if (err < 0)
                        return err;
                if (copy_compat_msqid_to_user(uptr, &msqid64, version))
                        err = -EFAULT;
                return err;
        case IPC_SET:
                if (copy_compat_msqid_from_user(&msqid64, uptr, version))
                        return -EFAULT;
                return msgctl_down(ns, msqid, cmd, &msqid64.msg_perm, msqid64.msg_qbytes);
        case IPC_RMID:
                return msgctl_down(ns, msqid, cmd, NULL, 0);
        default:
                return -EINVAL;
        }
}

COMPAT_SYSCALL_DEFINE3(msgctl, int, msqid, int, cmd, void __user *, uptr)
{
        return compat_ksys_msgctl(msqid, cmd, uptr, IPC_64);
}

#ifdef CONFIG_ARCH_WANT_COMPAT_IPC_PARSE_VERSION
long compat_ksys_old_msgctl(int msqid, int cmd, void __user *uptr)
{
        int version = compat_ipc_parse_version(&cmd);

        return compat_ksys_msgctl(msqid, cmd, uptr, version);
}

COMPAT_SYSCALL_DEFINE3(old_msgctl, int, msqid, int, cmd, void __user *, uptr)
{
        return compat_ksys_old_msgctl(msqid, cmd, uptr);
}
#endif
#endif

static int testmsg(struct msg_msg *msg, long type, int mode)
{
        switch (mode) {
        case SEARCH_ANY:
        case SEARCH_NUMBER:
                return 1;
        case SEARCH_LESSEQUAL:
                if (msg->m_type <= type)
                        return 1;
                break;
        case SEARCH_EQUAL:
                if (msg->m_type == type)
                        return 1;
                break;
        case SEARCH_NOTEQUAL:
                if (msg->m_type != type)
                        return 1;
                break;
        }
        return 0;
}

static inline int pipelined_send(struct msg_queue *msq, struct msg_msg *msg,
                                 struct wake_q_head *wake_q)
{
        struct msg_receiver *msr, *t;

        list_for_each_entry_safe(msr, t, &msq->q_receivers, r_list) {
                if (testmsg(msg, msr->r_msgtype, msr->r_mode) &&
                    !security_msg_queue_msgrcv(&msq->q_perm, msg, msr->r_tsk,
                                               msr->r_msgtype, msr->r_mode)) {

                        list_del(&msr->r_list);
                        if (msr->r_maxsize < msg->m_ts) {
                                wake_q_add(wake_q, msr->r_tsk);

                                /* See expunge_all regarding memory barrier */
                                smp_store_release(&msr->r_msg, ERR_PTR(-E2BIG));
                        } else {
                                ipc_update_pid(&msq->q_lrpid, task_pid(msr->r_tsk));
                                msq->q_rtime = ktime_get_real_seconds();

                                wake_q_add(wake_q, msr->r_tsk);

                                /* See expunge_all regarding memory barrier */
                                smp_store_release(&msr->r_msg, msg);
                                return 1;
                        }
                }
        }

        return 0;
}

static long do_msgsnd(int msqid, long mtype, void __user *mtext,
                size_t msgsz, int msgflg)
{
        struct msg_queue *msq;
        struct msg_msg *msg;
        int err;
        struct ipc_namespace *ns;
        DEFINE_WAKE_Q(wake_q);

        ns = current->nsproxy->ipc_ns;

        if (msgsz > ns->msg_ctlmax || (long) msgsz < 0 || msqid < 0)
                return -EINVAL;
        if (mtype < 1)
                return -EINVAL;

        msg = load_msg(mtext, msgsz);
        if (IS_ERR(msg))
                return PTR_ERR(msg);

        msg->m_type = mtype;
        msg->m_ts = msgsz;

        rcu_read_lock();
        msq = msq_obtain_object_check(ns, msqid);
        if (IS_ERR(msq)) {
                err = PTR_ERR(msq);
                goto out_unlock1;
        }

        ipc_lock_object(&msq->q_perm);

        for (;;) {
                struct msg_sender s;

                err = -EACCES;
                if (ipcperms(ns, &msq->q_perm, S_IWUGO))
                        goto out_unlock0;

                /* raced with RMID? */
                if (!ipc_valid_object(&msq->q_perm)) {
                        err = -EIDRM;
                        goto out_unlock0;
                }

                err = security_msg_queue_msgsnd(&msq->q_perm, msg, msgflg);
                if (err)
                        goto out_unlock0;

                if (msg_fits_inqueue(msq, msgsz))
                        break;

                /* queue full, wait: */
                if (msgflg & IPC_NOWAIT) {
                        err = -EAGAIN;
                        goto out_unlock0;
                }

                /* enqueue the sender and prepare to block */
                ss_add(msq, &s, msgsz);

                if (!ipc_rcu_getref(&msq->q_perm)) {
                        err = -EIDRM;
                        goto out_unlock0;
                }

                ipc_unlock_object(&msq->q_perm);
                rcu_read_unlock();
                schedule();

                rcu_read_lock();
                ipc_lock_object(&msq->q_perm);

                ipc_rcu_putref(&msq->q_perm, msg_rcu_free);
                /* raced with RMID? */
                if (!ipc_valid_object(&msq->q_perm)) {
                        err = -EIDRM;
                        goto out_unlock0;
                }
                ss_del(&s);

                if (signal_pending(current)) {
                        err = -ERESTARTNOHAND;
                        goto out_unlock0;
                }

        }

        ipc_update_pid(&msq->q_lspid, task_tgid(current));
        msq->q_stime = ktime_get_real_seconds();

        if (!pipelined_send(msq, msg, &wake_q)) {
                /* no one is waiting for this message, enqueue it */
                list_add_tail(&msg->m_list, &msq->q_messages);
                msq->q_cbytes += msgsz;
                msq->q_qnum++;
                atomic_add(msgsz, &ns->msg_bytes);
                atomic_inc(&ns->msg_hdrs);
        }

        err = 0;
        msg = NULL;

out_unlock0:
        ipc_unlock_object(&msq->q_perm);
        wake_up_q(&wake_q);
out_unlock1:
        rcu_read_unlock();
        if (msg != NULL)
                free_msg(msg);
        return err;
}

long ksys_msgsnd(int msqid, struct msgbuf __user *msgp, size_t msgsz,
                 int msgflg)
{
        long mtype;

        if (get_user(mtype, &msgp->mtype))
                return -EFAULT;
        return do_msgsnd(msqid, mtype, msgp->mtext, msgsz, msgflg);
}

SYSCALL_DEFINE4(msgsnd, int, msqid, struct msgbuf __user *, msgp, size_t, msgsz,
                int, msgflg)
{
        return ksys_msgsnd(msqid, msgp, msgsz, msgflg);
}

#ifdef CONFIG_COMPAT

struct compat_msgbuf {
        compat_long_t mtype;
        char mtext[1];
};

long compat_ksys_msgsnd(int msqid, compat_uptr_t msgp,
                       compat_ssize_t msgsz, int msgflg)
{
        struct compat_msgbuf __user *up = compat_ptr(msgp);
        compat_long_t mtype;

        if (get_user(mtype, &up->mtype))
                return -EFAULT;
        return do_msgsnd(msqid, mtype, up->mtext, (ssize_t)msgsz, msgflg);
}

COMPAT_SYSCALL_DEFINE4(msgsnd, int, msqid, compat_uptr_t, msgp,
                       compat_ssize_t, msgsz, int, msgflg)
{
        return compat_ksys_msgsnd(msqid, msgp, msgsz, msgflg);
}
#endif

static inline int convert_mode(long *msgtyp, int msgflg)
{
        if (msgflg & MSG_COPY)
                return SEARCH_NUMBER;
        /*
         *  find message of correct type.
         *  msgtyp = 0 => get first.
         *  msgtyp > 0 => get first message of matching type.
         *  msgtyp < 0 => get message with least type must be < abs(msgtype).
         */
        if (*msgtyp == 0)
                return SEARCH_ANY;
        if (*msgtyp < 0) {
                if (*msgtyp == LONG_MIN) /* -LONG_MIN is undefined */
                        *msgtyp = LONG_MAX;
                else
                        *msgtyp = -*msgtyp;
                return SEARCH_LESSEQUAL;
        }
        if (msgflg & MSG_EXCEPT)
                return SEARCH_NOTEQUAL;
        return SEARCH_EQUAL;
}

static long do_msg_fill(void __user *dest, struct msg_msg *msg, size_t bufsz)
{
        struct msgbuf __user *msgp = dest;
        size_t msgsz;

        if (put_user(msg->m_type, &msgp->mtype))
                return -EFAULT;

        msgsz = (bufsz > msg->m_ts) ? msg->m_ts : bufsz;
        if (store_msg(msgp->mtext, msg, msgsz))
                return -EFAULT;
        return msgsz;
}

#ifdef CONFIG_CHECKPOINT_RESTORE
/*
 * This function creates new kernel message structure, large enough to store
 * bufsz message bytes.
 */
static inline struct msg_msg *prepare_copy(void __user *buf, size_t bufsz)
{
        struct msg_msg *copy;

        /*
         * Create dummy message to copy real message to.
         */
        copy = load_msg(buf, bufsz);
        if (!IS_ERR(copy))
                copy->m_ts = bufsz;
        return copy;
}

static inline void free_copy(struct msg_msg *copy)
{
        if (copy)
                free_msg(copy);
}
#else
static inline struct msg_msg *prepare_copy(void __user *buf, size_t bufsz)
{
        return ERR_PTR(-ENOSYS);
}

static inline void free_copy(struct msg_msg *copy)
{
}
#endif

static struct msg_msg *find_msg(struct msg_queue *msq, long *msgtyp, int mode)
{
        struct msg_msg *msg, *found = NULL;
        long count = 0;

        list_for_each_entry(msg, &msq->q_messages, m_list) {
                if (testmsg(msg, *msgtyp, mode) &&
                    !security_msg_queue_msgrcv(&msq->q_perm, msg, current,
                                               *msgtyp, mode)) {
                        if (mode == SEARCH_LESSEQUAL && msg->m_type != 1) {
                                *msgtyp = msg->m_type - 1;
                                found = msg;
                        } else if (mode == SEARCH_NUMBER) {
                                if (*msgtyp == count)
                                        return msg;
                        } else
                                return msg;
                        count++;
                }
        }

        return found ?: ERR_PTR(-EAGAIN);
}

static long do_msgrcv(int msqid, void __user *buf, size_t bufsz, long msgtyp, int msgflg,
               long (*msg_handler)(void __user *, struct msg_msg *, size_t))
{
        int mode;
        struct msg_queue *msq;
        struct ipc_namespace *ns;
        struct msg_msg *msg, *copy = NULL;
        DEFINE_WAKE_Q(wake_q);

        ns = current->nsproxy->ipc_ns;

        if (msqid < 0 || (long) bufsz < 0)
                return -EINVAL;

        if (msgflg & MSG_COPY) {
                if ((msgflg & MSG_EXCEPT) || !(msgflg & IPC_NOWAIT))
                        return -EINVAL;
                copy = prepare_copy(buf, min_t(size_t, bufsz, ns->msg_ctlmax));
                if (IS_ERR(copy))
                        return PTR_ERR(copy);
        }
        mode = convert_mode(&msgtyp, msgflg);

        rcu_read_lock();
        msq = msq_obtain_object_check(ns, msqid);
        if (IS_ERR(msq)) {
                rcu_read_unlock();
                free_copy(copy);
                return PTR_ERR(msq);
        }

        for (;;) {
                struct msg_receiver msr_d;

                msg = ERR_PTR(-EACCES);
                if (ipcperms(ns, &msq->q_perm, S_IRUGO))
                        goto out_unlock1;

                ipc_lock_object(&msq->q_perm);

                /* raced with RMID? */
                if (!ipc_valid_object(&msq->q_perm)) {
                        msg = ERR_PTR(-EIDRM);
                        goto out_unlock0;
                }

                msg = find_msg(msq, &msgtyp, mode);
                if (!IS_ERR(msg)) {
                        /*
                         * Found a suitable message.
                         * Unlink it from the queue.
                         */
                        if ((bufsz < msg->m_ts) && !(msgflg & MSG_NOERROR)) {
                                msg = ERR_PTR(-E2BIG);
                                goto out_unlock0;
                        }
                        /*
                         * If we are copying, then do not unlink message and do
                         * not update queue parameters.
                         */
                        if (msgflg & MSG_COPY) {
                                msg = copy_msg(msg, copy);
                                goto out_unlock0;
                        }

                        list_del(&msg->m_list);
                        msq->q_qnum--;
                        msq->q_rtime = ktime_get_real_seconds();
                        ipc_update_pid(&msq->q_lrpid, task_tgid(current));
                        msq->q_cbytes -= msg->m_ts;
                        atomic_sub(msg->m_ts, &ns->msg_bytes);
                        atomic_dec(&ns->msg_hdrs);
                        ss_wakeup(msq, &wake_q, false);

                        goto out_unlock0;
                }

                /* No message waiting. Wait for a message */
                if (msgflg & IPC_NOWAIT) {
                        msg = ERR_PTR(-ENOMSG);
                        goto out_unlock0;
                }

                list_add_tail(&msr_d.r_list, &msq->q_receivers);
                msr_d.r_tsk = current;
                msr_d.r_msgtype = msgtyp;
                msr_d.r_mode = mode;
                if (msgflg & MSG_NOERROR)
                        msr_d.r_maxsize = INT_MAX;
                else
                        msr_d.r_maxsize = bufsz;

                /* memory barrier not require due to ipc_lock_object() */
                WRITE_ONCE(msr_d.r_msg, ERR_PTR(-EAGAIN));

                /* memory barrier not required, we own ipc_lock_object() */
                __set_current_state(TASK_INTERRUPTIBLE);

                ipc_unlock_object(&msq->q_perm);
                rcu_read_unlock();
                schedule();

                /*
                 * Lockless receive, part 1:
                 * We don't hold a reference to the queue and getting a
                 * reference would defeat the idea of a lockless operation,
                 * thus the code relies on rcu to guarantee the existence of
                 * msq:
                 * Prior to destruction, expunge_all(-EIRDM) changes r_msg.
                 * Thus if r_msg is -EAGAIN, then the queue not yet destroyed.
                 */
                rcu_read_lock();

                /*
                 * Lockless receive, part 2:
                 * The work in pipelined_send() and expunge_all():
                 * - Set pointer to message
                 * - Queue the receiver task for later wakeup
                 * - Wake up the process after the lock is dropped.
                 *
                 * Should the process wake up before this wakeup (due to a
                 * signal) it will either see the message and continue ...
                 */
                msg = READ_ONCE(msr_d.r_msg);
                if (msg != ERR_PTR(-EAGAIN)) {
                        /* see MSG_BARRIER for purpose/pairing */
                        smp_acquire__after_ctrl_dep();

                        goto out_unlock1;
                }

                 /*
                  * ... or see -EAGAIN, acquire the lock to check the message
                  * again.
                  */
                ipc_lock_object(&msq->q_perm);

                msg = READ_ONCE(msr_d.r_msg);
                if (msg != ERR_PTR(-EAGAIN))
                        goto out_unlock0;

                list_del(&msr_d.r_list);
                if (signal_pending(current)) {
                        msg = ERR_PTR(-ERESTARTNOHAND);
                        goto out_unlock0;
                }

                ipc_unlock_object(&msq->q_perm);
        }

out_unlock0:
        ipc_unlock_object(&msq->q_perm);
        wake_up_q(&wake_q);
out_unlock1:
        rcu_read_unlock();
        if (IS_ERR(msg)) {
                free_copy(copy);
                return PTR_ERR(msg);
        }

        bufsz = msg_handler(buf, msg, bufsz);
        free_msg(msg);

        return bufsz;
}

long ksys_msgrcv(int msqid, struct msgbuf __user *msgp, size_t msgsz,
                 long msgtyp, int msgflg)
{
        return do_msgrcv(msqid, msgp, msgsz, msgtyp, msgflg, do_msg_fill);
}

SYSCALL_DEFINE5(msgrcv, int, msqid, struct msgbuf __user *, msgp, size_t, msgsz,
                long, msgtyp, int, msgflg)
{
        return ksys_msgrcv(msqid, msgp, msgsz, msgtyp, msgflg);
}

#ifdef CONFIG_COMPAT
static long compat_do_msg_fill(void __user *dest, struct msg_msg *msg, size_t bufsz)
{
        struct compat_msgbuf __user *msgp = dest;
        size_t msgsz;

        if (put_user(msg->m_type, &msgp->mtype))
                return -EFAULT;

        msgsz = (bufsz > msg->m_ts) ? msg->m_ts : bufsz;
        if (store_msg(msgp->mtext, msg, msgsz))
                return -EFAULT;
        return msgsz;
}

long compat_ksys_msgrcv(int msqid, compat_uptr_t msgp, compat_ssize_t msgsz,
                        compat_long_t msgtyp, int msgflg)
{
        return do_msgrcv(msqid, compat_ptr(msgp), (ssize_t)msgsz, (long)msgtyp,
                         msgflg, compat_do_msg_fill);
}

COMPAT_SYSCALL_DEFINE5(msgrcv, int, msqid, compat_uptr_t, msgp,
                       compat_ssize_t, msgsz, compat_long_t, msgtyp,
                       int, msgflg)
{
        return compat_ksys_msgrcv(msqid, msgp, msgsz, msgtyp, msgflg);
}
#endif

void msg_init_ns(struct ipc_namespace *ns)
{
        ns->msg_ctlmax = MSGMAX;
        ns->msg_ctlmnb = MSGMNB;
        ns->msg_ctlmni = MSGMNI;

        atomic_set(&ns->msg_bytes, 0);
        atomic_set(&ns->msg_hdrs, 0);
        ipc_init_ids(&ns->ids[IPC_MSG_IDS]);
}

#ifdef CONFIG_IPC_NS
void msg_exit_ns(struct ipc_namespace *ns)
{
        free_ipcs(ns, &msg_ids(ns), freeque);
        idr_destroy(&ns->ids[IPC_MSG_IDS].ipcs_idr);
        rhashtable_destroy(&ns->ids[IPC_MSG_IDS].key_ht);
}
#endif

#ifdef CONFIG_PROC_FS
static int sysvipc_msg_proc_show(struct seq_file *s, void *it)
{
        struct pid_namespace *pid_ns = ipc_seq_pid_ns(s);
        struct user_namespace *user_ns = seq_user_ns(s);
        struct kern_ipc_perm *ipcp = it;
        struct msg_queue *msq = container_of(ipcp, struct msg_queue, q_perm);

        seq_printf(s,
                   "%10d %10d  %4o  %10lu %10lu %5u %5u %5u %5u %5u %5u %10llu %10llu %10llu\n",
                   msq->q_perm.key,
                   msq->q_perm.id,
                   msq->q_perm.mode,
                   msq->q_cbytes,
                   msq->q_qnum,
                   pid_nr_ns(msq->q_lspid, pid_ns),
                   pid_nr_ns(msq->q_lrpid, pid_ns),
                   from_kuid_munged(user_ns, msq->q_perm.uid),
                   from_kgid_munged(user_ns, msq->q_perm.gid),
                   from_kuid_munged(user_ns, msq->q_perm.cuid),
                   from_kgid_munged(user_ns, msq->q_perm.cgid),
                   msq->q_stime,
                   msq->q_rtime,
                   msq->q_ctime);

        return 0;
}
#endif

void __init msg_init(void)
{
        msg_init_ns(&init_ipc_ns);

        ipc_init_proc_interface("sysvipc/msg",
                                "       key      msqid perms      cbytes       qnum lspid lrpid   uid   gid  cuid  cgid      stime      rtime      ctime\n",
                                IPC_MSG_IDS, sysvipc_msg_proc_show);
}






























   48 
















   49 



   49 



   47 
    5 

    1 
























    9 


    4 

    5 















   32 





























































   29 
   32 










   32 













































































































   29 



























   32 
   28 
   29 














































   18 


   19 



















    5 


    5 
    4 













   32 
   32 
   32 




   44 




























   50 


   50 


   19 











   18 




   53 



   51 
   34 









   50 









   50 








   14 


   50 
































































    3 


    1 





    2 





    2 


    1 





    1 







   26 









   28 














    2 



    2 




























    9 



    9 














































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 *        Spanning tree protocol; generic parts
 *        Linux ethernet bridge
 *
 *        Authors:
 *        Lennert Buytenhek                <buytenh@gnu.org>
 */
#include <linux/kernel.h>
#include <linux/rculist.h>
#include <net/switchdev.h>

#include "br_private.h"
#include "br_private_stp.h"

/* since time values in bpdu are in jiffies and then scaled (1/256)
 * before sending, make sure that is at least one STP tick.
 */
#define MESSAGE_AGE_INCR        ((HZ / 256) + 1)

static const char *const br_port_state_names[] = {
        [BR_STATE_DISABLED] = "disabled",
        [BR_STATE_LISTENING] = "listening",
        [BR_STATE_LEARNING] = "learning",
        [BR_STATE_FORWARDING] = "forwarding",
        [BR_STATE_BLOCKING] = "blocking",
};

void br_set_state(struct net_bridge_port *p, unsigned int state)
{
        struct switchdev_attr attr = {
                .orig_dev = p->dev,
                .id = SWITCHDEV_ATTR_ID_PORT_STP_STATE,
                .flags = SWITCHDEV_F_DEFER,
                .u.stp_state = state,
        };
        int err;

        /* Don't change the state of the ports if they are driven by a different
         * protocol.
         */
        if (p->flags & BR_MRP_AWARE)
                return;

        p->state = state;
        err = switchdev_port_attr_set(p->dev, &attr, NULL);
        if (err && err != -EOPNOTSUPP)
                br_warn(p->br, "error setting offload STP state on port %u(%s)\n",
                                (unsigned int) p->port_no, p->dev->name);
        else
                br_info(p->br, "port %u(%s) entered %s state\n",
                                (unsigned int) p->port_no, p->dev->name,
                                br_port_state_names[p->state]);

        if (p->br->stp_enabled == BR_KERNEL_STP) {
                switch (p->state) {
                case BR_STATE_BLOCKING:
                        p->stp_xstats.transition_blk++;
                        break;
                case BR_STATE_FORWARDING:
                        p->stp_xstats.transition_fwd++;
                        break;
                }
        }
}

u8 br_port_get_stp_state(const struct net_device *dev)
{
        struct net_bridge_port *p;

        ASSERT_RTNL();

        p = br_port_get_rtnl(dev);
        if (!p)
                return BR_STATE_DISABLED;

        return p->state;
}
EXPORT_SYMBOL_GPL(br_port_get_stp_state);

/* called under bridge lock */
struct net_bridge_port *br_get_port(struct net_bridge *br, u16 port_no)
{
        struct net_bridge_port *p;

        list_for_each_entry_rcu(p, &br->port_list, list,
                                lockdep_is_held(&br->lock)) {
                if (p->port_no == port_no)
                        return p;
        }

        return NULL;
}

/* called under bridge lock */
static int br_should_become_root_port(const struct net_bridge_port *p,
                                      u16 root_port)
{
        struct net_bridge *br;
        struct net_bridge_port *rp;
        int t;

        br = p->br;
        if (p->state == BR_STATE_DISABLED ||
            br_is_designated_port(p))
                return 0;

        if (memcmp(&br->bridge_id, &p->designated_root, 8) <= 0)
                return 0;

        if (!root_port)
                return 1;

        rp = br_get_port(br, root_port);

        t = memcmp(&p->designated_root, &rp->designated_root, 8);
        if (t < 0)
                return 1;
        else if (t > 0)
                return 0;

        if (p->designated_cost + p->path_cost <
            rp->designated_cost + rp->path_cost)
                return 1;
        else if (p->designated_cost + p->path_cost >
                 rp->designated_cost + rp->path_cost)
                return 0;

        t = memcmp(&p->designated_bridge, &rp->designated_bridge, 8);
        if (t < 0)
                return 1;
        else if (t > 0)
                return 0;

        if (p->designated_port < rp->designated_port)
                return 1;
        else if (p->designated_port > rp->designated_port)
                return 0;

        if (p->port_id < rp->port_id)
                return 1;

        return 0;
}

static void br_root_port_block(const struct net_bridge *br,
                               struct net_bridge_port *p)
{

        br_notice(br, "port %u(%s) tried to become root port (blocked)",
                  (unsigned int) p->port_no, p->dev->name);

        br_set_state(p, BR_STATE_LISTENING);
        br_ifinfo_notify(RTM_NEWLINK, NULL, p);

        if (br->forward_delay > 0)
                mod_timer(&p->forward_delay_timer, jiffies + br->forward_delay);
}

/* called under bridge lock */
static void br_root_selection(struct net_bridge *br)
{
        struct net_bridge_port *p;
        u16 root_port = 0;

        list_for_each_entry(p, &br->port_list, list) {
                if (!br_should_become_root_port(p, root_port))
                        continue;

                if (p->flags & BR_ROOT_BLOCK)
                        br_root_port_block(br, p);
                else
                        root_port = p->port_no;
        }

        br->root_port = root_port;

        if (!root_port) {
                br->designated_root = br->bridge_id;
                br->root_path_cost = 0;
        } else {
                p = br_get_port(br, root_port);
                br->designated_root = p->designated_root;
                br->root_path_cost = p->designated_cost + p->path_cost;
        }
}

/* called under bridge lock */
void br_become_root_bridge(struct net_bridge *br)
{
        br->max_age = br->bridge_max_age;
        br->hello_time = br->bridge_hello_time;
        br->forward_delay = br->bridge_forward_delay;
        br_topology_change_detection(br);
        del_timer(&br->tcn_timer);

        if (br->dev->flags & IFF_UP) {
                br_config_bpdu_generation(br);
                mod_timer(&br->hello_timer, jiffies + br->hello_time);
        }
}

/* called under bridge lock */
void br_transmit_config(struct net_bridge_port *p)
{
        struct br_config_bpdu bpdu;
        struct net_bridge *br;

        if (timer_pending(&p->hold_timer)) {
                p->config_pending = 1;
                return;
        }

        br = p->br;

        bpdu.topology_change = br->topology_change;
        bpdu.topology_change_ack = p->topology_change_ack;
        bpdu.root = br->designated_root;
        bpdu.root_path_cost = br->root_path_cost;
        bpdu.bridge_id = br->bridge_id;
        bpdu.port_id = p->port_id;
        if (br_is_root_bridge(br))
                bpdu.message_age = 0;
        else {
                struct net_bridge_port *root
                        = br_get_port(br, br->root_port);
                bpdu.message_age = (jiffies - root->designated_age)
                        + MESSAGE_AGE_INCR;
        }
        bpdu.max_age = br->max_age;
        bpdu.hello_time = br->hello_time;
        bpdu.forward_delay = br->forward_delay;

        if (bpdu.message_age < br->max_age) {
                br_send_config_bpdu(p, &bpdu);
                p->topology_change_ack = 0;
                p->config_pending = 0;
                if (p->br->stp_enabled == BR_KERNEL_STP)
                        mod_timer(&p->hold_timer,
                                  round_jiffies(jiffies + BR_HOLD_TIME));
        }
}

/* called under bridge lock */
static void br_record_config_information(struct net_bridge_port *p,
                                         const struct br_config_bpdu *bpdu)
{
        p->designated_root = bpdu->root;
        p->designated_cost = bpdu->root_path_cost;
        p->designated_bridge = bpdu->bridge_id;
        p->designated_port = bpdu->port_id;
        p->designated_age = jiffies - bpdu->message_age;

        mod_timer(&p->message_age_timer, jiffies
                  + (bpdu->max_age - bpdu->message_age));
}

/* called under bridge lock */
static void br_record_config_timeout_values(struct net_bridge *br,
                                            const struct br_config_bpdu *bpdu)
{
        br->max_age = bpdu->max_age;
        br->hello_time = bpdu->hello_time;
        br->forward_delay = bpdu->forward_delay;
        __br_set_topology_change(br, bpdu->topology_change);
}

/* called under bridge lock */
void br_transmit_tcn(struct net_bridge *br)
{
        struct net_bridge_port *p;

        p = br_get_port(br, br->root_port);
        if (p)
                br_send_tcn_bpdu(p);
        else
                br_notice(br, "root port %u not found for topology notice\n",
                          br->root_port);
}

/* called under bridge lock */
static int br_should_become_designated_port(const struct net_bridge_port *p)
{
        struct net_bridge *br;
        int t;

        br = p->br;
        if (br_is_designated_port(p))
                return 1;

        if (memcmp(&p->designated_root, &br->designated_root, 8))
                return 1;

        if (br->root_path_cost < p->designated_cost)
                return 1;
        else if (br->root_path_cost > p->designated_cost)
                return 0;

        t = memcmp(&br->bridge_id, &p->designated_bridge, 8);
        if (t < 0)
                return 1;
        else if (t > 0)
                return 0;

        if (p->port_id < p->designated_port)
                return 1;

        return 0;
}

/* called under bridge lock */
static void br_designated_port_selection(struct net_bridge *br)
{
        struct net_bridge_port *p;

        list_for_each_entry(p, &br->port_list, list) {
                if (p->state != BR_STATE_DISABLED &&
                    br_should_become_designated_port(p))
                        br_become_designated_port(p);

        }
}

/* called under bridge lock */
static int br_supersedes_port_info(const struct net_bridge_port *p,
                                   const struct br_config_bpdu *bpdu)
{
        int t;

        t = memcmp(&bpdu->root, &p->designated_root, 8);
        if (t < 0)
                return 1;
        else if (t > 0)
                return 0;

        if (bpdu->root_path_cost < p->designated_cost)
                return 1;
        else if (bpdu->root_path_cost > p->designated_cost)
                return 0;

        t = memcmp(&bpdu->bridge_id, &p->designated_bridge, 8);
        if (t < 0)
                return 1;
        else if (t > 0)
                return 0;

        if (memcmp(&bpdu->bridge_id, &p->br->bridge_id, 8))
                return 1;

        if (bpdu->port_id <= p->designated_port)
                return 1;

        return 0;
}

/* called under bridge lock */
static void br_topology_change_acknowledged(struct net_bridge *br)
{
        br->topology_change_detected = 0;
        del_timer(&br->tcn_timer);
}

/* called under bridge lock */
void br_topology_change_detection(struct net_bridge *br)
{
        int isroot = br_is_root_bridge(br);

        if (br->stp_enabled != BR_KERNEL_STP)
                return;

        br_info(br, "topology change detected, %s\n",
                isroot ? "propagating" : "sending tcn bpdu");

        if (isroot) {
                __br_set_topology_change(br, 1);
                mod_timer(&br->topology_change_timer, jiffies
                          + br->bridge_forward_delay + br->bridge_max_age);
        } else if (!br->topology_change_detected) {
                br_transmit_tcn(br);
                mod_timer(&br->tcn_timer, jiffies + br->bridge_hello_time);
        }

        br->topology_change_detected = 1;
}

/* called under bridge lock */
void br_config_bpdu_generation(struct net_bridge *br)
{
        struct net_bridge_port *p;

        list_for_each_entry(p, &br->port_list, list) {
                if (p->state != BR_STATE_DISABLED &&
                    br_is_designated_port(p))
                        br_transmit_config(p);
        }
}

/* called under bridge lock */
static void br_reply(struct net_bridge_port *p)
{
        br_transmit_config(p);
}

/* called under bridge lock */
void br_configuration_update(struct net_bridge *br)
{
        br_root_selection(br);
        br_designated_port_selection(br);
}

/* called under bridge lock */
void br_become_designated_port(struct net_bridge_port *p)
{
        struct net_bridge *br;

        br = p->br;
        p->designated_root = br->designated_root;
        p->designated_cost = br->root_path_cost;
        p->designated_bridge = br->bridge_id;
        p->designated_port = p->port_id;
}


/* called under bridge lock */
static void br_make_blocking(struct net_bridge_port *p)
{
        if (p->state != BR_STATE_DISABLED &&
            p->state != BR_STATE_BLOCKING) {
                if (p->state == BR_STATE_FORWARDING ||
                    p->state == BR_STATE_LEARNING)
                        br_topology_change_detection(p->br);

                br_set_state(p, BR_STATE_BLOCKING);
                br_ifinfo_notify(RTM_NEWLINK, NULL, p);

                del_timer(&p->forward_delay_timer);
        }
}

/* called under bridge lock */
static void br_make_forwarding(struct net_bridge_port *p)
{
        struct net_bridge *br = p->br;

        if (p->state != BR_STATE_BLOCKING)
                return;

        if (br->stp_enabled == BR_NO_STP || br->forward_delay == 0) {
                br_set_state(p, BR_STATE_FORWARDING);
                br_topology_change_detection(br);
                del_timer(&p->forward_delay_timer);
        } else if (br->stp_enabled == BR_KERNEL_STP)
                br_set_state(p, BR_STATE_LISTENING);
        else
                br_set_state(p, BR_STATE_LEARNING);

        br_ifinfo_notify(RTM_NEWLINK, NULL, p);

        if (br->forward_delay != 0)
                mod_timer(&p->forward_delay_timer, jiffies + br->forward_delay);
}

/* called under bridge lock */
void br_port_state_selection(struct net_bridge *br)
{
        struct net_bridge_port *p;
        unsigned int liveports = 0;

        list_for_each_entry(p, &br->port_list, list) {
                if (p->state == BR_STATE_DISABLED)
                        continue;

                /* Don't change port states if userspace is handling STP */
                if (br->stp_enabled != BR_USER_STP) {
                        if (p->port_no == br->root_port) {
                                p->config_pending = 0;
                                p->topology_change_ack = 0;
                                br_make_forwarding(p);
                        } else if (br_is_designated_port(p)) {
                                del_timer(&p->message_age_timer);
                                br_make_forwarding(p);
                        } else {
                                p->config_pending = 0;
                                p->topology_change_ack = 0;
                                br_make_blocking(p);
                        }
                }

                if (p->state != BR_STATE_BLOCKING)
                        br_multicast_enable_port(p);
                /* Multicast is not disabled for the port when it goes in
                 * blocking state because the timers will expire and stop by
                 * themselves without sending more queries.
                 */
                if (p->state == BR_STATE_FORWARDING)
                        ++liveports;
        }

        if (liveports == 0)
                netif_carrier_off(br->dev);
        else
                netif_carrier_on(br->dev);
}

/* called under bridge lock */
static void br_topology_change_acknowledge(struct net_bridge_port *p)
{
        p->topology_change_ack = 1;
        br_transmit_config(p);
}

/* called under bridge lock */
void br_received_config_bpdu(struct net_bridge_port *p,
                             const struct br_config_bpdu *bpdu)
{
        struct net_bridge *br;
        int was_root;

        p->stp_xstats.rx_bpdu++;

        br = p->br;
        was_root = br_is_root_bridge(br);

        if (br_supersedes_port_info(p, bpdu)) {
                br_record_config_information(p, bpdu);
                br_configuration_update(br);
                br_port_state_selection(br);

                if (!br_is_root_bridge(br) && was_root) {
                        del_timer(&br->hello_timer);
                        if (br->topology_change_detected) {
                                del_timer(&br->topology_change_timer);
                                br_transmit_tcn(br);

                                mod_timer(&br->tcn_timer,
                                          jiffies + br->bridge_hello_time);
                        }
                }

                if (p->port_no == br->root_port) {
                        br_record_config_timeout_values(br, bpdu);
                        br_config_bpdu_generation(br);
                        if (bpdu->topology_change_ack)
                                br_topology_change_acknowledged(br);
                }
        } else if (br_is_designated_port(p)) {
                br_reply(p);
        }
}

/* called under bridge lock */
void br_received_tcn_bpdu(struct net_bridge_port *p)
{
        p->stp_xstats.rx_tcn++;

        if (br_is_designated_port(p)) {
                br_info(p->br, "port %u(%s) received tcn bpdu\n",
                        (unsigned int) p->port_no, p->dev->name);

                br_topology_change_detection(p->br);
                br_topology_change_acknowledge(p);
        }
}

/* Change bridge STP parameter */
int br_set_hello_time(struct net_bridge *br, unsigned long val)
{
        unsigned long t = clock_t_to_jiffies(val);

        if (t < BR_MIN_HELLO_TIME || t > BR_MAX_HELLO_TIME)
                return -ERANGE;

        spin_lock_bh(&br->lock);
        br->bridge_hello_time = t;
        if (br_is_root_bridge(br))
                br->hello_time = br->bridge_hello_time;
        spin_unlock_bh(&br->lock);
        return 0;
}

int br_set_max_age(struct net_bridge *br, unsigned long val)
{
        unsigned long t = clock_t_to_jiffies(val);

        if (t < BR_MIN_MAX_AGE || t > BR_MAX_MAX_AGE)
                return -ERANGE;

        spin_lock_bh(&br->lock);
        br->bridge_max_age = t;
        if (br_is_root_bridge(br))
                br->max_age = br->bridge_max_age;
        spin_unlock_bh(&br->lock);
        return 0;

}

/* called under bridge lock */
int __set_ageing_time(struct net_device *dev, unsigned long t)
{
        struct switchdev_attr attr = {
                .orig_dev = dev,
                .id = SWITCHDEV_ATTR_ID_BRIDGE_AGEING_TIME,
                .flags = SWITCHDEV_F_SKIP_EOPNOTSUPP | SWITCHDEV_F_DEFER,
                .u.ageing_time = jiffies_to_clock_t(t),
        };
        int err;

        err = switchdev_port_attr_set(dev, &attr, NULL);
        if (err && err != -EOPNOTSUPP)
                return err;

        return 0;
}

/* Set time interval that dynamic forwarding entries live
 * For pure software bridge, allow values outside the 802.1
 * standard specification for special cases:
 *  0 - entry never ages (all permanent)
 *  1 - entry disappears (no persistence)
 *
 * Offloaded switch entries maybe more restrictive
 */
int br_set_ageing_time(struct net_bridge *br, clock_t ageing_time)
{
        unsigned long t = clock_t_to_jiffies(ageing_time);
        int err;

        err = __set_ageing_time(br->dev, t);
        if (err)
                return err;

        spin_lock_bh(&br->lock);
        br->bridge_ageing_time = t;
        br->ageing_time = t;
        spin_unlock_bh(&br->lock);

        mod_delayed_work(system_long_wq, &br->gc_work, 0);

        return 0;
}

clock_t br_get_ageing_time(const struct net_device *br_dev)
{
        const struct net_bridge *br;

        if (!netif_is_bridge_master(br_dev))
                return 0;

        br = netdev_priv(br_dev);

        return jiffies_to_clock_t(br->ageing_time);
}
EXPORT_SYMBOL_GPL(br_get_ageing_time);

/* called under bridge lock */
void __br_set_topology_change(struct net_bridge *br, unsigned char val)
{
        unsigned long t;
        int err;

        if (br->stp_enabled == BR_KERNEL_STP && br->topology_change != val) {
                /* On topology change, set the bridge ageing time to twice the
                 * forward delay. Otherwise, restore its default ageing time.
                 */

                if (val) {
                        t = 2 * br->forward_delay;
                        br_debug(br, "decreasing ageing time to %lu\n", t);
                } else {
                        t = br->bridge_ageing_time;
                        br_debug(br, "restoring ageing time to %lu\n", t);
                }

                err = __set_ageing_time(br->dev, t);
                if (err)
                        br_warn(br, "error offloading ageing time\n");
                else
                        br->ageing_time = t;
        }

        br->topology_change = val;
}

void __br_set_forward_delay(struct net_bridge *br, unsigned long t)
{
        br->bridge_forward_delay = t;
        if (br_is_root_bridge(br))
                br->forward_delay = br->bridge_forward_delay;
}

int br_set_forward_delay(struct net_bridge *br, unsigned long val)
{
        unsigned long t = clock_t_to_jiffies(val);
        int err = -ERANGE;

        spin_lock_bh(&br->lock);
        if (br->stp_enabled != BR_NO_STP &&
            (t < BR_MIN_FORWARD_DELAY || t > BR_MAX_FORWARD_DELAY))
                goto unlock;

        __br_set_forward_delay(br, t);
        err = 0;

unlock:
        spin_unlock_bh(&br->lock);
        return err;
}












































































































































































   16 


   16 



































    6 



    6 

    6 



    6 






    7 





    7 
    7 






    7 




























    2 





    2 
    2 






    2 
























































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































   47 


















































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
// SPDX-License-Identifier: GPL-2.0-only
/* L2TP core.
 *
 * Copyright (c) 2008,2009,2010 Katalix Systems Ltd
 *
 * This file contains some code of the original L2TPv2 pppol2tp
 * driver, which has the following copyright:
 *
 * Authors:        Martijn van Oosterhout <kleptog@svana.org>
 *                James Chapman (jchapman@katalix.com)
 * Contributors:
 *                Michal Ostrowski <mostrows@speakeasy.net>
 *                Arnaldo Carvalho de Melo <acme@xconectiva.com.br>
 *                David S. Miller (davem@redhat.com)
 */

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#include <linux/module.h>
#include <linux/string.h>
#include <linux/list.h>
#include <linux/rculist.h>
#include <linux/uaccess.h>

#include <linux/kernel.h>
#include <linux/spinlock.h>
#include <linux/kthread.h>
#include <linux/sched.h>
#include <linux/slab.h>
#include <linux/errno.h>
#include <linux/jiffies.h>

#include <linux/netdevice.h>
#include <linux/net.h>
#include <linux/inetdevice.h>
#include <linux/skbuff.h>
#include <linux/init.h>
#include <linux/in.h>
#include <linux/ip.h>
#include <linux/udp.h>
#include <linux/l2tp.h>
#include <linux/hash.h>
#include <linux/sort.h>
#include <linux/file.h>
#include <linux/nsproxy.h>
#include <net/net_namespace.h>
#include <net/netns/generic.h>
#include <net/dst.h>
#include <net/ip.h>
#include <net/udp.h>
#include <net/udp_tunnel.h>
#include <net/inet_common.h>
#include <net/xfrm.h>
#include <net/protocol.h>
#include <net/inet6_connection_sock.h>
#include <net/inet_ecn.h>
#include <net/ip6_route.h>
#include <net/ip6_checksum.h>

#include <asm/byteorder.h>
#include <linux/atomic.h>

#include "l2tp_core.h"
#include "trace.h"

#define CREATE_TRACE_POINTS
#include "trace.h"

#define L2TP_DRV_VERSION        "V2.0"

/* L2TP header constants */
#define L2TP_HDRFLAG_T           0x8000
#define L2TP_HDRFLAG_L           0x4000
#define L2TP_HDRFLAG_S           0x0800
#define L2TP_HDRFLAG_O           0x0200
#define L2TP_HDRFLAG_P           0x0100

#define L2TP_HDR_VER_MASK  0x000F
#define L2TP_HDR_VER_2           0x0002
#define L2TP_HDR_VER_3           0x0003

/* L2TPv3 default L2-specific sublayer */
#define L2TP_SLFLAG_S           0x40000000
#define L2TP_SL_SEQ_MASK   0x00ffffff

#define L2TP_HDR_SIZE_MAX                14

/* Default trace flags */
#define L2TP_DEFAULT_DEBUG_FLAGS        0

#define L2TP_DEPTH_NESTING                2
#if L2TP_DEPTH_NESTING == SINGLE_DEPTH_NESTING
#error "L2TP requires its own lockdep subclass"
#endif

/* Private data stored for received packets in the skb.
 */
struct l2tp_skb_cb {
        u32                        ns;
        u16                        has_seq;
        u16                        length;
        unsigned long                expires;
};

#define L2TP_SKB_CB(skb)        ((struct l2tp_skb_cb *)&(skb)->cb[sizeof(struct inet_skb_parm)])

static struct workqueue_struct *l2tp_wq;

/* per-net private data for this module */
static unsigned int l2tp_net_id;
struct l2tp_net {
        /* Lock for write access to l2tp_tunnel_idr */
        spinlock_t l2tp_tunnel_idr_lock;
        struct idr l2tp_tunnel_idr;
        struct hlist_head l2tp_session_hlist[L2TP_HASH_SIZE_2];
        /* Lock for write access to l2tp_session_hlist */
        spinlock_t l2tp_session_hlist_lock;
};

#if IS_ENABLED(CONFIG_IPV6)
static bool l2tp_sk_is_v6(struct sock *sk)
{
        return sk->sk_family == PF_INET6 &&
               !ipv6_addr_v4mapped(&sk->sk_v6_daddr);
}
#endif

static inline struct l2tp_net *l2tp_pernet(const struct net *net)
{
        return net_generic(net, l2tp_net_id);
}

/* Session hash global list for L2TPv3.
 * The session_id SHOULD be random according to RFC3931, but several
 * L2TP implementations use incrementing session_ids.  So we do a real
 * hash on the session_id, rather than a simple bitmask.
 */
static inline struct hlist_head *
l2tp_session_id_hash_2(struct l2tp_net *pn, u32 session_id)
{
        return &pn->l2tp_session_hlist[hash_32(session_id, L2TP_HASH_BITS_2)];
}

/* Session hash list.
 * The session_id SHOULD be random according to RFC2661, but several
 * L2TP implementations (Cisco and Microsoft) use incrementing
 * session_ids.  So we do a real hash on the session_id, rather than a
 * simple bitmask.
 */
static inline struct hlist_head *
l2tp_session_id_hash(struct l2tp_tunnel *tunnel, u32 session_id)
{
        return &tunnel->session_hlist[hash_32(session_id, L2TP_HASH_BITS)];
}

static void l2tp_tunnel_free(struct l2tp_tunnel *tunnel)
{
        trace_free_tunnel(tunnel);
        sock_put(tunnel->sock);
        /* the tunnel is freed in the socket destructor */
}

static void l2tp_session_free(struct l2tp_session *session)
{
        trace_free_session(session);
        if (session->tunnel)
                l2tp_tunnel_dec_refcount(session->tunnel);
        kfree(session);
}

struct l2tp_tunnel *l2tp_sk_to_tunnel(struct sock *sk)
{
        struct l2tp_tunnel *tunnel = sk->sk_user_data;

        if (tunnel)
                if (WARN_ON(tunnel->magic != L2TP_TUNNEL_MAGIC))
                        return NULL;

        return tunnel;
}
EXPORT_SYMBOL_GPL(l2tp_sk_to_tunnel);

void l2tp_tunnel_inc_refcount(struct l2tp_tunnel *tunnel)
{
        refcount_inc(&tunnel->ref_count);
}
EXPORT_SYMBOL_GPL(l2tp_tunnel_inc_refcount);

void l2tp_tunnel_dec_refcount(struct l2tp_tunnel *tunnel)
{
        if (refcount_dec_and_test(&tunnel->ref_count))
                l2tp_tunnel_free(tunnel);
}
EXPORT_SYMBOL_GPL(l2tp_tunnel_dec_refcount);

void l2tp_session_inc_refcount(struct l2tp_session *session)
{
        refcount_inc(&session->ref_count);
}
EXPORT_SYMBOL_GPL(l2tp_session_inc_refcount);

void l2tp_session_dec_refcount(struct l2tp_session *session)
{
        if (refcount_dec_and_test(&session->ref_count))
                l2tp_session_free(session);
}
EXPORT_SYMBOL_GPL(l2tp_session_dec_refcount);

/* Lookup a tunnel. A new reference is held on the returned tunnel. */
struct l2tp_tunnel *l2tp_tunnel_get(const struct net *net, u32 tunnel_id)
{
        const struct l2tp_net *pn = l2tp_pernet(net);
        struct l2tp_tunnel *tunnel;

        rcu_read_lock_bh();
        tunnel = idr_find(&pn->l2tp_tunnel_idr, tunnel_id);
        if (tunnel && refcount_inc_not_zero(&tunnel->ref_count)) {
                rcu_read_unlock_bh();
                return tunnel;
        }
        rcu_read_unlock_bh();

        return NULL;
}
EXPORT_SYMBOL_GPL(l2tp_tunnel_get);

struct l2tp_tunnel *l2tp_tunnel_get_nth(const struct net *net, int nth)
{
        struct l2tp_net *pn = l2tp_pernet(net);
        unsigned long tunnel_id, tmp;
        struct l2tp_tunnel *tunnel;
        int count = 0;

        rcu_read_lock_bh();
        idr_for_each_entry_ul(&pn->l2tp_tunnel_idr, tunnel, tmp, tunnel_id) {
                if (tunnel && ++count > nth &&
                    refcount_inc_not_zero(&tunnel->ref_count)) {
                        rcu_read_unlock_bh();
                        return tunnel;
                }
        }
        rcu_read_unlock_bh();

        return NULL;
}
EXPORT_SYMBOL_GPL(l2tp_tunnel_get_nth);

struct l2tp_session *l2tp_tunnel_get_session(struct l2tp_tunnel *tunnel,
                                             u32 session_id)
{
        struct hlist_head *session_list;
        struct l2tp_session *session;

        session_list = l2tp_session_id_hash(tunnel, session_id);

        read_lock_bh(&tunnel->hlist_lock);
        hlist_for_each_entry(session, session_list, hlist)
                if (session->session_id == session_id) {
                        l2tp_session_inc_refcount(session);
                        read_unlock_bh(&tunnel->hlist_lock);

                        return session;
                }
        read_unlock_bh(&tunnel->hlist_lock);

        return NULL;
}
EXPORT_SYMBOL_GPL(l2tp_tunnel_get_session);

struct l2tp_session *l2tp_session_get(const struct net *net, u32 session_id)
{
        struct hlist_head *session_list;
        struct l2tp_session *session;

        session_list = l2tp_session_id_hash_2(l2tp_pernet(net), session_id);

        rcu_read_lock_bh();
        hlist_for_each_entry_rcu(session, session_list, global_hlist)
                if (session->session_id == session_id) {
                        l2tp_session_inc_refcount(session);
                        rcu_read_unlock_bh();

                        return session;
                }
        rcu_read_unlock_bh();

        return NULL;
}
EXPORT_SYMBOL_GPL(l2tp_session_get);

struct l2tp_session *l2tp_session_get_nth(struct l2tp_tunnel *tunnel, int nth)
{
        int hash;
        struct l2tp_session *session;
        int count = 0;

        read_lock_bh(&tunnel->hlist_lock);
        for (hash = 0; hash < L2TP_HASH_SIZE; hash++) {
                hlist_for_each_entry(session, &tunnel->session_hlist[hash], hlist) {
                        if (++count > nth) {
                                l2tp_session_inc_refcount(session);
                                read_unlock_bh(&tunnel->hlist_lock);
                                return session;
                        }
                }
        }

        read_unlock_bh(&tunnel->hlist_lock);

        return NULL;
}
EXPORT_SYMBOL_GPL(l2tp_session_get_nth);

/* Lookup a session by interface name.
 * This is very inefficient but is only used by management interfaces.
 */
struct l2tp_session *l2tp_session_get_by_ifname(const struct net *net,
                                                const char *ifname)
{
        struct l2tp_net *pn = l2tp_pernet(net);
        int hash;
        struct l2tp_session *session;

        rcu_read_lock_bh();
        for (hash = 0; hash < L2TP_HASH_SIZE_2; hash++) {
                hlist_for_each_entry_rcu(session, &pn->l2tp_session_hlist[hash], global_hlist) {
                        if (!strcmp(session->ifname, ifname)) {
                                l2tp_session_inc_refcount(session);
                                rcu_read_unlock_bh();

                                return session;
                        }
                }
        }

        rcu_read_unlock_bh();

        return NULL;
}
EXPORT_SYMBOL_GPL(l2tp_session_get_by_ifname);

int l2tp_session_register(struct l2tp_session *session,
                          struct l2tp_tunnel *tunnel)
{
        struct l2tp_session *session_walk;
        struct hlist_head *g_head;
        struct hlist_head *head;
        struct l2tp_net *pn;
        int err;

        head = l2tp_session_id_hash(tunnel, session->session_id);

        write_lock_bh(&tunnel->hlist_lock);
        if (!tunnel->acpt_newsess) {
                err = -ENODEV;
                goto err_tlock;
        }

        hlist_for_each_entry(session_walk, head, hlist)
                if (session_walk->session_id == session->session_id) {
                        err = -EEXIST;
                        goto err_tlock;
                }

        if (tunnel->version == L2TP_HDR_VER_3) {
                pn = l2tp_pernet(tunnel->l2tp_net);
                g_head = l2tp_session_id_hash_2(pn, session->session_id);

                spin_lock_bh(&pn->l2tp_session_hlist_lock);

                /* IP encap expects session IDs to be globally unique, while
                 * UDP encap doesn't.
                 */
                hlist_for_each_entry(session_walk, g_head, global_hlist)
                        if (session_walk->session_id == session->session_id &&
                            (session_walk->tunnel->encap == L2TP_ENCAPTYPE_IP ||
                             tunnel->encap == L2TP_ENCAPTYPE_IP)) {
                                err = -EEXIST;
                                goto err_tlock_pnlock;
                        }

                l2tp_tunnel_inc_refcount(tunnel);
                hlist_add_head_rcu(&session->global_hlist, g_head);

                spin_unlock_bh(&pn->l2tp_session_hlist_lock);
        } else {
                l2tp_tunnel_inc_refcount(tunnel);
        }

        hlist_add_head(&session->hlist, head);
        write_unlock_bh(&tunnel->hlist_lock);

        trace_register_session(session);

        return 0;

err_tlock_pnlock:
        spin_unlock_bh(&pn->l2tp_session_hlist_lock);
err_tlock:
        write_unlock_bh(&tunnel->hlist_lock);

        return err;
}
EXPORT_SYMBOL_GPL(l2tp_session_register);

/*****************************************************************************
 * Receive data handling
 *****************************************************************************/

/* Queue a skb in order. We come here only if the skb has an L2TP sequence
 * number.
 */
static void l2tp_recv_queue_skb(struct l2tp_session *session, struct sk_buff *skb)
{
        struct sk_buff *skbp;
        struct sk_buff *tmp;
        u32 ns = L2TP_SKB_CB(skb)->ns;

        spin_lock_bh(&session->reorder_q.lock);
        skb_queue_walk_safe(&session->reorder_q, skbp, tmp) {
                if (L2TP_SKB_CB(skbp)->ns > ns) {
                        __skb_queue_before(&session->reorder_q, skbp, skb);
                        atomic_long_inc(&session->stats.rx_oos_packets);
                        goto out;
                }
        }

        __skb_queue_tail(&session->reorder_q, skb);

out:
        spin_unlock_bh(&session->reorder_q.lock);
}

/* Dequeue a single skb.
 */
static void l2tp_recv_dequeue_skb(struct l2tp_session *session, struct sk_buff *skb)
{
        struct l2tp_tunnel *tunnel = session->tunnel;
        int length = L2TP_SKB_CB(skb)->length;

        /* We're about to requeue the skb, so return resources
         * to its current owner (a socket receive buffer).
         */
        skb_orphan(skb);

        atomic_long_inc(&tunnel->stats.rx_packets);
        atomic_long_add(length, &tunnel->stats.rx_bytes);
        atomic_long_inc(&session->stats.rx_packets);
        atomic_long_add(length, &session->stats.rx_bytes);

        if (L2TP_SKB_CB(skb)->has_seq) {
                /* Bump our Nr */
                session->nr++;
                session->nr &= session->nr_max;
                trace_session_seqnum_update(session);
        }

        /* call private receive handler */
        if (session->recv_skb)
                (*session->recv_skb)(session, skb, L2TP_SKB_CB(skb)->length);
        else
                kfree_skb(skb);
}

/* Dequeue skbs from the session's reorder_q, subject to packet order.
 * Skbs that have been in the queue for too long are simply discarded.
 */
static void l2tp_recv_dequeue(struct l2tp_session *session)
{
        struct sk_buff *skb;
        struct sk_buff *tmp;

        /* If the pkt at the head of the queue has the nr that we
         * expect to send up next, dequeue it and any other
         * in-sequence packets behind it.
         */
start:
        spin_lock_bh(&session->reorder_q.lock);
        skb_queue_walk_safe(&session->reorder_q, skb, tmp) {
                struct l2tp_skb_cb *cb = L2TP_SKB_CB(skb);

                /* If the packet has been pending on the queue for too long, discard it */
                if (time_after(jiffies, cb->expires)) {
                        atomic_long_inc(&session->stats.rx_seq_discards);
                        atomic_long_inc(&session->stats.rx_errors);
                        trace_session_pkt_expired(session, cb->ns);
                        session->reorder_skip = 1;
                        __skb_unlink(skb, &session->reorder_q);
                        kfree_skb(skb);
                        continue;
                }

                if (cb->has_seq) {
                        if (session->reorder_skip) {
                                session->reorder_skip = 0;
                                session->nr = cb->ns;
                                trace_session_seqnum_reset(session);
                        }
                        if (cb->ns != session->nr)
                                goto out;
                }
                __skb_unlink(skb, &session->reorder_q);

                /* Process the skb. We release the queue lock while we
                 * do so to let other contexts process the queue.
                 */
                spin_unlock_bh(&session->reorder_q.lock);
                l2tp_recv_dequeue_skb(session, skb);
                goto start;
        }

out:
        spin_unlock_bh(&session->reorder_q.lock);
}

static int l2tp_seq_check_rx_window(struct l2tp_session *session, u32 nr)
{
        u32 nws;

        if (nr >= session->nr)
                nws = nr - session->nr;
        else
                nws = (session->nr_max + 1) - (session->nr - nr);

        return nws < session->nr_window_size;
}

/* If packet has sequence numbers, queue it if acceptable. Returns 0 if
 * acceptable, else non-zero.
 */
static int l2tp_recv_data_seq(struct l2tp_session *session, struct sk_buff *skb)
{
        struct l2tp_skb_cb *cb = L2TP_SKB_CB(skb);

        if (!l2tp_seq_check_rx_window(session, cb->ns)) {
                /* Packet sequence number is outside allowed window.
                 * Discard it.
                 */
                trace_session_pkt_outside_rx_window(session, cb->ns);
                goto discard;
        }

        if (session->reorder_timeout != 0) {
                /* Packet reordering enabled. Add skb to session's
                 * reorder queue, in order of ns.
                 */
                l2tp_recv_queue_skb(session, skb);
                goto out;
        }

        /* Packet reordering disabled. Discard out-of-sequence packets, while
         * tracking the number if in-sequence packets after the first OOS packet
         * is seen. After nr_oos_count_max in-sequence packets, reset the
         * sequence number to re-enable packet reception.
         */
        if (cb->ns == session->nr) {
                skb_queue_tail(&session->reorder_q, skb);
        } else {
                u32 nr_oos = cb->ns;
                u32 nr_next = (session->nr_oos + 1) & session->nr_max;

                if (nr_oos == nr_next)
                        session->nr_oos_count++;
                else
                        session->nr_oos_count = 0;

                session->nr_oos = nr_oos;
                if (session->nr_oos_count > session->nr_oos_count_max) {
                        session->reorder_skip = 1;
                }
                if (!session->reorder_skip) {
                        atomic_long_inc(&session->stats.rx_seq_discards);
                        trace_session_pkt_oos(session, cb->ns);
                        goto discard;
                }
                skb_queue_tail(&session->reorder_q, skb);
        }

out:
        return 0;

discard:
        return 1;
}

/* Do receive processing of L2TP data frames. We handle both L2TPv2
 * and L2TPv3 data frames here.
 *
 * L2TPv2 Data Message Header
 *
 *  0                   1                   2                   3
 *  0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
 * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 * |T|L|x|x|S|x|O|P|x|x|x|x|  Ver  |          Length (opt)         |
 * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 * |           Tunnel ID           |           Session ID          |
 * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 * |             Ns (opt)          |             Nr (opt)          |
 * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 * |      Offset Size (opt)        |    Offset pad... (opt)
 * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 *
 * Data frames are marked by T=0. All other fields are the same as
 * those in L2TP control frames.
 *
 * L2TPv3 Data Message Header
 *
 * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 * |                      L2TP Session Header                      |
 * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 * |                      L2-Specific Sublayer                     |
 * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 * |                        Tunnel Payload                      ...
 * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 *
 * L2TPv3 Session Header Over IP
 *
 *  0                   1                   2                   3
 *  0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
 * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 * |                           Session ID                          |
 * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 * |               Cookie (optional, maximum 64 bits)...
 * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 *                                                                 |
 * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 *
 * L2TPv3 L2-Specific Sublayer Format
 *
 *  0                   1                   2                   3
 *  0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
 * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 * |x|S|x|x|x|x|x|x|              Sequence Number                  |
 * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 *
 * Cookie value and sublayer format are negotiated with the peer when
 * the session is set up. Unlike L2TPv2, we do not need to parse the
 * packet header to determine if optional fields are present.
 *
 * Caller must already have parsed the frame and determined that it is
 * a data (not control) frame before coming here. Fields up to the
 * session-id have already been parsed and ptr points to the data
 * after the session-id.
 */
void l2tp_recv_common(struct l2tp_session *session, struct sk_buff *skb,
                      unsigned char *ptr, unsigned char *optr, u16 hdrflags,
                      int length)
{
        struct l2tp_tunnel *tunnel = session->tunnel;
        int offset;

        /* Parse and check optional cookie */
        if (session->peer_cookie_len > 0) {
                if (memcmp(ptr, &session->peer_cookie[0], session->peer_cookie_len)) {
                        pr_debug_ratelimited("%s: cookie mismatch (%u/%u). Discarding.\n",
                                             tunnel->name, tunnel->tunnel_id,
                                             session->session_id);
                        atomic_long_inc(&session->stats.rx_cookie_discards);
                        goto discard;
                }
                ptr += session->peer_cookie_len;
        }

        /* Handle the optional sequence numbers. Sequence numbers are
         * in different places for L2TPv2 and L2TPv3.
         *
         * If we are the LAC, enable/disable sequence numbers under
         * the control of the LNS.  If no sequence numbers present but
         * we were expecting them, discard frame.
         */
        L2TP_SKB_CB(skb)->has_seq = 0;
        if (tunnel->version == L2TP_HDR_VER_2) {
                if (hdrflags & L2TP_HDRFLAG_S) {
                        /* Store L2TP info in the skb */
                        L2TP_SKB_CB(skb)->ns = ntohs(*(__be16 *)ptr);
                        L2TP_SKB_CB(skb)->has_seq = 1;
                        ptr += 2;
                        /* Skip past nr in the header */
                        ptr += 2;

                }
        } else if (session->l2specific_type == L2TP_L2SPECTYPE_DEFAULT) {
                u32 l2h = ntohl(*(__be32 *)ptr);

                if (l2h & 0x40000000) {
                        /* Store L2TP info in the skb */
                        L2TP_SKB_CB(skb)->ns = l2h & 0x00ffffff;
                        L2TP_SKB_CB(skb)->has_seq = 1;
                }
                ptr += 4;
        }

        if (L2TP_SKB_CB(skb)->has_seq) {
                /* Received a packet with sequence numbers. If we're the LAC,
                 * check if we sre sending sequence numbers and if not,
                 * configure it so.
                 */
                if (!session->lns_mode && !session->send_seq) {
                        trace_session_seqnum_lns_enable(session);
                        session->send_seq = 1;
                        l2tp_session_set_header_len(session, tunnel->version);
                }
        } else {
                /* No sequence numbers.
                 * If user has configured mandatory sequence numbers, discard.
                 */
                if (session->recv_seq) {
                        pr_debug_ratelimited("%s: recv data has no seq numbers when required. Discarding.\n",
                                             session->name);
                        atomic_long_inc(&session->stats.rx_seq_discards);
                        goto discard;
                }

                /* If we're the LAC and we're sending sequence numbers, the
                 * LNS has requested that we no longer send sequence numbers.
                 * If we're the LNS and we're sending sequence numbers, the
                 * LAC is broken. Discard the frame.
                 */
                if (!session->lns_mode && session->send_seq) {
                        trace_session_seqnum_lns_disable(session);
                        session->send_seq = 0;
                        l2tp_session_set_header_len(session, tunnel->version);
                } else if (session->send_seq) {
                        pr_debug_ratelimited("%s: recv data has no seq numbers when required. Discarding.\n",
                                             session->name);
                        atomic_long_inc(&session->stats.rx_seq_discards);
                        goto discard;
                }
        }

        /* Session data offset is defined only for L2TPv2 and is
         * indicated by an optional 16-bit value in the header.
         */
        if (tunnel->version == L2TP_HDR_VER_2) {
                /* If offset bit set, skip it. */
                if (hdrflags & L2TP_HDRFLAG_O) {
                        offset = ntohs(*(__be16 *)ptr);
                        ptr += 2 + offset;
                }
        }

        offset = ptr - optr;
        if (!pskb_may_pull(skb, offset))
                goto discard;

        __skb_pull(skb, offset);

        /* Prepare skb for adding to the session's reorder_q.  Hold
         * packets for max reorder_timeout or 1 second if not
         * reordering.
         */
        L2TP_SKB_CB(skb)->length = length;
        L2TP_SKB_CB(skb)->expires = jiffies +
                (session->reorder_timeout ? session->reorder_timeout : HZ);

        /* Add packet to the session's receive queue. Reordering is done here, if
         * enabled. Saved L2TP protocol info is stored in skb->sb[].
         */
        if (L2TP_SKB_CB(skb)->has_seq) {
                if (l2tp_recv_data_seq(session, skb))
                        goto discard;
        } else {
                /* No sequence numbers. Add the skb to the tail of the
                 * reorder queue. This ensures that it will be
                 * delivered after all previous sequenced skbs.
                 */
                skb_queue_tail(&session->reorder_q, skb);
        }

        /* Try to dequeue as many skbs from reorder_q as we can. */
        l2tp_recv_dequeue(session);

        return;

discard:
        atomic_long_inc(&session->stats.rx_errors);
        kfree_skb(skb);
}
EXPORT_SYMBOL_GPL(l2tp_recv_common);

/* Drop skbs from the session's reorder_q
 */
static void l2tp_session_queue_purge(struct l2tp_session *session)
{
        struct sk_buff *skb = NULL;

        while ((skb = skb_dequeue(&session->reorder_q))) {
                atomic_long_inc(&session->stats.rx_errors);
                kfree_skb(skb);
        }
}

/* Internal UDP receive frame. Do the real work of receiving an L2TP data frame
 * here. The skb is not on a list when we get here.
 * Returns 0 if the packet was a data packet and was successfully passed on.
 * Returns 1 if the packet was not a good data packet and could not be
 * forwarded.  All such packets are passed up to userspace to deal with.
 */
static int l2tp_udp_recv_core(struct l2tp_tunnel *tunnel, struct sk_buff *skb)
{
        struct l2tp_session *session = NULL;
        unsigned char *ptr, *optr;
        u16 hdrflags;
        u32 tunnel_id, session_id;
        u16 version;
        int length;

        /* UDP has verified checksum */

        /* UDP always verifies the packet length. */
        __skb_pull(skb, sizeof(struct udphdr));

        /* Short packet? */
        if (!pskb_may_pull(skb, L2TP_HDR_SIZE_MAX)) {
                pr_debug_ratelimited("%s: recv short packet (len=%d)\n",
                                     tunnel->name, skb->len);
                goto invalid;
        }

        /* Point to L2TP header */
        optr = skb->data;
        ptr = skb->data;

        /* Get L2TP header flags */
        hdrflags = ntohs(*(__be16 *)ptr);

        /* Check protocol version */
        version = hdrflags & L2TP_HDR_VER_MASK;
        if (version != tunnel->version) {
                pr_debug_ratelimited("%s: recv protocol version mismatch: got %d expected %d\n",
                                     tunnel->name, version, tunnel->version);
                goto invalid;
        }

        /* Get length of L2TP packet */
        length = skb->len;

        /* If type is control packet, it is handled by userspace. */
        if (hdrflags & L2TP_HDRFLAG_T)
                goto pass;

        /* Skip flags */
        ptr += 2;

        if (tunnel->version == L2TP_HDR_VER_2) {
                /* If length is present, skip it */
                if (hdrflags & L2TP_HDRFLAG_L)
                        ptr += 2;

                /* Extract tunnel and session ID */
                tunnel_id = ntohs(*(__be16 *)ptr);
                ptr += 2;
                session_id = ntohs(*(__be16 *)ptr);
                ptr += 2;
        } else {
                ptr += 2;        /* skip reserved bits */
                tunnel_id = tunnel->tunnel_id;
                session_id = ntohl(*(__be32 *)ptr);
                ptr += 4;
        }

        /* Find the session context */
        session = l2tp_tunnel_get_session(tunnel, session_id);
        if (!session || !session->recv_skb) {
                if (session)
                        l2tp_session_dec_refcount(session);

                /* Not found? Pass to userspace to deal with */
                pr_debug_ratelimited("%s: no session found (%u/%u). Passing up.\n",
                                     tunnel->name, tunnel_id, session_id);
                goto pass;
        }

        if (tunnel->version == L2TP_HDR_VER_3 &&
            l2tp_v3_ensure_opt_in_linear(session, skb, &ptr, &optr)) {
                l2tp_session_dec_refcount(session);
                goto invalid;
        }

        l2tp_recv_common(session, skb, ptr, optr, hdrflags, length);
        l2tp_session_dec_refcount(session);

        return 0;

invalid:
        atomic_long_inc(&tunnel->stats.rx_invalid);

pass:
        /* Put UDP header back */
        __skb_push(skb, sizeof(struct udphdr));

        return 1;
}

/* UDP encapsulation receive handler. See net/ipv4/udp.c.
 * Return codes:
 * 0 : success.
 * <0: error
 * >0: skb should be passed up to userspace as UDP.
 */
int l2tp_udp_encap_recv(struct sock *sk, struct sk_buff *skb)
{
        struct l2tp_tunnel *tunnel;

        /* Note that this is called from the encap_rcv hook inside an
         * RCU-protected region, but without the socket being locked.
         * Hence we use rcu_dereference_sk_user_data to access the
         * tunnel data structure rather the usual l2tp_sk_to_tunnel
         * accessor function.
         */
        tunnel = rcu_dereference_sk_user_data(sk);
        if (!tunnel)
                goto pass_up;
        if (WARN_ON(tunnel->magic != L2TP_TUNNEL_MAGIC))
                goto pass_up;

        if (l2tp_udp_recv_core(tunnel, skb))
                goto pass_up;

        return 0;

pass_up:
        return 1;
}
EXPORT_SYMBOL_GPL(l2tp_udp_encap_recv);

/************************************************************************
 * Transmit handling
 ***********************************************************************/

/* Build an L2TP header for the session into the buffer provided.
 */
static int l2tp_build_l2tpv2_header(struct l2tp_session *session, void *buf)
{
        struct l2tp_tunnel *tunnel = session->tunnel;
        __be16 *bufp = buf;
        __be16 *optr = buf;
        u16 flags = L2TP_HDR_VER_2;
        u32 tunnel_id = tunnel->peer_tunnel_id;
        u32 session_id = session->peer_session_id;

        if (session->send_seq)
                flags |= L2TP_HDRFLAG_S;

        /* Setup L2TP header. */
        *bufp++ = htons(flags);
        *bufp++ = htons(tunnel_id);
        *bufp++ = htons(session_id);
        if (session->send_seq) {
                *bufp++ = htons(session->ns);
                *bufp++ = 0;
                session->ns++;
                session->ns &= 0xffff;
                trace_session_seqnum_update(session);
        }

        return bufp - optr;
}

static int l2tp_build_l2tpv3_header(struct l2tp_session *session, void *buf)
{
        struct l2tp_tunnel *tunnel = session->tunnel;
        char *bufp = buf;
        char *optr = bufp;

        /* Setup L2TP header. The header differs slightly for UDP and
         * IP encapsulations. For UDP, there is 4 bytes of flags.
         */
        if (tunnel->encap == L2TP_ENCAPTYPE_UDP) {
                u16 flags = L2TP_HDR_VER_3;
                *((__be16 *)bufp) = htons(flags);
                bufp += 2;
                *((__be16 *)bufp) = 0;
                bufp += 2;
        }

        *((__be32 *)bufp) = htonl(session->peer_session_id);
        bufp += 4;
        if (session->cookie_len) {
                memcpy(bufp, &session->cookie[0], session->cookie_len);
                bufp += session->cookie_len;
        }
        if (session->l2specific_type == L2TP_L2SPECTYPE_DEFAULT) {
                u32 l2h = 0;

                if (session->send_seq) {
                        l2h = 0x40000000 | session->ns;
                        session->ns++;
                        session->ns &= 0xffffff;
                        trace_session_seqnum_update(session);
                }

                *((__be32 *)bufp) = htonl(l2h);
                bufp += 4;
        }

        return bufp - optr;
}

/* Queue the packet to IP for output: tunnel socket lock must be held */
static int l2tp_xmit_queue(struct l2tp_tunnel *tunnel, struct sk_buff *skb, struct flowi *fl)
{
        int err;

        skb->ignore_df = 1;
        skb_dst_drop(skb);
#if IS_ENABLED(CONFIG_IPV6)
        if (l2tp_sk_is_v6(tunnel->sock))
                err = inet6_csk_xmit(tunnel->sock, skb, NULL);
        else
#endif
                err = ip_queue_xmit(tunnel->sock, skb, fl);

        return err >= 0 ? NET_XMIT_SUCCESS : NET_XMIT_DROP;
}

static int l2tp_xmit_core(struct l2tp_session *session, struct sk_buff *skb, unsigned int *len)
{
        struct l2tp_tunnel *tunnel = session->tunnel;
        unsigned int data_len = skb->len;
        struct sock *sk = tunnel->sock;
        int headroom, uhlen, udp_len;
        int ret = NET_XMIT_SUCCESS;
        struct inet_sock *inet;
        struct udphdr *uh;

        /* Check that there's enough headroom in the skb to insert IP,
         * UDP and L2TP headers. If not enough, expand it to
         * make room. Adjust truesize.
         */
        uhlen = (tunnel->encap == L2TP_ENCAPTYPE_UDP) ? sizeof(*uh) : 0;
        headroom = NET_SKB_PAD + sizeof(struct iphdr) + uhlen + session->hdr_len;
        if (skb_cow_head(skb, headroom)) {
                kfree_skb(skb);
                return NET_XMIT_DROP;
        }

        /* Setup L2TP header */
        if (tunnel->version == L2TP_HDR_VER_2)
                l2tp_build_l2tpv2_header(session, __skb_push(skb, session->hdr_len));
        else
                l2tp_build_l2tpv3_header(session, __skb_push(skb, session->hdr_len));

        /* Reset skb netfilter state */
        memset(&(IPCB(skb)->opt), 0, sizeof(IPCB(skb)->opt));
        IPCB(skb)->flags &= ~(IPSKB_XFRM_TUNNEL_SIZE | IPSKB_XFRM_TRANSFORMED | IPSKB_REROUTED);
        nf_reset_ct(skb);

        /* L2TP uses its own lockdep subclass to avoid lockdep splats caused by
         * nested socket calls on the same lockdep socket class. This can
         * happen when data from a user socket is routed over l2tp, which uses
         * another userspace socket.
         */
        spin_lock_nested(&sk->sk_lock.slock, L2TP_DEPTH_NESTING);

        if (sock_owned_by_user(sk)) {
                kfree_skb(skb);
                ret = NET_XMIT_DROP;
                goto out_unlock;
        }

        /* The user-space may change the connection status for the user-space
         * provided socket at run time: we must check it under the socket lock
         */
        if (tunnel->fd >= 0 && sk->sk_state != TCP_ESTABLISHED) {
                kfree_skb(skb);
                ret = NET_XMIT_DROP;
                goto out_unlock;
        }

        /* Report transmitted length before we add encap header, which keeps
         * statistics consistent for both UDP and IP encap tx/rx paths.
         */
        *len = skb->len;

        inet = inet_sk(sk);
        switch (tunnel->encap) {
        case L2TP_ENCAPTYPE_UDP:
                /* Setup UDP header */
                __skb_push(skb, sizeof(*uh));
                skb_reset_transport_header(skb);
                uh = udp_hdr(skb);
                uh->source = inet->inet_sport;
                uh->dest = inet->inet_dport;
                udp_len = uhlen + session->hdr_len + data_len;
                uh->len = htons(udp_len);

                /* Calculate UDP checksum if configured to do so */
#if IS_ENABLED(CONFIG_IPV6)
                if (l2tp_sk_is_v6(sk))
                        udp6_set_csum(udp_get_no_check6_tx(sk),
                                      skb, &inet6_sk(sk)->saddr,
                                      &sk->sk_v6_daddr, udp_len);
                else
#endif
                        udp_set_csum(sk->sk_no_check_tx, skb, inet->inet_saddr,
                                     inet->inet_daddr, udp_len);
                break;

        case L2TP_ENCAPTYPE_IP:
                break;
        }

        ret = l2tp_xmit_queue(tunnel, skb, &inet->cork.fl);

out_unlock:
        spin_unlock(&sk->sk_lock.slock);

        return ret;
}

/* If caller requires the skb to have a ppp header, the header must be
 * inserted in the skb data before calling this function.
 */
int l2tp_xmit_skb(struct l2tp_session *session, struct sk_buff *skb)
{
        unsigned int len = 0;
        int ret;

        ret = l2tp_xmit_core(session, skb, &len);
        if (ret == NET_XMIT_SUCCESS) {
                atomic_long_inc(&session->tunnel->stats.tx_packets);
                atomic_long_add(len, &session->tunnel->stats.tx_bytes);
                atomic_long_inc(&session->stats.tx_packets);
                atomic_long_add(len, &session->stats.tx_bytes);
        } else {
                atomic_long_inc(&session->tunnel->stats.tx_errors);
                atomic_long_inc(&session->stats.tx_errors);
        }
        return ret;
}
EXPORT_SYMBOL_GPL(l2tp_xmit_skb);

/*****************************************************************************
 * Tinnel and session create/destroy.
 *****************************************************************************/

/* Tunnel socket destruct hook.
 * The tunnel context is deleted only when all session sockets have been
 * closed.
 */
static void l2tp_tunnel_destruct(struct sock *sk)
{
        struct l2tp_tunnel *tunnel = l2tp_sk_to_tunnel(sk);

        if (!tunnel)
                goto end;

        /* Disable udp encapsulation */
        switch (tunnel->encap) {
        case L2TP_ENCAPTYPE_UDP:
                /* No longer an encapsulation socket. See net/ipv4/udp.c */
                (udp_sk(sk))->encap_type = 0;
                (udp_sk(sk))->encap_rcv = NULL;
                (udp_sk(sk))->encap_destroy = NULL;
                break;
        case L2TP_ENCAPTYPE_IP:
                break;
        }

        /* Remove hooks into tunnel socket */
        write_lock_bh(&sk->sk_callback_lock);
        sk->sk_destruct = tunnel->old_sk_destruct;
        sk->sk_user_data = NULL;
        write_unlock_bh(&sk->sk_callback_lock);

        /* Call the original destructor */
        if (sk->sk_destruct)
                (*sk->sk_destruct)(sk);

        kfree_rcu(tunnel, rcu);
end:
        return;
}

/* Remove an l2tp session from l2tp_core's hash lists. */
static void l2tp_session_unhash(struct l2tp_session *session)
{
        struct l2tp_tunnel *tunnel = session->tunnel;

        /* Remove the session from core hashes */
        if (tunnel) {
                /* Remove from the per-tunnel hash */
                write_lock_bh(&tunnel->hlist_lock);
                hlist_del_init(&session->hlist);
                write_unlock_bh(&tunnel->hlist_lock);

                /* For L2TPv3 we have a per-net hash: remove from there, too */
                if (tunnel->version != L2TP_HDR_VER_2) {
                        struct l2tp_net *pn = l2tp_pernet(tunnel->l2tp_net);

                        spin_lock_bh(&pn->l2tp_session_hlist_lock);
                        hlist_del_init_rcu(&session->global_hlist);
                        spin_unlock_bh(&pn->l2tp_session_hlist_lock);
                        synchronize_rcu();
                }
        }
}

/* When the tunnel is closed, all the attached sessions need to go too.
 */
static void l2tp_tunnel_closeall(struct l2tp_tunnel *tunnel)
{
        int hash;
        struct hlist_node *walk;
        struct hlist_node *tmp;
        struct l2tp_session *session;

        write_lock_bh(&tunnel->hlist_lock);
        tunnel->acpt_newsess = false;
        for (hash = 0; hash < L2TP_HASH_SIZE; hash++) {
again:
                hlist_for_each_safe(walk, tmp, &tunnel->session_hlist[hash]) {
                        session = hlist_entry(walk, struct l2tp_session, hlist);
                        hlist_del_init(&session->hlist);

                        write_unlock_bh(&tunnel->hlist_lock);
                        l2tp_session_delete(session);
                        write_lock_bh(&tunnel->hlist_lock);

                        /* Now restart from the beginning of this hash
                         * chain.  We always remove a session from the
                         * list so we are guaranteed to make forward
                         * progress.
                         */
                        goto again;
                }
        }
        write_unlock_bh(&tunnel->hlist_lock);
}

/* Tunnel socket destroy hook for UDP encapsulation */
static void l2tp_udp_encap_destroy(struct sock *sk)
{
        struct l2tp_tunnel *tunnel = l2tp_sk_to_tunnel(sk);

        if (tunnel)
                l2tp_tunnel_delete(tunnel);
}

static void l2tp_tunnel_remove(struct net *net, struct l2tp_tunnel *tunnel)
{
        struct l2tp_net *pn = l2tp_pernet(net);

        spin_lock_bh(&pn->l2tp_tunnel_idr_lock);
        idr_remove(&pn->l2tp_tunnel_idr, tunnel->tunnel_id);
        spin_unlock_bh(&pn->l2tp_tunnel_idr_lock);
}

/* Workqueue tunnel deletion function */
static void l2tp_tunnel_del_work(struct work_struct *work)
{
        struct l2tp_tunnel *tunnel = container_of(work, struct l2tp_tunnel,
                                                  del_work);
        struct sock *sk = tunnel->sock;
        struct socket *sock = sk->sk_socket;

        l2tp_tunnel_closeall(tunnel);

        /* If the tunnel socket was created within the kernel, use
         * the sk API to release it here.
         */
        if (tunnel->fd < 0) {
                if (sock) {
                        kernel_sock_shutdown(sock, SHUT_RDWR);
                        sock_release(sock);
                }
        }

        l2tp_tunnel_remove(tunnel->l2tp_net, tunnel);
        /* drop initial ref */
        l2tp_tunnel_dec_refcount(tunnel);

        /* drop workqueue ref */
        l2tp_tunnel_dec_refcount(tunnel);
}

/* Create a socket for the tunnel, if one isn't set up by
 * userspace. This is used for static tunnels where there is no
 * managing L2TP daemon.
 *
 * Since we don't want these sockets to keep a namespace alive by
 * themselves, we drop the socket's namespace refcount after creation.
 * These sockets are freed when the namespace exits using the pernet
 * exit hook.
 */
static int l2tp_tunnel_sock_create(struct net *net,
                                   u32 tunnel_id,
                                   u32 peer_tunnel_id,
                                   struct l2tp_tunnel_cfg *cfg,
                                   struct socket **sockp)
{
        int err = -EINVAL;
        struct socket *sock = NULL;
        struct udp_port_cfg udp_conf;

        switch (cfg->encap) {
        case L2TP_ENCAPTYPE_UDP:
                memset(&udp_conf, 0, sizeof(udp_conf));

#if IS_ENABLED(CONFIG_IPV6)
                if (cfg->local_ip6 && cfg->peer_ip6) {
                        udp_conf.family = AF_INET6;
                        memcpy(&udp_conf.local_ip6, cfg->local_ip6,
                               sizeof(udp_conf.local_ip6));
                        memcpy(&udp_conf.peer_ip6, cfg->peer_ip6,
                               sizeof(udp_conf.peer_ip6));
                        udp_conf.use_udp6_tx_checksums =
                          !cfg->udp6_zero_tx_checksums;
                        udp_conf.use_udp6_rx_checksums =
                          !cfg->udp6_zero_rx_checksums;
                } else
#endif
                {
                        udp_conf.family = AF_INET;
                        udp_conf.local_ip = cfg->local_ip;
                        udp_conf.peer_ip = cfg->peer_ip;
                        udp_conf.use_udp_checksums = cfg->use_udp_checksums;
                }

                udp_conf.local_udp_port = htons(cfg->local_udp_port);
                udp_conf.peer_udp_port = htons(cfg->peer_udp_port);

                err = udp_sock_create(net, &udp_conf, &sock);
                if (err < 0)
                        goto out;

                break;

        case L2TP_ENCAPTYPE_IP:
#if IS_ENABLED(CONFIG_IPV6)
                if (cfg->local_ip6 && cfg->peer_ip6) {
                        struct sockaddr_l2tpip6 ip6_addr = {0};

                        err = sock_create_kern(net, AF_INET6, SOCK_DGRAM,
                                               IPPROTO_L2TP, &sock);
                        if (err < 0)
                                goto out;

                        ip6_addr.l2tp_family = AF_INET6;
                        memcpy(&ip6_addr.l2tp_addr, cfg->local_ip6,
                               sizeof(ip6_addr.l2tp_addr));
                        ip6_addr.l2tp_conn_id = tunnel_id;
                        err = kernel_bind(sock, (struct sockaddr *)&ip6_addr,
                                          sizeof(ip6_addr));
                        if (err < 0)
                                goto out;

                        ip6_addr.l2tp_family = AF_INET6;
                        memcpy(&ip6_addr.l2tp_addr, cfg->peer_ip6,
                               sizeof(ip6_addr.l2tp_addr));
                        ip6_addr.l2tp_conn_id = peer_tunnel_id;
                        err = kernel_connect(sock,
                                             (struct sockaddr *)&ip6_addr,
                                             sizeof(ip6_addr), 0);
                        if (err < 0)
                                goto out;
                } else
#endif
                {
                        struct sockaddr_l2tpip ip_addr = {0};

                        err = sock_create_kern(net, AF_INET, SOCK_DGRAM,
                                               IPPROTO_L2TP, &sock);
                        if (err < 0)
                                goto out;

                        ip_addr.l2tp_family = AF_INET;
                        ip_addr.l2tp_addr = cfg->local_ip;
                        ip_addr.l2tp_conn_id = tunnel_id;
                        err = kernel_bind(sock, (struct sockaddr *)&ip_addr,
                                          sizeof(ip_addr));
                        if (err < 0)
                                goto out;

                        ip_addr.l2tp_family = AF_INET;
                        ip_addr.l2tp_addr = cfg->peer_ip;
                        ip_addr.l2tp_conn_id = peer_tunnel_id;
                        err = kernel_connect(sock, (struct sockaddr *)&ip_addr,
                                             sizeof(ip_addr), 0);
                        if (err < 0)
                                goto out;
                }
                break;

        default:
                goto out;
        }

out:
        *sockp = sock;
        if (err < 0 && sock) {
                kernel_sock_shutdown(sock, SHUT_RDWR);
                sock_release(sock);
                *sockp = NULL;
        }

        return err;
}

int l2tp_tunnel_create(int fd, int version, u32 tunnel_id, u32 peer_tunnel_id,
                       struct l2tp_tunnel_cfg *cfg, struct l2tp_tunnel **tunnelp)
{
        struct l2tp_tunnel *tunnel = NULL;
        int err;
        enum l2tp_encap_type encap = L2TP_ENCAPTYPE_UDP;

        if (cfg)
                encap = cfg->encap;

        tunnel = kzalloc(sizeof(*tunnel), GFP_KERNEL);
        if (!tunnel) {
                err = -ENOMEM;
                goto err;
        }

        tunnel->version = version;
        tunnel->tunnel_id = tunnel_id;
        tunnel->peer_tunnel_id = peer_tunnel_id;

        tunnel->magic = L2TP_TUNNEL_MAGIC;
        sprintf(&tunnel->name[0], "tunl %u", tunnel_id);
        rwlock_init(&tunnel->hlist_lock);
        tunnel->acpt_newsess = true;

        tunnel->encap = encap;

        refcount_set(&tunnel->ref_count, 1);
        tunnel->fd = fd;

        /* Init delete workqueue struct */
        INIT_WORK(&tunnel->del_work, l2tp_tunnel_del_work);

        INIT_LIST_HEAD(&tunnel->list);

        err = 0;
err:
        if (tunnelp)
                *tunnelp = tunnel;

        return err;
}
EXPORT_SYMBOL_GPL(l2tp_tunnel_create);

static int l2tp_validate_socket(const struct sock *sk, const struct net *net,
                                enum l2tp_encap_type encap)
{
        if (!net_eq(sock_net(sk), net))
                return -EINVAL;

        if (sk->sk_type != SOCK_DGRAM)
                return -EPROTONOSUPPORT;

        if (sk->sk_family != PF_INET && sk->sk_family != PF_INET6)
                return -EPROTONOSUPPORT;

        if ((encap == L2TP_ENCAPTYPE_UDP && sk->sk_protocol != IPPROTO_UDP) ||
            (encap == L2TP_ENCAPTYPE_IP && sk->sk_protocol != IPPROTO_L2TP))
                return -EPROTONOSUPPORT;

        if (sk->sk_user_data)
                return -EBUSY;

        return 0;
}

int l2tp_tunnel_register(struct l2tp_tunnel *tunnel, struct net *net,
                         struct l2tp_tunnel_cfg *cfg)
{
        struct l2tp_net *pn = l2tp_pernet(net);
        u32 tunnel_id = tunnel->tunnel_id;
        struct socket *sock;
        struct sock *sk;
        int ret;

        spin_lock_bh(&pn->l2tp_tunnel_idr_lock);
        ret = idr_alloc_u32(&pn->l2tp_tunnel_idr, NULL, &tunnel_id, tunnel_id,
                            GFP_ATOMIC);
        spin_unlock_bh(&pn->l2tp_tunnel_idr_lock);
        if (ret)
                return ret == -ENOSPC ? -EEXIST : ret;

        if (tunnel->fd < 0) {
                ret = l2tp_tunnel_sock_create(net, tunnel->tunnel_id,
                                              tunnel->peer_tunnel_id, cfg,
                                              &sock);
                if (ret < 0)
                        goto err;
        } else {
                sock = sockfd_lookup(tunnel->fd, &ret);
                if (!sock)
                        goto err;
        }

        sk = sock->sk;
        lock_sock(sk);
        write_lock_bh(&sk->sk_callback_lock);
        ret = l2tp_validate_socket(sk, net, tunnel->encap);
        if (ret < 0)
                goto err_inval_sock;
        rcu_assign_sk_user_data(sk, tunnel);
        write_unlock_bh(&sk->sk_callback_lock);

        if (tunnel->encap == L2TP_ENCAPTYPE_UDP) {
                struct udp_tunnel_sock_cfg udp_cfg = {
                        .sk_user_data = tunnel,
                        .encap_type = UDP_ENCAP_L2TPINUDP,
                        .encap_rcv = l2tp_udp_encap_recv,
                        .encap_destroy = l2tp_udp_encap_destroy,
                };

                setup_udp_tunnel_sock(net, sock, &udp_cfg);
        }

        tunnel->old_sk_destruct = sk->sk_destruct;
        sk->sk_destruct = &l2tp_tunnel_destruct;
        sk->sk_allocation = GFP_ATOMIC;
        release_sock(sk);

        sock_hold(sk);
        tunnel->sock = sk;
        tunnel->l2tp_net = net;

        spin_lock_bh(&pn->l2tp_tunnel_idr_lock);
        idr_replace(&pn->l2tp_tunnel_idr, tunnel, tunnel->tunnel_id);
        spin_unlock_bh(&pn->l2tp_tunnel_idr_lock);

        trace_register_tunnel(tunnel);

        if (tunnel->fd >= 0)
                sockfd_put(sock);

        return 0;

err_inval_sock:
        write_unlock_bh(&sk->sk_callback_lock);
        release_sock(sk);

        if (tunnel->fd < 0)
                sock_release(sock);
        else
                sockfd_put(sock);
err:
        l2tp_tunnel_remove(net, tunnel);
        return ret;
}
EXPORT_SYMBOL_GPL(l2tp_tunnel_register);

/* This function is used by the netlink TUNNEL_DELETE command.
 */
void l2tp_tunnel_delete(struct l2tp_tunnel *tunnel)
{
        if (!test_and_set_bit(0, &tunnel->dead)) {
                trace_delete_tunnel(tunnel);
                l2tp_tunnel_inc_refcount(tunnel);
                queue_work(l2tp_wq, &tunnel->del_work);
        }
}
EXPORT_SYMBOL_GPL(l2tp_tunnel_delete);

void l2tp_session_delete(struct l2tp_session *session)
{
        if (test_and_set_bit(0, &session->dead))
                return;

        trace_delete_session(session);
        l2tp_session_unhash(session);
        l2tp_session_queue_purge(session);
        if (session->session_close)
                (*session->session_close)(session);

        l2tp_session_dec_refcount(session);
}
EXPORT_SYMBOL_GPL(l2tp_session_delete);

/* We come here whenever a session's send_seq, cookie_len or
 * l2specific_type parameters are set.
 */
void l2tp_session_set_header_len(struct l2tp_session *session, int version)
{
        if (version == L2TP_HDR_VER_2) {
                session->hdr_len = 6;
                if (session->send_seq)
                        session->hdr_len += 4;
        } else {
                session->hdr_len = 4 + session->cookie_len;
                session->hdr_len += l2tp_get_l2specific_len(session);
                if (session->tunnel->encap == L2TP_ENCAPTYPE_UDP)
                        session->hdr_len += 4;
        }
}
EXPORT_SYMBOL_GPL(l2tp_session_set_header_len);

struct l2tp_session *l2tp_session_create(int priv_size, struct l2tp_tunnel *tunnel, u32 session_id,
                                         u32 peer_session_id, struct l2tp_session_cfg *cfg)
{
        struct l2tp_session *session;

        session = kzalloc(sizeof(*session) + priv_size, GFP_KERNEL);
        if (session) {
                session->magic = L2TP_SESSION_MAGIC;
                session->tunnel = tunnel;

                session->session_id = session_id;
                session->peer_session_id = peer_session_id;
                session->nr = 0;
                if (tunnel->version == L2TP_HDR_VER_2)
                        session->nr_max = 0xffff;
                else
                        session->nr_max = 0xffffff;
                session->nr_window_size = session->nr_max / 2;
                session->nr_oos_count_max = 4;

                /* Use NR of first received packet */
                session->reorder_skip = 1;

                sprintf(&session->name[0], "sess %u/%u",
                        tunnel->tunnel_id, session->session_id);

                skb_queue_head_init(&session->reorder_q);

                INIT_HLIST_NODE(&session->hlist);
                INIT_HLIST_NODE(&session->global_hlist);

                if (cfg) {
                        session->pwtype = cfg->pw_type;
                        session->send_seq = cfg->send_seq;
                        session->recv_seq = cfg->recv_seq;
                        session->lns_mode = cfg->lns_mode;
                        session->reorder_timeout = cfg->reorder_timeout;
                        session->l2specific_type = cfg->l2specific_type;
                        session->cookie_len = cfg->cookie_len;
                        memcpy(&session->cookie[0], &cfg->cookie[0], cfg->cookie_len);
                        session->peer_cookie_len = cfg->peer_cookie_len;
                        memcpy(&session->peer_cookie[0], &cfg->peer_cookie[0], cfg->peer_cookie_len);
                }

                l2tp_session_set_header_len(session, tunnel->version);

                refcount_set(&session->ref_count, 1);

                return session;
        }

        return ERR_PTR(-ENOMEM);
}
EXPORT_SYMBOL_GPL(l2tp_session_create);

/*****************************************************************************
 * Init and cleanup
 *****************************************************************************/

static __net_init int l2tp_init_net(struct net *net)
{
        struct l2tp_net *pn = net_generic(net, l2tp_net_id);
        int hash;

        idr_init(&pn->l2tp_tunnel_idr);
        spin_lock_init(&pn->l2tp_tunnel_idr_lock);

        for (hash = 0; hash < L2TP_HASH_SIZE_2; hash++)
                INIT_HLIST_HEAD(&pn->l2tp_session_hlist[hash]);

        spin_lock_init(&pn->l2tp_session_hlist_lock);

        return 0;
}

static __net_exit void l2tp_exit_net(struct net *net)
{
        struct l2tp_net *pn = l2tp_pernet(net);
        struct l2tp_tunnel *tunnel = NULL;
        unsigned long tunnel_id, tmp;
        int hash;

        rcu_read_lock_bh();
        idr_for_each_entry_ul(&pn->l2tp_tunnel_idr, tunnel, tmp, tunnel_id) {
                if (tunnel)
                        l2tp_tunnel_delete(tunnel);
        }
        rcu_read_unlock_bh();

        if (l2tp_wq)
                flush_workqueue(l2tp_wq);
        rcu_barrier();

        for (hash = 0; hash < L2TP_HASH_SIZE_2; hash++)
                WARN_ON_ONCE(!hlist_empty(&pn->l2tp_session_hlist[hash]));
        idr_destroy(&pn->l2tp_tunnel_idr);
}

static struct pernet_operations l2tp_net_ops = {
        .init = l2tp_init_net,
        .exit = l2tp_exit_net,
        .id   = &l2tp_net_id,
        .size = sizeof(struct l2tp_net),
};

static int __init l2tp_init(void)
{
        int rc = 0;

        rc = register_pernet_device(&l2tp_net_ops);
        if (rc)
                goto out;

        l2tp_wq = alloc_workqueue("l2tp", WQ_UNBOUND, 0);
        if (!l2tp_wq) {
                pr_err("alloc_workqueue failed\n");
                unregister_pernet_device(&l2tp_net_ops);
                rc = -ENOMEM;
                goto out;
        }

        pr_info("L2TP core driver, %s\n", L2TP_DRV_VERSION);

out:
        return rc;
}

static void __exit l2tp_exit(void)
{
        unregister_pernet_device(&l2tp_net_ops);
        if (l2tp_wq) {
                destroy_workqueue(l2tp_wq);
                l2tp_wq = NULL;
        }
}

module_init(l2tp_init);
module_exit(l2tp_exit);

MODULE_AUTHOR("James Chapman <jchapman@katalix.com>");
MODULE_DESCRIPTION("L2TP core");
MODULE_LICENSE("GPL");
MODULE_VERSION(L2TP_DRV_VERSION);

































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_GENERIC_BITOPS_LE_H_
#define _ASM_GENERIC_BITOPS_LE_H_

#include <asm-generic/bitops/find.h>
#include <asm/types.h>
#include <asm/byteorder.h>
#include <linux/swab.h>

#if defined(__LITTLE_ENDIAN)

#define BITOP_LE_SWIZZLE        0

static inline unsigned long find_next_zero_bit_le(const void *addr,
                unsigned long size, unsigned long offset)
{
        return find_next_zero_bit(addr, size, offset);
}

static inline unsigned long find_next_bit_le(const void *addr,
                unsigned long size, unsigned long offset)
{
        return find_next_bit(addr, size, offset);
}

static inline unsigned long find_first_zero_bit_le(const void *addr,
                unsigned long size)
{
        return find_first_zero_bit(addr, size);
}

#elif defined(__BIG_ENDIAN)

#define BITOP_LE_SWIZZLE        ((BITS_PER_LONG-1) & ~0x7)

#ifndef find_next_zero_bit_le
static inline
unsigned long find_next_zero_bit_le(const void *addr, unsigned
                long size, unsigned long offset)
{
        if (small_const_nbits(size)) {
                unsigned long val = *(const unsigned long *)addr;

                if (unlikely(offset >= size))
                        return size;

                val = swab(val) | ~GENMASK(size - 1, offset);
                return val == ~0UL ? size : ffz(val);
        }

        return _find_next_bit(addr, NULL, size, offset, ~0UL, 1);
}
#endif

#ifndef find_next_bit_le
static inline
unsigned long find_next_bit_le(const void *addr, unsigned
                long size, unsigned long offset)
{
        if (small_const_nbits(size)) {
                unsigned long val = *(const unsigned long *)addr;

                if (unlikely(offset >= size))
                        return size;

                val = swab(val) & GENMASK(size - 1, offset);
                return val ? __ffs(val) : size;
        }

        return _find_next_bit(addr, NULL, size, offset, 0UL, 1);
}
#endif

#ifndef find_first_zero_bit_le
#define find_first_zero_bit_le(addr, size) \
        find_next_zero_bit_le((addr), (size), 0)
#endif

#else
#error "Please fix <asm/byteorder.h>"
#endif

static inline int test_bit_le(int nr, const void *addr)
{
        return test_bit(nr ^ BITOP_LE_SWIZZLE, addr);
}

static inline void set_bit_le(int nr, void *addr)
{
        set_bit(nr ^ BITOP_LE_SWIZZLE, addr);
}

static inline void clear_bit_le(int nr, void *addr)
{
        clear_bit(nr ^ BITOP_LE_SWIZZLE, addr);
}

static inline void __set_bit_le(int nr, void *addr)
{
        __set_bit(nr ^ BITOP_LE_SWIZZLE, addr);
}

static inline void __clear_bit_le(int nr, void *addr)
{
        __clear_bit(nr ^ BITOP_LE_SWIZZLE, addr);
}

static inline int test_and_set_bit_le(int nr, void *addr)
{
        return test_and_set_bit(nr ^ BITOP_LE_SWIZZLE, addr);
}

static inline int test_and_clear_bit_le(int nr, void *addr)
{
        return test_and_clear_bit(nr ^ BITOP_LE_SWIZZLE, addr);
}

static inline int __test_and_set_bit_le(int nr, void *addr)
{
        return __test_and_set_bit(nr ^ BITOP_LE_SWIZZLE, addr);
}

static inline int __test_and_clear_bit_le(int nr, void *addr)
{
        return __test_and_clear_bit(nr ^ BITOP_LE_SWIZZLE, addr);
}

#endif /* _ASM_GENERIC_BITOPS_LE_H_ */



























































   95 


   68 



















































































































    3 







   93 
   75 

















































































































































    7 


    7 





   11 


   11 





































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __NET_UDP_TUNNEL_H
#define __NET_UDP_TUNNEL_H

#include <net/ip_tunnels.h>
#include <net/udp.h>

#if IS_ENABLED(CONFIG_IPV6)
#include <net/ipv6.h>
#include <net/ipv6_stubs.h>
#endif

struct udp_port_cfg {
        u8                        family;

        /* Used only for kernel-created sockets */
        union {
                struct in_addr                local_ip;
#if IS_ENABLED(CONFIG_IPV6)
                struct in6_addr                local_ip6;
#endif
        };

        union {
                struct in_addr                peer_ip;
#if IS_ENABLED(CONFIG_IPV6)
                struct in6_addr                peer_ip6;
#endif
        };

        __be16                        local_udp_port;
        __be16                        peer_udp_port;
        int                        bind_ifindex;
        unsigned int                use_udp_checksums:1,
                                use_udp6_tx_checksums:1,
                                use_udp6_rx_checksums:1,
                                ipv6_v6only:1;
};

int udp_sock_create4(struct net *net, struct udp_port_cfg *cfg,
                     struct socket **sockp);

#if IS_ENABLED(CONFIG_IPV6)
int udp_sock_create6(struct net *net, struct udp_port_cfg *cfg,
                     struct socket **sockp);
#else
static inline int udp_sock_create6(struct net *net, struct udp_port_cfg *cfg,
                                   struct socket **sockp)
{
        return 0;
}
#endif

static inline int udp_sock_create(struct net *net,
                                  struct udp_port_cfg *cfg,
                                  struct socket **sockp)
{
        if (cfg->family == AF_INET)
                return udp_sock_create4(net, cfg, sockp);

        if (cfg->family == AF_INET6)
                return udp_sock_create6(net, cfg, sockp);

        return -EPFNOSUPPORT;
}

typedef int (*udp_tunnel_encap_rcv_t)(struct sock *sk, struct sk_buff *skb);
typedef int (*udp_tunnel_encap_err_lookup_t)(struct sock *sk,
                                             struct sk_buff *skb);
typedef void (*udp_tunnel_encap_err_rcv_t)(struct sock *sk,
                                           struct sk_buff *skb,
                                           unsigned int udp_offset);
typedef void (*udp_tunnel_encap_destroy_t)(struct sock *sk);
typedef struct sk_buff *(*udp_tunnel_gro_receive_t)(struct sock *sk,
                                                    struct list_head *head,
                                                    struct sk_buff *skb);
typedef int (*udp_tunnel_gro_complete_t)(struct sock *sk, struct sk_buff *skb,
                                         int nhoff);

struct udp_tunnel_sock_cfg {
        void *sk_user_data;     /* user data used by encap_rcv call back */
        /* Used for setting up udp_sock fields, see udp.h for details */
        __u8  encap_type;
        udp_tunnel_encap_rcv_t encap_rcv;
        udp_tunnel_encap_err_lookup_t encap_err_lookup;
        udp_tunnel_encap_err_rcv_t encap_err_rcv;
        udp_tunnel_encap_destroy_t encap_destroy;
        udp_tunnel_gro_receive_t gro_receive;
        udp_tunnel_gro_complete_t gro_complete;
};

/* Setup the given (UDP) sock to receive UDP encapsulated packets */
void setup_udp_tunnel_sock(struct net *net, struct socket *sock,
                           struct udp_tunnel_sock_cfg *sock_cfg);

/* -- List of parsable UDP tunnel types --
 *
 * Adding to this list will result in serious debate.  The main issue is
 * that this list is essentially a list of workarounds for either poorly
 * designed tunnels, or poorly designed device offloads.
 *
 * The parsing supported via these types should really be used for Rx
 * traffic only as the network stack will have already inserted offsets for
 * the location of the headers in the skb.  In addition any ports that are
 * pushed should be kept within the namespace without leaking to other
 * devices such as VFs or other ports on the same device.
 *
 * It is strongly encouraged to use CHECKSUM_COMPLETE for Rx to avoid the
 * need to use this for Rx checksum offload.  It should not be necessary to
 * call this function to perform Tx offloads on outgoing traffic.
 */
enum udp_parsable_tunnel_type {
        UDP_TUNNEL_TYPE_VXLAN          = BIT(0), /* RFC 7348 */
        UDP_TUNNEL_TYPE_GENEVE          = BIT(1), /* draft-ietf-nvo3-geneve */
        UDP_TUNNEL_TYPE_VXLAN_GPE = BIT(2), /* draft-ietf-nvo3-vxlan-gpe */
};

struct udp_tunnel_info {
        unsigned short type;
        sa_family_t sa_family;
        __be16 port;
        u8 hw_priv;
};

/* Notify network devices of offloadable types */
void udp_tunnel_push_rx_port(struct net_device *dev, struct socket *sock,
                             unsigned short type);
void udp_tunnel_drop_rx_port(struct net_device *dev, struct socket *sock,
                             unsigned short type);
void udp_tunnel_notify_add_rx_port(struct socket *sock, unsigned short type);
void udp_tunnel_notify_del_rx_port(struct socket *sock, unsigned short type);

static inline void udp_tunnel_get_rx_info(struct net_device *dev)
{
        ASSERT_RTNL();
        if (!(dev->features & NETIF_F_RX_UDP_TUNNEL_PORT))
                return;
        call_netdevice_notifiers(NETDEV_UDP_TUNNEL_PUSH_INFO, dev);
}

static inline void udp_tunnel_drop_rx_info(struct net_device *dev)
{
        ASSERT_RTNL();
        if (!(dev->features & NETIF_F_RX_UDP_TUNNEL_PORT))
                return;
        call_netdevice_notifiers(NETDEV_UDP_TUNNEL_DROP_INFO, dev);
}

/* Transmit the skb using UDP encapsulation. */
void udp_tunnel_xmit_skb(struct rtable *rt, struct sock *sk, struct sk_buff *skb,
                         __be32 src, __be32 dst, __u8 tos, __u8 ttl,
                         __be16 df, __be16 src_port, __be16 dst_port,
                         bool xnet, bool nocheck);

int udp_tunnel6_xmit_skb(struct dst_entry *dst, struct sock *sk,
                         struct sk_buff *skb,
                         struct net_device *dev, struct in6_addr *saddr,
                         struct in6_addr *daddr,
                         __u8 prio, __u8 ttl, __be32 label,
                         __be16 src_port, __be16 dst_port, bool nocheck);

void udp_tunnel_sock_release(struct socket *sock);

struct metadata_dst *udp_tun_rx_dst(struct sk_buff *skb, unsigned short family,
                                    __be16 flags, __be64 tunnel_id,
                                    int md_size);

#ifdef CONFIG_INET
static inline int udp_tunnel_handle_offloads(struct sk_buff *skb, bool udp_csum)
{
        int type = udp_csum ? SKB_GSO_UDP_TUNNEL_CSUM : SKB_GSO_UDP_TUNNEL;

        return iptunnel_handle_offloads(skb, type);
}
#endif

static inline void udp_tunnel_encap_enable(struct socket *sock)
{
        struct udp_sock *up = udp_sk(sock->sk);

        if (up->encap_enabled)
                return;

        up->encap_enabled = 1;
#if IS_ENABLED(CONFIG_IPV6)
        if (sock->sk->sk_family == PF_INET6)
                ipv6_stub->udpv6_encap_enable();
#endif
        udp_encap_enable();
}

#define UDP_TUNNEL_NIC_MAX_TABLES        4

enum udp_tunnel_nic_info_flags {
        /* Device callbacks may sleep */
        UDP_TUNNEL_NIC_INFO_MAY_SLEEP        = BIT(0),
        /* Device only supports offloads when it's open, all ports
         * will be removed before close and re-added after open.
         */
        UDP_TUNNEL_NIC_INFO_OPEN_ONLY        = BIT(1),
        /* Device supports only IPv4 tunnels */
        UDP_TUNNEL_NIC_INFO_IPV4_ONLY        = BIT(2),
        /* Device has hard-coded the IANA VXLAN port (4789) as VXLAN.
         * This port must not be counted towards n_entries of any table.
         * Driver will not receive any callback associated with port 4789.
         */
        UDP_TUNNEL_NIC_INFO_STATIC_IANA_VXLAN        = BIT(3),
};

struct udp_tunnel_nic;

#define UDP_TUNNEL_NIC_MAX_SHARING_DEVICES        (U16_MAX / 2)

struct udp_tunnel_nic_shared {
        struct udp_tunnel_nic *udp_tunnel_nic_info;

        struct list_head devices;
};

struct udp_tunnel_nic_shared_node {
        struct net_device *dev;
        struct list_head list;
};

/**
 * struct udp_tunnel_nic_info - driver UDP tunnel offload information
 * @set_port:        callback for adding a new port
 * @unset_port:        callback for removing a port
 * @sync_table:        callback for syncing the entire port table at once
 * @shared:        reference to device global state (optional)
 * @flags:        device flags from enum udp_tunnel_nic_info_flags
 * @tables:        UDP port tables this device has
 * @tables.n_entries:                number of entries in this table
 * @tables.tunnel_types:        types of tunnels this table accepts
 *
 * Drivers are expected to provide either @set_port and @unset_port callbacks
 * or the @sync_table callback. Callbacks are invoked with rtnl lock held.
 *
 * Devices which (misguidedly) share the UDP tunnel port table across multiple
 * netdevs should allocate an instance of struct udp_tunnel_nic_shared and
 * point @shared at it.
 * There must never be more than %UDP_TUNNEL_NIC_MAX_SHARING_DEVICES devices
 * sharing a table.
 *
 * Known limitations:
 *  - UDP tunnel port notifications are fundamentally best-effort -
 *    it is likely the driver will both see skbs which use a UDP tunnel port,
 *    while not being a tunneled skb, and tunnel skbs from other ports -
 *    drivers should only use these ports for non-critical RX-side offloads,
 *    e.g. the checksum offload;
 *  - none of the devices care about the socket family at present, so we don't
 *    track it. Please extend this code if you care.
 */
struct udp_tunnel_nic_info {
        /* one-by-one */
        int (*set_port)(struct net_device *dev,
                        unsigned int table, unsigned int entry,
                        struct udp_tunnel_info *ti);
        int (*unset_port)(struct net_device *dev,
                          unsigned int table, unsigned int entry,
                          struct udp_tunnel_info *ti);

        /* all at once */
        int (*sync_table)(struct net_device *dev, unsigned int table);

        struct udp_tunnel_nic_shared *shared;

        unsigned int flags;

        struct udp_tunnel_nic_table_info {
                unsigned int n_entries;
                unsigned int tunnel_types;
        } tables[UDP_TUNNEL_NIC_MAX_TABLES];
};

/* UDP tunnel module dependencies
 *
 * Tunnel drivers are expected to have a hard dependency on the udp_tunnel
 * module. NIC drivers are not, they just attach their
 * struct udp_tunnel_nic_info to the netdev and wait for callbacks to come.
 * Loading a tunnel driver will cause the udp_tunnel module to be loaded
 * and only then will all the required state structures be allocated.
 * Since we want a weak dependency from the drivers and the core to udp_tunnel
 * we call things through the following stubs.
 */
struct udp_tunnel_nic_ops {
        void (*get_port)(struct net_device *dev, unsigned int table,
                         unsigned int idx, struct udp_tunnel_info *ti);
        void (*set_port_priv)(struct net_device *dev, unsigned int table,
                              unsigned int idx, u8 priv);
        void (*add_port)(struct net_device *dev, struct udp_tunnel_info *ti);
        void (*del_port)(struct net_device *dev, struct udp_tunnel_info *ti);
        void (*reset_ntf)(struct net_device *dev);

        size_t (*dump_size)(struct net_device *dev, unsigned int table);
        int (*dump_write)(struct net_device *dev, unsigned int table,
                          struct sk_buff *skb);
};

#ifdef CONFIG_INET
extern const struct udp_tunnel_nic_ops *udp_tunnel_nic_ops;
#else
#define udp_tunnel_nic_ops        ((struct udp_tunnel_nic_ops *)NULL)
#endif

static inline void
udp_tunnel_nic_get_port(struct net_device *dev, unsigned int table,
                        unsigned int idx, struct udp_tunnel_info *ti)
{
        /* This helper is used from .sync_table, we indicate empty entries
         * by zero'ed @ti. Drivers which need to know the details of a port
         * when it gets deleted should use the .set_port / .unset_port
         * callbacks.
         * Zero out here, otherwise !CONFIG_INET causes uninitilized warnings.
         */
        memset(ti, 0, sizeof(*ti));

        if (udp_tunnel_nic_ops)
                udp_tunnel_nic_ops->get_port(dev, table, idx, ti);
}

static inline void
udp_tunnel_nic_set_port_priv(struct net_device *dev, unsigned int table,
                             unsigned int idx, u8 priv)
{
        if (udp_tunnel_nic_ops)
                udp_tunnel_nic_ops->set_port_priv(dev, table, idx, priv);
}

static inline void
udp_tunnel_nic_add_port(struct net_device *dev, struct udp_tunnel_info *ti)
{
        if (!(dev->features & NETIF_F_RX_UDP_TUNNEL_PORT))
                return;
        if (udp_tunnel_nic_ops)
                udp_tunnel_nic_ops->add_port(dev, ti);
}

static inline void
udp_tunnel_nic_del_port(struct net_device *dev, struct udp_tunnel_info *ti)
{
        if (!(dev->features & NETIF_F_RX_UDP_TUNNEL_PORT))
                return;
        if (udp_tunnel_nic_ops)
                udp_tunnel_nic_ops->del_port(dev, ti);
}

/**
 * udp_tunnel_nic_reset_ntf() - device-originating reset notification
 * @dev: network interface device structure
 *
 * Called by the driver to inform the core that the entire UDP tunnel port
 * state has been lost, usually due to device reset. Core will assume device
 * forgot all the ports and issue .set_port and .sync_table callbacks as
 * necessary.
 *
 * This function must be called with rtnl lock held, and will issue all
 * the callbacks before returning.
 */
static inline void udp_tunnel_nic_reset_ntf(struct net_device *dev)
{
        if (udp_tunnel_nic_ops)
                udp_tunnel_nic_ops->reset_ntf(dev);
}

static inline size_t
udp_tunnel_nic_dump_size(struct net_device *dev, unsigned int table)
{
        if (!udp_tunnel_nic_ops)
                return 0;
        return udp_tunnel_nic_ops->dump_size(dev, table);
}

static inline int
udp_tunnel_nic_dump_write(struct net_device *dev, unsigned int table,
                          struct sk_buff *skb)
{
        if (!udp_tunnel_nic_ops)
                return 0;
        return udp_tunnel_nic_ops->dump_write(dev, table, skb);
}
#endif































































































































  530 
   48 


















  601 
  605 




  606 




































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_LIST_BL_H
#define _LINUX_LIST_BL_H

#include <linux/list.h>
#include <linux/bit_spinlock.h>

/*
 * Special version of lists, where head of the list has a lock in the lowest
 * bit. This is useful for scalable hash tables without increasing memory
 * footprint overhead.
 *
 * For modification operations, the 0 bit of hlist_bl_head->first
 * pointer must be set.
 *
 * With some small modifications, this can easily be adapted to store several
 * arbitrary bits (not just a single lock bit), if the need arises to store
 * some fast and compact auxiliary data.
 */

#if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK)
#define LIST_BL_LOCKMASK        1UL
#else
#define LIST_BL_LOCKMASK        0UL
#endif

#ifdef CONFIG_DEBUG_LIST
#define LIST_BL_BUG_ON(x) BUG_ON(x)
#else
#define LIST_BL_BUG_ON(x)
#endif


struct hlist_bl_head {
        struct hlist_bl_node *first;
};

struct hlist_bl_node {
        struct hlist_bl_node *next, **pprev;
};
#define INIT_HLIST_BL_HEAD(ptr) \
        ((ptr)->first = NULL)

static inline void INIT_HLIST_BL_NODE(struct hlist_bl_node *h)
{
        h->next = NULL;
        h->pprev = NULL;
}

#define hlist_bl_entry(ptr, type, member) container_of(ptr,type,member)

static inline bool  hlist_bl_unhashed(const struct hlist_bl_node *h)
{
        return !h->pprev;
}

static inline struct hlist_bl_node *hlist_bl_first(struct hlist_bl_head *h)
{
        return (struct hlist_bl_node *)
                ((unsigned long)h->first & ~LIST_BL_LOCKMASK);
}

static inline void hlist_bl_set_first(struct hlist_bl_head *h,
                                        struct hlist_bl_node *n)
{
        LIST_BL_BUG_ON((unsigned long)n & LIST_BL_LOCKMASK);
        LIST_BL_BUG_ON(((unsigned long)h->first & LIST_BL_LOCKMASK) !=
                                                        LIST_BL_LOCKMASK);
        h->first = (struct hlist_bl_node *)((unsigned long)n | LIST_BL_LOCKMASK);
}

static inline bool hlist_bl_empty(const struct hlist_bl_head *h)
{
        return !((unsigned long)READ_ONCE(h->first) & ~LIST_BL_LOCKMASK);
}

static inline void hlist_bl_add_head(struct hlist_bl_node *n,
                                        struct hlist_bl_head *h)
{
        struct hlist_bl_node *first = hlist_bl_first(h);

        n->next = first;
        if (first)
                first->pprev = &n->next;
        n->pprev = &h->first;
        hlist_bl_set_first(h, n);
}

static inline void hlist_bl_add_before(struct hlist_bl_node *n,
                                       struct hlist_bl_node *next)
{
        struct hlist_bl_node **pprev = next->pprev;

        n->pprev = pprev;
        n->next = next;
        next->pprev = &n->next;

        /* pprev may be `first`, so be careful not to lose the lock bit */
        WRITE_ONCE(*pprev,
                   (struct hlist_bl_node *)
                        ((uintptr_t)n | ((uintptr_t)*pprev & LIST_BL_LOCKMASK)));
}

static inline void hlist_bl_add_behind(struct hlist_bl_node *n,
                                       struct hlist_bl_node *prev)
{
        n->next = prev->next;
        n->pprev = &prev->next;
        prev->next = n;

        if (n->next)
                n->next->pprev = &n->next;
}

static inline void __hlist_bl_del(struct hlist_bl_node *n)
{
        struct hlist_bl_node *next = n->next;
        struct hlist_bl_node **pprev = n->pprev;

        LIST_BL_BUG_ON((unsigned long)n & LIST_BL_LOCKMASK);

        /* pprev may be `first`, so be careful not to lose the lock bit */
        WRITE_ONCE(*pprev,
                   (struct hlist_bl_node *)
                        ((unsigned long)next |
                         ((unsigned long)*pprev & LIST_BL_LOCKMASK)));
        if (next)
                next->pprev = pprev;
}

static inline void hlist_bl_del(struct hlist_bl_node *n)
{
        __hlist_bl_del(n);
        n->next = LIST_POISON1;
        n->pprev = LIST_POISON2;
}

static inline void hlist_bl_del_init(struct hlist_bl_node *n)
{
        if (!hlist_bl_unhashed(n)) {
                __hlist_bl_del(n);
                INIT_HLIST_BL_NODE(n);
        }
}

static inline void hlist_bl_lock(struct hlist_bl_head *b)
{
        bit_spin_lock(0, (unsigned long *)b);
}

static inline void hlist_bl_unlock(struct hlist_bl_head *b)
{
        __bit_spin_unlock(0, (unsigned long *)b);
}

static inline bool hlist_bl_is_locked(struct hlist_bl_head *b)
{
        return bit_spin_is_locked(0, (unsigned long *)b);
}

/**
 * hlist_bl_for_each_entry        - iterate over list of given type
 * @tpos:        the type * to use as a loop cursor.
 * @pos:        the &struct hlist_node to use as a loop cursor.
 * @head:        the head for your list.
 * @member:        the name of the hlist_node within the struct.
 *
 */
#define hlist_bl_for_each_entry(tpos, pos, head, member)                \
        for (pos = hlist_bl_first(head);                                \
             pos &&                                                        \
                ({ tpos = hlist_bl_entry(pos, typeof(*tpos), member); 1;}); \
             pos = pos->next)

/**
 * hlist_bl_for_each_entry_safe - iterate over list of given type safe against removal of list entry
 * @tpos:        the type * to use as a loop cursor.
 * @pos:        the &struct hlist_node to use as a loop cursor.
 * @n:                another &struct hlist_node to use as temporary storage
 * @head:        the head for your list.
 * @member:        the name of the hlist_node within the struct.
 */
#define hlist_bl_for_each_entry_safe(tpos, pos, n, head, member)         \
        for (pos = hlist_bl_first(head);                                 \
             pos && ({ n = pos->next; 1; }) &&                                  \
                ({ tpos = hlist_bl_entry(pos, typeof(*tpos), member); 1;}); \
             pos = n)

#endif







































































































































































































































































































































































































































































































































































































































































































































































































































   49 

















   49 


























1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * Copyright(c) 2004-2005 Intel Corporation. All rights reserved.
 */

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/device.h>
#include <linux/sched/signal.h>
#include <linux/fs.h>
#include <linux/types.h>
#include <linux/string.h>
#include <linux/netdevice.h>
#include <linux/inetdevice.h>
#include <linux/in.h>
#include <linux/sysfs.h>
#include <linux/ctype.h>
#include <linux/inet.h>
#include <linux/rtnetlink.h>
#include <linux/etherdevice.h>
#include <net/net_namespace.h>
#include <net/netns/generic.h>
#include <linux/nsproxy.h>

#include <net/bonding.h>

#define to_bond(cd)        ((struct bonding *)(netdev_priv(to_net_dev(cd))))

/* "show" function for the bond_masters attribute.
 * The class parameter is ignored.
 */
static ssize_t bonding_show_bonds(struct class *cls,
                                  struct class_attribute *attr,
                                  char *buf)
{
        struct bond_net *bn =
                container_of(attr, struct bond_net, class_attr_bonding_masters);
        int res = 0;
        struct bonding *bond;

        rtnl_lock();

        list_for_each_entry(bond, &bn->dev_list, bond_list) {
                if (res > (PAGE_SIZE - IFNAMSIZ)) {
                        /* not enough space for another interface name */
                        if ((PAGE_SIZE - res) > 10)
                                res = PAGE_SIZE - 10;
                        res += sprintf(buf + res, "++more++ ");
                        break;
                }
                res += sprintf(buf + res, "%s ", bond->dev->name);
        }
        if (res)
                buf[res-1] = '\n'; /* eat the leftover space */

        rtnl_unlock();
        return res;
}

static struct net_device *bond_get_by_name(struct bond_net *bn, const char *ifname)
{
        struct bonding *bond;

        list_for_each_entry(bond, &bn->dev_list, bond_list) {
                if (strncmp(bond->dev->name, ifname, IFNAMSIZ) == 0)
                        return bond->dev;
        }
        return NULL;
}

/* "store" function for the bond_masters attribute.  This is what
 * creates and deletes entire bonds.
 *
 * The class parameter is ignored.
 */
static ssize_t bonding_store_bonds(struct class *cls,
                                   struct class_attribute *attr,
                                   const char *buffer, size_t count)
{
        struct bond_net *bn =
                container_of(attr, struct bond_net, class_attr_bonding_masters);
        char command[IFNAMSIZ + 1] = {0, };
        char *ifname;
        int rv, res = count;

        sscanf(buffer, "%16s", command); /* IFNAMSIZ*/
        ifname = command + 1;
        if ((strlen(command) <= 1) ||
            !dev_valid_name(ifname))
                goto err_no_cmd;

        if (command[0] == '+') {
                pr_info("%s is being created...\n", ifname);
                rv = bond_create(bn->net, ifname);
                if (rv) {
                        if (rv == -EEXIST)
                                pr_info("%s already exists\n", ifname);
                        else
                                pr_info("%s creation failed\n", ifname);
                        res = rv;
                }
        } else if (command[0] == '-') {
                struct net_device *bond_dev;

                rtnl_lock();
                bond_dev = bond_get_by_name(bn, ifname);
                if (bond_dev) {
                        pr_info("%s is being deleted...\n", ifname);
                        unregister_netdevice(bond_dev);
                } else {
                        pr_err("unable to delete non-existent %s\n", ifname);
                        res = -ENODEV;
                }
                rtnl_unlock();
        } else
                goto err_no_cmd;

        /* Always return either count or an error.  If you return 0, you'll
         * get called forever, which is bad.
         */
        return res;

err_no_cmd:
        pr_err("no command found in bonding_masters - use +ifname or -ifname\n");
        return -EPERM;
}

/* class attribute for bond_masters file.  This ends up in /sys/class/net */
static const struct class_attribute class_attr_bonding_masters = {
        .attr = {
                .name = "bonding_masters",
                .mode = 0644,
        },
        .show = bonding_show_bonds,
        .store = bonding_store_bonds,
};

/* Generic "store" method for bonding sysfs option setting */
static ssize_t bonding_sysfs_store_option(struct device *d,
                                          struct device_attribute *attr,
                                          const char *buffer, size_t count)
{
        struct bonding *bond = to_bond(d);
        const struct bond_option *opt;
        char *buffer_clone;
        int ret;

        opt = bond_opt_get_by_name(attr->attr.name);
        if (WARN_ON(!opt))
                return -ENOENT;
        buffer_clone = kstrndup(buffer, count, GFP_KERNEL);
        if (!buffer_clone)
                return -ENOMEM;
        ret = bond_opt_tryset_rtnl(bond, opt->id, buffer_clone);
        if (!ret)
                ret = count;
        kfree(buffer_clone);

        return ret;
}

/* Show the slaves in the current bond. */
static ssize_t bonding_show_slaves(struct device *d,
                                   struct device_attribute *attr, char *buf)
{
        struct bonding *bond = to_bond(d);
        struct list_head *iter;
        struct slave *slave;
        int res = 0;

        if (!rtnl_trylock())
                return restart_syscall();

        bond_for_each_slave(bond, slave, iter) {
                if (res > (PAGE_SIZE - IFNAMSIZ)) {
                        /* not enough space for another interface name */
                        if ((PAGE_SIZE - res) > 10)
                                res = PAGE_SIZE - 10;
                        res += sprintf(buf + res, "++more++ ");
                        break;
                }
                res += sprintf(buf + res, "%s ", slave->dev->name);
        }

        rtnl_unlock();

        if (res)
                buf[res-1] = '\n'; /* eat the leftover space */

        return res;
}
static DEVICE_ATTR(slaves, 0644, bonding_show_slaves,
                   bonding_sysfs_store_option);

/* Show the bonding mode. */
static ssize_t bonding_show_mode(struct device *d,
                                 struct device_attribute *attr, char *buf)
{
        struct bonding *bond = to_bond(d);
        const struct bond_opt_value *val;

        val = bond_opt_get_val(BOND_OPT_MODE, BOND_MODE(bond));

        return sprintf(buf, "%s %d\n", val->string, BOND_MODE(bond));
}
static DEVICE_ATTR(mode, 0644, bonding_show_mode, bonding_sysfs_store_option);

/* Show the bonding transmit hash method. */
static ssize_t bonding_show_xmit_hash(struct device *d,
                                      struct device_attribute *attr,
                                      char *buf)
{
        struct bonding *bond = to_bond(d);
        const struct bond_opt_value *val;

        val = bond_opt_get_val(BOND_OPT_XMIT_HASH, bond->params.xmit_policy);

        return sprintf(buf, "%s %d\n", val->string, bond->params.xmit_policy);
}
static DEVICE_ATTR(xmit_hash_policy, 0644,
                   bonding_show_xmit_hash, bonding_sysfs_store_option);

/* Show arp_validate. */
static ssize_t bonding_show_arp_validate(struct device *d,
                                         struct device_attribute *attr,
                                         char *buf)
{
        struct bonding *bond = to_bond(d);
        const struct bond_opt_value *val;

        val = bond_opt_get_val(BOND_OPT_ARP_VALIDATE,
                               bond->params.arp_validate);

        return sprintf(buf, "%s %d\n", val->string, bond->params.arp_validate);
}
static DEVICE_ATTR(arp_validate, 0644, bonding_show_arp_validate,
                   bonding_sysfs_store_option);

/* Show arp_all_targets. */
static ssize_t bonding_show_arp_all_targets(struct device *d,
                                         struct device_attribute *attr,
                                         char *buf)
{
        struct bonding *bond = to_bond(d);
        const struct bond_opt_value *val;

        val = bond_opt_get_val(BOND_OPT_ARP_ALL_TARGETS,
                               bond->params.arp_all_targets);
        return sprintf(buf, "%s %d\n",
                       val->string, bond->params.arp_all_targets);
}
static DEVICE_ATTR(arp_all_targets, 0644,
                   bonding_show_arp_all_targets, bonding_sysfs_store_option);

/* Show fail_over_mac. */
static ssize_t bonding_show_fail_over_mac(struct device *d,
                                          struct device_attribute *attr,
                                          char *buf)
{
        struct bonding *bond = to_bond(d);
        const struct bond_opt_value *val;

        val = bond_opt_get_val(BOND_OPT_FAIL_OVER_MAC,
                               bond->params.fail_over_mac);

        return sprintf(buf, "%s %d\n", val->string, bond->params.fail_over_mac);
}
static DEVICE_ATTR(fail_over_mac, 0644,
                   bonding_show_fail_over_mac, bonding_sysfs_store_option);

/* Show the arp timer interval. */
static ssize_t bonding_show_arp_interval(struct device *d,
                                         struct device_attribute *attr,
                                         char *buf)
{
        struct bonding *bond = to_bond(d);

        return sprintf(buf, "%d\n", bond->params.arp_interval);
}
static DEVICE_ATTR(arp_interval, 0644,
                   bonding_show_arp_interval, bonding_sysfs_store_option);

/* Show the arp targets. */
static ssize_t bonding_show_arp_targets(struct device *d,
                                        struct device_attribute *attr,
                                        char *buf)
{
        struct bonding *bond = to_bond(d);
        int i, res = 0;

        for (i = 0; i < BOND_MAX_ARP_TARGETS; i++) {
                if (bond->params.arp_targets[i])
                        res += sprintf(buf + res, "%pI4 ",
                                       &bond->params.arp_targets[i]);
        }
        if (res)
                buf[res-1] = '\n'; /* eat the leftover space */

        return res;
}
static DEVICE_ATTR(arp_ip_target, 0644,
                   bonding_show_arp_targets, bonding_sysfs_store_option);

/* Show the arp missed max. */
static ssize_t bonding_show_missed_max(struct device *d,
                                       struct device_attribute *attr,
                                       char *buf)
{
        struct bonding *bond = to_bond(d);

        return sprintf(buf, "%u\n", bond->params.missed_max);
}
static DEVICE_ATTR(arp_missed_max, 0644,
                   bonding_show_missed_max, bonding_sysfs_store_option);

/* Show the up and down delays. */
static ssize_t bonding_show_downdelay(struct device *d,
                                      struct device_attribute *attr,
                                      char *buf)
{
        struct bonding *bond = to_bond(d);

        return sprintf(buf, "%d\n", bond->params.downdelay * bond->params.miimon);
}
static DEVICE_ATTR(downdelay, 0644,
                   bonding_show_downdelay, bonding_sysfs_store_option);

static ssize_t bonding_show_updelay(struct device *d,
                                    struct device_attribute *attr,
                                    char *buf)
{
        struct bonding *bond = to_bond(d);

        return sprintf(buf, "%d\n", bond->params.updelay * bond->params.miimon);

}
static DEVICE_ATTR(updelay, 0644,
                   bonding_show_updelay, bonding_sysfs_store_option);

static ssize_t bonding_show_peer_notif_delay(struct device *d,
                                             struct device_attribute *attr,
                                             char *buf)
{
        struct bonding *bond = to_bond(d);

        return sprintf(buf, "%d\n",
                       bond->params.peer_notif_delay * bond->params.miimon);
}
static DEVICE_ATTR(peer_notif_delay, 0644,
                   bonding_show_peer_notif_delay, bonding_sysfs_store_option);

/* Show the LACP activity and interval. */
static ssize_t bonding_show_lacp_active(struct device *d,
                                        struct device_attribute *attr,
                                        char *buf)
{
        struct bonding *bond = to_bond(d);
        const struct bond_opt_value *val;

        val = bond_opt_get_val(BOND_OPT_LACP_ACTIVE, bond->params.lacp_active);

        return sprintf(buf, "%s %d\n", val->string, bond->params.lacp_active);
}
static DEVICE_ATTR(lacp_active, 0644,
                   bonding_show_lacp_active, bonding_sysfs_store_option);

static ssize_t bonding_show_lacp_rate(struct device *d,
                                      struct device_attribute *attr,
                                      char *buf)
{
        struct bonding *bond = to_bond(d);
        const struct bond_opt_value *val;

        val = bond_opt_get_val(BOND_OPT_LACP_RATE, bond->params.lacp_fast);

        return sprintf(buf, "%s %d\n", val->string, bond->params.lacp_fast);
}
static DEVICE_ATTR(lacp_rate, 0644,
                   bonding_show_lacp_rate, bonding_sysfs_store_option);

static ssize_t bonding_show_min_links(struct device *d,
                                      struct device_attribute *attr,
                                      char *buf)
{
        struct bonding *bond = to_bond(d);

        return sprintf(buf, "%u\n", bond->params.min_links);
}
static DEVICE_ATTR(min_links, 0644,
                   bonding_show_min_links, bonding_sysfs_store_option);

static ssize_t bonding_show_ad_select(struct device *d,
                                      struct device_attribute *attr,
                                      char *buf)
{
        struct bonding *bond = to_bond(d);
        const struct bond_opt_value *val;

        val = bond_opt_get_val(BOND_OPT_AD_SELECT, bond->params.ad_select);

        return sprintf(buf, "%s %d\n", val->string, bond->params.ad_select);
}
static DEVICE_ATTR(ad_select, 0644,
                   bonding_show_ad_select, bonding_sysfs_store_option);

/* Show the number of peer notifications to send after a failover event. */
static ssize_t bonding_show_num_peer_notif(struct device *d,
                                           struct device_attribute *attr,
                                           char *buf)
{
        struct bonding *bond = to_bond(d);

        return sprintf(buf, "%d\n", bond->params.num_peer_notif);
}
static DEVICE_ATTR(num_grat_arp, 0644,
                   bonding_show_num_peer_notif, bonding_sysfs_store_option);
static DEVICE_ATTR(num_unsol_na, 0644,
                   bonding_show_num_peer_notif, bonding_sysfs_store_option);

/* Show the MII monitor interval. */
static ssize_t bonding_show_miimon(struct device *d,
                                   struct device_attribute *attr,
                                   char *buf)
{
        struct bonding *bond = to_bond(d);

        return sprintf(buf, "%d\n", bond->params.miimon);
}
static DEVICE_ATTR(miimon, 0644,
                   bonding_show_miimon, bonding_sysfs_store_option);

/* Show the primary slave. */
static ssize_t bonding_show_primary(struct device *d,
                                    struct device_attribute *attr,
                                    char *buf)
{
        struct bonding *bond = to_bond(d);
        struct slave *primary;
        int count = 0;

        rcu_read_lock();
        primary = rcu_dereference(bond->primary_slave);
        if (primary)
                count = sprintf(buf, "%s\n", primary->dev->name);
        rcu_read_unlock();

        return count;
}
static DEVICE_ATTR(primary, 0644,
                   bonding_show_primary, bonding_sysfs_store_option);

/* Show the primary_reselect flag. */
static ssize_t bonding_show_primary_reselect(struct device *d,
                                             struct device_attribute *attr,
                                             char *buf)
{
        struct bonding *bond = to_bond(d);
        const struct bond_opt_value *val;

        val = bond_opt_get_val(BOND_OPT_PRIMARY_RESELECT,
                               bond->params.primary_reselect);

        return sprintf(buf, "%s %d\n",
                       val->string, bond->params.primary_reselect);
}
static DEVICE_ATTR(primary_reselect, 0644,
                   bonding_show_primary_reselect, bonding_sysfs_store_option);

/* Show the use_carrier flag. */
static ssize_t bonding_show_carrier(struct device *d,
                                    struct device_attribute *attr,
                                    char *buf)
{
        struct bonding *bond = to_bond(d);

        return sprintf(buf, "%d\n", bond->params.use_carrier);
}
static DEVICE_ATTR(use_carrier, 0644,
                   bonding_show_carrier, bonding_sysfs_store_option);


/* Show currently active_slave. */
static ssize_t bonding_show_active_slave(struct device *d,
                                         struct device_attribute *attr,
                                         char *buf)
{
        struct bonding *bond = to_bond(d);
        struct net_device *slave_dev;
        int count = 0;

        rcu_read_lock();
        slave_dev = bond_option_active_slave_get_rcu(bond);
        if (slave_dev)
                count = sprintf(buf, "%s\n", slave_dev->name);
        rcu_read_unlock();

        return count;
}
static DEVICE_ATTR(active_slave, 0644,
                   bonding_show_active_slave, bonding_sysfs_store_option);

/* Show link status of the bond interface. */
static ssize_t bonding_show_mii_status(struct device *d,
                                       struct device_attribute *attr,
                                       char *buf)
{
        struct bonding *bond = to_bond(d);
        bool active = netif_carrier_ok(bond->dev);

        return sprintf(buf, "%s\n", active ? "up" : "down");
}
static DEVICE_ATTR(mii_status, 0444, bonding_show_mii_status, NULL);

/* Show current 802.3ad aggregator ID. */
static ssize_t bonding_show_ad_aggregator(struct device *d,
                                          struct device_attribute *attr,
                                          char *buf)
{
        int count = 0;
        struct bonding *bond = to_bond(d);

        if (BOND_MODE(bond) == BOND_MODE_8023AD) {
                struct ad_info ad_info;

                count = sprintf(buf, "%d\n",
                                bond_3ad_get_active_agg_info(bond, &ad_info)
                                ?  0 : ad_info.aggregator_id);
        }

        return count;
}
static DEVICE_ATTR(ad_aggregator, 0444, bonding_show_ad_aggregator, NULL);


/* Show number of active 802.3ad ports. */
static ssize_t bonding_show_ad_num_ports(struct device *d,
                                         struct device_attribute *attr,
                                         char *buf)
{
        int count = 0;
        struct bonding *bond = to_bond(d);

        if (BOND_MODE(bond) == BOND_MODE_8023AD) {
                struct ad_info ad_info;

                count = sprintf(buf, "%d\n",
                                bond_3ad_get_active_agg_info(bond, &ad_info)
                                ?  0 : ad_info.ports);
        }

        return count;
}
static DEVICE_ATTR(ad_num_ports, 0444, bonding_show_ad_num_ports, NULL);


/* Show current 802.3ad actor key. */
static ssize_t bonding_show_ad_actor_key(struct device *d,
                                         struct device_attribute *attr,
                                         char *buf)
{
        int count = 0;
        struct bonding *bond = to_bond(d);

        if (BOND_MODE(bond) == BOND_MODE_8023AD && capable(CAP_NET_ADMIN)) {
                struct ad_info ad_info;

                count = sprintf(buf, "%d\n",
                                bond_3ad_get_active_agg_info(bond, &ad_info)
                                ?  0 : ad_info.actor_key);
        }

        return count;
}
static DEVICE_ATTR(ad_actor_key, 0444, bonding_show_ad_actor_key, NULL);


/* Show current 802.3ad partner key. */
static ssize_t bonding_show_ad_partner_key(struct device *d,
                                           struct device_attribute *attr,
                                           char *buf)
{
        int count = 0;
        struct bonding *bond = to_bond(d);

        if (BOND_MODE(bond) == BOND_MODE_8023AD && capable(CAP_NET_ADMIN)) {
                struct ad_info ad_info;

                count = sprintf(buf, "%d\n",
                                bond_3ad_get_active_agg_info(bond, &ad_info)
                                ?  0 : ad_info.partner_key);
        }

        return count;
}
static DEVICE_ATTR(ad_partner_key, 0444, bonding_show_ad_partner_key, NULL);


/* Show current 802.3ad partner mac. */
static ssize_t bonding_show_ad_partner_mac(struct device *d,
                                           struct device_attribute *attr,
                                           char *buf)
{
        int count = 0;
        struct bonding *bond = to_bond(d);

        if (BOND_MODE(bond) == BOND_MODE_8023AD && capable(CAP_NET_ADMIN)) {
                struct ad_info ad_info;

                if (!bond_3ad_get_active_agg_info(bond, &ad_info))
                        count = sprintf(buf, "%pM\n", ad_info.partner_system);
        }

        return count;
}
static DEVICE_ATTR(ad_partner_mac, 0444, bonding_show_ad_partner_mac, NULL);

/* Show the queue_ids of the slaves in the current bond. */
static ssize_t bonding_show_queue_id(struct device *d,
                                     struct device_attribute *attr,
                                     char *buf)
{
        struct bonding *bond = to_bond(d);
        struct list_head *iter;
        struct slave *slave;
        int res = 0;

        if (!rtnl_trylock())
                return restart_syscall();

        bond_for_each_slave(bond, slave, iter) {
                if (res > (PAGE_SIZE - IFNAMSIZ - 6)) {
                        /* not enough space for another interface_name:queue_id pair */
                        if ((PAGE_SIZE - res) > 10)
                                res = PAGE_SIZE - 10;
                        res += sprintf(buf + res, "++more++ ");
                        break;
                }
                res += sprintf(buf + res, "%s:%d ",
                               slave->dev->name, slave->queue_id);
        }
        if (res)
                buf[res-1] = '\n'; /* eat the leftover space */

        rtnl_unlock();

        return res;
}
static DEVICE_ATTR(queue_id, 0644, bonding_show_queue_id,
                   bonding_sysfs_store_option);


/* Show the all_slaves_active flag. */
static ssize_t bonding_show_slaves_active(struct device *d,
                                          struct device_attribute *attr,
                                          char *buf)
{
        struct bonding *bond = to_bond(d);

        return sprintf(buf, "%d\n", bond->params.all_slaves_active);
}
static DEVICE_ATTR(all_slaves_active, 0644,
                   bonding_show_slaves_active, bonding_sysfs_store_option);

/* Show the number of IGMP membership reports to send on link failure */
static ssize_t bonding_show_resend_igmp(struct device *d,
                                        struct device_attribute *attr,
                                        char *buf)
{
        struct bonding *bond = to_bond(d);

        return sprintf(buf, "%d\n", bond->params.resend_igmp);
}
static DEVICE_ATTR(resend_igmp, 0644,
                   bonding_show_resend_igmp, bonding_sysfs_store_option);


static ssize_t bonding_show_lp_interval(struct device *d,
                                        struct device_attribute *attr,
                                        char *buf)
{
        struct bonding *bond = to_bond(d);

        return sprintf(buf, "%d\n", bond->params.lp_interval);
}
static DEVICE_ATTR(lp_interval, 0644,
                   bonding_show_lp_interval, bonding_sysfs_store_option);

static ssize_t bonding_show_tlb_dynamic_lb(struct device *d,
                                           struct device_attribute *attr,
                                           char *buf)
{
        struct bonding *bond = to_bond(d);

        return sprintf(buf, "%d\n", bond->params.tlb_dynamic_lb);
}
static DEVICE_ATTR(tlb_dynamic_lb, 0644,
                   bonding_show_tlb_dynamic_lb, bonding_sysfs_store_option);

static ssize_t bonding_show_packets_per_slave(struct device *d,
                                              struct device_attribute *attr,
                                              char *buf)
{
        struct bonding *bond = to_bond(d);
        unsigned int packets_per_slave = bond->params.packets_per_slave;

        return sprintf(buf, "%u\n", packets_per_slave);
}
static DEVICE_ATTR(packets_per_slave, 0644,
                   bonding_show_packets_per_slave, bonding_sysfs_store_option);

static ssize_t bonding_show_ad_actor_sys_prio(struct device *d,
                                              struct device_attribute *attr,
                                              char *buf)
{
        struct bonding *bond = to_bond(d);

        if (BOND_MODE(bond) == BOND_MODE_8023AD && capable(CAP_NET_ADMIN))
                return sprintf(buf, "%hu\n", bond->params.ad_actor_sys_prio);

        return 0;
}
static DEVICE_ATTR(ad_actor_sys_prio, 0644,
                   bonding_show_ad_actor_sys_prio, bonding_sysfs_store_option);

static ssize_t bonding_show_ad_actor_system(struct device *d,
                                            struct device_attribute *attr,
                                            char *buf)
{
        struct bonding *bond = to_bond(d);

        if (BOND_MODE(bond) == BOND_MODE_8023AD && capable(CAP_NET_ADMIN))
                return sprintf(buf, "%pM\n", bond->params.ad_actor_system);

        return 0;
}

static DEVICE_ATTR(ad_actor_system, 0644,
                   bonding_show_ad_actor_system, bonding_sysfs_store_option);

static ssize_t bonding_show_ad_user_port_key(struct device *d,
                                             struct device_attribute *attr,
                                             char *buf)
{
        struct bonding *bond = to_bond(d);

        if (BOND_MODE(bond) == BOND_MODE_8023AD && capable(CAP_NET_ADMIN))
                return sprintf(buf, "%hu\n", bond->params.ad_user_port_key);

        return 0;
}
static DEVICE_ATTR(ad_user_port_key, 0644,
                   bonding_show_ad_user_port_key, bonding_sysfs_store_option);

static struct attribute *per_bond_attrs[] = {
        &dev_attr_slaves.attr,
        &dev_attr_mode.attr,
        &dev_attr_fail_over_mac.attr,
        &dev_attr_arp_validate.attr,
        &dev_attr_arp_all_targets.attr,
        &dev_attr_arp_interval.attr,
        &dev_attr_arp_ip_target.attr,
        &dev_attr_downdelay.attr,
        &dev_attr_updelay.attr,
        &dev_attr_peer_notif_delay.attr,
        &dev_attr_lacp_active.attr,
        &dev_attr_lacp_rate.attr,
        &dev_attr_ad_select.attr,
        &dev_attr_xmit_hash_policy.attr,
        &dev_attr_num_grat_arp.attr,
        &dev_attr_num_unsol_na.attr,
        &dev_attr_miimon.attr,
        &dev_attr_primary.attr,
        &dev_attr_primary_reselect.attr,
        &dev_attr_use_carrier.attr,
        &dev_attr_active_slave.attr,
        &dev_attr_mii_status.attr,
        &dev_attr_ad_aggregator.attr,
        &dev_attr_ad_num_ports.attr,
        &dev_attr_ad_actor_key.attr,
        &dev_attr_ad_partner_key.attr,
        &dev_attr_ad_partner_mac.attr,
        &dev_attr_queue_id.attr,
        &dev_attr_all_slaves_active.attr,
        &dev_attr_resend_igmp.attr,
        &dev_attr_min_links.attr,
        &dev_attr_lp_interval.attr,
        &dev_attr_packets_per_slave.attr,
        &dev_attr_tlb_dynamic_lb.attr,
        &dev_attr_ad_actor_sys_prio.attr,
        &dev_attr_ad_actor_system.attr,
        &dev_attr_ad_user_port_key.attr,
        &dev_attr_arp_missed_max.attr,
        NULL,
};

static const struct attribute_group bonding_group = {
        .name = "bonding",
        .attrs = per_bond_attrs,
};

/* Initialize sysfs.  This sets up the bonding_masters file in
 * /sys/class/net.
 */
int bond_create_sysfs(struct bond_net *bn)
{
        int ret;

        bn->class_attr_bonding_masters = class_attr_bonding_masters;
        sysfs_attr_init(&bn->class_attr_bonding_masters.attr);

        ret = netdev_class_create_file_ns(&bn->class_attr_bonding_masters,
                                          bn->net);
        /* Permit multiple loads of the module by ignoring failures to
         * create the bonding_masters sysfs file.  Bonding devices
         * created by second or subsequent loads of the module will
         * not be listed in, or controllable by, bonding_masters, but
         * will have the usual "bonding" sysfs directory.
         *
         * This is done to preserve backwards compatibility for
         * initscripts/sysconfig, which load bonding multiple times to
         * configure multiple bonding devices.
         */
        if (ret == -EEXIST) {
                /* Is someone being kinky and naming a device bonding_master? */
                if (__dev_get_by_name(bn->net,
                                      class_attr_bonding_masters.attr.name))
                        pr_err("network device named %s already exists in sysfs\n",
                               class_attr_bonding_masters.attr.name);
                ret = 0;
        }

        return ret;

}

/* Remove /sys/class/net/bonding_masters. */
void bond_destroy_sysfs(struct bond_net *bn)
{
        netdev_class_remove_file_ns(&bn->class_attr_bonding_masters, bn->net);
}

/* Initialize sysfs for each bond.  This sets up and registers
 * the 'bondctl' directory for each individual bond under /sys/class/net.
 */
void bond_prepare_sysfs_group(struct bonding *bond)
{
        bond->dev->sysfs_groups[0] = &bonding_group;
}


























































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































   49 










   49 




   49 




   49 









   49 













































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
// SPDX-License-Identifier: GPL-2.0
/*
 * sysctl_net_ipv4.c: sysctl interface to net IPV4 subsystem.
 *
 * Begun April 1, 1996, Mike Shaver.
 * Added /proc/sys/net/ipv4 directory entry (empty =) ). [MS]
 */

#include <linux/mm.h>
#include <linux/module.h>
#include <linux/sysctl.h>
#include <linux/igmp.h>
#include <linux/inetdevice.h>
#include <linux/seqlock.h>
#include <linux/init.h>
#include <linux/slab.h>
#include <linux/nsproxy.h>
#include <linux/swap.h>
#include <net/snmp.h>
#include <net/icmp.h>
#include <net/ip.h>
#include <net/ip_fib.h>
#include <net/route.h>
#include <net/tcp.h>
#include <net/udp.h>
#include <net/cipso_ipv4.h>
#include <net/inet_frag.h>
#include <net/ping.h>
#include <net/protocol.h>
#include <net/netevent.h>

static int two = 2;
static int three __maybe_unused = 3;
static int four = 4;
static int thousand = 1000;
static int tcp_retr1_max = 255;
static int ip_local_port_range_min[] = { 1, 1 };
static int ip_local_port_range_max[] = { 65535, 65535 };
static int tcp_adv_win_scale_min = -31;
static int tcp_adv_win_scale_max = 31;
static int tcp_app_win_max = 31;
static int tcp_min_snd_mss_min = TCP_MIN_SND_MSS;
static int tcp_min_snd_mss_max = 65535;
static int ip_privileged_port_min;
static int ip_privileged_port_max = 65535;
static int ip_ttl_min = 1;
static int ip_ttl_max = 255;
static int tcp_syn_retries_min = 1;
static int tcp_syn_retries_max = MAX_TCP_SYNCNT;
static int ip_ping_group_range_min[] = { 0, 0 };
static int ip_ping_group_range_max[] = { GID_T_MAX, GID_T_MAX };
static u32 u32_max_div_HZ = UINT_MAX / HZ;
static int one_day_secs = 24 * 3600;
static u32 fib_multipath_hash_fields_all_mask __maybe_unused =
        FIB_MULTIPATH_HASH_FIELD_ALL_MASK;

/* obsolete */
static int sysctl_tcp_low_latency __read_mostly;

/* Update system visible IP port range */
static void set_local_port_range(struct net *net, int range[2])
{
        bool same_parity = !((range[0] ^ range[1]) & 1);

        write_seqlock_bh(&net->ipv4.ip_local_ports.lock);
        if (same_parity && !net->ipv4.ip_local_ports.warned) {
                net->ipv4.ip_local_ports.warned = true;
                pr_err_ratelimited("ip_local_port_range: prefer different parity for start/end values.\n");
        }
        net->ipv4.ip_local_ports.range[0] = range[0];
        net->ipv4.ip_local_ports.range[1] = range[1];
        write_sequnlock_bh(&net->ipv4.ip_local_ports.lock);
}

/* Validate changes from /proc interface. */
static int ipv4_local_port_range(struct ctl_table *table, int write,
                                 void *buffer, size_t *lenp, loff_t *ppos)
{
        struct net *net =
                container_of(table->data, struct net, ipv4.ip_local_ports.range);
        int ret;
        int range[2];
        struct ctl_table tmp = {
                .data = &range,
                .maxlen = sizeof(range),
                .mode = table->mode,
                .extra1 = &ip_local_port_range_min,
                .extra2 = &ip_local_port_range_max,
        };

        inet_get_local_port_range(net, &range[0], &range[1]);

        ret = proc_dointvec_minmax(&tmp, write, buffer, lenp, ppos);

        if (write && ret == 0) {
                /* Ensure that the upper limit is not smaller than the lower,
                 * and that the lower does not encroach upon the privileged
                 * port limit.
                 */
                if ((range[1] < range[0]) ||
                    (range[0] < READ_ONCE(net->ipv4.sysctl_ip_prot_sock)))
                        ret = -EINVAL;
                else
                        set_local_port_range(net, range);
        }

        return ret;
}

/* Validate changes from /proc interface. */
static int ipv4_privileged_ports(struct ctl_table *table, int write,
                                void *buffer, size_t *lenp, loff_t *ppos)
{
        struct net *net = container_of(table->data, struct net,
            ipv4.sysctl_ip_prot_sock);
        int ret;
        int pports;
        int range[2];
        struct ctl_table tmp = {
                .data = &pports,
                .maxlen = sizeof(pports),
                .mode = table->mode,
                .extra1 = &ip_privileged_port_min,
                .extra2 = &ip_privileged_port_max,
        };

        pports = READ_ONCE(net->ipv4.sysctl_ip_prot_sock);

        ret = proc_dointvec_minmax(&tmp, write, buffer, lenp, ppos);

        if (write && ret == 0) {
                inet_get_local_port_range(net, &range[0], &range[1]);
                /* Ensure that the local port range doesn't overlap with the
                 * privileged port range.
                 */
                if (range[0] < pports)
                        ret = -EINVAL;
                else
                        WRITE_ONCE(net->ipv4.sysctl_ip_prot_sock, pports);
        }

        return ret;
}

static void inet_get_ping_group_range_table(struct ctl_table *table, kgid_t *low, kgid_t *high)
{
        kgid_t *data = table->data;
        struct net *net =
                container_of(table->data, struct net, ipv4.ping_group_range.range);
        unsigned int seq;
        do {
                seq = read_seqbegin(&net->ipv4.ping_group_range.lock);

                *low = data[0];
                *high = data[1];
        } while (read_seqretry(&net->ipv4.ping_group_range.lock, seq));
}

/* Update system visible IP port range */
static void set_ping_group_range(struct ctl_table *table, kgid_t low, kgid_t high)
{
        kgid_t *data = table->data;
        struct net *net =
                container_of(table->data, struct net, ipv4.ping_group_range.range);
        write_seqlock(&net->ipv4.ping_group_range.lock);
        data[0] = low;
        data[1] = high;
        write_sequnlock(&net->ipv4.ping_group_range.lock);
}

/* Validate changes from /proc interface. */
static int ipv4_ping_group_range(struct ctl_table *table, int write,
                                 void *buffer, size_t *lenp, loff_t *ppos)
{
        struct user_namespace *user_ns = current_user_ns();
        int ret;
        gid_t urange[2];
        kgid_t low, high;
        struct ctl_table tmp = {
                .data = &urange,
                .maxlen = sizeof(urange),
                .mode = table->mode,
                .extra1 = &ip_ping_group_range_min,
                .extra2 = &ip_ping_group_range_max,
        };

        inet_get_ping_group_range_table(table, &low, &high);
        urange[0] = from_kgid_munged(user_ns, low);
        urange[1] = from_kgid_munged(user_ns, high);
        ret = proc_dointvec_minmax(&tmp, write, buffer, lenp, ppos);

        if (write && ret == 0) {
                low = make_kgid(user_ns, urange[0]);
                high = make_kgid(user_ns, urange[1]);
                if (!gid_valid(low) || !gid_valid(high))
                        return -EINVAL;
                if (urange[1] < urange[0] || gid_lt(high, low)) {
                        low = make_kgid(&init_user_ns, 1);
                        high = make_kgid(&init_user_ns, 0);
                }
                set_ping_group_range(table, low, high);
        }

        return ret;
}

static int ipv4_fwd_update_priority(struct ctl_table *table, int write,
                                    void *buffer, size_t *lenp, loff_t *ppos)
{
        struct net *net;
        int ret;

        net = container_of(table->data, struct net,
                           ipv4.sysctl_ip_fwd_update_priority);
        ret = proc_dou8vec_minmax(table, write, buffer, lenp, ppos);
        if (write && ret == 0)
                call_netevent_notifiers(NETEVENT_IPV4_FWD_UPDATE_PRIORITY_UPDATE,
                                        net);

        return ret;
}

static int proc_tcp_congestion_control(struct ctl_table *ctl, int write,
                                       void *buffer, size_t *lenp, loff_t *ppos)
{
        struct net *net = container_of(ctl->data, struct net,
                                       ipv4.tcp_congestion_control);
        char val[TCP_CA_NAME_MAX];
        struct ctl_table tbl = {
                .data = val,
                .maxlen = TCP_CA_NAME_MAX,
        };
        int ret;

        tcp_get_default_congestion_control(net, val);

        ret = proc_dostring(&tbl, write, buffer, lenp, ppos);
        if (write && ret == 0)
                ret = tcp_set_default_congestion_control(net, val);
        return ret;
}

static int proc_tcp_available_congestion_control(struct ctl_table *ctl,
                                                 int write, void *buffer,
                                                 size_t *lenp, loff_t *ppos)
{
        struct ctl_table tbl = { .maxlen = TCP_CA_BUF_MAX, };
        int ret;

        tbl.data = kmalloc(tbl.maxlen, GFP_USER);
        if (!tbl.data)
                return -ENOMEM;
        tcp_get_available_congestion_control(tbl.data, TCP_CA_BUF_MAX);
        ret = proc_dostring(&tbl, write, buffer, lenp, ppos);
        kfree(tbl.data);
        return ret;
}

static int proc_allowed_congestion_control(struct ctl_table *ctl,
                                           int write, void *buffer,
                                           size_t *lenp, loff_t *ppos)
{
        struct ctl_table tbl = { .maxlen = TCP_CA_BUF_MAX };
        int ret;

        tbl.data = kmalloc(tbl.maxlen, GFP_USER);
        if (!tbl.data)
                return -ENOMEM;

        tcp_get_allowed_congestion_control(tbl.data, tbl.maxlen);
        ret = proc_dostring(&tbl, write, buffer, lenp, ppos);
        if (write && ret == 0)
                ret = tcp_set_allowed_congestion_control(tbl.data);
        kfree(tbl.data);
        return ret;
}

static int sscanf_key(char *buf, __le32 *key)
{
        u32 user_key[4];
        int i, ret = 0;

        if (sscanf(buf, "%x-%x-%x-%x", user_key, user_key + 1,
                   user_key + 2, user_key + 3) != 4) {
                ret = -EINVAL;
        } else {
                for (i = 0; i < ARRAY_SIZE(user_key); i++)
                        key[i] = cpu_to_le32(user_key[i]);
        }
        pr_debug("proc TFO key set 0x%x-%x-%x-%x <- 0x%s: %u\n",
                 user_key[0], user_key[1], user_key[2], user_key[3], buf, ret);

        return ret;
}

static int proc_tcp_fastopen_key(struct ctl_table *table, int write,
                                 void *buffer, size_t *lenp, loff_t *ppos)
{
        struct net *net = container_of(table->data, struct net,
            ipv4.sysctl_tcp_fastopen);
        /* maxlen to print the list of keys in hex (*2), with dashes
         * separating doublewords and a comma in between keys.
         */
        struct ctl_table tbl = { .maxlen = ((TCP_FASTOPEN_KEY_LENGTH *
                                            2 * TCP_FASTOPEN_KEY_MAX) +
                                            (TCP_FASTOPEN_KEY_MAX * 5)) };
        u32 user_key[TCP_FASTOPEN_KEY_BUF_LENGTH / sizeof(u32)];
        __le32 key[TCP_FASTOPEN_KEY_BUF_LENGTH / sizeof(__le32)];
        char *backup_data;
        int ret, i = 0, off = 0, n_keys;

        tbl.data = kmalloc(tbl.maxlen, GFP_KERNEL);
        if (!tbl.data)
                return -ENOMEM;

        n_keys = tcp_fastopen_get_cipher(net, NULL, (u64 *)key);
        if (!n_keys) {
                memset(&key[0], 0, TCP_FASTOPEN_KEY_LENGTH);
                n_keys = 1;
        }

        for (i = 0; i < n_keys * 4; i++)
                user_key[i] = le32_to_cpu(key[i]);

        for (i = 0; i < n_keys; i++) {
                off += snprintf(tbl.data + off, tbl.maxlen - off,
                                "%08x-%08x-%08x-%08x",
                                user_key[i * 4],
                                user_key[i * 4 + 1],
                                user_key[i * 4 + 2],
                                user_key[i * 4 + 3]);

                if (WARN_ON_ONCE(off >= tbl.maxlen - 1))
                        break;

                if (i + 1 < n_keys)
                        off += snprintf(tbl.data + off, tbl.maxlen - off, ",");
        }

        ret = proc_dostring(&tbl, write, buffer, lenp, ppos);

        if (write && ret == 0) {
                backup_data = strchr(tbl.data, ',');
                if (backup_data) {
                        *backup_data = '\0';
                        backup_data++;
                }
                if (sscanf_key(tbl.data, key)) {
                        ret = -EINVAL;
                        goto bad_key;
                }
                if (backup_data) {
                        if (sscanf_key(backup_data, key + 4)) {
                                ret = -EINVAL;
                                goto bad_key;
                        }
                }
                tcp_fastopen_reset_cipher(net, NULL, key,
                                          backup_data ? key + 4 : NULL);
        }

bad_key:
        kfree(tbl.data);
        return ret;
}

static int proc_tfo_blackhole_detect_timeout(struct ctl_table *table,
                                             int write, void *buffer,
                                             size_t *lenp, loff_t *ppos)
{
        struct net *net = container_of(table->data, struct net,
            ipv4.sysctl_tcp_fastopen_blackhole_timeout);
        int ret;

        ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
        if (write && ret == 0)
                atomic_set(&net->ipv4.tfo_active_disable_times, 0);

        return ret;
}

static int proc_tcp_available_ulp(struct ctl_table *ctl,
                                  int write, void *buffer, size_t *lenp,
                                  loff_t *ppos)
{
        struct ctl_table tbl = { .maxlen = TCP_ULP_BUF_MAX, };
        int ret;

        tbl.data = kmalloc(tbl.maxlen, GFP_USER);
        if (!tbl.data)
                return -ENOMEM;
        tcp_get_available_ulp(tbl.data, TCP_ULP_BUF_MAX);
        ret = proc_dostring(&tbl, write, buffer, lenp, ppos);
        kfree(tbl.data);

        return ret;
}

#ifdef CONFIG_IP_ROUTE_MULTIPATH
static int proc_fib_multipath_hash_policy(struct ctl_table *table, int write,
                                          void *buffer, size_t *lenp,
                                          loff_t *ppos)
{
        struct net *net = container_of(table->data, struct net,
            ipv4.sysctl_fib_multipath_hash_policy);
        int ret;

        ret = proc_dou8vec_minmax(table, write, buffer, lenp, ppos);
        if (write && ret == 0)
                call_netevent_notifiers(NETEVENT_IPV4_MPATH_HASH_UPDATE, net);

        return ret;
}

static int proc_fib_multipath_hash_fields(struct ctl_table *table, int write,
                                          void *buffer, size_t *lenp,
                                          loff_t *ppos)
{
        struct net *net;
        int ret;

        net = container_of(table->data, struct net,
                           ipv4.sysctl_fib_multipath_hash_fields);
        ret = proc_douintvec_minmax(table, write, buffer, lenp, ppos);
        if (write && ret == 0)
                call_netevent_notifiers(NETEVENT_IPV4_MPATH_HASH_UPDATE, net);

        return ret;
}
#endif

static struct ctl_table ipv4_table[] = {
        {
                .procname        = "tcp_max_orphans",
                .data                = &sysctl_tcp_max_orphans,
                .maxlen                = sizeof(int),
                .mode                = 0644,
                .proc_handler        = proc_dointvec
        },
        {
                .procname        = "inet_peer_threshold",
                .data                = &inet_peer_threshold,
                .maxlen                = sizeof(int),
                .mode                = 0644,
                .proc_handler        = proc_dointvec
        },
        {
                .procname        = "inet_peer_minttl",
                .data                = &inet_peer_minttl,
                .maxlen                = sizeof(int),
                .mode                = 0644,
                .proc_handler        = proc_dointvec_jiffies,
        },
        {
                .procname        = "inet_peer_maxttl",
                .data                = &inet_peer_maxttl,
                .maxlen                = sizeof(int),
                .mode                = 0644,
                .proc_handler        = proc_dointvec_jiffies,
        },
        {
                .procname        = "tcp_mem",
                .maxlen                = sizeof(sysctl_tcp_mem),
                .data                = &sysctl_tcp_mem,
                .mode                = 0644,
                .proc_handler        = proc_doulongvec_minmax,
        },
        {
                .procname        = "tcp_low_latency",
                .data                = &sysctl_tcp_low_latency,
                .maxlen                = sizeof(int),
                .mode                = 0644,
                .proc_handler        = proc_dointvec
        },
#ifdef CONFIG_NETLABEL
        {
                .procname        = "cipso_cache_enable",
                .data                = &cipso_v4_cache_enabled,
                .maxlen                = sizeof(int),
                .mode                = 0644,
                .proc_handler        = proc_dointvec,
        },
        {
                .procname        = "cipso_cache_bucket_size",
                .data                = &cipso_v4_cache_bucketsize,
                .maxlen                = sizeof(int),
                .mode                = 0644,
                .proc_handler        = proc_dointvec,
        },
        {
                .procname        = "cipso_rbm_optfmt",
                .data                = &cipso_v4_rbm_optfmt,
                .maxlen                = sizeof(int),
                .mode                = 0644,
                .proc_handler        = proc_dointvec,
        },
        {
                .procname        = "cipso_rbm_strictvalid",
                .data                = &cipso_v4_rbm_strictvalid,
                .maxlen                = sizeof(int),
                .mode                = 0644,
                .proc_handler        = proc_dointvec,
        },
#endif /* CONFIG_NETLABEL */
        {
                .procname        = "tcp_available_ulp",
                .maxlen                = TCP_ULP_BUF_MAX,
                .mode                = 0444,
                .proc_handler   = proc_tcp_available_ulp,
        },
        {
                .procname        = "icmp_msgs_per_sec",
                .data                = &sysctl_icmp_msgs_per_sec,
                .maxlen                = sizeof(int),
                .mode                = 0644,
                .proc_handler        = proc_dointvec_minmax,
                .extra1                = SYSCTL_ZERO,
        },
        {
                .procname        = "icmp_msgs_burst",
                .data                = &sysctl_icmp_msgs_burst,
                .maxlen                = sizeof(int),
                .mode                = 0644,
                .proc_handler        = proc_dointvec_minmax,
                .extra1                = SYSCTL_ZERO,
        },
        {
                .procname        = "udp_mem",
                .data                = &sysctl_udp_mem,
                .maxlen                = sizeof(sysctl_udp_mem),
                .mode                = 0644,
                .proc_handler        = proc_doulongvec_minmax,
        },
        {
                .procname        = "fib_sync_mem",
                .data                = &sysctl_fib_sync_mem,
                .maxlen                = sizeof(sysctl_fib_sync_mem),
                .mode                = 0644,
                .proc_handler        = proc_douintvec_minmax,
                .extra1                = &sysctl_fib_sync_mem_min,
                .extra2                = &sysctl_fib_sync_mem_max,
        },
        {
                .procname        = "tcp_rx_skb_cache",
                .data                = &tcp_rx_skb_cache_key.key,
                .mode                = 0644,
                .proc_handler        = proc_do_static_key,
        },
        {
                .procname        = "tcp_tx_skb_cache",
                .data                = &tcp_tx_skb_cache_key.key,
                .mode                = 0644,
                .proc_handler        = proc_do_static_key,
        },
        { }
};

static struct ctl_table ipv4_net_table[] = {
        {
                .procname        = "icmp_echo_ignore_all",
                .data                = &init_net.ipv4.sysctl_icmp_echo_ignore_all,
                .maxlen                = sizeof(u8),
                .mode                = 0644,
                .proc_handler        = proc_dou8vec_minmax,
        },
        {
                .procname        = "icmp_echo_enable_probe",
                .data                = &init_net.ipv4.sysctl_icmp_echo_enable_probe,
                .maxlen                = sizeof(u8),
                .mode                = 0644,
                .proc_handler        = proc_dou8vec_minmax,
                .extra1                = SYSCTL_ZERO,
                .extra2                = SYSCTL_ONE
        },
        {
                .procname        = "icmp_echo_ignore_broadcasts",
                .data                = &init_net.ipv4.sysctl_icmp_echo_ignore_broadcasts,
                .maxlen                = sizeof(u8),
                .mode                = 0644,
                .proc_handler        = proc_dou8vec_minmax,
        },
        {
                .procname        = "icmp_ignore_bogus_error_responses",
                .data                = &init_net.ipv4.sysctl_icmp_ignore_bogus_error_responses,
                .maxlen                = sizeof(u8),
                .mode                = 0644,
                .proc_handler        = proc_dou8vec_minmax,
                .extra1                = SYSCTL_ZERO,
                .extra2                = SYSCTL_ONE
        },
        {
                .procname        = "icmp_errors_use_inbound_ifaddr",
                .data                = &init_net.ipv4.sysctl_icmp_errors_use_inbound_ifaddr,
                .maxlen                = sizeof(u8),
                .mode                = 0644,
                .proc_handler        = proc_dou8vec_minmax,
                .extra1                = SYSCTL_ZERO,
                .extra2                = SYSCTL_ONE
        },
        {
                .procname        = "icmp_ratelimit",
                .data                = &init_net.ipv4.sysctl_icmp_ratelimit,
                .maxlen                = sizeof(int),
                .mode                = 0644,
                .proc_handler        = proc_dointvec_ms_jiffies,
        },
        {
                .procname        = "icmp_ratemask",
                .data                = &init_net.ipv4.sysctl_icmp_ratemask,
                .maxlen                = sizeof(int),
                .mode                = 0644,
                .proc_handler        = proc_dointvec
        },
        {
                .procname        = "ping_group_range",
                .data                = &init_net.ipv4.ping_group_range.range,
                .maxlen                = sizeof(gid_t)*2,
                .mode                = 0644,
                .proc_handler        = ipv4_ping_group_range,
        },
#ifdef CONFIG_NET_L3_MASTER_DEV
        {
                .procname        = "raw_l3mdev_accept",
                .data                = &init_net.ipv4.sysctl_raw_l3mdev_accept,
                .maxlen                = sizeof(u8),
                .mode                = 0644,
                .proc_handler        = proc_dou8vec_minmax,
                .extra1                = SYSCTL_ZERO,
                .extra2                = SYSCTL_ONE,
        },
#endif
        {
                .procname        = "tcp_ecn",
                .data                = &init_net.ipv4.sysctl_tcp_ecn,
                .maxlen                = sizeof(u8),
                .mode                = 0644,
                .proc_handler        = proc_dou8vec_minmax,
                .extra1                = SYSCTL_ZERO,
                .extra2                = SYSCTL_TWO,
        },
        {
                .procname        = "tcp_ecn_fallback",
                .data                = &init_net.ipv4.sysctl_tcp_ecn_fallback,
                .maxlen                = sizeof(u8),
                .mode                = 0644,
                .proc_handler        = proc_dou8vec_minmax,
                .extra1                = SYSCTL_ZERO,
                .extra2                = SYSCTL_ONE,
        },
        {
                .procname        = "ip_dynaddr",
                .data                = &init_net.ipv4.sysctl_ip_dynaddr,
                .maxlen                = sizeof(u8),
                .mode                = 0644,
                .proc_handler        = proc_dou8vec_minmax,
        },
        {
                .procname        = "ip_early_demux",
                .data                = &init_net.ipv4.sysctl_ip_early_demux,
                .maxlen                = sizeof(u8),
                .mode                = 0644,
                .proc_handler        = proc_dou8vec_minmax,
        },
        {
                .procname       = "udp_early_demux",
                .data           = &init_net.ipv4.sysctl_udp_early_demux,
                .maxlen         = sizeof(u8),
                .mode           = 0644,
                .proc_handler   = proc_dou8vec_minmax,
        },
        {
                .procname       = "tcp_early_demux",
                .data           = &init_net.ipv4.sysctl_tcp_early_demux,
                .maxlen         = sizeof(u8),
                .mode           = 0644,
                .proc_handler   = proc_dou8vec_minmax,
        },
        {
                .procname       = "nexthop_compat_mode",
                .data           = &init_net.ipv4.sysctl_nexthop_compat_mode,
                .maxlen         = sizeof(u8),
                .mode           = 0644,
                .proc_handler   = proc_dou8vec_minmax,
                .extra1                = SYSCTL_ZERO,
                .extra2                = SYSCTL_ONE,
        },
        {
                .procname        = "ip_default_ttl",
                .data                = &init_net.ipv4.sysctl_ip_default_ttl,
                .maxlen                = sizeof(u8),
                .mode                = 0644,
                .proc_handler        = proc_dou8vec_minmax,
                .extra1                = &ip_ttl_min,
                .extra2                = &ip_ttl_max,
        },
        {
                .procname        = "ip_local_port_range",
                .maxlen                = sizeof(init_net.ipv4.ip_local_ports.range),
                .data                = &init_net.ipv4.ip_local_ports.range,
                .mode                = 0644,
                .proc_handler        = ipv4_local_port_range,
        },
        {
                .procname        = "ip_local_reserved_ports",
                .data                = &init_net.ipv4.sysctl_local_reserved_ports,
                .maxlen                = 65536,
                .mode                = 0644,
                .proc_handler        = proc_do_large_bitmap,
        },
        {
                .procname        = "ip_no_pmtu_disc",
                .data                = &init_net.ipv4.sysctl_ip_no_pmtu_disc,
                .maxlen                = sizeof(u8),
                .mode                = 0644,
                .proc_handler        = proc_dou8vec_minmax,
        },
        {
                .procname        = "ip_forward_use_pmtu",
                .data                = &init_net.ipv4.sysctl_ip_fwd_use_pmtu,
                .maxlen                = sizeof(u8),
                .mode                = 0644,
                .proc_handler        = proc_dou8vec_minmax,
        },
        {
                .procname        = "ip_forward_update_priority",
                .data                = &init_net.ipv4.sysctl_ip_fwd_update_priority,
                .maxlen                = sizeof(u8),
                .mode                = 0644,
                .proc_handler   = ipv4_fwd_update_priority,
                .extra1                = SYSCTL_ZERO,
                .extra2                = SYSCTL_ONE,
        },
        {
                .procname        = "ip_nonlocal_bind",
                .data                = &init_net.ipv4.sysctl_ip_nonlocal_bind,
                .maxlen                = sizeof(u8),
                .mode                = 0644,
                .proc_handler        = proc_dou8vec_minmax,
        },
        {
                .procname        = "ip_autobind_reuse",
                .data                = &init_net.ipv4.sysctl_ip_autobind_reuse,
                .maxlen                = sizeof(u8),
                .mode                = 0644,
                .proc_handler        = proc_dou8vec_minmax,
                .extra1         = SYSCTL_ZERO,
                .extra2         = SYSCTL_ONE,
        },
        {
                .procname        = "fwmark_reflect",
                .data                = &init_net.ipv4.sysctl_fwmark_reflect,
                .maxlen                = sizeof(u8),
                .mode                = 0644,
                .proc_handler        = proc_dou8vec_minmax,
        },
        {
                .procname        = "tcp_fwmark_accept",
                .data                = &init_net.ipv4.sysctl_tcp_fwmark_accept,
                .maxlen                = sizeof(u8),
                .mode                = 0644,
                .proc_handler        = proc_dou8vec_minmax,
        },
#ifdef CONFIG_NET_L3_MASTER_DEV
        {
                .procname        = "tcp_l3mdev_accept",
                .data                = &init_net.ipv4.sysctl_tcp_l3mdev_accept,
                .maxlen                = sizeof(u8),
                .mode                = 0644,
                .proc_handler        = proc_dou8vec_minmax,
                .extra1                = SYSCTL_ZERO,
                .extra2                = SYSCTL_ONE,
        },
#endif
        {
                .procname        = "tcp_mtu_probing",
                .data                = &init_net.ipv4.sysctl_tcp_mtu_probing,
                .maxlen                = sizeof(u8),
                .mode                = 0644,
                .proc_handler        = proc_dou8vec_minmax,
        },
        {
                .procname        = "tcp_base_mss",
                .data                = &init_net.ipv4.sysctl_tcp_base_mss,
                .maxlen                = sizeof(int),
                .mode                = 0644,
                .proc_handler        = proc_dointvec,
        },
        {
                .procname        = "tcp_min_snd_mss",
                .data                = &init_net.ipv4.sysctl_tcp_min_snd_mss,
                .maxlen                = sizeof(int),
                .mode                = 0644,
                .proc_handler        = proc_dointvec_minmax,
                .extra1                = &tcp_min_snd_mss_min,
                .extra2                = &tcp_min_snd_mss_max,
        },
        {
                .procname        = "tcp_mtu_probe_floor",
                .data                = &init_net.ipv4.sysctl_tcp_mtu_probe_floor,
                .maxlen                = sizeof(int),
                .mode                = 0644,
                .proc_handler        = proc_dointvec_minmax,
                .extra1                = &tcp_min_snd_mss_min,
                .extra2                = &tcp_min_snd_mss_max,
        },
        {
                .procname        = "tcp_probe_threshold",
                .data                = &init_net.ipv4.sysctl_tcp_probe_threshold,
                .maxlen                = sizeof(int),
                .mode                = 0644,
                .proc_handler        = proc_dointvec,
        },
        {
                .procname        = "tcp_probe_interval",
                .data                = &init_net.ipv4.sysctl_tcp_probe_interval,
                .maxlen                = sizeof(u32),
                .mode                = 0644,
                .proc_handler        = proc_douintvec_minmax,
                .extra2                = &u32_max_div_HZ,
        },
        {
                .procname        = "igmp_link_local_mcast_reports",
                .data                = &init_net.ipv4.sysctl_igmp_llm_reports,
                .maxlen                = sizeof(u8),
                .mode                = 0644,
                .proc_handler        = proc_dou8vec_minmax,
        },
        {
                .procname        = "igmp_max_memberships",
                .data                = &init_net.ipv4.sysctl_igmp_max_memberships,
                .maxlen                = sizeof(int),
                .mode                = 0644,
                .proc_handler        = proc_dointvec
        },
        {
                .procname        = "igmp_max_msf",
                .data                = &init_net.ipv4.sysctl_igmp_max_msf,
                .maxlen                = sizeof(int),
                .mode                = 0644,
                .proc_handler        = proc_dointvec
        },
#ifdef CONFIG_IP_MULTICAST
        {
                .procname        = "igmp_qrv",
                .data                = &init_net.ipv4.sysctl_igmp_qrv,
                .maxlen                = sizeof(int),
                .mode                = 0644,
                .proc_handler        = proc_dointvec_minmax,
                .extra1                = SYSCTL_ONE
        },
#endif
        {
                .procname        = "tcp_congestion_control",
                .data                = &init_net.ipv4.tcp_congestion_control,
                .mode                = 0644,
                .maxlen                = TCP_CA_NAME_MAX,
                .proc_handler        = proc_tcp_congestion_control,
        },
        {
                .procname        = "tcp_available_congestion_control",
                .maxlen                = TCP_CA_BUF_MAX,
                .mode                = 0444,
                .proc_handler   = proc_tcp_available_congestion_control,
        },
        {
                .procname        = "tcp_allowed_congestion_control",
                .maxlen                = TCP_CA_BUF_MAX,
                .mode                = 0644,
                .proc_handler   = proc_allowed_congestion_control,
        },
        {
                .procname        = "tcp_keepalive_time",
                .data                = &init_net.ipv4.sysctl_tcp_keepalive_time,
                .maxlen                = sizeof(int),
                .mode                = 0644,
                .proc_handler        = proc_dointvec_jiffies,
        },
        {
                .procname        = "tcp_keepalive_probes",
                .data                = &init_net.ipv4.sysctl_tcp_keepalive_probes,
                .maxlen                = sizeof(u8),
                .mode                = 0644,
                .proc_handler        = proc_dou8vec_minmax,
        },
        {
                .procname        = "tcp_keepalive_intvl",
                .data                = &init_net.ipv4.sysctl_tcp_keepalive_intvl,
                .maxlen                = sizeof(int),
                .mode                = 0644,
                .proc_handler        = proc_dointvec_jiffies,
        },
        {
                .procname        = "tcp_syn_retries",
                .data                = &init_net.ipv4.sysctl_tcp_syn_retries,
                .maxlen                = sizeof(u8),
                .mode                = 0644,
                .proc_handler        = proc_dou8vec_minmax,
                .extra1                = &tcp_syn_retries_min,
                .extra2                = &tcp_syn_retries_max
        },
        {
                .procname        = "tcp_synack_retries",
                .data                = &init_net.ipv4.sysctl_tcp_synack_retries,
                .maxlen                = sizeof(u8),
                .mode                = 0644,
                .proc_handler        = proc_dou8vec_minmax,
        },
#ifdef CONFIG_SYN_COOKIES
        {
                .procname        = "tcp_syncookies",
                .data                = &init_net.ipv4.sysctl_tcp_syncookies,
                .maxlen                = sizeof(u8),
                .mode                = 0644,
                .proc_handler        = proc_dou8vec_minmax,
        },
#endif
        {
                .procname        = "tcp_migrate_req",
                .data                = &init_net.ipv4.sysctl_tcp_migrate_req,
                .maxlen                = sizeof(u8),
                .mode                = 0644,
                .proc_handler        = proc_dou8vec_minmax,
                .extra1                = SYSCTL_ZERO,
                .extra2                = SYSCTL_ONE
        },
        {
                .procname        = "tcp_reordering",
                .data                = &init_net.ipv4.sysctl_tcp_reordering,
                .maxlen                = sizeof(int),
                .mode                = 0644,
                .proc_handler        = proc_dointvec
        },
        {
                .procname        = "tcp_retries1",
                .data                = &init_net.ipv4.sysctl_tcp_retries1,
                .maxlen                = sizeof(u8),
                .mode                = 0644,
                .proc_handler        = proc_dou8vec_minmax,
                .extra2                = &tcp_retr1_max
        },
        {
                .procname        = "tcp_retries2",
                .data                = &init_net.ipv4.sysctl_tcp_retries2,
                .maxlen                = sizeof(u8),
                .mode                = 0644,
                .proc_handler        = proc_dou8vec_minmax,
        },
        {
                .procname        = "tcp_orphan_retries",
                .data                = &init_net.ipv4.sysctl_tcp_orphan_retries,
                .maxlen                = sizeof(u8),
                .mode                = 0644,
                .proc_handler        = proc_dou8vec_minmax,
        },
        {
                .procname        = "tcp_fin_timeout",
                .data                = &init_net.ipv4.sysctl_tcp_fin_timeout,
                .maxlen                = sizeof(int),
                .mode                = 0644,
                .proc_handler        = proc_dointvec_jiffies,
        },
        {
                .procname        = "tcp_notsent_lowat",
                .data                = &init_net.ipv4.sysctl_tcp_notsent_lowat,
                .maxlen                = sizeof(unsigned int),
                .mode                = 0644,
                .proc_handler        = proc_douintvec,
        },
        {
                .procname        = "tcp_tw_reuse",
                .data                = &init_net.ipv4.sysctl_tcp_tw_reuse,
                .maxlen                = sizeof(u8),
                .mode                = 0644,
                .proc_handler        = proc_dou8vec_minmax,
                .extra1                = SYSCTL_ZERO,
                .extra2                = &two,
        },
        {
                .procname        = "tcp_max_tw_buckets",
                .data                = &init_net.ipv4.tcp_death_row.sysctl_max_tw_buckets,
                .maxlen                = sizeof(int),
                .mode                = 0644,
                .proc_handler        = proc_dointvec
        },
        {
                .procname        = "tcp_max_syn_backlog",
                .data                = &init_net.ipv4.sysctl_max_syn_backlog,
                .maxlen                = sizeof(int),
                .mode                = 0644,
                .proc_handler        = proc_dointvec
        },
        {
                .procname        = "tcp_fastopen",
                .data                = &init_net.ipv4.sysctl_tcp_fastopen,
                .maxlen                = sizeof(int),
                .mode                = 0644,
                .proc_handler        = proc_dointvec,
        },
        {
                .procname        = "tcp_fastopen_key",
                .mode                = 0600,
                .data                = &init_net.ipv4.sysctl_tcp_fastopen,
                /* maxlen to print the list of keys in hex (*2), with dashes
                 * separating doublewords and a comma in between keys.
                 */
                .maxlen                = ((TCP_FASTOPEN_KEY_LENGTH *
                                   2 * TCP_FASTOPEN_KEY_MAX) +
                                   (TCP_FASTOPEN_KEY_MAX * 5)),
                .proc_handler        = proc_tcp_fastopen_key,
        },
        {
                .procname        = "tcp_fastopen_blackhole_timeout_sec",
                .data                = &init_net.ipv4.sysctl_tcp_fastopen_blackhole_timeout,
                .maxlen                = sizeof(int),
                .mode                = 0644,
                .proc_handler        = proc_tfo_blackhole_detect_timeout,
                .extra1                = SYSCTL_ZERO,
        },
#ifdef CONFIG_IP_ROUTE_MULTIPATH
        {
                .procname        = "fib_multipath_use_neigh",
                .data                = &init_net.ipv4.sysctl_fib_multipath_use_neigh,
                .maxlen                = sizeof(u8),
                .mode                = 0644,
                .proc_handler        = proc_dou8vec_minmax,
                .extra1                = SYSCTL_ZERO,
                .extra2                = SYSCTL_ONE,
        },
        {
                .procname        = "fib_multipath_hash_policy",
                .data                = &init_net.ipv4.sysctl_fib_multipath_hash_policy,
                .maxlen                = sizeof(u8),
                .mode                = 0644,
                .proc_handler        = proc_fib_multipath_hash_policy,
                .extra1                = SYSCTL_ZERO,
                .extra2                = &three,
        },
        {
                .procname        = "fib_multipath_hash_fields",
                .data                = &init_net.ipv4.sysctl_fib_multipath_hash_fields,
                .maxlen                = sizeof(u32),
                .mode                = 0644,
                .proc_handler        = proc_fib_multipath_hash_fields,
                .extra1                = SYSCTL_ONE,
                .extra2                = &fib_multipath_hash_fields_all_mask,
        },
#endif
        {
                .procname        = "ip_unprivileged_port_start",
                .maxlen                = sizeof(int),
                .data                = &init_net.ipv4.sysctl_ip_prot_sock,
                .mode                = 0644,
                .proc_handler        = ipv4_privileged_ports,
        },
#ifdef CONFIG_NET_L3_MASTER_DEV
        {
                .procname        = "udp_l3mdev_accept",
                .data                = &init_net.ipv4.sysctl_udp_l3mdev_accept,
                .maxlen                = sizeof(u8),
                .mode                = 0644,
                .proc_handler        = proc_dou8vec_minmax,
                .extra1                = SYSCTL_ZERO,
                .extra2                = SYSCTL_ONE,
        },
#endif
        {
                .procname        = "tcp_sack",
                .data                = &init_net.ipv4.sysctl_tcp_sack,
                .maxlen                = sizeof(u8),
                .mode                = 0644,
                .proc_handler        = proc_dou8vec_minmax,
        },
        {
                .procname        = "tcp_window_scaling",
                .data                = &init_net.ipv4.sysctl_tcp_window_scaling,
                .maxlen                = sizeof(u8),
                .mode                = 0644,
                .proc_handler        = proc_dou8vec_minmax,
        },
        {
                .procname        = "tcp_timestamps",
                .data                = &init_net.ipv4.sysctl_tcp_timestamps,
                .maxlen                = sizeof(u8),
                .mode                = 0644,
                .proc_handler        = proc_dou8vec_minmax,
        },
        {
                .procname        = "tcp_early_retrans",
                .data                = &init_net.ipv4.sysctl_tcp_early_retrans,
                .maxlen                = sizeof(u8),
                .mode                = 0644,
                .proc_handler        = proc_dou8vec_minmax,
                .extra1                = SYSCTL_ZERO,
                .extra2                = &four,
        },
        {
                .procname        = "tcp_recovery",
                .data                = &init_net.ipv4.sysctl_tcp_recovery,
                .maxlen                = sizeof(u8),
                .mode                = 0644,
                .proc_handler        = proc_dou8vec_minmax,
        },
        {
                .procname       = "tcp_thin_linear_timeouts",
                .data           = &init_net.ipv4.sysctl_tcp_thin_linear_timeouts,
                .maxlen         = sizeof(u8),
                .mode           = 0644,
                .proc_handler   = proc_dou8vec_minmax,
        },
        {
                .procname        = "tcp_slow_start_after_idle",
                .data                = &init_net.ipv4.sysctl_tcp_slow_start_after_idle,
                .maxlen                = sizeof(u8),
                .mode                = 0644,
                .proc_handler        = proc_dou8vec_minmax,
        },
        {
                .procname        = "tcp_retrans_collapse",
                .data                = &init_net.ipv4.sysctl_tcp_retrans_collapse,
                .maxlen                = sizeof(u8),
                .mode                = 0644,
                .proc_handler        = proc_dou8vec_minmax,
        },
        {
                .procname        = "tcp_stdurg",
                .data                = &init_net.ipv4.sysctl_tcp_stdurg,
                .maxlen                = sizeof(u8),
                .mode                = 0644,
                .proc_handler        = proc_dou8vec_minmax,
        },
        {
                .procname        = "tcp_rfc1337",
                .data                = &init_net.ipv4.sysctl_tcp_rfc1337,
                .maxlen                = sizeof(u8),
                .mode                = 0644,
                .proc_handler        = proc_dou8vec_minmax,
        },
        {
                .procname        = "tcp_abort_on_overflow",
                .data                = &init_net.ipv4.sysctl_tcp_abort_on_overflow,
                .maxlen                = sizeof(u8),
                .mode                = 0644,
                .proc_handler        = proc_dou8vec_minmax,
        },
        {
                .procname        = "tcp_fack",
                .data                = &init_net.ipv4.sysctl_tcp_fack,
                .maxlen                = sizeof(u8),
                .mode                = 0644,
                .proc_handler        = proc_dou8vec_minmax,
        },
        {
                .procname        = "tcp_max_reordering",
                .data                = &init_net.ipv4.sysctl_tcp_max_reordering,
                .maxlen                = sizeof(int),
                .mode                = 0644,
                .proc_handler        = proc_dointvec
        },
        {
                .procname        = "tcp_dsack",
                .data                = &init_net.ipv4.sysctl_tcp_dsack,
                .maxlen                = sizeof(u8),
                .mode                = 0644,
                .proc_handler        = proc_dou8vec_minmax,
        },
        {
                .procname        = "tcp_app_win",
                .data                = &init_net.ipv4.sysctl_tcp_app_win,
                .maxlen                = sizeof(u8),
                .mode                = 0644,
                .proc_handler        = proc_dou8vec_minmax,
                .extra1                = SYSCTL_ZERO,
                .extra2                = &tcp_app_win_max,
        },
        {
                .procname        = "tcp_adv_win_scale",
                .data                = &init_net.ipv4.sysctl_tcp_adv_win_scale,
                .maxlen                = sizeof(int),
                .mode                = 0644,
                .proc_handler        = proc_dointvec_minmax,
                .extra1                = &tcp_adv_win_scale_min,
                .extra2                = &tcp_adv_win_scale_max,
        },
        {
                .procname        = "tcp_frto",
                .data                = &init_net.ipv4.sysctl_tcp_frto,
                .maxlen                = sizeof(u8),
                .mode                = 0644,
                .proc_handler        = proc_dou8vec_minmax,
        },
        {
                .procname        = "tcp_no_metrics_save",
                .data                = &init_net.ipv4.sysctl_tcp_nometrics_save,
                .maxlen                = sizeof(u8),
                .mode                = 0644,
                .proc_handler        = proc_dou8vec_minmax,
        },
        {
                .procname        = "tcp_no_ssthresh_metrics_save",
                .data                = &init_net.ipv4.sysctl_tcp_no_ssthresh_metrics_save,
                .maxlen                = sizeof(u8),
                .mode                = 0644,
                .proc_handler        = proc_dou8vec_minmax,
                .extra1                = SYSCTL_ZERO,
                .extra2                = SYSCTL_ONE,
        },
        {
                .procname        = "tcp_moderate_rcvbuf",
                .data                = &init_net.ipv4.sysctl_tcp_moderate_rcvbuf,
                .maxlen                = sizeof(u8),
                .mode                = 0644,
                .proc_handler        = proc_dou8vec_minmax,
        },
        {
                .procname        = "tcp_tso_win_divisor",
                .data                = &init_net.ipv4.sysctl_tcp_tso_win_divisor,
                .maxlen                = sizeof(u8),
                .mode                = 0644,
                .proc_handler        = proc_dou8vec_minmax,
        },
        {
                .procname        = "tcp_workaround_signed_windows",
                .data                = &init_net.ipv4.sysctl_tcp_workaround_signed_windows,
                .maxlen                = sizeof(u8),
                .mode                = 0644,
                .proc_handler        = proc_dou8vec_minmax,
        },
        {
                .procname        = "tcp_limit_output_bytes",
                .data                = &init_net.ipv4.sysctl_tcp_limit_output_bytes,
                .maxlen                = sizeof(int),
                .mode                = 0644,
                .proc_handler        = proc_dointvec
        },
        {
                .procname        = "tcp_challenge_ack_limit",
                .data                = &init_net.ipv4.sysctl_tcp_challenge_ack_limit,
                .maxlen                = sizeof(int),
                .mode                = 0644,
                .proc_handler        = proc_dointvec
        },
        {
                .procname        = "tcp_min_tso_segs",
                .data                = &init_net.ipv4.sysctl_tcp_min_tso_segs,
                .maxlen                = sizeof(u8),
                .mode                = 0644,
                .proc_handler        = proc_dou8vec_minmax,
                .extra1                = SYSCTL_ONE,
        },
        {
                .procname        = "tcp_min_rtt_wlen",
                .data                = &init_net.ipv4.sysctl_tcp_min_rtt_wlen,
                .maxlen                = sizeof(int),
                .mode                = 0644,
                .proc_handler        = proc_dointvec_minmax,
                .extra1                = SYSCTL_ZERO,
                .extra2                = &one_day_secs
        },
        {
                .procname        = "tcp_autocorking",
                .data                = &init_net.ipv4.sysctl_tcp_autocorking,
                .maxlen                = sizeof(u8),
                .mode                = 0644,
                .proc_handler        = proc_dou8vec_minmax,
                .extra1                = SYSCTL_ZERO,
                .extra2                = SYSCTL_ONE,
        },
        {
                .procname        = "tcp_invalid_ratelimit",
                .data                = &init_net.ipv4.sysctl_tcp_invalid_ratelimit,
                .maxlen                = sizeof(int),
                .mode                = 0644,
                .proc_handler        = proc_dointvec_ms_jiffies,
        },
        {
                .procname        = "tcp_pacing_ss_ratio",
                .data                = &init_net.ipv4.sysctl_tcp_pacing_ss_ratio,
                .maxlen                = sizeof(int),
                .mode                = 0644,
                .proc_handler        = proc_dointvec_minmax,
                .extra1                = SYSCTL_ZERO,
                .extra2                = &thousand,
        },
        {
                .procname        = "tcp_pacing_ca_ratio",
                .data                = &init_net.ipv4.sysctl_tcp_pacing_ca_ratio,
                .maxlen                = sizeof(int),
                .mode                = 0644,
                .proc_handler        = proc_dointvec_minmax,
                .extra1                = SYSCTL_ZERO,
                .extra2                = &thousand,
        },
        {
                .procname        = "tcp_wmem",
                .data                = &init_net.ipv4.sysctl_tcp_wmem,
                .maxlen                = sizeof(init_net.ipv4.sysctl_tcp_wmem),
                .mode                = 0644,
                .proc_handler        = proc_dointvec_minmax,
                .extra1                = SYSCTL_ONE,
        },
        {
                .procname        = "tcp_rmem",
                .data                = &init_net.ipv4.sysctl_tcp_rmem,
                .maxlen                = sizeof(init_net.ipv4.sysctl_tcp_rmem),
                .mode                = 0644,
                .proc_handler        = proc_dointvec_minmax,
                .extra1                = SYSCTL_ONE,
        },
        {
                .procname        = "tcp_comp_sack_delay_ns",
                .data                = &init_net.ipv4.sysctl_tcp_comp_sack_delay_ns,
                .maxlen                = sizeof(unsigned long),
                .mode                = 0644,
                .proc_handler        = proc_doulongvec_minmax,
        },
        {
                .procname        = "tcp_comp_sack_slack_ns",
                .data                = &init_net.ipv4.sysctl_tcp_comp_sack_slack_ns,
                .maxlen                = sizeof(unsigned long),
                .mode                = 0644,
                .proc_handler        = proc_doulongvec_minmax,
        },
        {
                .procname        = "tcp_comp_sack_nr",
                .data                = &init_net.ipv4.sysctl_tcp_comp_sack_nr,
                .maxlen                = sizeof(u8),
                .mode                = 0644,
                .proc_handler        = proc_dou8vec_minmax,
                .extra1                = SYSCTL_ZERO,
        },
        {
                .procname       = "tcp_reflect_tos",
                .data           = &init_net.ipv4.sysctl_tcp_reflect_tos,
                .maxlen         = sizeof(u8),
                .mode           = 0644,
                .proc_handler   = proc_dou8vec_minmax,
                .extra1         = SYSCTL_ZERO,
                .extra2         = SYSCTL_ONE,
        },
        {
                .procname        = "udp_rmem_min",
                .data                = &init_net.ipv4.sysctl_udp_rmem_min,
                .maxlen                = sizeof(init_net.ipv4.sysctl_udp_rmem_min),
                .mode                = 0644,
                .proc_handler        = proc_dointvec_minmax,
                .extra1                = SYSCTL_ONE
        },
        {
                .procname        = "udp_wmem_min",
                .data                = &init_net.ipv4.sysctl_udp_wmem_min,
                .maxlen                = sizeof(init_net.ipv4.sysctl_udp_wmem_min),
                .mode                = 0644,
                .proc_handler        = proc_dointvec_minmax,
                .extra1                = SYSCTL_ONE
        },
        {
                .procname        = "fib_notify_on_flag_change",
                .data                = &init_net.ipv4.sysctl_fib_notify_on_flag_change,
                .maxlen                = sizeof(u8),
                .mode                = 0644,
                .proc_handler        = proc_dou8vec_minmax,
                .extra1                = SYSCTL_ZERO,
                .extra2                = &two,
        },
        { }
};

static __net_init int ipv4_sysctl_init_net(struct net *net)
{
        struct ctl_table *table;

        table = ipv4_net_table;
        if (!net_eq(net, &init_net)) {
                int i;

                table = kmemdup(table, sizeof(ipv4_net_table), GFP_KERNEL);
                if (!table)
                        goto err_alloc;

                for (i = 0; i < ARRAY_SIZE(ipv4_net_table) - 1; i++) {
                        if (table[i].data) {
                                /* Update the variables to point into
                                 * the current struct net
                                 */
                                table[i].data += (void *)net - (void *)&init_net;
                        } else {
                                /* Entries without data pointer are global;
                                 * Make them read-only in non-init_net ns
                                 */
                                table[i].mode &= ~0222;
                        }
                }
        }

        net->ipv4.ipv4_hdr = register_net_sysctl(net, "net/ipv4", table);
        if (!net->ipv4.ipv4_hdr)
                goto err_reg;

        net->ipv4.sysctl_local_reserved_ports = kzalloc(65536 / 8, GFP_KERNEL);
        if (!net->ipv4.sysctl_local_reserved_ports)
                goto err_ports;

        return 0;

err_ports:
        unregister_net_sysctl_table(net->ipv4.ipv4_hdr);
err_reg:
        if (!net_eq(net, &init_net))
                kfree(table);
err_alloc:
        return -ENOMEM;
}

static __net_exit void ipv4_sysctl_exit_net(struct net *net)
{
        struct ctl_table *table;

        kfree(net->ipv4.sysctl_local_reserved_ports);
        table = net->ipv4.ipv4_hdr->ctl_table_arg;
        unregister_net_sysctl_table(net->ipv4.ipv4_hdr);
        kfree(table);
}

static __net_initdata struct pernet_operations ipv4_sysctl_ops = {
        .init = ipv4_sysctl_init_net,
        .exit = ipv4_sysctl_exit_net,
};

static __init int sysctl_ipv4_init(void)
{
        struct ctl_table_header *hdr;

        hdr = register_net_sysctl(&init_net, "net/ipv4", ipv4_table);
        if (!hdr)
                return -ENOMEM;

        if (register_pernet_subsys(&ipv4_sysctl_ops)) {
                unregister_net_sysctl_table(hdr);
                return -ENOMEM;
        }

        return 0;
}

__initcall(sysctl_ipv4_init);


































   78 


   78 












   72 

   14 

















   56 









  142 









    1 
   42 






   56 

  142 
















   43 


   43 




   43 

   43 





















































   47 




   47 
















    1 




    2 


















    3 



    5 






















    6 




    6 
    3 
    1 
    2 










































    6 





    6 












    1 



    5 














   16 
   16 






















    6 












    6 


















    6 































































































































   86 






   86 

   86 








   85 






   86 







   86 



   86 


































   47 










   47 



   47 





   47 



   47 
   47 



















   93 


   52 





























   41 
   41 






















   86 


















   86 







   86 













   86 

   86 








   86 













   86 





























   86 

   86 


   86 


   86 










   72 



   72 
















































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * INET                An implementation of the TCP/IP protocol suite for the LINUX
 *                operating system.  INET is implemented using the BSD Socket
 *                interface as the means of communication with the user level.
 *
 *                Generic INET transport hashtables
 *
 * Authors:        Lotsa people, from code originally in tcp
 */

#include <linux/module.h>
#include <linux/random.h>
#include <linux/sched.h>
#include <linux/slab.h>
#include <linux/wait.h>
#include <linux/vmalloc.h>
#include <linux/memblock.h>

#include <net/addrconf.h>
#include <net/inet_connection_sock.h>
#include <net/inet_hashtables.h>
#if IS_ENABLED(CONFIG_IPV6)
#include <net/inet6_hashtables.h>
#endif
#include <net/secure_seq.h>
#include <net/ip.h>
#include <net/tcp.h>
#include <net/sock_reuseport.h>

u32 inet_ehashfn(const struct net *net, const __be32 laddr,
                 const __u16 lport, const __be32 faddr,
                 const __be16 fport)
{
        static u32 inet_ehash_secret __read_mostly;

        net_get_random_once(&inet_ehash_secret, sizeof(inet_ehash_secret));

        return __inet_ehashfn(laddr, lport, faddr, fport,
                              inet_ehash_secret + net_hash_mix(net));
}
EXPORT_SYMBOL_GPL(inet_ehashfn);

/* This function handles inet_sock, but also timewait and request sockets
 * for IPv4/IPv6.
 */
static u32 sk_ehashfn(const struct sock *sk)
{
#if IS_ENABLED(CONFIG_IPV6)
        if (sk->sk_family == AF_INET6 &&
            !ipv6_addr_v4mapped(&sk->sk_v6_daddr))
                return inet6_ehashfn(sock_net(sk),
                                     &sk->sk_v6_rcv_saddr, sk->sk_num,
                                     &sk->sk_v6_daddr, sk->sk_dport);
#endif
        return inet_ehashfn(sock_net(sk),
                            sk->sk_rcv_saddr, sk->sk_num,
                            sk->sk_daddr, sk->sk_dport);
}

/*
 * Allocate and initialize a new local port bind bucket.
 * The bindhash mutex for snum's hash chain must be held here.
 */
struct inet_bind_bucket *inet_bind_bucket_create(struct kmem_cache *cachep,
                                                 struct net *net,
                                                 struct inet_bind_hashbucket *head,
                                                 const unsigned short snum,
                                                 int l3mdev)
{
        struct inet_bind_bucket *tb = kmem_cache_alloc(cachep, GFP_ATOMIC);

        if (tb) {
                write_pnet(&tb->ib_net, net);
                tb->l3mdev    = l3mdev;
                tb->port      = snum;
                tb->fastreuse = 0;
                tb->fastreuseport = 0;
                INIT_HLIST_HEAD(&tb->owners);
                hlist_add_head(&tb->node, &head->chain);
        }
        return tb;
}

/*
 * Caller must hold hashbucket lock for this tb with local BH disabled
 */
void inet_bind_bucket_destroy(struct kmem_cache *cachep, struct inet_bind_bucket *tb)
{
        if (hlist_empty(&tb->owners)) {
                __hlist_del(&tb->node);
                kmem_cache_free(cachep, tb);
        }
}

void inet_bind_hash(struct sock *sk, struct inet_bind_bucket *tb,
                    const unsigned short snum)
{
        inet_sk(sk)->inet_num = snum;
        sk_add_bind_node(sk, &tb->owners);
        inet_csk(sk)->icsk_bind_hash = tb;
}

/*
 * Get rid of any references to a local port held by the given sock.
 */
static void __inet_put_port(struct sock *sk)
{
        struct inet_hashinfo *hashinfo = sk->sk_prot->h.hashinfo;
        const int bhash = inet_bhashfn(sock_net(sk), inet_sk(sk)->inet_num,
                        hashinfo->bhash_size);
        struct inet_bind_hashbucket *head = &hashinfo->bhash[bhash];
        struct inet_bind_bucket *tb;

        spin_lock(&head->lock);
        tb = inet_csk(sk)->icsk_bind_hash;
        __sk_del_bind_node(sk);
        inet_csk(sk)->icsk_bind_hash = NULL;
        inet_sk(sk)->inet_num = 0;
        inet_bind_bucket_destroy(hashinfo->bind_bucket_cachep, tb);
        spin_unlock(&head->lock);
}

void inet_put_port(struct sock *sk)
{
        local_bh_disable();
        __inet_put_port(sk);
        local_bh_enable();
}
EXPORT_SYMBOL(inet_put_port);

int __inet_inherit_port(const struct sock *sk, struct sock *child)
{
        struct inet_hashinfo *table = sk->sk_prot->h.hashinfo;
        unsigned short port = inet_sk(child)->inet_num;
        const int bhash = inet_bhashfn(sock_net(sk), port,
                        table->bhash_size);
        struct inet_bind_hashbucket *head = &table->bhash[bhash];
        struct inet_bind_bucket *tb;
        int l3mdev;

        spin_lock(&head->lock);
        tb = inet_csk(sk)->icsk_bind_hash;
        if (unlikely(!tb)) {
                spin_unlock(&head->lock);
                return -ENOENT;
        }
        if (tb->port != port) {
                l3mdev = inet_sk_bound_l3mdev(sk);

                /* NOTE: using tproxy and redirecting skbs to a proxy
                 * on a different listener port breaks the assumption
                 * that the listener socket's icsk_bind_hash is the same
                 * as that of the child socket. We have to look up or
                 * create a new bind bucket for the child here. */
                inet_bind_bucket_for_each(tb, &head->chain) {
                        if (net_eq(ib_net(tb), sock_net(sk)) &&
                            tb->l3mdev == l3mdev && tb->port == port)
                                break;
                }
                if (!tb) {
                        tb = inet_bind_bucket_create(table->bind_bucket_cachep,
                                                     sock_net(sk), head, port,
                                                     l3mdev);
                        if (!tb) {
                                spin_unlock(&head->lock);
                                return -ENOMEM;
                        }
                }
                inet_csk_update_fastreuse(tb, child);
        }
        inet_bind_hash(child, tb, port);
        spin_unlock(&head->lock);

        return 0;
}
EXPORT_SYMBOL_GPL(__inet_inherit_port);

static struct inet_listen_hashbucket *
inet_lhash2_bucket_sk(struct inet_hashinfo *h, struct sock *sk)
{
        u32 hash;

#if IS_ENABLED(CONFIG_IPV6)
        if (sk->sk_family == AF_INET6)
                hash = ipv6_portaddr_hash(sock_net(sk),
                                          &sk->sk_v6_rcv_saddr,
                                          inet_sk(sk)->inet_num);
        else
#endif
                hash = ipv4_portaddr_hash(sock_net(sk),
                                          inet_sk(sk)->inet_rcv_saddr,
                                          inet_sk(sk)->inet_num);
        return inet_lhash2_bucket(h, hash);
}

static inline int compute_score(struct sock *sk, struct net *net,
                                const unsigned short hnum, const __be32 daddr,
                                const int dif, const int sdif)
{
        int score = -1;

        if (net_eq(sock_net(sk), net) && sk->sk_num == hnum &&
                        !ipv6_only_sock(sk)) {
                if (sk->sk_rcv_saddr != daddr)
                        return -1;

                if (!inet_sk_bound_dev_eq(net, sk->sk_bound_dev_if, dif, sdif))
                        return -1;
                score =  sk->sk_bound_dev_if ? 2 : 1;

                if (sk->sk_family == PF_INET)
                        score++;
                if (READ_ONCE(sk->sk_incoming_cpu) == raw_smp_processor_id())
                        score++;
        }
        return score;
}

INDIRECT_CALLABLE_DECLARE(inet_ehashfn_t udp_ehashfn);

struct sock *inet_lookup_reuseport(struct net *net, struct sock *sk,
                                   struct sk_buff *skb, int doff,
                                   __be32 saddr, __be16 sport,
                                   __be32 daddr, unsigned short hnum,
                                   inet_ehashfn_t *ehashfn)
{
        struct sock *reuse_sk = NULL;
        u32 phash;

        if (sk->sk_reuseport) {
                phash = INDIRECT_CALL_2(ehashfn, udp_ehashfn, inet_ehashfn,
                                        net, daddr, hnum, saddr, sport);
                reuse_sk = reuseport_select_sock(sk, phash, skb, doff);
        }
        return reuse_sk;
}
EXPORT_SYMBOL_GPL(inet_lookup_reuseport);

/*
 * Here are some nice properties to exploit here. The BSD API
 * does not allow a listening sock to specify the remote port nor the
 * remote address for the connection. So always assume those are both
 * wildcarded during the search since they can never be otherwise.
 */

/* called with rcu_read_lock() : No refcount taken on the socket */
static struct sock *inet_lhash2_lookup(struct net *net,
                                struct inet_listen_hashbucket *ilb2,
                                struct sk_buff *skb, int doff,
                                const __be32 saddr, __be16 sport,
                                const __be32 daddr, const unsigned short hnum,
                                const int dif, const int sdif)
{
        struct sock *sk, *result = NULL;
        struct hlist_nulls_node *node;
        int score, hiscore = 0;

        sk_nulls_for_each_rcu(sk, node, &ilb2->nulls_head) {
                score = compute_score(sk, net, hnum, daddr, dif, sdif);
                if (score > hiscore) {
                        result = inet_lookup_reuseport(net, sk, skb, doff,
                                                       saddr, sport, daddr, hnum, inet_ehashfn);
                        if (result)
                                return result;

                        result = sk;
                        hiscore = score;
                }
        }

        return result;
}

static inline struct sock *inet_lookup_run_bpf(struct net *net,
                                               struct inet_hashinfo *hashinfo,
                                               struct sk_buff *skb, int doff,
                                               __be32 saddr, __be16 sport,
                                               __be32 daddr, u16 hnum)
{
        struct sock *sk, *reuse_sk;
        bool no_reuseport;

        if (hashinfo != &tcp_hashinfo)
                return NULL; /* only TCP is supported */

        no_reuseport = bpf_sk_lookup_run_v4(net, IPPROTO_TCP,
                                            saddr, sport, daddr, hnum, &sk);
        if (no_reuseport || IS_ERR_OR_NULL(sk))
                return sk;

        reuse_sk = inet_lookup_reuseport(net, sk, skb, doff, saddr, sport, daddr, hnum,
                                         inet_ehashfn);
        if (reuse_sk)
                sk = reuse_sk;
        return sk;
}

struct sock *__inet_lookup_listener(struct net *net,
                                    struct inet_hashinfo *hashinfo,
                                    struct sk_buff *skb, int doff,
                                    const __be32 saddr, __be16 sport,
                                    const __be32 daddr, const unsigned short hnum,
                                    const int dif, const int sdif)
{
        struct inet_listen_hashbucket *ilb2;
        struct sock *result = NULL;
        unsigned int hash2;

        /* Lookup redirect from BPF */
        if (static_branch_unlikely(&bpf_sk_lookup_enabled)) {
                result = inet_lookup_run_bpf(net, hashinfo, skb, doff,
                                             saddr, sport, daddr, hnum);
                if (result)
                        goto done;
        }

        hash2 = ipv4_portaddr_hash(net, daddr, hnum);
        ilb2 = inet_lhash2_bucket(hashinfo, hash2);

        result = inet_lhash2_lookup(net, ilb2, skb, doff,
                                    saddr, sport, daddr, hnum,
                                    dif, sdif);
        if (result)
                goto done;

        /* Lookup lhash2 with INADDR_ANY */
        hash2 = ipv4_portaddr_hash(net, htonl(INADDR_ANY), hnum);
        ilb2 = inet_lhash2_bucket(hashinfo, hash2);

        result = inet_lhash2_lookup(net, ilb2, skb, doff,
                                    saddr, sport, htonl(INADDR_ANY), hnum,
                                    dif, sdif);
done:
        if (IS_ERR(result))
                return NULL;
        return result;
}
EXPORT_SYMBOL_GPL(__inet_lookup_listener);

/* All sockets share common refcount, but have different destructors */
void sock_gen_put(struct sock *sk)
{
        if (!refcount_dec_and_test(&sk->sk_refcnt))
                return;

        if (sk->sk_state == TCP_TIME_WAIT)
                inet_twsk_free(inet_twsk(sk));
        else if (sk->sk_state == TCP_NEW_SYN_RECV)
                reqsk_free(inet_reqsk(sk));
        else
                sk_free(sk);
}
EXPORT_SYMBOL_GPL(sock_gen_put);

void sock_edemux(struct sk_buff *skb)
{
        sock_gen_put(skb->sk);
}
EXPORT_SYMBOL(sock_edemux);

struct sock *__inet_lookup_established(struct net *net,
                                  struct inet_hashinfo *hashinfo,
                                  const __be32 saddr, const __be16 sport,
                                  const __be32 daddr, const u16 hnum,
                                  const int dif, const int sdif)
{
        INET_ADDR_COOKIE(acookie, saddr, daddr);
        const __portpair ports = INET_COMBINED_PORTS(sport, hnum);
        struct sock *sk;
        const struct hlist_nulls_node *node;
        /* Optimize here for direct hit, only listening connections can
         * have wildcards anyways.
         */
        unsigned int hash = inet_ehashfn(net, daddr, hnum, saddr, sport);
        unsigned int slot = hash & hashinfo->ehash_mask;
        struct inet_ehash_bucket *head = &hashinfo->ehash[slot];

begin:
        sk_nulls_for_each_rcu(sk, node, &head->chain) {
                if (sk->sk_hash != hash)
                        continue;
                if (likely(INET_MATCH(net, sk, acookie, ports, dif, sdif))) {
                        if (unlikely(!refcount_inc_not_zero(&sk->sk_refcnt)))
                                goto out;
                        if (unlikely(!INET_MATCH(net, sk, acookie,
                                                 ports, dif, sdif))) {
                                sock_gen_put(sk);
                                goto begin;
                        }
                        goto found;
                }
        }
        /*
         * if the nulls value we got at the end of this lookup is
         * not the expected one, we must restart lookup.
         * We probably met an item that was moved to another chain.
         */
        if (get_nulls_value(node) != slot)
                goto begin;
out:
        sk = NULL;
found:
        return sk;
}
EXPORT_SYMBOL_GPL(__inet_lookup_established);

/* called with local bh disabled */
static int __inet_check_established(struct inet_timewait_death_row *death_row,
                                    struct sock *sk, __u16 lport,
                                    struct inet_timewait_sock **twp)
{
        struct inet_hashinfo *hinfo = death_row->hashinfo;
        struct inet_sock *inet = inet_sk(sk);
        __be32 daddr = inet->inet_rcv_saddr;
        __be32 saddr = inet->inet_daddr;
        int dif = sk->sk_bound_dev_if;
        struct net *net = sock_net(sk);
        int sdif = l3mdev_master_ifindex_by_index(net, dif);
        INET_ADDR_COOKIE(acookie, saddr, daddr);
        const __portpair ports = INET_COMBINED_PORTS(inet->inet_dport, lport);
        unsigned int hash = inet_ehashfn(net, daddr, lport,
                                         saddr, inet->inet_dport);
        struct inet_ehash_bucket *head = inet_ehash_bucket(hinfo, hash);
        spinlock_t *lock = inet_ehash_lockp(hinfo, hash);
        struct sock *sk2;
        const struct hlist_nulls_node *node;
        struct inet_timewait_sock *tw = NULL;

        spin_lock(lock);

        sk_nulls_for_each(sk2, node, &head->chain) {
                if (sk2->sk_hash != hash)
                        continue;

                if (likely(INET_MATCH(net, sk2, acookie, ports, dif, sdif))) {
                        if (sk2->sk_state == TCP_TIME_WAIT) {
                                tw = inet_twsk(sk2);
                                if (twsk_unique(sk, sk2, twp))
                                        break;
                        }
                        goto not_unique;
                }
        }

        /* Must record num and sport now. Otherwise we will see
         * in hash table socket with a funny identity.
         */
        inet->inet_num = lport;
        inet->inet_sport = htons(lport);
        sk->sk_hash = hash;
        WARN_ON(!sk_unhashed(sk));
        __sk_nulls_add_node_rcu(sk, &head->chain);
        if (tw) {
                sk_nulls_del_node_init_rcu((struct sock *)tw);
                __NET_INC_STATS(net, LINUX_MIB_TIMEWAITRECYCLED);
        }
        spin_unlock(lock);
        sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);

        if (twp) {
                *twp = tw;
        } else if (tw) {
                /* Silly. Should hash-dance instead... */
                inet_twsk_deschedule_put(tw);
        }
        return 0;

not_unique:
        spin_unlock(lock);
        return -EADDRNOTAVAIL;
}

static u64 inet_sk_port_offset(const struct sock *sk)
{
        const struct inet_sock *inet = inet_sk(sk);

        return secure_ipv4_port_ephemeral(inet->inet_rcv_saddr,
                                          inet->inet_daddr,
                                          inet->inet_dport);
}

/* Searches for an exsiting socket in the ehash bucket list.
 * Returns true if found, false otherwise.
 */
static bool inet_ehash_lookup_by_sk(struct sock *sk,
                                    struct hlist_nulls_head *list)
{
        const __portpair ports = INET_COMBINED_PORTS(sk->sk_dport, sk->sk_num);
        const int sdif = sk->sk_bound_dev_if;
        const int dif = sk->sk_bound_dev_if;
        const struct hlist_nulls_node *node;
        struct net *net = sock_net(sk);
        struct sock *esk;

        INET_ADDR_COOKIE(acookie, sk->sk_daddr, sk->sk_rcv_saddr);

        sk_nulls_for_each_rcu(esk, node, list) {
                if (esk->sk_hash != sk->sk_hash)
                        continue;
                if (sk->sk_family == AF_INET) {
                        if (unlikely(INET_MATCH(net, esk, acookie,
                                                ports, dif, sdif))) {
                                return true;
                        }
                }
#if IS_ENABLED(CONFIG_IPV6)
                else if (sk->sk_family == AF_INET6) {
                        if (unlikely(inet6_match(net, esk,
                                                 &sk->sk_v6_daddr,
                                                 &sk->sk_v6_rcv_saddr,
                                                 ports, dif, sdif))) {
                                return true;
                        }
                }
#endif
        }
        return false;
}

/* Insert a socket into ehash, and eventually remove another one
 * (The another one can be a SYN_RECV or TIMEWAIT)
 * If an existing socket already exists, socket sk is not inserted,
 * and sets found_dup_sk parameter to true.
 */
bool inet_ehash_insert(struct sock *sk, struct sock *osk, bool *found_dup_sk)
{
        struct inet_hashinfo *hashinfo = sk->sk_prot->h.hashinfo;
        struct hlist_nulls_head *list;
        struct inet_ehash_bucket *head;
        spinlock_t *lock;
        bool ret = true;

        WARN_ON_ONCE(!sk_unhashed(sk));

        sk->sk_hash = sk_ehashfn(sk);
        head = inet_ehash_bucket(hashinfo, sk->sk_hash);
        list = &head->chain;
        lock = inet_ehash_lockp(hashinfo, sk->sk_hash);

        spin_lock(lock);
        if (osk) {
                WARN_ON_ONCE(sk->sk_hash != osk->sk_hash);
                ret = sk_nulls_del_node_init_rcu(osk);
        } else if (found_dup_sk) {
                *found_dup_sk = inet_ehash_lookup_by_sk(sk, list);
                if (*found_dup_sk)
                        ret = false;
        }

        if (ret)
                __sk_nulls_add_node_rcu(sk, list);

        spin_unlock(lock);

        return ret;
}

bool inet_ehash_nolisten(struct sock *sk, struct sock *osk, bool *found_dup_sk)
{
        bool ok = inet_ehash_insert(sk, osk, found_dup_sk);

        if (ok) {
                sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);
        } else {
                this_cpu_inc(*sk->sk_prot->orphan_count);
                inet_sk_set_state(sk, TCP_CLOSE);
                sock_set_flag(sk, SOCK_DEAD);
                inet_csk_destroy_sock(sk);
        }
        return ok;
}
EXPORT_SYMBOL_GPL(inet_ehash_nolisten);

static int inet_reuseport_add_sock(struct sock *sk,
                                   struct inet_listen_hashbucket *ilb)
{
        struct inet_bind_bucket *tb = inet_csk(sk)->icsk_bind_hash;
        const struct hlist_nulls_node *node;
        struct sock *sk2;
        kuid_t uid = sock_i_uid(sk);

        sk_nulls_for_each_rcu(sk2, node, &ilb->nulls_head) {
                if (sk2 != sk &&
                    sk2->sk_family == sk->sk_family &&
                    ipv6_only_sock(sk2) == ipv6_only_sock(sk) &&
                    sk2->sk_bound_dev_if == sk->sk_bound_dev_if &&
                    inet_csk(sk2)->icsk_bind_hash == tb &&
                    sk2->sk_reuseport && uid_eq(uid, sock_i_uid(sk2)) &&
                    inet_rcv_saddr_equal(sk, sk2, false))
                        return reuseport_add_sock(sk, sk2,
                                                  inet_rcv_saddr_any(sk));
        }

        return reuseport_alloc(sk, inet_rcv_saddr_any(sk));
}

int __inet_hash(struct sock *sk, struct sock *osk)
{
        struct inet_hashinfo *hashinfo = sk->sk_prot->h.hashinfo;
        struct inet_listen_hashbucket *ilb2;
        int err = 0;

        if (sk->sk_state != TCP_LISTEN) {
                local_bh_disable();
                inet_ehash_nolisten(sk, osk, NULL);
                local_bh_enable();
                return 0;
        }
        WARN_ON(!sk_unhashed(sk));
        ilb2 = inet_lhash2_bucket_sk(hashinfo, sk);

        spin_lock(&ilb2->lock);
        if (sk->sk_reuseport) {
                err = inet_reuseport_add_sock(sk, ilb2);
                if (err)
                        goto unlock;
        }
        sock_set_flag(sk, SOCK_RCU_FREE);
        if (IS_ENABLED(CONFIG_IPV6) && sk->sk_reuseport &&
                sk->sk_family == AF_INET6)
                __sk_nulls_add_node_tail_rcu(sk, &ilb2->nulls_head);
        else
                __sk_nulls_add_node_rcu(sk, &ilb2->nulls_head);
        sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);
unlock:
        spin_unlock(&ilb2->lock);

        return err;
}
EXPORT_SYMBOL(__inet_hash);

int inet_hash(struct sock *sk)
{
        int err = 0;

        if (sk->sk_state != TCP_CLOSE)
                err = __inet_hash(sk, NULL);

        return err;
}
EXPORT_SYMBOL_GPL(inet_hash);

void inet_unhash(struct sock *sk)
{
        struct inet_hashinfo *hashinfo = sk->sk_prot->h.hashinfo;

        if (sk_unhashed(sk))
                return;

        if (sk->sk_state == TCP_LISTEN) {
                struct inet_listen_hashbucket *ilb2;

                ilb2 = inet_lhash2_bucket_sk(hashinfo, sk);
                /* Don't disable bottom halves while acquiring the lock to
                 * avoid circular locking dependency on PREEMPT_RT.
                 */
                spin_lock(&ilb2->lock);
                if (sk_unhashed(sk)) {
                        spin_unlock(&ilb2->lock);
                        return;
                }

                if (rcu_access_pointer(sk->sk_reuseport_cb))
                        reuseport_stop_listen_sock(sk);

                __sk_nulls_del_node_init_rcu(sk);
                sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1);
                spin_unlock(&ilb2->lock);
        } else {
                spinlock_t *lock = inet_ehash_lockp(hashinfo, sk->sk_hash);

                spin_lock_bh(lock);
                if (sk_unhashed(sk)) {
                        spin_unlock_bh(lock);
                        return;
                }
                __sk_nulls_del_node_init_rcu(sk);
                sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1);
                spin_unlock_bh(lock);
        }
}
EXPORT_SYMBOL_GPL(inet_unhash);

/* RFC 6056 3.3.4.  Algorithm 4: Double-Hash Port Selection Algorithm
 * Note that we use 32bit integers (vs RFC 'short integers')
 * because 2^16 is not a multiple of num_ephemeral and this
 * property might be used by clever attacker.
 *
 * RFC claims using TABLE_LENGTH=10 buckets gives an improvement, though
 * attacks were since demonstrated, thus we use 65536 by default instead
 * to really give more isolation and privacy, at the expense of 256kB
 * of kernel memory.
 */
#define INET_TABLE_PERTURB_SIZE (1 << CONFIG_INET_TABLE_PERTURB_ORDER)
static u32 *table_perturb;

int __inet_hash_connect(struct inet_timewait_death_row *death_row,
                struct sock *sk, u64 port_offset,
                int (*check_established)(struct inet_timewait_death_row *,
                        struct sock *, __u16, struct inet_timewait_sock **))
{
        struct inet_hashinfo *hinfo = death_row->hashinfo;
        struct inet_timewait_sock *tw = NULL;
        struct inet_bind_hashbucket *head;
        int port = inet_sk(sk)->inet_num;
        struct net *net = sock_net(sk);
        struct inet_bind_bucket *tb;
        u32 remaining, offset;
        int ret, i, low, high;
        int l3mdev;
        u32 index;

        if (port) {
                local_bh_disable();
                ret = check_established(death_row, sk, port, NULL);
                local_bh_enable();
                return ret;
        }

        l3mdev = inet_sk_bound_l3mdev(sk);

        inet_get_local_port_range(net, &low, &high);
        high++; /* [32768, 60999] -> [32768, 61000[ */
        remaining = high - low;
        if (likely(remaining > 1))
                remaining &= ~1U;

        get_random_slow_once(table_perturb,
                             INET_TABLE_PERTURB_SIZE * sizeof(*table_perturb));
        index = port_offset & (INET_TABLE_PERTURB_SIZE - 1);

        offset = READ_ONCE(table_perturb[index]) + (port_offset >> 32);
        offset %= remaining;

        /* In first pass we try ports of @low parity.
         * inet_csk_get_port() does the opposite choice.
         */
        offset &= ~1U;
other_parity_scan:
        port = low + offset;
        for (i = 0; i < remaining; i += 2, port += 2) {
                if (unlikely(port >= high))
                        port -= remaining;
                if (inet_is_local_reserved_port(net, port))
                        continue;
                head = &hinfo->bhash[inet_bhashfn(net, port,
                                                  hinfo->bhash_size)];
                spin_lock_bh(&head->lock);

                /* Does not bother with rcv_saddr checks, because
                 * the established check is already unique enough.
                 */
                inet_bind_bucket_for_each(tb, &head->chain) {
                        if (net_eq(ib_net(tb), net) && tb->l3mdev == l3mdev &&
                            tb->port == port) {
                                if (tb->fastreuse >= 0 ||
                                    tb->fastreuseport >= 0)
                                        goto next_port;
                                WARN_ON(hlist_empty(&tb->owners));
                                if (!check_established(death_row, sk,
                                                       port, &tw))
                                        goto ok;
                                goto next_port;
                        }
                }

                tb = inet_bind_bucket_create(hinfo->bind_bucket_cachep,
                                             net, head, port, l3mdev);
                if (!tb) {
                        spin_unlock_bh(&head->lock);
                        return -ENOMEM;
                }
                tb->fastreuse = -1;
                tb->fastreuseport = -1;
                goto ok;
next_port:
                spin_unlock_bh(&head->lock);
                cond_resched();
        }

        offset++;
        if ((offset & 1) && remaining > 1)
                goto other_parity_scan;

        return -EADDRNOTAVAIL;

ok:
        /* Here we want to add a little bit of randomness to the next source
         * port that will be chosen. We use a max() with a random here so that
         * on low contention the randomness is maximal and on high contention
         * it may be inexistent.
         */
        i = max_t(int, i, (prandom_u32() & 7) * 2);
        WRITE_ONCE(table_perturb[index], READ_ONCE(table_perturb[index]) + i + 2);

        /* Head lock still held and bh's disabled */
        inet_bind_hash(sk, tb, port);
        if (sk_unhashed(sk)) {
                inet_sk(sk)->inet_sport = htons(port);
                inet_ehash_nolisten(sk, (struct sock *)tw, NULL);
        }
        if (tw)
                inet_twsk_bind_unhash(tw, hinfo);
        spin_unlock(&head->lock);
        if (tw)
                inet_twsk_deschedule_put(tw);
        local_bh_enable();
        return 0;
}

/*
 * Bind a port for a connect operation and hash it.
 */
int inet_hash_connect(struct inet_timewait_death_row *death_row,
                      struct sock *sk)
{
        u64 port_offset = 0;

        if (!inet_sk(sk)->inet_num)
                port_offset = inet_sk_port_offset(sk);
        return __inet_hash_connect(death_row, sk, port_offset,
                                   __inet_check_established);
}
EXPORT_SYMBOL_GPL(inet_hash_connect);

static void init_hashinfo_lhash2(struct inet_hashinfo *h)
{
        int i;

        for (i = 0; i <= h->lhash2_mask; i++) {
                spin_lock_init(&h->lhash2[i].lock);
                INIT_HLIST_NULLS_HEAD(&h->lhash2[i].nulls_head,
                                      i + LISTENING_NULLS_BASE);
        }
}

void __init inet_hashinfo2_init(struct inet_hashinfo *h, const char *name,
                                unsigned long numentries, int scale,
                                unsigned long low_limit,
                                unsigned long high_limit)
{
        h->lhash2 = alloc_large_system_hash(name,
                                            sizeof(*h->lhash2),
                                            numentries,
                                            scale,
                                            0,
                                            NULL,
                                            &h->lhash2_mask,
                                            low_limit,
                                            high_limit);
        init_hashinfo_lhash2(h);

        /* this one is used for source ports of outgoing connections */
        table_perturb = alloc_large_system_hash("Table-perturb",
                                                sizeof(*table_perturb),
                                                INET_TABLE_PERTURB_SIZE,
                                                0, 0, NULL, NULL,
                                                INET_TABLE_PERTURB_SIZE,
                                                INET_TABLE_PERTURB_SIZE);
}

int inet_hashinfo2_init_mod(struct inet_hashinfo *h)
{
        h->lhash2 = kmalloc_array(INET_LHTABLE_SIZE, sizeof(*h->lhash2), GFP_KERNEL);
        if (!h->lhash2)
                return -ENOMEM;

        h->lhash2_mask = INET_LHTABLE_SIZE - 1;
        /* INET_LHTABLE_SIZE must be a power of 2 */
        BUG_ON(INET_LHTABLE_SIZE & h->lhash2_mask);

        init_hashinfo_lhash2(h);
        return 0;
}
EXPORT_SYMBOL_GPL(inet_hashinfo2_init_mod);

int inet_ehash_locks_alloc(struct inet_hashinfo *hashinfo)
{
        unsigned int locksz = sizeof(spinlock_t);
        unsigned int i, nblocks = 1;

        if (locksz != 0) {
                /* allocate 2 cache lines or at least one spinlock per cpu */
                nblocks = max(2U * L1_CACHE_BYTES / locksz, 1U);
                nblocks = roundup_pow_of_two(nblocks * num_possible_cpus());

                /* no more locks than number of hash buckets */
                nblocks = min(nblocks, hashinfo->ehash_mask + 1);

                hashinfo->ehash_locks = kvmalloc_array(nblocks, locksz, GFP_KERNEL);
                if (!hashinfo->ehash_locks)
                        return -ENOMEM;

                for (i = 0; i < nblocks; i++)
                        spin_lock_init(&hashinfo->ehash_locks[i]);
        }
        hashinfo->ehash_locks_mask = nblocks - 1;
        return 0;
}
EXPORT_SYMBOL_GPL(inet_ehash_locks_alloc);


















































































    2 













































    2 







    2 



























































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
// SPDX-License-Identifier: GPL-2.0
#include <linux/swap_cgroup.h>
#include <linux/vmalloc.h>
#include <linux/mm.h>

#include <linux/swapops.h> /* depends on mm.h include */

static DEFINE_MUTEX(swap_cgroup_mutex);
struct swap_cgroup_ctrl {
        struct page **map;
        unsigned long length;
        spinlock_t        lock;
};

static struct swap_cgroup_ctrl swap_cgroup_ctrl[MAX_SWAPFILES];

struct swap_cgroup {
        unsigned short                id;
};
#define SC_PER_PAGE        (PAGE_SIZE/sizeof(struct swap_cgroup))

/*
 * SwapCgroup implements "lookup" and "exchange" operations.
 * In typical usage, this swap_cgroup is accessed via memcg's charge/uncharge
 * against SwapCache. At swap_free(), this is accessed directly from swap.
 *
 * This means,
 *  - we have no race in "exchange" when we're accessed via SwapCache because
 *    SwapCache(and its swp_entry) is under lock.
 *  - When called via swap_free(), there is no user of this entry and no race.
 * Then, we don't need lock around "exchange".
 *
 * TODO: we can push these buffers out to HIGHMEM.
 */

/*
 * allocate buffer for swap_cgroup.
 */
static int swap_cgroup_prepare(int type)
{
        struct page *page;
        struct swap_cgroup_ctrl *ctrl;
        unsigned long idx, max;

        ctrl = &swap_cgroup_ctrl[type];

        for (idx = 0; idx < ctrl->length; idx++) {
                page = alloc_page(GFP_KERNEL | __GFP_ZERO);
                if (!page)
                        goto not_enough_page;
                ctrl->map[idx] = page;

                if (!(idx % SWAP_CLUSTER_MAX))
                        cond_resched();
        }
        return 0;
not_enough_page:
        max = idx;
        for (idx = 0; idx < max; idx++)
                __free_page(ctrl->map[idx]);

        return -ENOMEM;
}

static struct swap_cgroup *__lookup_swap_cgroup(struct swap_cgroup_ctrl *ctrl,
                                                pgoff_t offset)
{
        struct page *mappage;
        struct swap_cgroup *sc;

        mappage = ctrl->map[offset / SC_PER_PAGE];
        sc = page_address(mappage);
        return sc + offset % SC_PER_PAGE;
}

static struct swap_cgroup *lookup_swap_cgroup(swp_entry_t ent,
                                        struct swap_cgroup_ctrl **ctrlp)
{
        pgoff_t offset = swp_offset(ent);
        struct swap_cgroup_ctrl *ctrl;

        ctrl = &swap_cgroup_ctrl[swp_type(ent)];
        if (ctrlp)
                *ctrlp = ctrl;
        return __lookup_swap_cgroup(ctrl, offset);
}

/**
 * swap_cgroup_cmpxchg - cmpxchg mem_cgroup's id for this swp_entry.
 * @ent: swap entry to be cmpxchged
 * @old: old id
 * @new: new id
 *
 * Returns old id at success, 0 at failure.
 * (There is no mem_cgroup using 0 as its id)
 */
unsigned short swap_cgroup_cmpxchg(swp_entry_t ent,
                                        unsigned short old, unsigned short new)
{
        struct swap_cgroup_ctrl *ctrl;
        struct swap_cgroup *sc;
        unsigned long flags;
        unsigned short retval;

        sc = lookup_swap_cgroup(ent, &ctrl);

        spin_lock_irqsave(&ctrl->lock, flags);
        retval = sc->id;
        if (retval == old)
                sc->id = new;
        else
                retval = 0;
        spin_unlock_irqrestore(&ctrl->lock, flags);
        return retval;
}

/**
 * swap_cgroup_record - record mem_cgroup for a set of swap entries
 * @ent: the first swap entry to be recorded into
 * @id: mem_cgroup to be recorded
 * @nr_ents: number of swap entries to be recorded
 *
 * Returns old value at success, 0 at failure.
 * (Of course, old value can be 0.)
 */
unsigned short swap_cgroup_record(swp_entry_t ent, unsigned short id,
                                  unsigned int nr_ents)
{
        struct swap_cgroup_ctrl *ctrl;
        struct swap_cgroup *sc;
        unsigned short old;
        unsigned long flags;
        pgoff_t offset = swp_offset(ent);
        pgoff_t end = offset + nr_ents;

        sc = lookup_swap_cgroup(ent, &ctrl);

        spin_lock_irqsave(&ctrl->lock, flags);
        old = sc->id;
        for (;;) {
                VM_BUG_ON(sc->id != old);
                sc->id = id;
                offset++;
                if (offset == end)
                        break;
                if (offset % SC_PER_PAGE)
                        sc++;
                else
                        sc = __lookup_swap_cgroup(ctrl, offset);
        }
        spin_unlock_irqrestore(&ctrl->lock, flags);

        return old;
}

/**
 * lookup_swap_cgroup_id - lookup mem_cgroup id tied to swap entry
 * @ent: swap entry to be looked up.
 *
 * Returns ID of mem_cgroup at success. 0 at failure. (0 is invalid ID)
 */
unsigned short lookup_swap_cgroup_id(swp_entry_t ent)
{
        return lookup_swap_cgroup(ent, NULL)->id;
}

int swap_cgroup_swapon(int type, unsigned long max_pages)
{
        void *array;
        unsigned long array_size;
        unsigned long length;
        struct swap_cgroup_ctrl *ctrl;

        length = DIV_ROUND_UP(max_pages, SC_PER_PAGE);
        array_size = length * sizeof(void *);

        array = vzalloc(array_size);
        if (!array)
                goto nomem;

        ctrl = &swap_cgroup_ctrl[type];
        mutex_lock(&swap_cgroup_mutex);
        ctrl->length = length;
        ctrl->map = array;
        spin_lock_init(&ctrl->lock);
        if (swap_cgroup_prepare(type)) {
                /* memory shortage */
                ctrl->map = NULL;
                ctrl->length = 0;
                mutex_unlock(&swap_cgroup_mutex);
                vfree(array);
                goto nomem;
        }
        mutex_unlock(&swap_cgroup_mutex);

        return 0;
nomem:
        pr_info("couldn't allocate enough memory for swap_cgroup\n");
        pr_info("swap_cgroup can be disabled by swapaccount=0 boot option\n");
        return -ENOMEM;
}

void swap_cgroup_swapoff(int type)
{
        struct page **map;
        unsigned long i, length;
        struct swap_cgroup_ctrl *ctrl;

        mutex_lock(&swap_cgroup_mutex);
        ctrl = &swap_cgroup_ctrl[type];
        map = ctrl->map;
        length = ctrl->length;
        ctrl->map = NULL;
        ctrl->length = 0;
        mutex_unlock(&swap_cgroup_mutex);

        if (map) {
                for (i = 0; i < length; i++) {
                        struct page *page = map[i];
                        if (page)
                                __free_page(page);
                        if (!(i % SWAP_CLUSTER_MAX))
                                cond_resched();
                }
                vfree(map);
        }
}

























































































































































































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
#ifndef __LINUX_MROUTE_BASE_H
#define __LINUX_MROUTE_BASE_H

#include <linux/netdevice.h>
#include <linux/rhashtable-types.h>
#include <linux/spinlock.h>
#include <net/net_namespace.h>
#include <net/sock.h>
#include <net/fib_notifier.h>
#include <net/ip_fib.h>

/**
 * struct vif_device - interface representor for multicast routing
 * @dev: network device being used
 * @bytes_in: statistic; bytes ingressing
 * @bytes_out: statistic; bytes egresing
 * @pkt_in: statistic; packets ingressing
 * @pkt_out: statistic; packets egressing
 * @rate_limit: Traffic shaping (NI)
 * @threshold: TTL threshold
 * @flags: Control flags
 * @link: Physical interface index
 * @dev_parent_id: device parent id
 * @local: Local address
 * @remote: Remote address for tunnels
 */
struct vif_device {
        struct net_device *dev;
        unsigned long bytes_in, bytes_out;
        unsigned long pkt_in, pkt_out;
        unsigned long rate_limit;
        unsigned char threshold;
        unsigned short flags;
        int link;

        /* Currently only used by ipmr */
        struct netdev_phys_item_id dev_parent_id;
        __be32 local, remote;
};

struct vif_entry_notifier_info {
        struct fib_notifier_info info;
        struct net_device *dev;
        unsigned short vif_index;
        unsigned short vif_flags;
        u32 tb_id;
};

static inline int mr_call_vif_notifier(struct notifier_block *nb,
                                       unsigned short family,
                                       enum fib_event_type event_type,
                                       struct vif_device *vif,
                                       unsigned short vif_index, u32 tb_id,
                                       struct netlink_ext_ack *extack)
{
        struct vif_entry_notifier_info info = {
                .info = {
                        .family = family,
                        .extack = extack,
                },
                .dev = vif->dev,
                .vif_index = vif_index,
                .vif_flags = vif->flags,
                .tb_id = tb_id,
        };

        return call_fib_notifier(nb, event_type, &info.info);
}

static inline int mr_call_vif_notifiers(struct net *net,
                                        unsigned short family,
                                        enum fib_event_type event_type,
                                        struct vif_device *vif,
                                        unsigned short vif_index, u32 tb_id,
                                        unsigned int *ipmr_seq)
{
        struct vif_entry_notifier_info info = {
                .info = {
                        .family = family,
                },
                .dev = vif->dev,
                .vif_index = vif_index,
                .vif_flags = vif->flags,
                .tb_id = tb_id,
        };

        ASSERT_RTNL();
        (*ipmr_seq)++;
        return call_fib_notifiers(net, event_type, &info.info);
}

#ifndef MAXVIFS
/* This one is nasty; value is defined in uapi using different symbols for
 * mroute and morute6 but both map into same 32.
 */
#define MAXVIFS        32
#endif

#define VIF_EXISTS(_mrt, _idx) (!!((_mrt)->vif_table[_idx].dev))

/* mfc_flags:
 * MFC_STATIC - the entry was added statically (not by a routing daemon)
 * MFC_OFFLOAD - the entry was offloaded to the hardware
 */
enum {
        MFC_STATIC = BIT(0),
        MFC_OFFLOAD = BIT(1),
};

/**
 * struct mr_mfc - common multicast routing entries
 * @mnode: rhashtable list
 * @mfc_parent: source interface (iif)
 * @mfc_flags: entry flags
 * @expires: unresolved entry expire time
 * @unresolved: unresolved cached skbs
 * @last_assert: time of last assert
 * @minvif: minimum VIF id
 * @maxvif: maximum VIF id
 * @bytes: bytes that have passed for this entry
 * @pkt: packets that have passed for this entry
 * @wrong_if: number of wrong source interface hits
 * @lastuse: time of last use of the group (traffic or update)
 * @ttls: OIF TTL threshold array
 * @refcount: reference count for this entry
 * @list: global entry list
 * @rcu: used for entry destruction
 * @free: Operation used for freeing an entry under RCU
 */
struct mr_mfc {
        struct rhlist_head mnode;
        unsigned short mfc_parent;
        int mfc_flags;

        union {
                struct {
                        unsigned long expires;
                        struct sk_buff_head unresolved;
                } unres;
                struct {
                        unsigned long last_assert;
                        int minvif;
                        int maxvif;
                        unsigned long bytes;
                        unsigned long pkt;
                        unsigned long wrong_if;
                        unsigned long lastuse;
                        unsigned char ttls[MAXVIFS];
                        refcount_t refcount;
                } res;
        } mfc_un;
        struct list_head list;
        struct rcu_head        rcu;
        void (*free)(struct rcu_head *head);
};

static inline void mr_cache_put(struct mr_mfc *c)
{
        if (refcount_dec_and_test(&c->mfc_un.res.refcount))
                call_rcu(&c->rcu, c->free);
}

static inline void mr_cache_hold(struct mr_mfc *c)
{
        refcount_inc(&c->mfc_un.res.refcount);
}

struct mfc_entry_notifier_info {
        struct fib_notifier_info info;
        struct mr_mfc *mfc;
        u32 tb_id;
};

static inline int mr_call_mfc_notifier(struct notifier_block *nb,
                                       unsigned short family,
                                       enum fib_event_type event_type,
                                       struct mr_mfc *mfc, u32 tb_id,
                                       struct netlink_ext_ack *extack)
{
        struct mfc_entry_notifier_info info = {
                .info = {
                        .family = family,
                        .extack = extack,
                },
                .mfc = mfc,
                .tb_id = tb_id
        };

        return call_fib_notifier(nb, event_type, &info.info);
}

static inline int mr_call_mfc_notifiers(struct net *net,
                                        unsigned short family,
                                        enum fib_event_type event_type,
                                        struct mr_mfc *mfc, u32 tb_id,
                                        unsigned int *ipmr_seq)
{
        struct mfc_entry_notifier_info info = {
                .info = {
                        .family = family,
                },
                .mfc = mfc,
                .tb_id = tb_id
        };

        ASSERT_RTNL();
        (*ipmr_seq)++;
        return call_fib_notifiers(net, event_type, &info.info);
}

struct mr_table;

/**
 * struct mr_table_ops - callbacks and info for protocol-specific ops
 * @rht_params: parameters for accessing the MFC hash
 * @cmparg_any: a hash key to be used for matching on (*,*) routes
 */
struct mr_table_ops {
        const struct rhashtable_params *rht_params;
        void *cmparg_any;
};

/**
 * struct mr_table - a multicast routing table
 * @list: entry within a list of multicast routing tables
 * @net: net where this table belongs
 * @ops: protocol specific operations
 * @id: identifier of the table
 * @mroute_sk: socket associated with the table
 * @ipmr_expire_timer: timer for handling unresolved routes
 * @mfc_unres_queue: list of unresolved MFC entries
 * @vif_table: array containing all possible vifs
 * @mfc_hash: Hash table of all resolved routes for easy lookup
 * @mfc_cache_list: list of resovled routes for possible traversal
 * @maxvif: Identifier of highest value vif currently in use
 * @cache_resolve_queue_len: current size of unresolved queue
 * @mroute_do_assert: Whether to inform userspace on wrong ingress
 * @mroute_do_pim: Whether to receive IGMP PIMv1
 * @mroute_reg_vif_num: PIM-device vif index
 */
struct mr_table {
        struct list_head        list;
        possible_net_t                net;
        struct mr_table_ops        ops;
        u32                        id;
        struct sock __rcu        *mroute_sk;
        struct timer_list        ipmr_expire_timer;
        struct list_head        mfc_unres_queue;
        struct vif_device        vif_table[MAXVIFS];
        struct rhltable                mfc_hash;
        struct list_head        mfc_cache_list;
        int                        maxvif;
        atomic_t                cache_resolve_queue_len;
        bool                        mroute_do_assert;
        bool                        mroute_do_pim;
        bool                        mroute_do_wrvifwhole;
        int                        mroute_reg_vif_num;
};

#ifdef CONFIG_IP_MROUTE_COMMON
void vif_device_init(struct vif_device *v,
                     struct net_device *dev,
                     unsigned long rate_limit,
                     unsigned char threshold,
                     unsigned short flags,
                     unsigned short get_iflink_mask);

struct mr_table *
mr_table_alloc(struct net *net, u32 id,
               struct mr_table_ops *ops,
               void (*expire_func)(struct timer_list *t),
               void (*table_set)(struct mr_table *mrt,
                                 struct net *net));

/* These actually return 'struct mr_mfc *', but to avoid need for explicit
 * castings they simply return void.
 */
void *mr_mfc_find_parent(struct mr_table *mrt,
                         void *hasharg, int parent);
void *mr_mfc_find_any_parent(struct mr_table *mrt, int vifi);
void *mr_mfc_find_any(struct mr_table *mrt, int vifi, void *hasharg);

int mr_fill_mroute(struct mr_table *mrt, struct sk_buff *skb,
                   struct mr_mfc *c, struct rtmsg *rtm);
int mr_table_dump(struct mr_table *mrt, struct sk_buff *skb,
                  struct netlink_callback *cb,
                  int (*fill)(struct mr_table *mrt, struct sk_buff *skb,
                              u32 portid, u32 seq, struct mr_mfc *c,
                              int cmd, int flags),
                  spinlock_t *lock, struct fib_dump_filter *filter);
int mr_rtm_dumproute(struct sk_buff *skb, struct netlink_callback *cb,
                     struct mr_table *(*iter)(struct net *net,
                                              struct mr_table *mrt),
                     int (*fill)(struct mr_table *mrt,
                                 struct sk_buff *skb,
                                 u32 portid, u32 seq, struct mr_mfc *c,
                                 int cmd, int flags),
                     spinlock_t *lock, struct fib_dump_filter *filter);

int mr_dump(struct net *net, struct notifier_block *nb, unsigned short family,
            int (*rules_dump)(struct net *net,
                              struct notifier_block *nb,
                              struct netlink_ext_ack *extack),
            struct mr_table *(*mr_iter)(struct net *net,
                                        struct mr_table *mrt),
            rwlock_t *mrt_lock, struct netlink_ext_ack *extack);
#else
static inline void vif_device_init(struct vif_device *v,
                                   struct net_device *dev,
                                   unsigned long rate_limit,
                                   unsigned char threshold,
                                   unsigned short flags,
                                   unsigned short get_iflink_mask)
{
}

static inline void *mr_mfc_find_parent(struct mr_table *mrt,
                                       void *hasharg, int parent)
{
        return NULL;
}

static inline void *mr_mfc_find_any_parent(struct mr_table *mrt,
                                           int vifi)
{
        return NULL;
}

static inline struct mr_mfc *mr_mfc_find_any(struct mr_table *mrt,
                                             int vifi, void *hasharg)
{
        return NULL;
}

static inline int mr_fill_mroute(struct mr_table *mrt, struct sk_buff *skb,
                                 struct mr_mfc *c, struct rtmsg *rtm)
{
        return -EINVAL;
}

static inline int
mr_rtm_dumproute(struct sk_buff *skb, struct netlink_callback *cb,
                 struct mr_table *(*iter)(struct net *net,
                                          struct mr_table *mrt),
                 int (*fill)(struct mr_table *mrt,
                             struct sk_buff *skb,
                             u32 portid, u32 seq, struct mr_mfc *c,
                             int cmd, int flags),
                 spinlock_t *lock, struct fib_dump_filter *filter)
{
        return -EINVAL;
}

static inline int mr_dump(struct net *net, struct notifier_block *nb,
                          unsigned short family,
                          int (*rules_dump)(struct net *net,
                                            struct notifier_block *nb,
                                            struct netlink_ext_ack *extack),
                          struct mr_table *(*mr_iter)(struct net *net,
                                                      struct mr_table *mrt),
                          rwlock_t *mrt_lock, struct netlink_ext_ack *extack)
{
        return -EINVAL;
}
#endif

static inline void *mr_mfc_find(struct mr_table *mrt, void *hasharg)
{
        return mr_mfc_find_parent(mrt, hasharg, -1);
}

#ifdef CONFIG_PROC_FS
struct mr_vif_iter {
        struct seq_net_private p;
        struct mr_table *mrt;
        int ct;
};

struct mr_mfc_iter {
        struct seq_net_private p;
        struct mr_table *mrt;
        struct list_head *cache;

        /* Lock protecting the mr_table's unresolved queue */
        spinlock_t *lock;
};

#ifdef CONFIG_IP_MROUTE_COMMON
void *mr_vif_seq_idx(struct net *net, struct mr_vif_iter *iter, loff_t pos);
void *mr_vif_seq_next(struct seq_file *seq, void *v, loff_t *pos);

static inline void *mr_vif_seq_start(struct seq_file *seq, loff_t *pos)
{
        return *pos ? mr_vif_seq_idx(seq_file_net(seq),
                                     seq->private, *pos - 1)
                    : SEQ_START_TOKEN;
}

/* These actually return 'struct mr_mfc *', but to avoid need for explicit
 * castings they simply return void.
 */
void *mr_mfc_seq_idx(struct net *net,
                     struct mr_mfc_iter *it, loff_t pos);
void *mr_mfc_seq_next(struct seq_file *seq, void *v,
                      loff_t *pos);

static inline void *mr_mfc_seq_start(struct seq_file *seq, loff_t *pos,
                                     struct mr_table *mrt, spinlock_t *lock)
{
        struct mr_mfc_iter *it = seq->private;

        it->mrt = mrt;
        it->cache = NULL;
        it->lock = lock;

        return *pos ? mr_mfc_seq_idx(seq_file_net(seq),
                                     seq->private, *pos - 1)
                    : SEQ_START_TOKEN;
}

static inline void mr_mfc_seq_stop(struct seq_file *seq, void *v)
{
        struct mr_mfc_iter *it = seq->private;
        struct mr_table *mrt = it->mrt;

        if (it->cache == &mrt->mfc_unres_queue)
                spin_unlock_bh(it->lock);
        else if (it->cache == &mrt->mfc_cache_list)
                rcu_read_unlock();
}
#else
static inline void *mr_vif_seq_idx(struct net *net, struct mr_vif_iter *iter,
                                   loff_t pos)
{
        return NULL;
}

static inline void *mr_vif_seq_next(struct seq_file *seq,
                                    void *v, loff_t *pos)
{
        return NULL;
}

static inline void *mr_vif_seq_start(struct seq_file *seq, loff_t *pos)
{
        return NULL;
}

static inline void *mr_mfc_seq_idx(struct net *net,
                                   struct mr_mfc_iter *it, loff_t pos)
{
        return NULL;
}

static inline void *mr_mfc_seq_next(struct seq_file *seq, void *v,
                                    loff_t *pos)
{
        return NULL;
}

static inline void *mr_mfc_seq_start(struct seq_file *seq, loff_t *pos,
                                     struct mr_table *mrt, spinlock_t *lock)
{
        return NULL;
}

static inline void mr_mfc_seq_stop(struct seq_file *seq, void *v)
{
}
#endif
#endif
#endif























































   22 
   22 































































































   24 




   16 




















































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 *        Bridge per vlan tunnel port dst_metadata handling code
 *
 *        Authors:
 *        Roopa Prabhu                <roopa@cumulusnetworks.com>
 */

#include <linux/kernel.h>
#include <linux/netdevice.h>
#include <linux/rtnetlink.h>
#include <linux/slab.h>
#include <net/switchdev.h>
#include <net/dst_metadata.h>

#include "br_private.h"
#include "br_private_tunnel.h"

static inline int br_vlan_tunid_cmp(struct rhashtable_compare_arg *arg,
                                    const void *ptr)
{
        const struct net_bridge_vlan *vle = ptr;
        __be64 tunid = *(__be64 *)arg->key;

        return vle->tinfo.tunnel_id != tunid;
}

static const struct rhashtable_params br_vlan_tunnel_rht_params = {
        .head_offset = offsetof(struct net_bridge_vlan, tnode),
        .key_offset = offsetof(struct net_bridge_vlan, tinfo.tunnel_id),
        .key_len = sizeof(__be64),
        .nelem_hint = 3,
        .obj_cmpfn = br_vlan_tunid_cmp,
        .automatic_shrinking = true,
};

static struct net_bridge_vlan *br_vlan_tunnel_lookup(struct rhashtable *tbl,
                                                     __be64 tunnel_id)
{
        return rhashtable_lookup_fast(tbl, &tunnel_id,
                                      br_vlan_tunnel_rht_params);
}

static void vlan_tunnel_info_release(struct net_bridge_vlan *vlan)
{
        struct metadata_dst *tdst = rtnl_dereference(vlan->tinfo.tunnel_dst);

        WRITE_ONCE(vlan->tinfo.tunnel_id, 0);
        RCU_INIT_POINTER(vlan->tinfo.tunnel_dst, NULL);
        dst_release(&tdst->dst);
}

void vlan_tunnel_info_del(struct net_bridge_vlan_group *vg,
                          struct net_bridge_vlan *vlan)
{
        if (!rcu_access_pointer(vlan->tinfo.tunnel_dst))
                return;
        rhashtable_remove_fast(&vg->tunnel_hash, &vlan->tnode,
                               br_vlan_tunnel_rht_params);
        vlan_tunnel_info_release(vlan);
}

static int __vlan_tunnel_info_add(struct net_bridge_vlan_group *vg,
                                  struct net_bridge_vlan *vlan, u32 tun_id)
{
        struct metadata_dst *metadata = rtnl_dereference(vlan->tinfo.tunnel_dst);
        __be64 key = key32_to_tunnel_id(cpu_to_be32(tun_id));
        int err;

        if (metadata)
                return -EEXIST;

        metadata = __ip_tun_set_dst(0, 0, 0, 0, 0, TUNNEL_KEY,
                                    key, 0);
        if (!metadata)
                return -EINVAL;

        metadata->u.tun_info.mode |= IP_TUNNEL_INFO_TX | IP_TUNNEL_INFO_BRIDGE;
        rcu_assign_pointer(vlan->tinfo.tunnel_dst, metadata);
        WRITE_ONCE(vlan->tinfo.tunnel_id, key);

        err = rhashtable_lookup_insert_fast(&vg->tunnel_hash, &vlan->tnode,
                                            br_vlan_tunnel_rht_params);
        if (err)
                goto out;

        return 0;
out:
        vlan_tunnel_info_release(vlan);

        return err;
}

/* Must be protected by RTNL.
 * Must be called with vid in range from 1 to 4094 inclusive.
 */
int nbp_vlan_tunnel_info_add(const struct net_bridge_port *port, u16 vid,
                             u32 tun_id)
{
        struct net_bridge_vlan_group *vg;
        struct net_bridge_vlan *vlan;

        ASSERT_RTNL();

        vg = nbp_vlan_group(port);
        vlan = br_vlan_find(vg, vid);
        if (!vlan)
                return -EINVAL;

        return __vlan_tunnel_info_add(vg, vlan, tun_id);
}

/* Must be protected by RTNL.
 * Must be called with vid in range from 1 to 4094 inclusive.
 */
int nbp_vlan_tunnel_info_delete(const struct net_bridge_port *port, u16 vid)
{
        struct net_bridge_vlan_group *vg;
        struct net_bridge_vlan *v;

        ASSERT_RTNL();

        vg = nbp_vlan_group(port);
        v = br_vlan_find(vg, vid);
        if (!v)
                return -ENOENT;

        vlan_tunnel_info_del(vg, v);

        return 0;
}

static void __vlan_tunnel_info_flush(struct net_bridge_vlan_group *vg)
{
        struct net_bridge_vlan *vlan, *tmp;

        list_for_each_entry_safe(vlan, tmp, &vg->vlan_list, vlist)
                vlan_tunnel_info_del(vg, vlan);
}

void nbp_vlan_tunnel_info_flush(struct net_bridge_port *port)
{
        struct net_bridge_vlan_group *vg;

        ASSERT_RTNL();

        vg = nbp_vlan_group(port);
        __vlan_tunnel_info_flush(vg);
}

int vlan_tunnel_init(struct net_bridge_vlan_group *vg)
{
        return rhashtable_init(&vg->tunnel_hash, &br_vlan_tunnel_rht_params);
}

void vlan_tunnel_deinit(struct net_bridge_vlan_group *vg)
{
        rhashtable_destroy(&vg->tunnel_hash);
}

void br_handle_ingress_vlan_tunnel(struct sk_buff *skb,
                                   struct net_bridge_port *p,
                                   struct net_bridge_vlan_group *vg)
{
        struct ip_tunnel_info *tinfo = skb_tunnel_info(skb);
        struct net_bridge_vlan *vlan;

        if (!vg || !tinfo)
                return;

        /* if already tagged, ignore */
        if (skb_vlan_tagged(skb))
                return;

        /* lookup vid, given tunnel id */
        vlan = br_vlan_tunnel_lookup(&vg->tunnel_hash, tinfo->key.tun_id);
        if (!vlan)
                return;

        skb_dst_drop(skb);

        __vlan_hwaccel_put_tag(skb, p->br->vlan_proto, vlan->vid);
}

int br_handle_egress_vlan_tunnel(struct sk_buff *skb,
                                 struct net_bridge_vlan *vlan)
{
        struct metadata_dst *tunnel_dst;
        __be64 tunnel_id;
        int err;

        if (!vlan)
                return 0;

        tunnel_id = READ_ONCE(vlan->tinfo.tunnel_id);
        if (!tunnel_id || unlikely(!skb_vlan_tag_present(skb)))
                return 0;

        skb_dst_drop(skb);
        err = skb_vlan_pop(skb);
        if (err)
                return err;

        tunnel_dst = rcu_dereference(vlan->tinfo.tunnel_dst);
        if (tunnel_dst && dst_hold_safe(&tunnel_dst->dst))
                skb_dst_set(skb, &tunnel_dst->dst);

        return 0;
}























































































































































































































































































































































































































































































































































































































































































































  416 












  413 


  414 




  416 






  415 











































































  132 









  133 




















  132 


  132 





1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
// SPDX-License-Identifier: GPL-2.0
/* sysfs entries for device PM */
#include <linux/device.h>
#include <linux/kobject.h>
#include <linux/string.h>
#include <linux/export.h>
#include <linux/pm_qos.h>
#include <linux/pm_runtime.h>
#include <linux/pm_wakeup.h>
#include <linux/atomic.h>
#include <linux/jiffies.h>
#include "power.h"

/*
 *        control - Report/change current runtime PM setting of the device
 *
 *        Runtime power management of a device can be blocked with the help of
 *        this attribute.  All devices have one of the following two values for
 *        the power/control file:
 *
 *         + "auto\n" to allow the device to be power managed at run time;
 *         + "on\n" to prevent the device from being power managed at run time;
 *
 *        The default for all devices is "auto", which means that devices may be
 *        subject to automatic power management, depending on their drivers.
 *        Changing this attribute to "on" prevents the driver from power managing
 *        the device at run time.  Doing that while the device is suspended causes
 *        it to be woken up.
 *
 *        wakeup - Report/change current wakeup option for device
 *
 *        Some devices support "wakeup" events, which are hardware signals
 *        used to activate devices from suspended or low power states.  Such
 *        devices have one of three values for the sysfs power/wakeup file:
 *
 *         + "enabled\n" to issue the events;
 *         + "disabled\n" not to do so; or
 *         + "\n" for temporary or permanent inability to issue wakeup.
 *
 *        (For example, unconfigured USB devices can't issue wakeups.)
 *
 *        Familiar examples of devices that can issue wakeup events include
 *        keyboards and mice (both PS2 and USB styles), power buttons, modems,
 *        "Wake-On-LAN" Ethernet links, GPIO lines, and more.  Some events
 *        will wake the entire system from a suspend state; others may just
 *        wake up the device (if the system as a whole is already active).
 *        Some wakeup events use normal IRQ lines; other use special out
 *        of band signaling.
 *
 *        It is the responsibility of device drivers to enable (or disable)
 *        wakeup signaling as part of changing device power states, respecting
 *        the policy choices provided through the driver model.
 *
 *        Devices may not be able to generate wakeup events from all power
 *        states.  Also, the events may be ignored in some configurations;
 *        for example, they might need help from other devices that aren't
 *        active, or which may have wakeup disabled.  Some drivers rely on
 *        wakeup events internally (unless they are disabled), keeping
 *        their hardware in low power modes whenever they're unused.  This
 *        saves runtime power, without requiring system-wide sleep states.
 *
 *        async - Report/change current async suspend setting for the device
 *
 *        Asynchronous suspend and resume of the device during system-wide power
 *        state transitions can be enabled by writing "enabled" to this file.
 *        Analogously, if "disabled" is written to this file, the device will be
 *        suspended and resumed synchronously.
 *
 *        All devices have one of the following two values for power/async:
 *
 *         + "enabled\n" to permit the asynchronous suspend/resume of the device;
 *         + "disabled\n" to forbid it;
 *
 *        NOTE: It generally is unsafe to permit the asynchronous suspend/resume
 *        of a device unless it is certain that all of the PM dependencies of the
 *        device are known to the PM core.  However, for some devices this
 *        attribute is set to "enabled" by bus type code or device drivers and in
 *        that cases it should be safe to leave the default value.
 *
 *        autosuspend_delay_ms - Report/change a device's autosuspend_delay value
 *
 *        Some drivers don't want to carry out a runtime suspend as soon as a
 *        device becomes idle; they want it always to remain idle for some period
 *        of time before suspending it.  This period is the autosuspend_delay
 *        value (expressed in milliseconds) and it can be controlled by the user.
 *        If the value is negative then the device will never be runtime
 *        suspended.
 *
 *        NOTE: The autosuspend_delay_ms attribute and the autosuspend_delay
 *        value are used only if the driver calls pm_runtime_use_autosuspend().
 *
 *        wakeup_count - Report the number of wakeup events related to the device
 */

const char power_group_name[] = "power";
EXPORT_SYMBOL_GPL(power_group_name);

static const char ctrl_auto[] = "auto";
static const char ctrl_on[] = "on";

static ssize_t control_show(struct device *dev, struct device_attribute *attr,
                            char *buf)
{
        return sysfs_emit(buf, "%s\n",
                          dev->power.runtime_auto ? ctrl_auto : ctrl_on);
}

static ssize_t control_store(struct device * dev, struct device_attribute *attr,
                             const char * buf, size_t n)
{
        device_lock(dev);
        if (sysfs_streq(buf, ctrl_auto))
                pm_runtime_allow(dev);
        else if (sysfs_streq(buf, ctrl_on))
                pm_runtime_forbid(dev);
        else
                n = -EINVAL;
        device_unlock(dev);
        return n;
}

static DEVICE_ATTR_RW(control);

static ssize_t runtime_active_time_show(struct device *dev,
                                        struct device_attribute *attr,
                                        char *buf)
{
        u64 tmp = pm_runtime_active_time(dev);

        do_div(tmp, NSEC_PER_MSEC);

        return sysfs_emit(buf, "%llu\n", tmp);
}

static DEVICE_ATTR_RO(runtime_active_time);

static ssize_t runtime_suspended_time_show(struct device *dev,
                                           struct device_attribute *attr,
                                           char *buf)
{
        u64 tmp = pm_runtime_suspended_time(dev);

        do_div(tmp, NSEC_PER_MSEC);

        return sysfs_emit(buf, "%llu\n", tmp);
}

static DEVICE_ATTR_RO(runtime_suspended_time);

static ssize_t runtime_status_show(struct device *dev,
                                   struct device_attribute *attr, char *buf)
{
        const char *output;

        if (dev->power.runtime_error) {
                output = "error";
        } else if (dev->power.disable_depth) {
                output = "unsupported";
        } else {
                switch (dev->power.runtime_status) {
                case RPM_SUSPENDED:
                        output = "suspended";
                        break;
                case RPM_SUSPENDING:
                        output = "suspending";
                        break;
                case RPM_RESUMING:
                        output = "resuming";
                        break;
                case RPM_ACTIVE:
                        output = "active";
                        break;
                default:
                        return -EIO;
                }
        }
        return sysfs_emit(buf, "%s\n", output);
}

static DEVICE_ATTR_RO(runtime_status);

static ssize_t autosuspend_delay_ms_show(struct device *dev,
                                         struct device_attribute *attr,
                                         char *buf)
{
        if (!dev->power.use_autosuspend)
                return -EIO;

        return sysfs_emit(buf, "%d\n", dev->power.autosuspend_delay);
}

static ssize_t autosuspend_delay_ms_store(struct device *dev,
                struct device_attribute *attr, const char *buf, size_t n)
{
        long delay;

        if (!dev->power.use_autosuspend)
                return -EIO;

        if (kstrtol(buf, 10, &delay) != 0 || delay != (int) delay)
                return -EINVAL;

        device_lock(dev);
        pm_runtime_set_autosuspend_delay(dev, delay);
        device_unlock(dev);
        return n;
}

static DEVICE_ATTR_RW(autosuspend_delay_ms);

static ssize_t pm_qos_resume_latency_us_show(struct device *dev,
                                             struct device_attribute *attr,
                                             char *buf)
{
        s32 value = dev_pm_qos_requested_resume_latency(dev);

        if (value == 0)
                return sysfs_emit(buf, "n/a\n");
        if (value == PM_QOS_RESUME_LATENCY_NO_CONSTRAINT)
                value = 0;

        return sysfs_emit(buf, "%d\n", value);
}

static ssize_t pm_qos_resume_latency_us_store(struct device *dev,
                                              struct device_attribute *attr,
                                              const char *buf, size_t n)
{
        s32 value;
        int ret;

        if (!kstrtos32(buf, 0, &value)) {
                /*
                 * Prevent users from writing negative or "no constraint" values
                 * directly.
                 */
                if (value < 0 || value == PM_QOS_RESUME_LATENCY_NO_CONSTRAINT)
                        return -EINVAL;

                if (value == 0)
                        value = PM_QOS_RESUME_LATENCY_NO_CONSTRAINT;
        } else if (sysfs_streq(buf, "n/a")) {
                value = 0;
        } else {
                return -EINVAL;
        }

        ret = dev_pm_qos_update_request(dev->power.qos->resume_latency_req,
                                        value);
        return ret < 0 ? ret : n;
}

static DEVICE_ATTR_RW(pm_qos_resume_latency_us);

static ssize_t pm_qos_latency_tolerance_us_show(struct device *dev,
                                                struct device_attribute *attr,
                                                char *buf)
{
        s32 value = dev_pm_qos_get_user_latency_tolerance(dev);

        if (value < 0)
                return sysfs_emit(buf, "%s\n", "auto");
        if (value == PM_QOS_LATENCY_ANY)
                return sysfs_emit(buf, "%s\n", "any");

        return sysfs_emit(buf, "%d\n", value);
}

static ssize_t pm_qos_latency_tolerance_us_store(struct device *dev,
                                                 struct device_attribute *attr,
                                                 const char *buf, size_t n)
{
        s32 value;
        int ret;

        if (kstrtos32(buf, 0, &value) == 0) {
                /* Users can't write negative values directly */
                if (value < 0)
                        return -EINVAL;
        } else {
                if (sysfs_streq(buf, "auto"))
                        value = PM_QOS_LATENCY_TOLERANCE_NO_CONSTRAINT;
                else if (sysfs_streq(buf, "any"))
                        value = PM_QOS_LATENCY_ANY;
                else
                        return -EINVAL;
        }
        ret = dev_pm_qos_update_user_latency_tolerance(dev, value);
        return ret < 0 ? ret : n;
}

static DEVICE_ATTR_RW(pm_qos_latency_tolerance_us);

static ssize_t pm_qos_no_power_off_show(struct device *dev,
                                        struct device_attribute *attr,
                                        char *buf)
{
        return sysfs_emit(buf, "%d\n", !!(dev_pm_qos_requested_flags(dev)
                                          & PM_QOS_FLAG_NO_POWER_OFF));
}

static ssize_t pm_qos_no_power_off_store(struct device *dev,
                                         struct device_attribute *attr,
                                         const char *buf, size_t n)
{
        int ret;

        if (kstrtoint(buf, 0, &ret))
                return -EINVAL;

        if (ret != 0 && ret != 1)
                return -EINVAL;

        ret = dev_pm_qos_update_flags(dev, PM_QOS_FLAG_NO_POWER_OFF, ret);
        return ret < 0 ? ret : n;
}

static DEVICE_ATTR_RW(pm_qos_no_power_off);

#ifdef CONFIG_PM_SLEEP
static const char _enabled[] = "enabled";
static const char _disabled[] = "disabled";

static ssize_t wakeup_show(struct device *dev, struct device_attribute *attr,
                           char *buf)
{
        return sysfs_emit(buf, "%s\n", device_can_wakeup(dev)
                          ? (device_may_wakeup(dev) ? _enabled : _disabled)
                          : "");
}

static ssize_t wakeup_store(struct device *dev, struct device_attribute *attr,
                            const char *buf, size_t n)
{
        if (!device_can_wakeup(dev))
                return -EINVAL;

        if (sysfs_streq(buf, _enabled))
                device_set_wakeup_enable(dev, 1);
        else if (sysfs_streq(buf, _disabled))
                device_set_wakeup_enable(dev, 0);
        else
                return -EINVAL;
        return n;
}

static DEVICE_ATTR_RW(wakeup);

static ssize_t wakeup_count_show(struct device *dev,
                                 struct device_attribute *attr, char *buf)
{
        unsigned long count;
        bool enabled = false;

        spin_lock_irq(&dev->power.lock);
        if (dev->power.wakeup) {
                count = dev->power.wakeup->wakeup_count;
                enabled = true;
        }
        spin_unlock_irq(&dev->power.lock);

        if (!enabled)
                return sysfs_emit(buf, "\n");
        return sysfs_emit(buf, "%lu\n", count);
}

static DEVICE_ATTR_RO(wakeup_count);

static ssize_t wakeup_active_count_show(struct device *dev,
                                        struct device_attribute *attr,
                                        char *buf)
{
        unsigned long count;
        bool enabled = false;

        spin_lock_irq(&dev->power.lock);
        if (dev->power.wakeup) {
                count = dev->power.wakeup->active_count;
                enabled = true;
        }
        spin_unlock_irq(&dev->power.lock);

        if (!enabled)
                return sysfs_emit(buf, "\n");
        return sysfs_emit(buf, "%lu\n", count);
}

static DEVICE_ATTR_RO(wakeup_active_count);

static ssize_t wakeup_abort_count_show(struct device *dev,
                                       struct device_attribute *attr,
                                       char *buf)
{
        unsigned long count;
        bool enabled = false;

        spin_lock_irq(&dev->power.lock);
        if (dev->power.wakeup) {
                count = dev->power.wakeup->wakeup_count;
                enabled = true;
        }
        spin_unlock_irq(&dev->power.lock);

        if (!enabled)
                return sysfs_emit(buf, "\n");
        return sysfs_emit(buf, "%lu\n", count);
}

static DEVICE_ATTR_RO(wakeup_abort_count);

static ssize_t wakeup_expire_count_show(struct device *dev,
                                        struct device_attribute *attr,
                                        char *buf)
{
        unsigned long count;
        bool enabled = false;

        spin_lock_irq(&dev->power.lock);
        if (dev->power.wakeup) {
                count = dev->power.wakeup->expire_count;
                enabled = true;
        }
        spin_unlock_irq(&dev->power.lock);

        if (!enabled)
                return sysfs_emit(buf, "\n");
        return sysfs_emit(buf, "%lu\n", count);
}

static DEVICE_ATTR_RO(wakeup_expire_count);

static ssize_t wakeup_active_show(struct device *dev,
                                  struct device_attribute *attr, char *buf)
{
        unsigned int active;
        bool enabled = false;

        spin_lock_irq(&dev->power.lock);
        if (dev->power.wakeup) {
                active = dev->power.wakeup->active;
                enabled = true;
        }
        spin_unlock_irq(&dev->power.lock);

        if (!enabled)
                return sysfs_emit(buf, "\n");
        return sysfs_emit(buf, "%u\n", active);
}

static DEVICE_ATTR_RO(wakeup_active);

static ssize_t wakeup_total_time_ms_show(struct device *dev,
                                         struct device_attribute *attr,
                                         char *buf)
{
        s64 msec;
        bool enabled = false;

        spin_lock_irq(&dev->power.lock);
        if (dev->power.wakeup) {
                msec = ktime_to_ms(dev->power.wakeup->total_time);
                enabled = true;
        }
        spin_unlock_irq(&dev->power.lock);

        if (!enabled)
                return sysfs_emit(buf, "\n");
        return sysfs_emit(buf, "%lld\n", msec);
}

static DEVICE_ATTR_RO(wakeup_total_time_ms);

static ssize_t wakeup_max_time_ms_show(struct device *dev,
                                       struct device_attribute *attr, char *buf)
{
        s64 msec;
        bool enabled = false;

        spin_lock_irq(&dev->power.lock);
        if (dev->power.wakeup) {
                msec = ktime_to_ms(dev->power.wakeup->max_time);
                enabled = true;
        }
        spin_unlock_irq(&dev->power.lock);

        if (!enabled)
                return sysfs_emit(buf, "\n");
        return sysfs_emit(buf, "%lld\n", msec);
}

static DEVICE_ATTR_RO(wakeup_max_time_ms);

static ssize_t wakeup_last_time_ms_show(struct device *dev,
                                        struct device_attribute *attr,
                                        char *buf)
{
        s64 msec;
        bool enabled = false;

        spin_lock_irq(&dev->power.lock);
        if (dev->power.wakeup) {
                msec = ktime_to_ms(dev->power.wakeup->last_time);
                enabled = true;
        }
        spin_unlock_irq(&dev->power.lock);

        if (!enabled)
                return sysfs_emit(buf, "\n");
        return sysfs_emit(buf, "%lld\n", msec);
}

static inline int dpm_sysfs_wakeup_change_owner(struct device *dev, kuid_t kuid,
                                                kgid_t kgid)
{
        if (dev->power.wakeup && dev->power.wakeup->dev)
                return device_change_owner(dev->power.wakeup->dev, kuid, kgid);
        return 0;
}

static DEVICE_ATTR_RO(wakeup_last_time_ms);

#ifdef CONFIG_PM_AUTOSLEEP
static ssize_t wakeup_prevent_sleep_time_ms_show(struct device *dev,
                                                 struct device_attribute *attr,
                                                 char *buf)
{
        s64 msec;
        bool enabled = false;

        spin_lock_irq(&dev->power.lock);
        if (dev->power.wakeup) {
                msec = ktime_to_ms(dev->power.wakeup->prevent_sleep_time);
                enabled = true;
        }
        spin_unlock_irq(&dev->power.lock);

        if (!enabled)
                return sysfs_emit(buf, "\n");
        return sysfs_emit(buf, "%lld\n", msec);
}

static DEVICE_ATTR_RO(wakeup_prevent_sleep_time_ms);
#endif /* CONFIG_PM_AUTOSLEEP */
#else /* CONFIG_PM_SLEEP */
static inline int dpm_sysfs_wakeup_change_owner(struct device *dev, kuid_t kuid,
                                                kgid_t kgid)
{
        return 0;
}
#endif

#ifdef CONFIG_PM_ADVANCED_DEBUG
static ssize_t runtime_usage_show(struct device *dev,
                                  struct device_attribute *attr, char *buf)
{
        return sysfs_emit(buf, "%d\n", atomic_read(&dev->power.usage_count));
}
static DEVICE_ATTR_RO(runtime_usage);

static ssize_t runtime_active_kids_show(struct device *dev,
                                        struct device_attribute *attr,
                                        char *buf)
{
        return sysfs_emit(buf, "%d\n", dev->power.ignore_children ?
                          0 : atomic_read(&dev->power.child_count));
}
static DEVICE_ATTR_RO(runtime_active_kids);

static ssize_t runtime_enabled_show(struct device *dev,
                                    struct device_attribute *attr, char *buf)
{
        const char *output;

        if (dev->power.disable_depth && !dev->power.runtime_auto)
                output = "disabled & forbidden";
        else if (dev->power.disable_depth)
                output = "disabled";
        else if (!dev->power.runtime_auto)
                output = "forbidden";
        else
                output = "enabled";

        return sysfs_emit(buf, "%s\n", output);
}
static DEVICE_ATTR_RO(runtime_enabled);

#ifdef CONFIG_PM_SLEEP
static ssize_t async_show(struct device *dev, struct device_attribute *attr,
                          char *buf)
{
        return sysfs_emit(buf, "%s\n",
                          device_async_suspend_enabled(dev) ?
                          _enabled : _disabled);
}

static ssize_t async_store(struct device *dev, struct device_attribute *attr,
                           const char *buf, size_t n)
{
        if (sysfs_streq(buf, _enabled))
                device_enable_async_suspend(dev);
        else if (sysfs_streq(buf, _disabled))
                device_disable_async_suspend(dev);
        else
                return -EINVAL;
        return n;
}

static DEVICE_ATTR_RW(async);

#endif /* CONFIG_PM_SLEEP */
#endif /* CONFIG_PM_ADVANCED_DEBUG */

static struct attribute *power_attrs[] = {
#ifdef CONFIG_PM_ADVANCED_DEBUG
#ifdef CONFIG_PM_SLEEP
        &dev_attr_async.attr,
#endif
        &dev_attr_runtime_status.attr,
        &dev_attr_runtime_usage.attr,
        &dev_attr_runtime_active_kids.attr,
        &dev_attr_runtime_enabled.attr,
#endif /* CONFIG_PM_ADVANCED_DEBUG */
        NULL,
};
static const struct attribute_group pm_attr_group = {
        .name        = power_group_name,
        .attrs        = power_attrs,
};

static struct attribute *wakeup_attrs[] = {
#ifdef CONFIG_PM_SLEEP
        &dev_attr_wakeup.attr,
        &dev_attr_wakeup_count.attr,
        &dev_attr_wakeup_active_count.attr,
        &dev_attr_wakeup_abort_count.attr,
        &dev_attr_wakeup_expire_count.attr,
        &dev_attr_wakeup_active.attr,
        &dev_attr_wakeup_total_time_ms.attr,
        &dev_attr_wakeup_max_time_ms.attr,
        &dev_attr_wakeup_last_time_ms.attr,
#ifdef CONFIG_PM_AUTOSLEEP
        &dev_attr_wakeup_prevent_sleep_time_ms.attr,
#endif
#endif
        NULL,
};
static const struct attribute_group pm_wakeup_attr_group = {
        .name        = power_group_name,
        .attrs        = wakeup_attrs,
};

static struct attribute *runtime_attrs[] = {
#ifndef CONFIG_PM_ADVANCED_DEBUG
        &dev_attr_runtime_status.attr,
#endif
        &dev_attr_control.attr,
        &dev_attr_runtime_suspended_time.attr,
        &dev_attr_runtime_active_time.attr,
        &dev_attr_autosuspend_delay_ms.attr,
        NULL,
};
static const struct attribute_group pm_runtime_attr_group = {
        .name        = power_group_name,
        .attrs        = runtime_attrs,
};

static struct attribute *pm_qos_resume_latency_attrs[] = {
        &dev_attr_pm_qos_resume_latency_us.attr,
        NULL,
};
static const struct attribute_group pm_qos_resume_latency_attr_group = {
        .name        = power_group_name,
        .attrs        = pm_qos_resume_latency_attrs,
};

static struct attribute *pm_qos_latency_tolerance_attrs[] = {
        &dev_attr_pm_qos_latency_tolerance_us.attr,
        NULL,
};
static const struct attribute_group pm_qos_latency_tolerance_attr_group = {
        .name        = power_group_name,
        .attrs        = pm_qos_latency_tolerance_attrs,
};

static struct attribute *pm_qos_flags_attrs[] = {
        &dev_attr_pm_qos_no_power_off.attr,
        NULL,
};
static const struct attribute_group pm_qos_flags_attr_group = {
        .name        = power_group_name,
        .attrs        = pm_qos_flags_attrs,
};

int dpm_sysfs_add(struct device *dev)
{
        int rc;

        /* No need to create PM sysfs if explicitly disabled. */
        if (device_pm_not_required(dev))
                return 0;

        rc = sysfs_create_group(&dev->kobj, &pm_attr_group);
        if (rc)
                return rc;

        if (!pm_runtime_has_no_callbacks(dev)) {
                rc = sysfs_merge_group(&dev->kobj, &pm_runtime_attr_group);
                if (rc)
                        goto err_out;
        }
        if (device_can_wakeup(dev)) {
                rc = sysfs_merge_group(&dev->kobj, &pm_wakeup_attr_group);
                if (rc)
                        goto err_runtime;
        }
        if (dev->power.set_latency_tolerance) {
                rc = sysfs_merge_group(&dev->kobj,
                                       &pm_qos_latency_tolerance_attr_group);
                if (rc)
                        goto err_wakeup;
        }
        rc = pm_wakeup_source_sysfs_add(dev);
        if (rc)
                goto err_latency;
        return 0;

 err_latency:
        sysfs_unmerge_group(&dev->kobj, &pm_qos_latency_tolerance_attr_group);
 err_wakeup:
        sysfs_unmerge_group(&dev->kobj, &pm_wakeup_attr_group);
 err_runtime:
        sysfs_unmerge_group(&dev->kobj, &pm_runtime_attr_group);
 err_out:
        sysfs_remove_group(&dev->kobj, &pm_attr_group);
        return rc;
}

int dpm_sysfs_change_owner(struct device *dev, kuid_t kuid, kgid_t kgid)
{
        int rc;

        if (device_pm_not_required(dev))
                return 0;

        rc = sysfs_group_change_owner(&dev->kobj, &pm_attr_group, kuid, kgid);
        if (rc)
                return rc;

        if (!pm_runtime_has_no_callbacks(dev)) {
                rc = sysfs_group_change_owner(
                        &dev->kobj, &pm_runtime_attr_group, kuid, kgid);
                if (rc)
                        return rc;
        }

        if (device_can_wakeup(dev)) {
                rc = sysfs_group_change_owner(&dev->kobj, &pm_wakeup_attr_group,
                                              kuid, kgid);
                if (rc)
                        return rc;

                rc = dpm_sysfs_wakeup_change_owner(dev, kuid, kgid);
                if (rc)
                        return rc;
        }

        if (dev->power.set_latency_tolerance) {
                rc = sysfs_group_change_owner(
                        &dev->kobj, &pm_qos_latency_tolerance_attr_group, kuid,
                        kgid);
                if (rc)
                        return rc;
        }
        return 0;
}

int wakeup_sysfs_add(struct device *dev)
{
        int ret = sysfs_merge_group(&dev->kobj, &pm_wakeup_attr_group);

        if (!ret)
                kobject_uevent(&dev->kobj, KOBJ_CHANGE);

        return ret;
}

void wakeup_sysfs_remove(struct device *dev)
{
        sysfs_unmerge_group(&dev->kobj, &pm_wakeup_attr_group);
        kobject_uevent(&dev->kobj, KOBJ_CHANGE);
}

int pm_qos_sysfs_add_resume_latency(struct device *dev)
{
        return sysfs_merge_group(&dev->kobj, &pm_qos_resume_latency_attr_group);
}

void pm_qos_sysfs_remove_resume_latency(struct device *dev)
{
        sysfs_unmerge_group(&dev->kobj, &pm_qos_resume_latency_attr_group);
}

int pm_qos_sysfs_add_flags(struct device *dev)
{
        return sysfs_merge_group(&dev->kobj, &pm_qos_flags_attr_group);
}

void pm_qos_sysfs_remove_flags(struct device *dev)
{
        sysfs_unmerge_group(&dev->kobj, &pm_qos_flags_attr_group);
}

int pm_qos_sysfs_add_latency_tolerance(struct device *dev)
{
        return sysfs_merge_group(&dev->kobj,
                                 &pm_qos_latency_tolerance_attr_group);
}

void pm_qos_sysfs_remove_latency_tolerance(struct device *dev)
{
        sysfs_unmerge_group(&dev->kobj, &pm_qos_latency_tolerance_attr_group);
}

void rpm_sysfs_remove(struct device *dev)
{
        sysfs_unmerge_group(&dev->kobj, &pm_runtime_attr_group);
}

void dpm_sysfs_remove(struct device *dev)
{
        if (device_pm_not_required(dev))
                return;
        sysfs_unmerge_group(&dev->kobj, &pm_qos_latency_tolerance_attr_group);
        dev_pm_qos_constraints_destroy(dev);
        rpm_sysfs_remove(dev);
        sysfs_unmerge_group(&dev->kobj, &pm_wakeup_attr_group);
        sysfs_remove_group(&dev->kobj, &pm_attr_group);
}










































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
/* SPDX-License-Identifier: GPL-2.0 */
/*
 *        pci.h
 *
 *        PCI defines and function prototypes
 *        Copyright 1994, Drew Eckhardt
 *        Copyright 1997--1999 Martin Mares <mj@ucw.cz>
 *
 *        PCI Express ASPM defines and function prototypes
 *        Copyright (c) 2007 Intel Corp.
 *                Zhang Yanmin (yanmin.zhang@intel.com)
 *                Shaohua Li (shaohua.li@intel.com)
 *
 *        For more information, please consult the following manuals (look at
 *        http://www.pcisig.com/ for how to get them):
 *
 *        PCI BIOS Specification
 *        PCI Local Bus Specification
 *        PCI to PCI Bridge Specification
 *        PCI Express Specification
 *        PCI System Design Guide
 */
#ifndef LINUX_PCI_H
#define LINUX_PCI_H


#include <linux/mod_devicetable.h>

#include <linux/types.h>
#include <linux/init.h>
#include <linux/ioport.h>
#include <linux/list.h>
#include <linux/compiler.h>
#include <linux/errno.h>
#include <linux/kobject.h>
#include <linux/atomic.h>
#include <linux/device.h>
#include <linux/interrupt.h>
#include <linux/io.h>
#include <linux/resource_ext.h>
#include <uapi/linux/pci.h>

#include <linux/pci_ids.h>

#define PCI_STATUS_ERROR_BITS (PCI_STATUS_DETECTED_PARITY  | \
                               PCI_STATUS_SIG_SYSTEM_ERROR | \
                               PCI_STATUS_REC_MASTER_ABORT | \
                               PCI_STATUS_REC_TARGET_ABORT | \
                               PCI_STATUS_SIG_TARGET_ABORT | \
                               PCI_STATUS_PARITY)

/* Number of reset methods used in pci_reset_fn_methods array in pci.c */
#define PCI_NUM_RESET_METHODS 7

#define PCI_RESET_PROBE                true
#define PCI_RESET_DO_RESET        false

/*
 * The PCI interface treats multi-function devices as independent
 * devices.  The slot/function address of each device is encoded
 * in a single byte as follows:
 *
 *        7:3 = slot
 *        2:0 = function
 *
 * PCI_DEVFN(), PCI_SLOT(), and PCI_FUNC() are defined in uapi/linux/pci.h.
 * In the interest of not exposing interfaces to user-space unnecessarily,
 * the following kernel-only defines are being added here.
 */
#define PCI_DEVID(bus, devfn)        ((((u16)(bus)) << 8) | (devfn))
/* return bus from PCI devid = ((u16)bus_number) << 8) | devfn */
#define PCI_BUS_NUM(x) (((x) >> 8) & 0xff)

/* pci_slot represents a physical slot */
struct pci_slot {
        struct pci_bus                *bus;                /* Bus this slot is on */
        struct list_head        list;                /* Node in list of slots */
        struct hotplug_slot        *hotplug;        /* Hotplug info (move here) */
        unsigned char                number;                /* PCI_SLOT(pci_dev->devfn) */
        struct kobject                kobj;
};

static inline const char *pci_slot_name(const struct pci_slot *slot)
{
        return kobject_name(&slot->kobj);
}

/* File state for mmap()s on /proc/bus/pci/X/Y */
enum pci_mmap_state {
        pci_mmap_io,
        pci_mmap_mem
};

/* For PCI devices, the region numbers are assigned this way: */
enum {
        /* #0-5: standard PCI resources */
        PCI_STD_RESOURCES,
        PCI_STD_RESOURCE_END = PCI_STD_RESOURCES + PCI_STD_NUM_BARS - 1,

        /* #6: expansion ROM resource */
        PCI_ROM_RESOURCE,

        /* Device-specific resources */
#ifdef CONFIG_PCI_IOV
        PCI_IOV_RESOURCES,
        PCI_IOV_RESOURCE_END = PCI_IOV_RESOURCES + PCI_SRIOV_NUM_BARS - 1,
#endif

/* PCI-to-PCI (P2P) bridge windows */
#define PCI_BRIDGE_IO_WINDOW                (PCI_BRIDGE_RESOURCES + 0)
#define PCI_BRIDGE_MEM_WINDOW                (PCI_BRIDGE_RESOURCES + 1)
#define PCI_BRIDGE_PREF_MEM_WINDOW        (PCI_BRIDGE_RESOURCES + 2)

/* CardBus bridge windows */
#define PCI_CB_BRIDGE_IO_0_WINDOW        (PCI_BRIDGE_RESOURCES + 0)
#define PCI_CB_BRIDGE_IO_1_WINDOW        (PCI_BRIDGE_RESOURCES + 1)
#define PCI_CB_BRIDGE_MEM_0_WINDOW        (PCI_BRIDGE_RESOURCES + 2)
#define PCI_CB_BRIDGE_MEM_1_WINDOW        (PCI_BRIDGE_RESOURCES + 3)

/* Total number of bridge resources for P2P and CardBus */
#define PCI_BRIDGE_RESOURCE_NUM 4

        /* Resources assigned to buses behind the bridge */
        PCI_BRIDGE_RESOURCES,
        PCI_BRIDGE_RESOURCE_END = PCI_BRIDGE_RESOURCES +
                                  PCI_BRIDGE_RESOURCE_NUM - 1,

        /* Total resources associated with a PCI device */
        PCI_NUM_RESOURCES,

        /* Preserve this for compatibility */
        DEVICE_COUNT_RESOURCE = PCI_NUM_RESOURCES,
};

/**
 * enum pci_interrupt_pin - PCI INTx interrupt values
 * @PCI_INTERRUPT_UNKNOWN: Unknown or unassigned interrupt
 * @PCI_INTERRUPT_INTA: PCI INTA pin
 * @PCI_INTERRUPT_INTB: PCI INTB pin
 * @PCI_INTERRUPT_INTC: PCI INTC pin
 * @PCI_INTERRUPT_INTD: PCI INTD pin
 *
 * Corresponds to values for legacy PCI INTx interrupts, as can be found in the
 * PCI_INTERRUPT_PIN register.
 */
enum pci_interrupt_pin {
        PCI_INTERRUPT_UNKNOWN,
        PCI_INTERRUPT_INTA,
        PCI_INTERRUPT_INTB,
        PCI_INTERRUPT_INTC,
        PCI_INTERRUPT_INTD,
};

/* The number of legacy PCI INTx interrupts */
#define PCI_NUM_INTX        4

/*
 * Reading from a device that doesn't respond typically returns ~0.  A
 * successful read from a device may also return ~0, so you need additional
 * information to reliably identify errors.
 */
#define PCI_ERROR_RESPONSE                (~0ULL)
#define PCI_SET_ERROR_RESPONSE(val)        (*(val) = ((typeof(*(val))) PCI_ERROR_RESPONSE))
#define PCI_POSSIBLE_ERROR(val)                ((val) == ((typeof(val)) PCI_ERROR_RESPONSE))

/*
 * pci_power_t values must match the bits in the Capabilities PME_Support
 * and Control/Status PowerState fields in the Power Management capability.
 */
typedef int __bitwise pci_power_t;

#define PCI_D0                ((pci_power_t __force) 0)
#define PCI_D1                ((pci_power_t __force) 1)
#define PCI_D2                ((pci_power_t __force) 2)
#define PCI_D3hot        ((pci_power_t __force) 3)
#define PCI_D3cold        ((pci_power_t __force) 4)
#define PCI_UNKNOWN        ((pci_power_t __force) 5)
#define PCI_POWER_ERROR        ((pci_power_t __force) -1)

/* Remember to update this when the list above changes! */
extern const char *pci_power_names[];

static inline const char *pci_power_name(pci_power_t state)
{
        return pci_power_names[1 + (__force int) state];
}

/**
 * typedef pci_channel_state_t
 *
 * The pci_channel state describes connectivity between the CPU and
 * the PCI device.  If some PCI bus between here and the PCI device
 * has crashed or locked up, this info is reflected here.
 */
typedef unsigned int __bitwise pci_channel_state_t;

enum {
        /* I/O channel is in normal state */
        pci_channel_io_normal = (__force pci_channel_state_t) 1,

        /* I/O to channel is blocked */
        pci_channel_io_frozen = (__force pci_channel_state_t) 2,

        /* PCI card is dead */
        pci_channel_io_perm_failure = (__force pci_channel_state_t) 3,
};

typedef unsigned int __bitwise pcie_reset_state_t;

enum pcie_reset_state {
        /* Reset is NOT asserted (Use to deassert reset) */
        pcie_deassert_reset = (__force pcie_reset_state_t) 1,

        /* Use #PERST to reset PCIe device */
        pcie_warm_reset = (__force pcie_reset_state_t) 2,

        /* Use PCIe Hot Reset to reset device */
        pcie_hot_reset = (__force pcie_reset_state_t) 3
};

typedef unsigned short __bitwise pci_dev_flags_t;
enum pci_dev_flags {
        /* INTX_DISABLE in PCI_COMMAND register disables MSI too */
        PCI_DEV_FLAGS_MSI_INTX_DISABLE_BUG = (__force pci_dev_flags_t) (1 << 0),
        /* Device configuration is irrevocably lost if disabled into D3 */
        PCI_DEV_FLAGS_NO_D3 = (__force pci_dev_flags_t) (1 << 1),
        /* Provide indication device is assigned by a Virtual Machine Manager */
        PCI_DEV_FLAGS_ASSIGNED = (__force pci_dev_flags_t) (1 << 2),
        /* Flag for quirk use to store if quirk-specific ACS is enabled */
        PCI_DEV_FLAGS_ACS_ENABLED_QUIRK = (__force pci_dev_flags_t) (1 << 3),
        /* Use a PCIe-to-PCI bridge alias even if !pci_is_pcie */
        PCI_DEV_FLAG_PCIE_BRIDGE_ALIAS = (__force pci_dev_flags_t) (1 << 5),
        /* Do not use bus resets for device */
        PCI_DEV_FLAGS_NO_BUS_RESET = (__force pci_dev_flags_t) (1 << 6),
        /* Do not use PM reset even if device advertises NoSoftRst- */
        PCI_DEV_FLAGS_NO_PM_RESET = (__force pci_dev_flags_t) (1 << 7),
        /* Get VPD from function 0 VPD */
        PCI_DEV_FLAGS_VPD_REF_F0 = (__force pci_dev_flags_t) (1 << 8),
        /* A non-root bridge where translation occurs, stop alias search here */
        PCI_DEV_FLAGS_BRIDGE_XLATE_ROOT = (__force pci_dev_flags_t) (1 << 9),
        /* Do not use FLR even if device advertises PCI_AF_CAP */
        PCI_DEV_FLAGS_NO_FLR_RESET = (__force pci_dev_flags_t) (1 << 10),
        /* Don't use Relaxed Ordering for TLPs directed at this device */
        PCI_DEV_FLAGS_NO_RELAXED_ORDERING = (__force pci_dev_flags_t) (1 << 11),
        /* Device does honor MSI masking despite saying otherwise */
        PCI_DEV_FLAGS_HAS_MSI_MASKING = (__force pci_dev_flags_t) (1 << 12),
};

enum pci_irq_reroute_variant {
        INTEL_IRQ_REROUTE_VARIANT = 1,
        MAX_IRQ_REROUTE_VARIANTS = 3
};

typedef unsigned short __bitwise pci_bus_flags_t;
enum pci_bus_flags {
        PCI_BUS_FLAGS_NO_MSI        = (__force pci_bus_flags_t) 1,
        PCI_BUS_FLAGS_NO_MMRBC        = (__force pci_bus_flags_t) 2,
        PCI_BUS_FLAGS_NO_AERSID        = (__force pci_bus_flags_t) 4,
        PCI_BUS_FLAGS_NO_EXTCFG        = (__force pci_bus_flags_t) 8,
};

/* Values from Link Status register, PCIe r3.1, sec 7.8.8 */
enum pcie_link_width {
        PCIE_LNK_WIDTH_RESRV        = 0x00,
        PCIE_LNK_X1                = 0x01,
        PCIE_LNK_X2                = 0x02,
        PCIE_LNK_X4                = 0x04,
        PCIE_LNK_X8                = 0x08,
        PCIE_LNK_X12                = 0x0c,
        PCIE_LNK_X16                = 0x10,
        PCIE_LNK_X32                = 0x20,
        PCIE_LNK_WIDTH_UNKNOWN        = 0xff,
};

/* See matching string table in pci_speed_string() */
enum pci_bus_speed {
        PCI_SPEED_33MHz                        = 0x00,
        PCI_SPEED_66MHz                        = 0x01,
        PCI_SPEED_66MHz_PCIX                = 0x02,
        PCI_SPEED_100MHz_PCIX                = 0x03,
        PCI_SPEED_133MHz_PCIX                = 0x04,
        PCI_SPEED_66MHz_PCIX_ECC        = 0x05,
        PCI_SPEED_100MHz_PCIX_ECC        = 0x06,
        PCI_SPEED_133MHz_PCIX_ECC        = 0x07,
        PCI_SPEED_66MHz_PCIX_266        = 0x09,
        PCI_SPEED_100MHz_PCIX_266        = 0x0a,
        PCI_SPEED_133MHz_PCIX_266        = 0x0b,
        AGP_UNKNOWN                        = 0x0c,
        AGP_1X                                = 0x0d,
        AGP_2X                                = 0x0e,
        AGP_4X                                = 0x0f,
        AGP_8X                                = 0x10,
        PCI_SPEED_66MHz_PCIX_533        = 0x11,
        PCI_SPEED_100MHz_PCIX_533        = 0x12,
        PCI_SPEED_133MHz_PCIX_533        = 0x13,
        PCIE_SPEED_2_5GT                = 0x14,
        PCIE_SPEED_5_0GT                = 0x15,
        PCIE_SPEED_8_0GT                = 0x16,
        PCIE_SPEED_16_0GT                = 0x17,
        PCIE_SPEED_32_0GT                = 0x18,
        PCIE_SPEED_64_0GT                = 0x19,
        PCI_SPEED_UNKNOWN                = 0xff,
};

enum pci_bus_speed pcie_get_speed_cap(struct pci_dev *dev);
enum pcie_link_width pcie_get_width_cap(struct pci_dev *dev);

struct pci_vpd {
        struct mutex        lock;
        unsigned int        len;
        u8                cap;
};

struct irq_affinity;
struct pcie_link_state;
struct pci_sriov;
struct pci_p2pdma;
struct rcec_ea;

/* The pci_dev structure describes PCI devices */
struct pci_dev {
        struct list_head bus_list;        /* Node in per-bus list */
        struct pci_bus        *bus;                /* Bus this device is on */
        struct pci_bus        *subordinate;        /* Bus this device bridges to */

        void                *sysdata;        /* Hook for sys-specific extension */
        struct proc_dir_entry *procent;        /* Device entry in /proc/bus/pci */
        struct pci_slot        *slot;                /* Physical slot this device is in */

        unsigned int        devfn;                /* Encoded device & function index */
        unsigned short        vendor;
        unsigned short        device;
        unsigned short        subsystem_vendor;
        unsigned short        subsystem_device;
        unsigned int        class;                /* 3 bytes: (base,sub,prog-if) */
        u8                revision;        /* PCI revision, low byte of class word */
        u8                hdr_type;        /* PCI header type (`multi' flag masked out) */
#ifdef CONFIG_PCIEAER
        u16                aer_cap;        /* AER capability offset */
        struct aer_stats *aer_stats;        /* AER stats for this device */
#endif
#ifdef CONFIG_PCIEPORTBUS
        struct rcec_ea        *rcec_ea;        /* RCEC cached endpoint association */
        struct pci_dev  *rcec;          /* Associated RCEC device */
#endif
        u32                devcap;                /* PCIe Device Capabilities */
        u8                pcie_cap;        /* PCIe capability offset */
        u8                msi_cap;        /* MSI capability offset */
        u8                msix_cap;        /* MSI-X capability offset */
        u8                pcie_mpss:3;        /* PCIe Max Payload Size Supported */
        u8                rom_base_reg;        /* Config register controlling ROM */
        u8                pin;                /* Interrupt pin this device uses */
        u16                pcie_flags_reg;        /* Cached PCIe Capabilities Register */
        unsigned long        *dma_alias_mask;/* Mask of enabled devfn aliases */

        struct pci_driver *driver;        /* Driver bound to this device */
        u64                dma_mask;        /* Mask of the bits of bus address this
                                           device implements.  Normally this is
                                           0xffffffff.  You only need to change
                                           this if your device has broken DMA
                                           or supports 64-bit transfers.  */

        struct device_dma_parameters dma_parms;

        pci_power_t        current_state;        /* Current operating state. In ACPI,
                                           this is D0-D3, D0 being fully
                                           functional, and D3 being off. */
        unsigned int        imm_ready:1;        /* Supports Immediate Readiness */
        u8                pm_cap;                /* PM capability offset */
        unsigned int        pme_support:5;        /* Bitmask of states from which PME#
                                           can be generated */
        unsigned int        pme_poll:1;        /* Poll device's PME status bit */
        unsigned int        d1_support:1;        /* Low power state D1 is supported */
        unsigned int        d2_support:1;        /* Low power state D2 is supported */
        unsigned int        no_d1d2:1;        /* D1 and D2 are forbidden */
        unsigned int        no_d3cold:1;        /* D3cold is forbidden */
        unsigned int        bridge_d3:1;        /* Allow D3 for bridge */
        unsigned int        d3cold_allowed:1;        /* D3cold is allowed by user */
        unsigned int        mmio_always_on:1;        /* Disallow turning off io/mem
                                                   decoding during BAR sizing */
        unsigned int        wakeup_prepared:1;
        unsigned int        runtime_d3cold:1;        /* Whether go through runtime
                                                   D3cold, not set for devices
                                                   powered on/off by the
                                                   corresponding bridge */
        unsigned int        skip_bus_pm:1;        /* Internal: Skip bus-level PM */
        unsigned int        ignore_hotplug:1;        /* Ignore hotplug events */
        unsigned int        hotplug_user_indicators:1; /* SlotCtl indicators
                                                      controlled exclusively by
                                                      user sysfs */
        unsigned int        clear_retrain_link:1;        /* Need to clear Retrain Link
                                                   bit manually */
        unsigned int        d3hot_delay;        /* D3hot->D0 transition time in ms */
        unsigned int        d3cold_delay;        /* D3cold->D0 transition time in ms */

#ifdef CONFIG_PCIEASPM
        struct pcie_link_state        *link_state;        /* ASPM link state */
        unsigned int        ltr_path:1;        /* Latency Tolerance Reporting
                                           supported from root to here */
        u16                l1ss;                /* L1SS Capability pointer */
#endif
        unsigned int        pasid_no_tlp:1;                /* PASID works without TLP Prefix */
        unsigned int        eetlp_prefix_path:1;        /* End-to-End TLP Prefix */

        pci_channel_state_t error_state;        /* Current connectivity state */
        struct device        dev;                        /* Generic device interface */

        int                cfg_size;                /* Size of config space */

        /*
         * Instead of touching interrupt line and base address registers
         * directly, use the values stored here. They might be different!
         */
        unsigned int        irq;
        struct resource resource[DEVICE_COUNT_RESOURCE]; /* I/O and memory regions + expansion ROMs */

        bool                match_driver;                /* Skip attaching driver */

        unsigned int        transparent:1;                /* Subtractive decode bridge */
        unsigned int        io_window:1;                /* Bridge has I/O window */
        unsigned int        pref_window:1;                /* Bridge has pref mem window */
        unsigned int        pref_64_window:1;        /* Pref mem window is 64-bit */
        unsigned int        multifunction:1;        /* Multi-function device */

        unsigned int        is_busmaster:1;                /* Is busmaster */
        unsigned int        no_msi:1;                /* May not use MSI */
        unsigned int        no_64bit_msi:1;                /* May only use 32-bit MSIs */
        unsigned int        block_cfg_access:1;        /* Config space access blocked */
        unsigned int        broken_parity_status:1;        /* Generates false positive parity */
        unsigned int        irq_reroute_variant:2;        /* Needs IRQ rerouting variant */
        unsigned int        msi_enabled:1;
        unsigned int        msix_enabled:1;
        unsigned int        ari_enabled:1;                /* ARI forwarding */
        unsigned int        ats_enabled:1;                /* Address Translation Svc */
        unsigned int        pasid_enabled:1;        /* Process Address Space ID */
        unsigned int        pri_enabled:1;                /* Page Request Interface */
        unsigned int        is_managed:1;
        unsigned int        needs_freset:1;                /* Requires fundamental reset */
        unsigned int        state_saved:1;
        unsigned int        is_physfn:1;
        unsigned int        is_virtfn:1;
        unsigned int        is_hotplug_bridge:1;
        unsigned int        shpc_managed:1;                /* SHPC owned by shpchp */
        unsigned int        is_thunderbolt:1;        /* Thunderbolt controller */
        /*
         * Devices marked being untrusted are the ones that can potentially
         * execute DMA attacks and similar. They are typically connected
         * through external ports such as Thunderbolt but not limited to
         * that. When an IOMMU is enabled they should be getting full
         * mappings to make sure they cannot access arbitrary memory.
         */
        unsigned int        untrusted:1;
        /*
         * Info from the platform, e.g., ACPI or device tree, may mark a
         * device as "external-facing".  An external-facing device is
         * itself internal but devices downstream from it are external.
         */
        unsigned int        external_facing:1;
        unsigned int        broken_intx_masking:1;        /* INTx masking can't be used */
        unsigned int        io_window_1k:1;                /* Intel bridge 1K I/O windows */
        unsigned int        irq_managed:1;
        unsigned int        non_compliant_bars:1;        /* Broken BARs; ignore them */
        unsigned int        is_probed:1;                /* Device probing in progress */
        unsigned int        link_active_reporting:1;/* Device capable of reporting link active */
        unsigned int        no_vf_scan:1;                /* Don't scan for VFs after IOV enablement */
        unsigned int        no_command_memory:1;        /* No PCI_COMMAND_MEMORY */
        unsigned int        rom_bar_overlap:1;        /* ROM BAR disable broken */
        pci_dev_flags_t dev_flags;
        atomic_t        enable_cnt;        /* pci_enable_device has been called */

        u32                saved_config_space[16]; /* Config space saved at suspend time */
        struct hlist_head saved_cap_space;
        int                rom_attr_enabled;        /* Display of ROM attribute enabled? */
        struct bin_attribute *res_attr[DEVICE_COUNT_RESOURCE]; /* sysfs file for resources */
        struct bin_attribute *res_attr_wc[DEVICE_COUNT_RESOURCE]; /* sysfs file for WC mapping of resources */

#ifdef CONFIG_HOTPLUG_PCI_PCIE
        unsigned int        broken_cmd_compl:1;        /* No compl for some cmds */
#endif
#ifdef CONFIG_PCIE_PTM
        unsigned int        ptm_root:1;
        unsigned int        ptm_enabled:1;
        u8                ptm_granularity;
#endif
#ifdef CONFIG_PCI_MSI
        const struct attribute_group **msi_irq_groups;
#endif
        struct pci_vpd        vpd;
#ifdef CONFIG_PCIE_DPC
        u16                dpc_cap;
        unsigned int        dpc_rp_extensions:1;
        u8                dpc_rp_log_size;
#endif
#ifdef CONFIG_PCI_ATS
        union {
                struct pci_sriov        *sriov;                /* PF: SR-IOV info */
                struct pci_dev                *physfn;        /* VF: related PF */
        };
        u16                ats_cap;        /* ATS Capability offset */
        u8                ats_stu;        /* ATS Smallest Translation Unit */
#endif
#ifdef CONFIG_PCI_PRI
        u16                pri_cap;        /* PRI Capability offset */
        u32                pri_reqs_alloc; /* Number of PRI requests allocated */
        unsigned int        pasid_required:1; /* PRG Response PASID Required */
#endif
#ifdef CONFIG_PCI_PASID
        u16                pasid_cap;        /* PASID Capability offset */
        u16                pasid_features;
#endif
#ifdef CONFIG_PCI_P2PDMA
        struct pci_p2pdma __rcu *p2pdma;
#endif
        u16                acs_cap;        /* ACS Capability offset */
        phys_addr_t        rom;                /* Physical address if not from BAR */
        size_t                romlen;                /* Length if not from BAR */
        char                *driver_override; /* Driver name to force a match */

        unsigned long        priv_flags;        /* Private flags for the PCI driver */

        /* These methods index pci_reset_fn_methods[] */
        u8 reset_methods[PCI_NUM_RESET_METHODS]; /* In priority order */
};

static inline struct pci_dev *pci_physfn(struct pci_dev *dev)
{
#ifdef CONFIG_PCI_IOV
        if (dev->is_virtfn)
                dev = dev->physfn;
#endif
        return dev;
}

struct pci_dev *pci_alloc_dev(struct pci_bus *bus);

#define        to_pci_dev(n) container_of(n, struct pci_dev, dev)
#define for_each_pci_dev(d) while ((d = pci_get_device(PCI_ANY_ID, PCI_ANY_ID, d)) != NULL)

static inline int pci_channel_offline(struct pci_dev *pdev)
{
        return (pdev->error_state != pci_channel_io_normal);
}

/*
 * Currently in ACPI spec, for each PCI host bridge, PCI Segment
 * Group number is limited to a 16-bit value, therefore (int)-1 is
 * not a valid PCI domain number, and can be used as a sentinel
 * value indicating ->domain_nr is not set by the driver (and
 * CONFIG_PCI_DOMAINS_GENERIC=y archs will set it with
 * pci_bus_find_domain_nr()).
 */
#define PCI_DOMAIN_NR_NOT_SET (-1)

struct pci_host_bridge {
        struct device        dev;
        struct pci_bus        *bus;                /* Root bus */
        struct pci_ops        *ops;
        struct pci_ops        *child_ops;
        void                *sysdata;
        int                busnr;
        int                domain_nr;
        struct list_head windows;        /* resource_entry */
        struct list_head dma_ranges;        /* dma ranges resource list */
        u8 (*swizzle_irq)(struct pci_dev *, u8 *); /* Platform IRQ swizzler */
        int (*map_irq)(const struct pci_dev *, u8, u8);
        void (*release_fn)(struct pci_host_bridge *);
        void                *release_data;
        unsigned int        ignore_reset_delay:1;        /* For entire hierarchy */
        unsigned int        no_ext_tags:1;                /* No Extended Tags */
        unsigned int        no_inc_mrrs:1;                /* No Increase MRRS */
        unsigned int        native_aer:1;                /* OS may use PCIe AER */
        unsigned int        native_pcie_hotplug:1;        /* OS may use PCIe hotplug */
        unsigned int        native_shpc_hotplug:1;        /* OS may use SHPC hotplug */
        unsigned int        native_pme:1;                /* OS may use PCIe PME */
        unsigned int        native_ltr:1;                /* OS may use PCIe LTR */
        unsigned int        native_dpc:1;                /* OS may use PCIe DPC */
        unsigned int        preserve_config:1;        /* Preserve FW resource setup */
        unsigned int        size_windows:1;                /* Enable root bus sizing */
        unsigned int        msi_domain:1;                /* Bridge wants MSI domain */

        /* Resource alignment requirements */
        resource_size_t (*align_resource)(struct pci_dev *dev,
                        const struct resource *res,
                        resource_size_t start,
                        resource_size_t size,
                        resource_size_t align);
        unsigned long        private[] ____cacheline_aligned;
};

#define        to_pci_host_bridge(n) container_of(n, struct pci_host_bridge, dev)

static inline void *pci_host_bridge_priv(struct pci_host_bridge *bridge)
{
        return (void *)bridge->private;
}

static inline struct pci_host_bridge *pci_host_bridge_from_priv(void *priv)
{
        return container_of(priv, struct pci_host_bridge, private);
}

struct pci_host_bridge *pci_alloc_host_bridge(size_t priv);
struct pci_host_bridge *devm_pci_alloc_host_bridge(struct device *dev,
                                                   size_t priv);
void pci_free_host_bridge(struct pci_host_bridge *bridge);
struct pci_host_bridge *pci_find_host_bridge(struct pci_bus *bus);

void pci_set_host_bridge_release(struct pci_host_bridge *bridge,
                                 void (*release_fn)(struct pci_host_bridge *),
                                 void *release_data);

int pcibios_root_bridge_prepare(struct pci_host_bridge *bridge);

/*
 * The first PCI_BRIDGE_RESOURCE_NUM PCI bus resources (those that correspond
 * to P2P or CardBus bridge windows) go in a table.  Additional ones (for
 * buses below host bridges or subtractive decode bridges) go in the list.
 * Use pci_bus_for_each_resource() to iterate through all the resources.
 */

/*
 * PCI_SUBTRACTIVE_DECODE means the bridge forwards the window implicitly
 * and there's no way to program the bridge with the details of the window.
 * This does not apply to ACPI _CRS windows, even with the _DEC subtractive-
 * decode bit set, because they are explicit and can be programmed with _SRS.
 */
#define PCI_SUBTRACTIVE_DECODE        0x1

struct pci_bus_resource {
        struct list_head        list;
        struct resource                *res;
        unsigned int                flags;
};

#define PCI_REGION_FLAG_MASK        0x0fU        /* These bits of resource flags tell us the PCI region flags */

struct pci_bus {
        struct list_head node;                /* Node in list of buses */
        struct pci_bus        *parent;        /* Parent bus this bridge is on */
        struct list_head children;        /* List of child buses */
        struct list_head devices;        /* List of devices on this bus */
        struct pci_dev        *self;                /* Bridge device as seen by parent */
        struct list_head slots;                /* List of slots on this bus;
                                           protected by pci_slot_mutex */
        struct resource *resource[PCI_BRIDGE_RESOURCE_NUM];
        struct list_head resources;        /* Address space routed to this bus */
        struct resource busn_res;        /* Bus numbers routed to this bus */

        struct pci_ops        *ops;                /* Configuration access functions */
        void                *sysdata;        /* Hook for sys-specific extension */
        struct proc_dir_entry *procdir;        /* Directory entry in /proc/bus/pci */

        unsigned char        number;                /* Bus number */
        unsigned char        primary;        /* Number of primary bridge */
        unsigned char        max_bus_speed;        /* enum pci_bus_speed */
        unsigned char        cur_bus_speed;        /* enum pci_bus_speed */
#ifdef CONFIG_PCI_DOMAINS_GENERIC
        int                domain_nr;
#endif

        char                name[48];

        unsigned short        bridge_ctl;        /* Manage NO_ISA/FBB/et al behaviors */
        pci_bus_flags_t bus_flags;        /* Inherited by child buses */
        struct device                *bridge;
        struct device                dev;
        struct bin_attribute        *legacy_io;        /* Legacy I/O for this bus */
        struct bin_attribute        *legacy_mem;        /* Legacy mem */
        unsigned int                is_added:1;
        unsigned int                unsafe_warn:1;        /* warned about RW1C config write */
};

#define to_pci_bus(n)        container_of(n, struct pci_bus, dev)

static inline u16 pci_dev_id(struct pci_dev *dev)
{
        return PCI_DEVID(dev->bus->number, dev->devfn);
}

/*
 * Returns true if the PCI bus is root (behind host-PCI bridge),
 * false otherwise
 *
 * Some code assumes that "bus->self == NULL" means that bus is a root bus.
 * This is incorrect because "virtual" buses added for SR-IOV (via
 * virtfn_add_bus()) have "bus->self == NULL" but are not root buses.
 */
static inline bool pci_is_root_bus(struct pci_bus *pbus)
{
        return !(pbus->parent);
}

/**
 * pci_is_bridge - check if the PCI device is a bridge
 * @dev: PCI device
 *
 * Return true if the PCI device is bridge whether it has subordinate
 * or not.
 */
static inline bool pci_is_bridge(struct pci_dev *dev)
{
        return dev->hdr_type == PCI_HEADER_TYPE_BRIDGE ||
                dev->hdr_type == PCI_HEADER_TYPE_CARDBUS;
}

#define for_each_pci_bridge(dev, bus)                                \
        list_for_each_entry(dev, &bus->devices, bus_list)        \
                if (!pci_is_bridge(dev)) {} else

static inline struct pci_dev *pci_upstream_bridge(struct pci_dev *dev)
{
        dev = pci_physfn(dev);
        if (pci_is_root_bus(dev->bus))
                return NULL;

        return dev->bus->self;
}

#ifdef CONFIG_PCI_MSI
static inline bool pci_dev_msi_enabled(struct pci_dev *pci_dev)
{
        return pci_dev->msi_enabled || pci_dev->msix_enabled;
}
#else
static inline bool pci_dev_msi_enabled(struct pci_dev *pci_dev) { return false; }
#endif

/* Error values that may be returned by PCI functions */
#define PCIBIOS_SUCCESSFUL                0x00
#define PCIBIOS_FUNC_NOT_SUPPORTED        0x81
#define PCIBIOS_BAD_VENDOR_ID                0x83
#define PCIBIOS_DEVICE_NOT_FOUND        0x86
#define PCIBIOS_BAD_REGISTER_NUMBER        0x87
#define PCIBIOS_SET_FAILED                0x88
#define PCIBIOS_BUFFER_TOO_SMALL        0x89

/* Translate above to generic errno for passing back through non-PCI code */
static inline int pcibios_err_to_errno(int err)
{
        if (err <= PCIBIOS_SUCCESSFUL)
                return err; /* Assume already errno */

        switch (err) {
        case PCIBIOS_FUNC_NOT_SUPPORTED:
                return -ENOENT;
        case PCIBIOS_BAD_VENDOR_ID:
                return -ENOTTY;
        case PCIBIOS_DEVICE_NOT_FOUND:
                return -ENODEV;
        case PCIBIOS_BAD_REGISTER_NUMBER:
                return -EFAULT;
        case PCIBIOS_SET_FAILED:
                return -EIO;
        case PCIBIOS_BUFFER_TOO_SMALL:
                return -ENOSPC;
        }

        return -ERANGE;
}

/* Low-level architecture-dependent routines */

struct pci_ops {
        int (*add_bus)(struct pci_bus *bus);
        void (*remove_bus)(struct pci_bus *bus);
        void __iomem *(*map_bus)(struct pci_bus *bus, unsigned int devfn, int where);
        int (*read)(struct pci_bus *bus, unsigned int devfn, int where, int size, u32 *val);
        int (*write)(struct pci_bus *bus, unsigned int devfn, int where, int size, u32 val);
};

/*
 * ACPI needs to be able to access PCI config space before we've done a
 * PCI bus scan and created pci_bus structures.
 */
int raw_pci_read(unsigned int domain, unsigned int bus, unsigned int devfn,
                 int reg, int len, u32 *val);
int raw_pci_write(unsigned int domain, unsigned int bus, unsigned int devfn,
                  int reg, int len, u32 val);

#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
typedef u64 pci_bus_addr_t;
#else
typedef u32 pci_bus_addr_t;
#endif

struct pci_bus_region {
        pci_bus_addr_t        start;
        pci_bus_addr_t        end;
};

struct pci_dynids {
        spinlock_t                lock;        /* Protects list, index */
        struct list_head        list;        /* For IDs added at runtime */
};


/*
 * PCI Error Recovery System (PCI-ERS).  If a PCI device driver provides
 * a set of callbacks in struct pci_error_handlers, that device driver
 * will be notified of PCI bus errors, and will be driven to recovery
 * when an error occurs.
 */

typedef unsigned int __bitwise pci_ers_result_t;

enum pci_ers_result {
        /* No result/none/not supported in device driver */
        PCI_ERS_RESULT_NONE = (__force pci_ers_result_t) 1,

        /* Device driver can recover without slot reset */
        PCI_ERS_RESULT_CAN_RECOVER = (__force pci_ers_result_t) 2,

        /* Device driver wants slot to be reset */
        PCI_ERS_RESULT_NEED_RESET = (__force pci_ers_result_t) 3,

        /* Device has completely failed, is unrecoverable */
        PCI_ERS_RESULT_DISCONNECT = (__force pci_ers_result_t) 4,

        /* Device driver is fully recovered and operational */
        PCI_ERS_RESULT_RECOVERED = (__force pci_ers_result_t) 5,

        /* No AER capabilities registered for the driver */
        PCI_ERS_RESULT_NO_AER_DRIVER = (__force pci_ers_result_t) 6,
};

/* PCI bus error event callbacks */
struct pci_error_handlers {
        /* PCI bus error detected on this device */
        pci_ers_result_t (*error_detected)(struct pci_dev *dev,
                                           pci_channel_state_t error);

        /* MMIO has been re-enabled, but not DMA */
        pci_ers_result_t (*mmio_enabled)(struct pci_dev *dev);

        /* PCI slot has been reset */
        pci_ers_result_t (*slot_reset)(struct pci_dev *dev);

        /* PCI function reset prepare or completed */
        void (*reset_prepare)(struct pci_dev *dev);
        void (*reset_done)(struct pci_dev *dev);

        /* Device driver may resume normal operations */
        void (*resume)(struct pci_dev *dev);
};


struct module;

/**
 * struct pci_driver - PCI driver structure
 * @node:        List of driver structures.
 * @name:        Driver name.
 * @id_table:        Pointer to table of device IDs the driver is
 *                interested in.  Most drivers should export this
 *                table using MODULE_DEVICE_TABLE(pci,...).
 * @probe:        This probing function gets called (during execution
 *                of pci_register_driver() for already existing
 *                devices or later if a new device gets inserted) for
 *                all PCI devices which match the ID table and are not
 *                "owned" by the other drivers yet. This function gets
 *                passed a "struct pci_dev \*" for each device whose
 *                entry in the ID table matches the device. The probe
 *                function returns zero when the driver chooses to
 *                take "ownership" of the device or an error code
 *                (negative number) otherwise.
 *                The probe function always gets called from process
 *                context, so it can sleep.
 * @remove:        The remove() function gets called whenever a device
 *                being handled by this driver is removed (either during
 *                deregistration of the driver or when it's manually
 *                pulled out of a hot-pluggable slot).
 *                The remove function always gets called from process
 *                context, so it can sleep.
 * @suspend:        Put device into low power state.
 * @resume:        Wake device from low power state.
 *                (Please see Documentation/power/pci.rst for descriptions
 *                of PCI Power Management and the related functions.)
 * @shutdown:        Hook into reboot_notifier_list (kernel/sys.c).
 *                Intended to stop any idling DMA operations.
 *                Useful for enabling wake-on-lan (NIC) or changing
 *                the power state of a device before reboot.
 *                e.g. drivers/net/e100.c.
 * @sriov_configure: Optional driver callback to allow configuration of
 *                number of VFs to enable via sysfs "sriov_numvfs" file.
 * @sriov_set_msix_vec_count: PF Driver callback to change number of MSI-X
 *              vectors on a VF. Triggered via sysfs "sriov_vf_msix_count".
 *              This will change MSI-X Table Size in the VF Message Control
 *              registers.
 * @sriov_get_vf_total_msix: PF driver callback to get the total number of
 *              MSI-X vectors available for distribution to the VFs.
 * @err_handler: See Documentation/PCI/pci-error-recovery.rst
 * @groups:        Sysfs attribute groups.
 * @dev_groups: Attributes attached to the device that will be
 *              created once it is bound to the driver.
 * @driver:        Driver model structure.
 * @dynids:        List of dynamically added device IDs.
 */
struct pci_driver {
        struct list_head        node;
        const char                *name;
        const struct pci_device_id *id_table;        /* Must be non-NULL for probe to be called */
        int  (*probe)(struct pci_dev *dev, const struct pci_device_id *id);        /* New device inserted */
        void (*remove)(struct pci_dev *dev);        /* Device removed (NULL if not a hot-plug capable driver) */
        int  (*suspend)(struct pci_dev *dev, pm_message_t state);        /* Device suspended */
        int  (*resume)(struct pci_dev *dev);        /* Device woken up */
        void (*shutdown)(struct pci_dev *dev);
        int  (*sriov_configure)(struct pci_dev *dev, int num_vfs); /* On PF */
        int  (*sriov_set_msix_vec_count)(struct pci_dev *vf, int msix_vec_count); /* On PF */
        u32  (*sriov_get_vf_total_msix)(struct pci_dev *pf);
        const struct pci_error_handlers *err_handler;
        const struct attribute_group **groups;
        const struct attribute_group **dev_groups;
        struct device_driver        driver;
        struct pci_dynids        dynids;
};

#define        to_pci_driver(drv) container_of(drv, struct pci_driver, driver)

/**
 * PCI_DEVICE - macro used to describe a specific PCI device
 * @vend: the 16 bit PCI Vendor ID
 * @dev: the 16 bit PCI Device ID
 *
 * This macro is used to create a struct pci_device_id that matches a
 * specific device.  The subvendor and subdevice fields will be set to
 * PCI_ANY_ID.
 */
#define PCI_DEVICE(vend,dev) \
        .vendor = (vend), .device = (dev), \
        .subvendor = PCI_ANY_ID, .subdevice = PCI_ANY_ID

/**
 * PCI_DEVICE_DRIVER_OVERRIDE - macro used to describe a PCI device with
 *                              override_only flags.
 * @vend: the 16 bit PCI Vendor ID
 * @dev: the 16 bit PCI Device ID
 * @driver_override: the 32 bit PCI Device override_only
 *
 * This macro is used to create a struct pci_device_id that matches only a
 * driver_override device. The subvendor and subdevice fields will be set to
 * PCI_ANY_ID.
 */
#define PCI_DEVICE_DRIVER_OVERRIDE(vend, dev, driver_override) \
        .vendor = (vend), .device = (dev), .subvendor = PCI_ANY_ID, \
        .subdevice = PCI_ANY_ID, .override_only = (driver_override)

/**
 * PCI_DRIVER_OVERRIDE_DEVICE_VFIO - macro used to describe a VFIO
 *                                   "driver_override" PCI device.
 * @vend: the 16 bit PCI Vendor ID
 * @dev: the 16 bit PCI Device ID
 *
 * This macro is used to create a struct pci_device_id that matches a
 * specific device. The subvendor and subdevice fields will be set to
 * PCI_ANY_ID and the driver_override will be set to
 * PCI_ID_F_VFIO_DRIVER_OVERRIDE.
 */
#define PCI_DRIVER_OVERRIDE_DEVICE_VFIO(vend, dev) \
        PCI_DEVICE_DRIVER_OVERRIDE(vend, dev, PCI_ID_F_VFIO_DRIVER_OVERRIDE)

/**
 * PCI_DEVICE_SUB - macro used to describe a specific PCI device with subsystem
 * @vend: the 16 bit PCI Vendor ID
 * @dev: the 16 bit PCI Device ID
 * @subvend: the 16 bit PCI Subvendor ID
 * @subdev: the 16 bit PCI Subdevice ID
 *
 * This macro is used to create a struct pci_device_id that matches a
 * specific device with subsystem information.
 */
#define PCI_DEVICE_SUB(vend, dev, subvend, subdev) \
        .vendor = (vend), .device = (dev), \
        .subvendor = (subvend), .subdevice = (subdev)

/**
 * PCI_DEVICE_CLASS - macro used to describe a specific PCI device class
 * @dev_class: the class, subclass, prog-if triple for this device
 * @dev_class_mask: the class mask for this device
 *
 * This macro is used to create a struct pci_device_id that matches a
 * specific PCI class.  The vendor, device, subvendor, and subdevice
 * fields will be set to PCI_ANY_ID.
 */
#define PCI_DEVICE_CLASS(dev_class,dev_class_mask) \
        .class = (dev_class), .class_mask = (dev_class_mask), \
        .vendor = PCI_ANY_ID, .device = PCI_ANY_ID, \
        .subvendor = PCI_ANY_ID, .subdevice = PCI_ANY_ID

/**
 * PCI_VDEVICE - macro used to describe a specific PCI device in short form
 * @vend: the vendor name
 * @dev: the 16 bit PCI Device ID
 *
 * This macro is used to create a struct pci_device_id that matches a
 * specific PCI device.  The subvendor, and subdevice fields will be set
 * to PCI_ANY_ID. The macro allows the next field to follow as the device
 * private data.
 */
#define PCI_VDEVICE(vend, dev) \
        .vendor = PCI_VENDOR_ID_##vend, .device = (dev), \
        .subvendor = PCI_ANY_ID, .subdevice = PCI_ANY_ID, 0, 0

/**
 * PCI_DEVICE_DATA - macro used to describe a specific PCI device in very short form
 * @vend: the vendor name (without PCI_VENDOR_ID_ prefix)
 * @dev: the device name (without PCI_DEVICE_ID_<vend>_ prefix)
 * @data: the driver data to be filled
 *
 * This macro is used to create a struct pci_device_id that matches a
 * specific PCI device.  The subvendor, and subdevice fields will be set
 * to PCI_ANY_ID.
 */
#define PCI_DEVICE_DATA(vend, dev, data) \
        .vendor = PCI_VENDOR_ID_##vend, .device = PCI_DEVICE_ID_##vend##_##dev, \
        .subvendor = PCI_ANY_ID, .subdevice = PCI_ANY_ID, 0, 0, \
        .driver_data = (kernel_ulong_t)(data)

enum {
        PCI_REASSIGN_ALL_RSRC        = 0x00000001,        /* Ignore firmware setup */
        PCI_REASSIGN_ALL_BUS        = 0x00000002,        /* Reassign all bus numbers */
        PCI_PROBE_ONLY                = 0x00000004,        /* Use existing setup */
        PCI_CAN_SKIP_ISA_ALIGN        = 0x00000008,        /* Don't do ISA alignment */
        PCI_ENABLE_PROC_DOMAINS        = 0x00000010,        /* Enable domains in /proc */
        PCI_COMPAT_DOMAIN_0        = 0x00000020,        /* ... except domain 0 */
        PCI_SCAN_ALL_PCIE_DEVS        = 0x00000040,        /* Scan all, not just dev 0 */
};

#define PCI_IRQ_LEGACY                (1 << 0) /* Allow legacy interrupts */
#define PCI_IRQ_MSI                (1 << 1) /* Allow MSI interrupts */
#define PCI_IRQ_MSIX                (1 << 2) /* Allow MSI-X interrupts */
#define PCI_IRQ_AFFINITY        (1 << 3) /* Auto-assign affinity */

/* These external functions are only available when PCI support is enabled */
#ifdef CONFIG_PCI

extern unsigned int pci_flags;

static inline void pci_set_flags(int flags) { pci_flags = flags; }
static inline void pci_add_flags(int flags) { pci_flags |= flags; }
static inline void pci_clear_flags(int flags) { pci_flags &= ~flags; }
static inline int pci_has_flag(int flag) { return pci_flags & flag; }

void pcie_bus_configure_settings(struct pci_bus *bus);

enum pcie_bus_config_types {
        PCIE_BUS_TUNE_OFF,        /* Don't touch MPS at all */
        PCIE_BUS_DEFAULT,        /* Ensure MPS matches upstream bridge */
        PCIE_BUS_SAFE,                /* Use largest MPS boot-time devices support */
        PCIE_BUS_PERFORMANCE,        /* Use MPS and MRRS for best performance */
        PCIE_BUS_PEER2PEER,        /* Set MPS = 128 for all devices */
};

extern enum pcie_bus_config_types pcie_bus_config;

extern struct bus_type pci_bus_type;

/* Do NOT directly access these two variables, unless you are arch-specific PCI
 * code, or PCI core code. */
extern struct list_head pci_root_buses;        /* List of all known PCI buses */
/* Some device drivers need know if PCI is initiated */
int no_pci_devices(void);

void pcibios_resource_survey_bus(struct pci_bus *bus);
void pcibios_bus_add_device(struct pci_dev *pdev);
void pcibios_add_bus(struct pci_bus *bus);
void pcibios_remove_bus(struct pci_bus *bus);
void pcibios_fixup_bus(struct pci_bus *);
int __must_check pcibios_enable_device(struct pci_dev *, int mask);
/* Architecture-specific versions may override this (weak) */
char *pcibios_setup(char *str);

/* Used only when drivers/pci/setup.c is used */
resource_size_t pcibios_align_resource(void *, const struct resource *,
                                resource_size_t,
                                resource_size_t);

/* Weak but can be overridden by arch */
void pci_fixup_cardbus(struct pci_bus *);

/* Generic PCI functions used internally */

void pcibios_resource_to_bus(struct pci_bus *bus, struct pci_bus_region *region,
                             struct resource *res);
void pcibios_bus_to_resource(struct pci_bus *bus, struct resource *res,
                             struct pci_bus_region *region);
void pcibios_scan_specific_bus(int busn);
struct pci_bus *pci_find_bus(int domain, int busnr);
void pci_bus_add_devices(const struct pci_bus *bus);
struct pci_bus *pci_scan_bus(int bus, struct pci_ops *ops, void *sysdata);
struct pci_bus *pci_create_root_bus(struct device *parent, int bus,
                                    struct pci_ops *ops, void *sysdata,
                                    struct list_head *resources);
int pci_host_probe(struct pci_host_bridge *bridge);
int pci_bus_insert_busn_res(struct pci_bus *b, int bus, int busmax);
int pci_bus_update_busn_res_end(struct pci_bus *b, int busmax);
void pci_bus_release_busn_res(struct pci_bus *b);
struct pci_bus *pci_scan_root_bus(struct device *parent, int bus,
                                  struct pci_ops *ops, void *sysdata,
                                  struct list_head *resources);
int pci_scan_root_bus_bridge(struct pci_host_bridge *bridge);
struct pci_bus *pci_add_new_bus(struct pci_bus *parent, struct pci_dev *dev,
                                int busnr);
struct pci_slot *pci_create_slot(struct pci_bus *parent, int slot_nr,
                                 const char *name,
                                 struct hotplug_slot *hotplug);
void pci_destroy_slot(struct pci_slot *slot);
#ifdef CONFIG_SYSFS
void pci_dev_assign_slot(struct pci_dev *dev);
#else
static inline void pci_dev_assign_slot(struct pci_dev *dev) { }
#endif
int pci_scan_slot(struct pci_bus *bus, int devfn);
struct pci_dev *pci_scan_single_device(struct pci_bus *bus, int devfn);
void pci_device_add(struct pci_dev *dev, struct pci_bus *bus);
unsigned int pci_scan_child_bus(struct pci_bus *bus);
void pci_bus_add_device(struct pci_dev *dev);
void pci_read_bridge_bases(struct pci_bus *child);
struct resource *pci_find_parent_resource(const struct pci_dev *dev,
                                          struct resource *res);
u8 pci_swizzle_interrupt_pin(const struct pci_dev *dev, u8 pin);
int pci_get_interrupt_pin(struct pci_dev *dev, struct pci_dev **bridge);
u8 pci_common_swizzle(struct pci_dev *dev, u8 *pinp);
struct pci_dev *pci_dev_get(struct pci_dev *dev);
void pci_dev_put(struct pci_dev *dev);
void pci_remove_bus(struct pci_bus *b);
void pci_stop_and_remove_bus_device(struct pci_dev *dev);
void pci_stop_and_remove_bus_device_locked(struct pci_dev *dev);
void pci_stop_root_bus(struct pci_bus *bus);
void pci_remove_root_bus(struct pci_bus *bus);
void pci_setup_cardbus(struct pci_bus *bus);
void pcibios_setup_bridge(struct pci_bus *bus, unsigned long type);
void pci_sort_breadthfirst(void);
#define dev_is_pci(d) ((d)->bus == &pci_bus_type)
#define dev_is_pf(d) ((dev_is_pci(d) ? to_pci_dev(d)->is_physfn : false))

/* Generic PCI functions exported to card drivers */

u8 pci_bus_find_capability(struct pci_bus *bus, unsigned int devfn, int cap);
u8 pci_find_capability(struct pci_dev *dev, int cap);
u8 pci_find_next_capability(struct pci_dev *dev, u8 pos, int cap);
u8 pci_find_ht_capability(struct pci_dev *dev, int ht_cap);
u8 pci_find_next_ht_capability(struct pci_dev *dev, u8 pos, int ht_cap);
u16 pci_find_ext_capability(struct pci_dev *dev, int cap);
u16 pci_find_next_ext_capability(struct pci_dev *dev, u16 pos, int cap);
struct pci_bus *pci_find_next_bus(const struct pci_bus *from);
u16 pci_find_vsec_capability(struct pci_dev *dev, u16 vendor, int cap);

u64 pci_get_dsn(struct pci_dev *dev);

struct pci_dev *pci_get_device(unsigned int vendor, unsigned int device,
                               struct pci_dev *from);
struct pci_dev *pci_get_subsys(unsigned int vendor, unsigned int device,
                               unsigned int ss_vendor, unsigned int ss_device,
                               struct pci_dev *from);
struct pci_dev *pci_get_slot(struct pci_bus *bus, unsigned int devfn);
struct pci_dev *pci_get_domain_bus_and_slot(int domain, unsigned int bus,
                                            unsigned int devfn);
struct pci_dev *pci_get_class(unsigned int class, struct pci_dev *from);
int pci_dev_present(const struct pci_device_id *ids);

int pci_bus_read_config_byte(struct pci_bus *bus, unsigned int devfn,
                             int where, u8 *val);
int pci_bus_read_config_word(struct pci_bus *bus, unsigned int devfn,
                             int where, u16 *val);
int pci_bus_read_config_dword(struct pci_bus *bus, unsigned int devfn,
                              int where, u32 *val);
int pci_bus_write_config_byte(struct pci_bus *bus, unsigned int devfn,
                              int where, u8 val);
int pci_bus_write_config_word(struct pci_bus *bus, unsigned int devfn,
                              int where, u16 val);
int pci_bus_write_config_dword(struct pci_bus *bus, unsigned int devfn,
                               int where, u32 val);

int pci_generic_config_read(struct pci_bus *bus, unsigned int devfn,
                            int where, int size, u32 *val);
int pci_generic_config_write(struct pci_bus *bus, unsigned int devfn,
                            int where, int size, u32 val);
int pci_generic_config_read32(struct pci_bus *bus, unsigned int devfn,
                              int where, int size, u32 *val);
int pci_generic_config_write32(struct pci_bus *bus, unsigned int devfn,
                               int where, int size, u32 val);

struct pci_ops *pci_bus_set_ops(struct pci_bus *bus, struct pci_ops *ops);

int pci_read_config_byte(const struct pci_dev *dev, int where, u8 *val);
int pci_read_config_word(const struct pci_dev *dev, int where, u16 *val);
int pci_read_config_dword(const struct pci_dev *dev, int where, u32 *val);
int pci_write_config_byte(const struct pci_dev *dev, int where, u8 val);
int pci_write_config_word(const struct pci_dev *dev, int where, u16 val);
int pci_write_config_dword(const struct pci_dev *dev, int where, u32 val);

int pcie_capability_read_word(struct pci_dev *dev, int pos, u16 *val);
int pcie_capability_read_dword(struct pci_dev *dev, int pos, u32 *val);
int pcie_capability_write_word(struct pci_dev *dev, int pos, u16 val);
int pcie_capability_write_dword(struct pci_dev *dev, int pos, u32 val);
int pcie_capability_clear_and_set_word(struct pci_dev *dev, int pos,
                                       u16 clear, u16 set);
int pcie_capability_clear_and_set_dword(struct pci_dev *dev, int pos,
                                        u32 clear, u32 set);

static inline int pcie_capability_set_word(struct pci_dev *dev, int pos,
                                           u16 set)
{
        return pcie_capability_clear_and_set_word(dev, pos, 0, set);
}

static inline int pcie_capability_set_dword(struct pci_dev *dev, int pos,
                                            u32 set)
{
        return pcie_capability_clear_and_set_dword(dev, pos, 0, set);
}

static inline int pcie_capability_clear_word(struct pci_dev *dev, int pos,
                                             u16 clear)
{
        return pcie_capability_clear_and_set_word(dev, pos, clear, 0);
}

static inline int pcie_capability_clear_dword(struct pci_dev *dev, int pos,
                                              u32 clear)
{
        return pcie_capability_clear_and_set_dword(dev, pos, clear, 0);
}

/* User-space driven config access */
int pci_user_read_config_byte(struct pci_dev *dev, int where, u8 *val);
int pci_user_read_config_word(struct pci_dev *dev, int where, u16 *val);
int pci_user_read_config_dword(struct pci_dev *dev, int where, u32 *val);
int pci_user_write_config_byte(struct pci_dev *dev, int where, u8 val);
int pci_user_write_config_word(struct pci_dev *dev, int where, u16 val);
int pci_user_write_config_dword(struct pci_dev *dev, int where, u32 val);

int __must_check pci_enable_device(struct pci_dev *dev);
int __must_check pci_enable_device_io(struct pci_dev *dev);
int __must_check pci_enable_device_mem(struct pci_dev *dev);
int __must_check pci_reenable_device(struct pci_dev *);
int __must_check pcim_enable_device(struct pci_dev *pdev);
void pcim_pin_device(struct pci_dev *pdev);

static inline bool pci_intx_mask_supported(struct pci_dev *pdev)
{
        /*
         * INTx masking is supported if PCI_COMMAND_INTX_DISABLE is
         * writable and no quirk has marked the feature broken.
         */
        return !pdev->broken_intx_masking;
}

static inline int pci_is_enabled(struct pci_dev *pdev)
{
        return (atomic_read(&pdev->enable_cnt) > 0);
}

static inline int pci_is_managed(struct pci_dev *pdev)
{
        return pdev->is_managed;
}

void pci_disable_device(struct pci_dev *dev);

extern unsigned int pcibios_max_latency;
void pci_set_master(struct pci_dev *dev);
void pci_clear_master(struct pci_dev *dev);

int pci_set_pcie_reset_state(struct pci_dev *dev, enum pcie_reset_state state);
int pci_set_cacheline_size(struct pci_dev *dev);
int __must_check pci_set_mwi(struct pci_dev *dev);
int __must_check pcim_set_mwi(struct pci_dev *dev);
int pci_try_set_mwi(struct pci_dev *dev);
void pci_clear_mwi(struct pci_dev *dev);
void pci_disable_parity(struct pci_dev *dev);
void pci_intx(struct pci_dev *dev, int enable);
bool pci_check_and_mask_intx(struct pci_dev *dev);
bool pci_check_and_unmask_intx(struct pci_dev *dev);
int pci_wait_for_pending(struct pci_dev *dev, int pos, u16 mask);
int pci_wait_for_pending_transaction(struct pci_dev *dev);
int pcix_get_max_mmrbc(struct pci_dev *dev);
int pcix_get_mmrbc(struct pci_dev *dev);
int pcix_set_mmrbc(struct pci_dev *dev, int mmrbc);
int pcie_get_readrq(struct pci_dev *dev);
int pcie_set_readrq(struct pci_dev *dev, int rq);
int pcie_get_mps(struct pci_dev *dev);
int pcie_set_mps(struct pci_dev *dev, int mps);
u32 pcie_bandwidth_available(struct pci_dev *dev, struct pci_dev **limiting_dev,
                             enum pci_bus_speed *speed,
                             enum pcie_link_width *width);
void pcie_print_link_status(struct pci_dev *dev);
int pcie_reset_flr(struct pci_dev *dev, bool probe);
int pcie_flr(struct pci_dev *dev);
int __pci_reset_function_locked(struct pci_dev *dev);
int pci_reset_function(struct pci_dev *dev);
int pci_reset_function_locked(struct pci_dev *dev);
int pci_try_reset_function(struct pci_dev *dev);
int pci_probe_reset_slot(struct pci_slot *slot);
int pci_probe_reset_bus(struct pci_bus *bus);
int pci_reset_bus(struct pci_dev *dev);
void pci_reset_secondary_bus(struct pci_dev *dev);
void pcibios_reset_secondary_bus(struct pci_dev *dev);
void pci_update_resource(struct pci_dev *dev, int resno);
int __must_check pci_assign_resource(struct pci_dev *dev, int i);
int __must_check pci_reassign_resource(struct pci_dev *dev, int i, resource_size_t add_size, resource_size_t align);
void pci_release_resource(struct pci_dev *dev, int resno);
static inline int pci_rebar_bytes_to_size(u64 bytes)
{
        bytes = roundup_pow_of_two(bytes);

        /* Return BAR size as defined in the resizable BAR specification */
        return max(ilog2(bytes), 20) - 20;
}

u32 pci_rebar_get_possible_sizes(struct pci_dev *pdev, int bar);
int __must_check pci_resize_resource(struct pci_dev *dev, int i, int size);
int pci_select_bars(struct pci_dev *dev, unsigned long flags);
bool pci_device_is_present(struct pci_dev *pdev);
void pci_ignore_hotplug(struct pci_dev *dev);
struct pci_dev *pci_real_dma_dev(struct pci_dev *dev);
int pci_status_get_and_clear_errors(struct pci_dev *pdev);

int __printf(6, 7) pci_request_irq(struct pci_dev *dev, unsigned int nr,
                irq_handler_t handler, irq_handler_t thread_fn, void *dev_id,
                const char *fmt, ...);
void pci_free_irq(struct pci_dev *dev, unsigned int nr, void *dev_id);

/* ROM control related routines */
int pci_enable_rom(struct pci_dev *pdev);
void pci_disable_rom(struct pci_dev *pdev);
void __iomem __must_check *pci_map_rom(struct pci_dev *pdev, size_t *size);
void pci_unmap_rom(struct pci_dev *pdev, void __iomem *rom);

/* Power management related routines */
int pci_save_state(struct pci_dev *dev);
void pci_restore_state(struct pci_dev *dev);
struct pci_saved_state *pci_store_saved_state(struct pci_dev *dev);
int pci_load_saved_state(struct pci_dev *dev,
                         struct pci_saved_state *state);
int pci_load_and_free_saved_state(struct pci_dev *dev,
                                  struct pci_saved_state **state);
int pci_platform_power_transition(struct pci_dev *dev, pci_power_t state);
int pci_set_power_state(struct pci_dev *dev, pci_power_t state);
pci_power_t pci_choose_state(struct pci_dev *dev, pm_message_t state);
bool pci_pme_capable(struct pci_dev *dev, pci_power_t state);
void pci_pme_active(struct pci_dev *dev, bool enable);
int pci_enable_wake(struct pci_dev *dev, pci_power_t state, bool enable);
int pci_wake_from_d3(struct pci_dev *dev, bool enable);
int pci_prepare_to_sleep(struct pci_dev *dev);
int pci_back_from_sleep(struct pci_dev *dev);
bool pci_dev_run_wake(struct pci_dev *dev);
void pci_d3cold_enable(struct pci_dev *dev);
void pci_d3cold_disable(struct pci_dev *dev);
bool pcie_relaxed_ordering_enabled(struct pci_dev *dev);
void pci_resume_bus(struct pci_bus *bus);
void pci_bus_set_current_state(struct pci_bus *bus, pci_power_t state);

/* For use by arch with custom probe code */
void set_pcie_port_type(struct pci_dev *pdev);
void set_pcie_hotplug_bridge(struct pci_dev *pdev);

/* Functions for PCI Hotplug drivers to use */
unsigned int pci_rescan_bus_bridge_resize(struct pci_dev *bridge);
unsigned int pci_rescan_bus(struct pci_bus *bus);
void pci_lock_rescan_remove(void);
void pci_unlock_rescan_remove(void);

/* Vital Product Data routines */
ssize_t pci_read_vpd(struct pci_dev *dev, loff_t pos, size_t count, void *buf);
ssize_t pci_write_vpd(struct pci_dev *dev, loff_t pos, size_t count, const void *buf);

/* Helper functions for low-level code (drivers/pci/setup-[bus,res].c) */
resource_size_t pcibios_retrieve_fw_addr(struct pci_dev *dev, int idx);
void pci_bus_assign_resources(const struct pci_bus *bus);
void pci_bus_claim_resources(struct pci_bus *bus);
void pci_bus_size_bridges(struct pci_bus *bus);
int pci_claim_resource(struct pci_dev *, int);
int pci_claim_bridge_resource(struct pci_dev *bridge, int i);
void pci_assign_unassigned_resources(void);
void pci_assign_unassigned_bridge_resources(struct pci_dev *bridge);
void pci_assign_unassigned_bus_resources(struct pci_bus *bus);
void pci_assign_unassigned_root_bus_resources(struct pci_bus *bus);
int pci_reassign_bridge_resources(struct pci_dev *bridge, unsigned long type);
void pdev_enable_device(struct pci_dev *);
int pci_enable_resources(struct pci_dev *, int mask);
void pci_assign_irq(struct pci_dev *dev);
struct resource *pci_find_resource(struct pci_dev *dev, struct resource *res);
#define HAVE_PCI_REQ_REGIONS        2
int __must_check pci_request_regions(struct pci_dev *, const char *);
int __must_check pci_request_regions_exclusive(struct pci_dev *, const char *);
void pci_release_regions(struct pci_dev *);
int __must_check pci_request_region(struct pci_dev *, int, const char *);
void pci_release_region(struct pci_dev *, int);
int pci_request_selected_regions(struct pci_dev *, int, const char *);
int pci_request_selected_regions_exclusive(struct pci_dev *, int, const char *);
void pci_release_selected_regions(struct pci_dev *, int);

/* drivers/pci/bus.c */
void pci_add_resource(struct list_head *resources, struct resource *res);
void pci_add_resource_offset(struct list_head *resources, struct resource *res,
                             resource_size_t offset);
void pci_free_resource_list(struct list_head *resources);
void pci_bus_add_resource(struct pci_bus *bus, struct resource *res,
                          unsigned int flags);
struct resource *pci_bus_resource_n(const struct pci_bus *bus, int n);
void pci_bus_remove_resources(struct pci_bus *bus);
void pci_bus_remove_resource(struct pci_bus *bus, struct resource *res);
int devm_request_pci_bus_resources(struct device *dev,
                                   struct list_head *resources);

/* Temporary until new and working PCI SBR API in place */
int pci_bridge_secondary_bus_reset(struct pci_dev *dev);

#define pci_bus_for_each_resource(bus, res, i)                                \
        for (i = 0;                                                        \
            (res = pci_bus_resource_n(bus, i)) || i < PCI_BRIDGE_RESOURCE_NUM; \
             i++)

int __must_check pci_bus_alloc_resource(struct pci_bus *bus,
                        struct resource *res, resource_size_t size,
                        resource_size_t align, resource_size_t min,
                        unsigned long type_mask,
                        resource_size_t (*alignf)(void *,
                                                  const struct resource *,
                                                  resource_size_t,
                                                  resource_size_t),
                        void *alignf_data);


int pci_register_io_range(struct fwnode_handle *fwnode, phys_addr_t addr,
                        resource_size_t size);
unsigned long pci_address_to_pio(phys_addr_t addr);
phys_addr_t pci_pio_to_address(unsigned long pio);
int pci_remap_iospace(const struct resource *res, phys_addr_t phys_addr);
int devm_pci_remap_iospace(struct device *dev, const struct resource *res,
                           phys_addr_t phys_addr);
void pci_unmap_iospace(struct resource *res);
void __iomem *devm_pci_remap_cfgspace(struct device *dev,
                                      resource_size_t offset,
                                      resource_size_t size);
void __iomem *devm_pci_remap_cfg_resource(struct device *dev,
                                          struct resource *res);

static inline pci_bus_addr_t pci_bus_address(struct pci_dev *pdev, int bar)
{
        struct pci_bus_region region;

        pcibios_resource_to_bus(pdev->bus, &region, &pdev->resource[bar]);
        return region.start;
}

/* Proper probing supporting hot-pluggable devices */
int __must_check __pci_register_driver(struct pci_driver *, struct module *,
                                       const char *mod_name);

/* pci_register_driver() must be a macro so KBUILD_MODNAME can be expanded */
#define pci_register_driver(driver)                \
        __pci_register_driver(driver, THIS_MODULE, KBUILD_MODNAME)

void pci_unregister_driver(struct pci_driver *dev);

/**
 * module_pci_driver() - Helper macro for registering a PCI driver
 * @__pci_driver: pci_driver struct
 *
 * Helper macro for PCI drivers which do not do anything special in module
 * init/exit. This eliminates a lot of boilerplate. Each module may only
 * use this macro once, and calling it replaces module_init() and module_exit()
 */
#define module_pci_driver(__pci_driver) \
        module_driver(__pci_driver, pci_register_driver, pci_unregister_driver)

/**
 * builtin_pci_driver() - Helper macro for registering a PCI driver
 * @__pci_driver: pci_driver struct
 *
 * Helper macro for PCI drivers which do not do anything special in their
 * init code. This eliminates a lot of boilerplate. Each driver may only
 * use this macro once, and calling it replaces device_initcall(...)
 */
#define builtin_pci_driver(__pci_driver) \
        builtin_driver(__pci_driver, pci_register_driver)

struct pci_driver *pci_dev_driver(const struct pci_dev *dev);
int pci_add_dynid(struct pci_driver *drv,
                  unsigned int vendor, unsigned int device,
                  unsigned int subvendor, unsigned int subdevice,
                  unsigned int class, unsigned int class_mask,
                  unsigned long driver_data);
const struct pci_device_id *pci_match_id(const struct pci_device_id *ids,
                                         struct pci_dev *dev);
int pci_scan_bridge(struct pci_bus *bus, struct pci_dev *dev, int max,
                    int pass);

void pci_walk_bus(struct pci_bus *top, int (*cb)(struct pci_dev *, void *),
                  void *userdata);
int pci_cfg_space_size(struct pci_dev *dev);
unsigned char pci_bus_max_busnr(struct pci_bus *bus);
void pci_setup_bridge(struct pci_bus *bus);
resource_size_t pcibios_window_alignment(struct pci_bus *bus,
                                         unsigned long type);

#define PCI_VGA_STATE_CHANGE_BRIDGE (1 << 0)
#define PCI_VGA_STATE_CHANGE_DECODES (1 << 1)

int pci_set_vga_state(struct pci_dev *pdev, bool decode,
                      unsigned int command_bits, u32 flags);

/*
 * Virtual interrupts allow for more interrupts to be allocated
 * than the device has interrupts for. These are not programmed
 * into the device's MSI-X table and must be handled by some
 * other driver means.
 */
#define PCI_IRQ_VIRTUAL                (1 << 4)

#define PCI_IRQ_ALL_TYPES \
        (PCI_IRQ_LEGACY | PCI_IRQ_MSI | PCI_IRQ_MSIX)

/* kmem_cache style wrapper around pci_alloc_consistent() */

#include <linux/dmapool.h>

#define        pci_pool dma_pool
#define pci_pool_create(name, pdev, size, align, allocation) \
                dma_pool_create(name, &pdev->dev, size, align, allocation)
#define        pci_pool_destroy(pool) dma_pool_destroy(pool)
#define        pci_pool_alloc(pool, flags, handle) dma_pool_alloc(pool, flags, handle)
#define        pci_pool_zalloc(pool, flags, handle) \
                dma_pool_zalloc(pool, flags, handle)
#define        pci_pool_free(pool, vaddr, addr) dma_pool_free(pool, vaddr, addr)

struct msix_entry {
        u32        vector;        /* Kernel uses to write allocated vector */
        u16        entry;        /* Driver uses to specify entry, OS writes */
};

#ifdef CONFIG_PCI_MSI
int pci_msi_vec_count(struct pci_dev *dev);
void pci_disable_msi(struct pci_dev *dev);
int pci_msix_vec_count(struct pci_dev *dev);
void pci_disable_msix(struct pci_dev *dev);
void pci_restore_msi_state(struct pci_dev *dev);
int pci_msi_enabled(void);
int pci_enable_msi(struct pci_dev *dev);
int pci_enable_msix_range(struct pci_dev *dev, struct msix_entry *entries,
                          int minvec, int maxvec);
static inline int pci_enable_msix_exact(struct pci_dev *dev,
                                        struct msix_entry *entries, int nvec)
{
        int rc = pci_enable_msix_range(dev, entries, nvec, nvec);
        if (rc < 0)
                return rc;
        return 0;
}
int pci_alloc_irq_vectors_affinity(struct pci_dev *dev, unsigned int min_vecs,
                                   unsigned int max_vecs, unsigned int flags,
                                   struct irq_affinity *affd);

void pci_free_irq_vectors(struct pci_dev *dev);
int pci_irq_vector(struct pci_dev *dev, unsigned int nr);
const struct cpumask *pci_irq_get_affinity(struct pci_dev *pdev, int vec);

#else
static inline int pci_msi_vec_count(struct pci_dev *dev) { return -ENOSYS; }
static inline void pci_disable_msi(struct pci_dev *dev) { }
static inline int pci_msix_vec_count(struct pci_dev *dev) { return -ENOSYS; }
static inline void pci_disable_msix(struct pci_dev *dev) { }
static inline void pci_restore_msi_state(struct pci_dev *dev) { }
static inline int pci_msi_enabled(void) { return 0; }
static inline int pci_enable_msi(struct pci_dev *dev)
{ return -ENOSYS; }
static inline int pci_enable_msix_range(struct pci_dev *dev,
                        struct msix_entry *entries, int minvec, int maxvec)
{ return -ENOSYS; }
static inline int pci_enable_msix_exact(struct pci_dev *dev,
                        struct msix_entry *entries, int nvec)
{ return -ENOSYS; }

static inline int
pci_alloc_irq_vectors_affinity(struct pci_dev *dev, unsigned int min_vecs,
                               unsigned int max_vecs, unsigned int flags,
                               struct irq_affinity *aff_desc)
{
        if ((flags & PCI_IRQ_LEGACY) && min_vecs == 1 && dev->irq)
                return 1;
        return -ENOSPC;
}

static inline void pci_free_irq_vectors(struct pci_dev *dev)
{
}

static inline int pci_irq_vector(struct pci_dev *dev, unsigned int nr)
{
        if (WARN_ON_ONCE(nr > 0))
                return -EINVAL;
        return dev->irq;
}
static inline const struct cpumask *pci_irq_get_affinity(struct pci_dev *pdev,
                int vec)
{
        return cpu_possible_mask;
}
#endif

/**
 * pci_irqd_intx_xlate() - Translate PCI INTx value to an IRQ domain hwirq
 * @d: the INTx IRQ domain
 * @node: the DT node for the device whose interrupt we're translating
 * @intspec: the interrupt specifier data from the DT
 * @intsize: the number of entries in @intspec
 * @out_hwirq: pointer at which to write the hwirq number
 * @out_type: pointer at which to write the interrupt type
 *
 * Translate a PCI INTx interrupt number from device tree in the range 1-4, as
 * stored in the standard PCI_INTERRUPT_PIN register, to a value in the range
 * 0-3 suitable for use in a 4 entry IRQ domain. That is, subtract one from the
 * INTx value to obtain the hwirq number.
 *
 * Returns 0 on success, or -EINVAL if the interrupt specifier is out of range.
 */
static inline int pci_irqd_intx_xlate(struct irq_domain *d,
                                      struct device_node *node,
                                      const u32 *intspec,
                                      unsigned int intsize,
                                      unsigned long *out_hwirq,
                                      unsigned int *out_type)
{
        const u32 intx = intspec[0];

        if (intx < PCI_INTERRUPT_INTA || intx > PCI_INTERRUPT_INTD)
                return -EINVAL;

        *out_hwirq = intx - PCI_INTERRUPT_INTA;
        return 0;
}

#ifdef CONFIG_PCIEPORTBUS
extern bool pcie_ports_disabled;
extern bool pcie_ports_native;
#else
#define pcie_ports_disabled        true
#define pcie_ports_native        false
#endif

#define PCIE_LINK_STATE_L0S                BIT(0)
#define PCIE_LINK_STATE_L1                BIT(1)
#define PCIE_LINK_STATE_CLKPM                BIT(2)
#define PCIE_LINK_STATE_L1_1                BIT(3)
#define PCIE_LINK_STATE_L1_2                BIT(4)
#define PCIE_LINK_STATE_L1_1_PCIPM        BIT(5)
#define PCIE_LINK_STATE_L1_2_PCIPM        BIT(6)

#ifdef CONFIG_PCIEASPM
int pci_disable_link_state(struct pci_dev *pdev, int state);
int pci_disable_link_state_locked(struct pci_dev *pdev, int state);
void pcie_no_aspm(void);
bool pcie_aspm_support_enabled(void);
bool pcie_aspm_enabled(struct pci_dev *pdev);
#else
static inline int pci_disable_link_state(struct pci_dev *pdev, int state)
{ return 0; }
static inline int pci_disable_link_state_locked(struct pci_dev *pdev, int state)
{ return 0; }
static inline void pcie_no_aspm(void) { }
static inline bool pcie_aspm_support_enabled(void) { return false; }
static inline bool pcie_aspm_enabled(struct pci_dev *pdev) { return false; }
#endif

#ifdef CONFIG_PCIEAER
bool pci_aer_available(void);
#else
static inline bool pci_aer_available(void) { return false; }
#endif

bool pci_ats_disabled(void);

#ifdef CONFIG_PCIE_PTM
int pci_enable_ptm(struct pci_dev *dev, u8 *granularity);
bool pcie_ptm_enabled(struct pci_dev *dev);
#else
static inline int pci_enable_ptm(struct pci_dev *dev, u8 *granularity)
{ return -EINVAL; }
static inline bool pcie_ptm_enabled(struct pci_dev *dev)
{ return false; }
#endif

void pci_cfg_access_lock(struct pci_dev *dev);
bool pci_cfg_access_trylock(struct pci_dev *dev);
void pci_cfg_access_unlock(struct pci_dev *dev);

int pci_dev_trylock(struct pci_dev *dev);
void pci_dev_unlock(struct pci_dev *dev);

/*
 * PCI domain support.  Sometimes called PCI segment (eg by ACPI),
 * a PCI domain is defined to be a set of PCI buses which share
 * configuration space.
 */
#ifdef CONFIG_PCI_DOMAINS
extern int pci_domains_supported;
#else
enum { pci_domains_supported = 0 };
static inline int pci_domain_nr(struct pci_bus *bus) { return 0; }
static inline int pci_proc_domain(struct pci_bus *bus) { return 0; }
#endif /* CONFIG_PCI_DOMAINS */

/*
 * Generic implementation for PCI domain support. If your
 * architecture does not need custom management of PCI
 * domains then this implementation will be used
 */
#ifdef CONFIG_PCI_DOMAINS_GENERIC
static inline int pci_domain_nr(struct pci_bus *bus)
{
        return bus->domain_nr;
}
#ifdef CONFIG_ACPI
int acpi_pci_bus_find_domain_nr(struct pci_bus *bus);
#else
static inline int acpi_pci_bus_find_domain_nr(struct pci_bus *bus)
{ return 0; }
#endif
int pci_bus_find_domain_nr(struct pci_bus *bus, struct device *parent);
#endif

/* Some architectures require additional setup to direct VGA traffic */
typedef int (*arch_set_vga_state_t)(struct pci_dev *pdev, bool decode,
                                    unsigned int command_bits, u32 flags);
void pci_register_set_vga_state(arch_set_vga_state_t func);

static inline int
pci_request_io_regions(struct pci_dev *pdev, const char *name)
{
        return pci_request_selected_regions(pdev,
                            pci_select_bars(pdev, IORESOURCE_IO), name);
}

static inline void
pci_release_io_regions(struct pci_dev *pdev)
{
        return pci_release_selected_regions(pdev,
                            pci_select_bars(pdev, IORESOURCE_IO));
}

static inline int
pci_request_mem_regions(struct pci_dev *pdev, const char *name)
{
        return pci_request_selected_regions(pdev,
                            pci_select_bars(pdev, IORESOURCE_MEM), name);
}

static inline void
pci_release_mem_regions(struct pci_dev *pdev)
{
        return pci_release_selected_regions(pdev,
                            pci_select_bars(pdev, IORESOURCE_MEM));
}

#else /* CONFIG_PCI is not enabled */

static inline void pci_set_flags(int flags) { }
static inline void pci_add_flags(int flags) { }
static inline void pci_clear_flags(int flags) { }
static inline int pci_has_flag(int flag) { return 0; }

/*
 * If the system does not have PCI, clearly these return errors.  Define
 * these as simple inline functions to avoid hair in drivers.
 */
#define _PCI_NOP(o, s, t) \
        static inline int pci_##o##_config_##s(struct pci_dev *dev, \
                                                int where, t val) \
                { return PCIBIOS_FUNC_NOT_SUPPORTED; }

#define _PCI_NOP_ALL(o, x)        _PCI_NOP(o, byte, u8 x) \
                                _PCI_NOP(o, word, u16 x) \
                                _PCI_NOP(o, dword, u32 x)
_PCI_NOP_ALL(read, *)
_PCI_NOP_ALL(write,)

static inline struct pci_dev *pci_get_device(unsigned int vendor,
                                             unsigned int device,
                                             struct pci_dev *from)
{ return NULL; }

static inline struct pci_dev *pci_get_subsys(unsigned int vendor,
                                             unsigned int device,
                                             unsigned int ss_vendor,
                                             unsigned int ss_device,
                                             struct pci_dev *from)
{ return NULL; }

static inline struct pci_dev *pci_get_class(unsigned int class,
                                            struct pci_dev *from)
{ return NULL; }

#define pci_dev_present(ids)        (0)
#define no_pci_devices()        (1)
#define pci_dev_put(dev)        do { } while (0)

static inline void pci_set_master(struct pci_dev *dev) { }
static inline void pci_clear_master(struct pci_dev *dev) { }
static inline int pci_enable_device(struct pci_dev *dev) { return -EIO; }
static inline void pci_disable_device(struct pci_dev *dev) { }
static inline int pcim_enable_device(struct pci_dev *pdev) { return -EIO; }
static inline int pci_assign_resource(struct pci_dev *dev, int i)
{ return -EBUSY; }
static inline int __must_check __pci_register_driver(struct pci_driver *drv,
                                                     struct module *owner,
                                                     const char *mod_name)
{ return 0; }
static inline int pci_register_driver(struct pci_driver *drv)
{ return 0; }
static inline void pci_unregister_driver(struct pci_driver *drv) { }
static inline u8 pci_find_capability(struct pci_dev *dev, int cap)
{ return 0; }
static inline int pci_find_next_capability(struct pci_dev *dev, u8 post,
                                           int cap)
{ return 0; }
static inline int pci_find_ext_capability(struct pci_dev *dev, int cap)
{ return 0; }

static inline u64 pci_get_dsn(struct pci_dev *dev)
{ return 0; }

/* Power management related routines */
static inline int pci_save_state(struct pci_dev *dev) { return 0; }
static inline void pci_restore_state(struct pci_dev *dev) { }
static inline int pci_set_power_state(struct pci_dev *dev, pci_power_t state)
{ return 0; }
static inline int pci_wake_from_d3(struct pci_dev *dev, bool enable)
{ return 0; }
static inline pci_power_t pci_choose_state(struct pci_dev *dev,
                                           pm_message_t state)
{ return PCI_D0; }
static inline int pci_enable_wake(struct pci_dev *dev, pci_power_t state,
                                  int enable)
{ return 0; }

static inline struct resource *pci_find_resource(struct pci_dev *dev,
                                                 struct resource *res)
{ return NULL; }
static inline int pci_request_regions(struct pci_dev *dev, const char *res_name)
{ return -EIO; }
static inline void pci_release_regions(struct pci_dev *dev) { }

static inline int pci_register_io_range(struct fwnode_handle *fwnode,
                                        phys_addr_t addr, resource_size_t size)
{ return -EINVAL; }

static inline unsigned long pci_address_to_pio(phys_addr_t addr) { return -1; }

static inline struct pci_bus *pci_find_next_bus(const struct pci_bus *from)
{ return NULL; }
static inline struct pci_dev *pci_get_slot(struct pci_bus *bus,
                                                unsigned int devfn)
{ return NULL; }
static inline struct pci_dev *pci_get_domain_bus_and_slot(int domain,
                                        unsigned int bus, unsigned int devfn)
{ return NULL; }

static inline int pci_domain_nr(struct pci_bus *bus) { return 0; }
static inline struct pci_dev *pci_dev_get(struct pci_dev *dev) { return NULL; }

#define dev_is_pci(d) (false)
#define dev_is_pf(d) (false)
static inline bool pci_acs_enabled(struct pci_dev *pdev, u16 acs_flags)
{ return false; }
static inline int pci_irqd_intx_xlate(struct irq_domain *d,
                                      struct device_node *node,
                                      const u32 *intspec,
                                      unsigned int intsize,
                                      unsigned long *out_hwirq,
                                      unsigned int *out_type)
{ return -EINVAL; }

static inline const struct pci_device_id *pci_match_id(const struct pci_device_id *ids,
                                                         struct pci_dev *dev)
{ return NULL; }
static inline bool pci_ats_disabled(void) { return true; }

static inline int pci_irq_vector(struct pci_dev *dev, unsigned int nr)
{
        return -EINVAL;
}

static inline int
pci_alloc_irq_vectors_affinity(struct pci_dev *dev, unsigned int min_vecs,
                               unsigned int max_vecs, unsigned int flags,
                               struct irq_affinity *aff_desc)
{
        return -ENOSPC;
}
#endif /* CONFIG_PCI */

static inline int
pci_alloc_irq_vectors(struct pci_dev *dev, unsigned int min_vecs,
                      unsigned int max_vecs, unsigned int flags)
{
        return pci_alloc_irq_vectors_affinity(dev, min_vecs, max_vecs, flags,
                                              NULL);
}

/* Include architecture-dependent settings and functions */

#include <asm/pci.h>

/* These two functions provide almost identical functionality. Depending
 * on the architecture, one will be implemented as a wrapper around the
 * other (in drivers/pci/mmap.c).
 *
 * pci_mmap_resource_range() maps a specific BAR, and vm->vm_pgoff
 * is expected to be an offset within that region.
 *
 * pci_mmap_page_range() is the legacy architecture-specific interface,
 * which accepts a "user visible" resource address converted by
 * pci_resource_to_user(), as used in the legacy mmap() interface in
 * /proc/bus/pci/.
 */
int pci_mmap_resource_range(struct pci_dev *dev, int bar,
                            struct vm_area_struct *vma,
                            enum pci_mmap_state mmap_state, int write_combine);
int pci_mmap_page_range(struct pci_dev *pdev, int bar,
                        struct vm_area_struct *vma,
                        enum pci_mmap_state mmap_state, int write_combine);

#ifndef arch_can_pci_mmap_wc
#define arch_can_pci_mmap_wc()                0
#endif

#ifndef arch_can_pci_mmap_io
#define arch_can_pci_mmap_io()                0
#define pci_iobar_pfn(pdev, bar, vma) (-EINVAL)
#else
int pci_iobar_pfn(struct pci_dev *pdev, int bar, struct vm_area_struct *vma);
#endif

#ifndef pci_root_bus_fwnode
#define pci_root_bus_fwnode(bus)        NULL
#endif

/*
 * These helpers provide future and backwards compatibility
 * for accessing popular PCI BAR info
 */
#define pci_resource_start(dev, bar)        ((dev)->resource[(bar)].start)
#define pci_resource_end(dev, bar)        ((dev)->resource[(bar)].end)
#define pci_resource_flags(dev, bar)        ((dev)->resource[(bar)].flags)
#define pci_resource_len(dev,bar) \
        ((pci_resource_end((dev), (bar)) == 0) ? 0 :        \
                                                        \
         (pci_resource_end((dev), (bar)) -                \
          pci_resource_start((dev), (bar)) + 1))

/*
 * Similar to the helpers above, these manipulate per-pci_dev
 * driver-specific data.  They are really just a wrapper around
 * the generic device structure functions of these calls.
 */
static inline void *pci_get_drvdata(struct pci_dev *pdev)
{
        return dev_get_drvdata(&pdev->dev);
}

static inline void pci_set_drvdata(struct pci_dev *pdev, void *data)
{
        dev_set_drvdata(&pdev->dev, data);
}

static inline const char *pci_name(const struct pci_dev *pdev)
{
        return dev_name(&pdev->dev);
}

void pci_resource_to_user(const struct pci_dev *dev, int bar,
                          const struct resource *rsrc,
                          resource_size_t *start, resource_size_t *end);

/*
 * The world is not perfect and supplies us with broken PCI devices.
 * For at least a part of these bugs we need a work-around, so both
 * generic (drivers/pci/quirks.c) and per-architecture code can define
 * fixup hooks to be called for particular buggy devices.
 */

struct pci_fixup {
        u16 vendor;                        /* Or PCI_ANY_ID */
        u16 device;                        /* Or PCI_ANY_ID */
        u32 class;                        /* Or PCI_ANY_ID */
        unsigned int class_shift;        /* should be 0, 8, 16 */
#ifdef CONFIG_HAVE_ARCH_PREL32_RELOCATIONS
        int hook_offset;
#else
        void (*hook)(struct pci_dev *dev);
#endif
};

enum pci_fixup_pass {
        pci_fixup_early,        /* Before probing BARs */
        pci_fixup_header,        /* After reading configuration header */
        pci_fixup_final,        /* Final phase of device fixups */
        pci_fixup_enable,        /* pci_enable_device() time */
        pci_fixup_resume,        /* pci_device_resume() */
        pci_fixup_suspend,        /* pci_device_suspend() */
        pci_fixup_resume_early, /* pci_device_resume_early() */
        pci_fixup_suspend_late,        /* pci_device_suspend_late() */
};

#ifdef CONFIG_HAVE_ARCH_PREL32_RELOCATIONS
#define ___DECLARE_PCI_FIXUP_SECTION(sec, name, vendor, device, class,        \
                                    class_shift, hook)                        \
        __ADDRESSABLE(hook)                                                \
        asm(".section "        #sec ", \"a\"                                \n"        \
            ".balign        16                                        \n"        \
            ".short "        #vendor ", " #device "                        \n"        \
            ".long "        #class ", " #class_shift "                \n"        \
            ".long "        #hook " - .                                \n"        \
            ".previous                                                \n");

/*
 * Clang's LTO may rename static functions in C, but has no way to
 * handle such renamings when referenced from inline asm. To work
 * around this, create global C stubs for these cases.
 */
#ifdef CONFIG_LTO_CLANG
#define __DECLARE_PCI_FIXUP_SECTION(sec, name, vendor, device, class,        \
                                  class_shift, hook, stub)                \
        void __cficanonical stub(struct pci_dev *dev);                        \
        void __cficanonical stub(struct pci_dev *dev)                        \
        {                                                                 \
                hook(dev);                                                 \
        }                                                                \
        ___DECLARE_PCI_FIXUP_SECTION(sec, name, vendor, device, class,        \
                                  class_shift, stub)
#else
#define __DECLARE_PCI_FIXUP_SECTION(sec, name, vendor, device, class,        \
                                  class_shift, hook, stub)                \
        ___DECLARE_PCI_FIXUP_SECTION(sec, name, vendor, device, class,        \
                                  class_shift, hook)
#endif

#define DECLARE_PCI_FIXUP_SECTION(sec, name, vendor, device, class,        \
                                  class_shift, hook)                        \
        __DECLARE_PCI_FIXUP_SECTION(sec, name, vendor, device, class,        \
                                  class_shift, hook, __UNIQUE_ID(hook))
#else
/* Anonymous variables would be nice... */
#define DECLARE_PCI_FIXUP_SECTION(section, name, vendor, device, class,        \
                                  class_shift, hook)                        \
        static const struct pci_fixup __PASTE(__pci_fixup_##name,__LINE__) __used        \
        __attribute__((__section__(#section), aligned((sizeof(void *)))))    \
                = { vendor, device, class, class_shift, hook };
#endif

#define DECLARE_PCI_FIXUP_CLASS_EARLY(vendor, device, class,                \
                                         class_shift, hook)                \
        DECLARE_PCI_FIXUP_SECTION(.pci_fixup_early,                        \
                hook, vendor, device, class, class_shift, hook)
#define DECLARE_PCI_FIXUP_CLASS_HEADER(vendor, device, class,                \
                                         class_shift, hook)                \
        DECLARE_PCI_FIXUP_SECTION(.pci_fixup_header,                        \
                hook, vendor, device, class, class_shift, hook)
#define DECLARE_PCI_FIXUP_CLASS_FINAL(vendor, device, class,                \
                                         class_shift, hook)                \
        DECLARE_PCI_FIXUP_SECTION(.pci_fixup_final,                        \
                hook, vendor, device, class, class_shift, hook)
#define DECLARE_PCI_FIXUP_CLASS_ENABLE(vendor, device, class,                \
                                         class_shift, hook)                \
        DECLARE_PCI_FIXUP_SECTION(.pci_fixup_enable,                        \
                hook, vendor, device, class, class_shift, hook)
#define DECLARE_PCI_FIXUP_CLASS_RESUME(vendor, device, class,                \
                                         class_shift, hook)                \
        DECLARE_PCI_FIXUP_SECTION(.pci_fixup_resume,                        \
                resume##hook, vendor, device, class, class_shift, hook)
#define DECLARE_PCI_FIXUP_CLASS_RESUME_EARLY(vendor, device, class,        \
                                         class_shift, hook)                \
        DECLARE_PCI_FIXUP_SECTION(.pci_fixup_resume_early,                \
                resume_early##hook, vendor, device, class, class_shift, hook)
#define DECLARE_PCI_FIXUP_CLASS_SUSPEND(vendor, device, class,                \
                                         class_shift, hook)                \
        DECLARE_PCI_FIXUP_SECTION(.pci_fixup_suspend,                        \
                suspend##hook, vendor, device, class, class_shift, hook)
#define DECLARE_PCI_FIXUP_CLASS_SUSPEND_LATE(vendor, device, class,        \
                                         class_shift, hook)                \
        DECLARE_PCI_FIXUP_SECTION(.pci_fixup_suspend_late,                \
                suspend_late##hook, vendor, device, class, class_shift, hook)

#define DECLARE_PCI_FIXUP_EARLY(vendor, device, hook)                        \
        DECLARE_PCI_FIXUP_SECTION(.pci_fixup_early,                        \
                hook, vendor, device, PCI_ANY_ID, 0, hook)
#define DECLARE_PCI_FIXUP_HEADER(vendor, device, hook)                        \
        DECLARE_PCI_FIXUP_SECTION(.pci_fixup_header,                        \
                hook, vendor, device, PCI_ANY_ID, 0, hook)
#define DECLARE_PCI_FIXUP_FINAL(vendor, device, hook)                        \
        DECLARE_PCI_FIXUP_SECTION(.pci_fixup_final,                        \
                hook, vendor, device, PCI_ANY_ID, 0, hook)
#define DECLARE_PCI_FIXUP_ENABLE(vendor, device, hook)                        \
        DECLARE_PCI_FIXUP_SECTION(.pci_fixup_enable,                        \
                hook, vendor, device, PCI_ANY_ID, 0, hook)
#define DECLARE_PCI_FIXUP_RESUME(vendor, device, hook)                        \
        DECLARE_PCI_FIXUP_SECTION(.pci_fixup_resume,                        \
                resume##hook, vendor, device, PCI_ANY_ID, 0, hook)
#define DECLARE_PCI_FIXUP_RESUME_EARLY(vendor, device, hook)                \
        DECLARE_PCI_FIXUP_SECTION(.pci_fixup_resume_early,                \
                resume_early##hook, vendor, device, PCI_ANY_ID, 0, hook)
#define DECLARE_PCI_FIXUP_SUSPEND(vendor, device, hook)                        \
        DECLARE_PCI_FIXUP_SECTION(.pci_fixup_suspend,                        \
                suspend##hook, vendor, device, PCI_ANY_ID, 0, hook)
#define DECLARE_PCI_FIXUP_SUSPEND_LATE(vendor, device, hook)                \
        DECLARE_PCI_FIXUP_SECTION(.pci_fixup_suspend_late,                \
                suspend_late##hook, vendor, device, PCI_ANY_ID, 0, hook)

#ifdef CONFIG_PCI_QUIRKS
void pci_fixup_device(enum pci_fixup_pass pass, struct pci_dev *dev);
#else
static inline void pci_fixup_device(enum pci_fixup_pass pass,
                                    struct pci_dev *dev) { }
#endif

void __iomem *pcim_iomap(struct pci_dev *pdev, int bar, unsigned long maxlen);
void pcim_iounmap(struct pci_dev *pdev, void __iomem *addr);
void __iomem * const *pcim_iomap_table(struct pci_dev *pdev);
int pcim_iomap_regions(struct pci_dev *pdev, int mask, const char *name);
int pcim_iomap_regions_request_all(struct pci_dev *pdev, int mask,
                                   const char *name);
void pcim_iounmap_regions(struct pci_dev *pdev, int mask);

extern int pci_pci_problems;
#define PCIPCI_FAIL                1        /* No PCI PCI DMA */
#define PCIPCI_TRITON                2
#define PCIPCI_NATOMA                4
#define PCIPCI_VIAETBF                8
#define PCIPCI_VSFX                16
#define PCIPCI_ALIMAGIK                32        /* Need low latency setting */
#define PCIAGP_FAIL                64        /* No PCI to AGP DMA */

extern unsigned long pci_cardbus_io_size;
extern unsigned long pci_cardbus_mem_size;
extern u8 pci_dfl_cache_line_size;
extern u8 pci_cache_line_size;

/* Architecture-specific versions may override these (weak) */
void pcibios_disable_device(struct pci_dev *dev);
void pcibios_set_master(struct pci_dev *dev);
int pcibios_set_pcie_reset_state(struct pci_dev *dev,
                                 enum pcie_reset_state state);
int pcibios_add_device(struct pci_dev *dev);
void pcibios_release_device(struct pci_dev *dev);
#ifdef CONFIG_PCI
void pcibios_penalize_isa_irq(int irq, int active);
#else
static inline void pcibios_penalize_isa_irq(int irq, int active) {}
#endif
int pcibios_alloc_irq(struct pci_dev *dev);
void pcibios_free_irq(struct pci_dev *dev);
resource_size_t pcibios_default_alignment(void);

#if defined(CONFIG_PCI_MMCONFIG) || defined(CONFIG_ACPI_MCFG)
void __init pci_mmcfg_early_init(void);
void __init pci_mmcfg_late_init(void);
#else
static inline void pci_mmcfg_early_init(void) { }
static inline void pci_mmcfg_late_init(void) { }
#endif

int pci_ext_cfg_avail(void);

void __iomem *pci_ioremap_bar(struct pci_dev *pdev, int bar);
void __iomem *pci_ioremap_wc_bar(struct pci_dev *pdev, int bar);

#ifdef CONFIG_PCI_IOV
int pci_iov_virtfn_bus(struct pci_dev *dev, int id);
int pci_iov_virtfn_devfn(struct pci_dev *dev, int id);

int pci_enable_sriov(struct pci_dev *dev, int nr_virtfn);
void pci_disable_sriov(struct pci_dev *dev);

int pci_iov_sysfs_link(struct pci_dev *dev, struct pci_dev *virtfn, int id);
int pci_iov_add_virtfn(struct pci_dev *dev, int id);
void pci_iov_remove_virtfn(struct pci_dev *dev, int id);
int pci_num_vf(struct pci_dev *dev);
int pci_vfs_assigned(struct pci_dev *dev);
int pci_sriov_set_totalvfs(struct pci_dev *dev, u16 numvfs);
int pci_sriov_get_totalvfs(struct pci_dev *dev);
int pci_sriov_configure_simple(struct pci_dev *dev, int nr_virtfn);
resource_size_t pci_iov_resource_size(struct pci_dev *dev, int resno);
void pci_vf_drivers_autoprobe(struct pci_dev *dev, bool probe);

/* Arch may override these (weak) */
int pcibios_sriov_enable(struct pci_dev *pdev, u16 num_vfs);
int pcibios_sriov_disable(struct pci_dev *pdev);
resource_size_t pcibios_iov_resource_alignment(struct pci_dev *dev, int resno);
#else
static inline int pci_iov_virtfn_bus(struct pci_dev *dev, int id)
{
        return -ENOSYS;
}
static inline int pci_iov_virtfn_devfn(struct pci_dev *dev, int id)
{
        return -ENOSYS;
}
static inline int pci_enable_sriov(struct pci_dev *dev, int nr_virtfn)
{ return -ENODEV; }

static inline int pci_iov_sysfs_link(struct pci_dev *dev,
                                     struct pci_dev *virtfn, int id)
{
        return -ENODEV;
}
static inline int pci_iov_add_virtfn(struct pci_dev *dev, int id)
{
        return -ENOSYS;
}
static inline void pci_iov_remove_virtfn(struct pci_dev *dev,
                                         int id) { }
static inline void pci_disable_sriov(struct pci_dev *dev) { }
static inline int pci_num_vf(struct pci_dev *dev) { return 0; }
static inline int pci_vfs_assigned(struct pci_dev *dev)
{ return 0; }
static inline int pci_sriov_set_totalvfs(struct pci_dev *dev, u16 numvfs)
{ return 0; }
static inline int pci_sriov_get_totalvfs(struct pci_dev *dev)
{ return 0; }
#define pci_sriov_configure_simple        NULL
static inline resource_size_t pci_iov_resource_size(struct pci_dev *dev, int resno)
{ return 0; }
static inline void pci_vf_drivers_autoprobe(struct pci_dev *dev, bool probe) { }
#endif

#if defined(CONFIG_HOTPLUG_PCI) || defined(CONFIG_HOTPLUG_PCI_MODULE)
void pci_hp_create_module_link(struct pci_slot *pci_slot);
void pci_hp_remove_module_link(struct pci_slot *pci_slot);
#endif

/**
 * pci_pcie_cap - get the saved PCIe capability offset
 * @dev: PCI device
 *
 * PCIe capability offset is calculated at PCI device initialization
 * time and saved in the data structure. This function returns saved
 * PCIe capability offset. Using this instead of pci_find_capability()
 * reduces unnecessary search in the PCI configuration space. If you
 * need to calculate PCIe capability offset from raw device for some
 * reasons, please use pci_find_capability() instead.
 */
static inline int pci_pcie_cap(struct pci_dev *dev)
{
        return dev->pcie_cap;
}

/**
 * pci_is_pcie - check if the PCI device is PCI Express capable
 * @dev: PCI device
 *
 * Returns: true if the PCI device is PCI Express capable, false otherwise.
 */
static inline bool pci_is_pcie(struct pci_dev *dev)
{
        return pci_pcie_cap(dev);
}

/**
 * pcie_caps_reg - get the PCIe Capabilities Register
 * @dev: PCI device
 */
static inline u16 pcie_caps_reg(const struct pci_dev *dev)
{
        return dev->pcie_flags_reg;
}

/**
 * pci_pcie_type - get the PCIe device/port type
 * @dev: PCI device
 */
static inline int pci_pcie_type(const struct pci_dev *dev)
{
        return (pcie_caps_reg(dev) & PCI_EXP_FLAGS_TYPE) >> 4;
}

/**
 * pcie_find_root_port - Get the PCIe root port device
 * @dev: PCI device
 *
 * Traverse up the parent chain and return the PCIe Root Port PCI Device
 * for a given PCI/PCIe Device.
 */
static inline struct pci_dev *pcie_find_root_port(struct pci_dev *dev)
{
        while (dev) {
                if (pci_is_pcie(dev) &&
                    pci_pcie_type(dev) == PCI_EXP_TYPE_ROOT_PORT)
                        return dev;
                dev = pci_upstream_bridge(dev);
        }

        return NULL;
}

static inline bool pci_dev_is_disconnected(const struct pci_dev *dev)
{
        return dev->error_state == pci_channel_io_perm_failure;
}

void pci_request_acs(void);
bool pci_acs_enabled(struct pci_dev *pdev, u16 acs_flags);
bool pci_acs_path_enabled(struct pci_dev *start,
                          struct pci_dev *end, u16 acs_flags);
int pci_enable_atomic_ops_to_root(struct pci_dev *dev, u32 cap_mask);

#define PCI_VPD_LRDT                        0x80        /* Large Resource Data Type */
#define PCI_VPD_LRDT_ID(x)                ((x) | PCI_VPD_LRDT)

/* Large Resource Data Type Tag Item Names */
#define PCI_VPD_LTIN_ID_STRING                0x02        /* Identifier String */
#define PCI_VPD_LTIN_RO_DATA                0x10        /* Read-Only Data */
#define PCI_VPD_LTIN_RW_DATA                0x11        /* Read-Write Data */

#define PCI_VPD_LRDT_ID_STRING                PCI_VPD_LRDT_ID(PCI_VPD_LTIN_ID_STRING)
#define PCI_VPD_LRDT_RO_DATA                PCI_VPD_LRDT_ID(PCI_VPD_LTIN_RO_DATA)
#define PCI_VPD_LRDT_RW_DATA                PCI_VPD_LRDT_ID(PCI_VPD_LTIN_RW_DATA)

#define PCI_VPD_RO_KEYWORD_PARTNO        "PN"
#define PCI_VPD_RO_KEYWORD_SERIALNO        "SN"
#define PCI_VPD_RO_KEYWORD_MFR_ID        "MN"
#define PCI_VPD_RO_KEYWORD_VENDOR0        "V0"
#define PCI_VPD_RO_KEYWORD_CHKSUM        "RV"

/**
 * pci_vpd_alloc - Allocate buffer and read VPD into it
 * @dev: PCI device
 * @size: pointer to field where VPD length is returned
 *
 * Returns pointer to allocated buffer or an ERR_PTR in case of failure
 */
void *pci_vpd_alloc(struct pci_dev *dev, unsigned int *size);

/**
 * pci_vpd_find_id_string - Locate id string in VPD
 * @buf: Pointer to buffered VPD data
 * @len: The length of the buffer area in which to search
 * @size: Pointer to field where length of id string is returned
 *
 * Returns the index of the id string or -ENOENT if not found.
 */
int pci_vpd_find_id_string(const u8 *buf, unsigned int len, unsigned int *size);

/**
 * pci_vpd_find_ro_info_keyword - Locate info field keyword in VPD RO section
 * @buf: Pointer to buffered VPD data
 * @len: The length of the buffer area in which to search
 * @kw: The keyword to search for
 * @size: Pointer to field where length of found keyword data is returned
 *
 * Returns the index of the information field keyword data or -ENOENT if
 * not found.
 */
int pci_vpd_find_ro_info_keyword(const void *buf, unsigned int len,
                                 const char *kw, unsigned int *size);

/**
 * pci_vpd_check_csum - Check VPD checksum
 * @buf: Pointer to buffered VPD data
 * @len: VPD size
 *
 * Returns 1 if VPD has no checksum, otherwise 0 or an errno
 */
int pci_vpd_check_csum(const void *buf, unsigned int len);

/* PCI <-> OF binding helpers */
#ifdef CONFIG_OF
struct device_node;
struct irq_domain;
struct irq_domain *pci_host_bridge_of_msi_domain(struct pci_bus *bus);
bool pci_host_of_has_msi_map(struct device *dev);

/* Arch may override this (weak) */
struct device_node *pcibios_get_phb_of_node(struct pci_bus *bus);

#else        /* CONFIG_OF */
static inline struct irq_domain *
pci_host_bridge_of_msi_domain(struct pci_bus *bus) { return NULL; }
static inline bool pci_host_of_has_msi_map(struct device *dev) { return false; }
#endif  /* CONFIG_OF */

static inline struct device_node *
pci_device_to_OF_node(const struct pci_dev *pdev)
{
        return pdev ? pdev->dev.of_node : NULL;
}

static inline struct device_node *pci_bus_to_OF_node(struct pci_bus *bus)
{
        return bus ? bus->dev.of_node : NULL;
}

#ifdef CONFIG_ACPI
struct irq_domain *pci_host_bridge_acpi_msi_domain(struct pci_bus *bus);

void
pci_msi_register_fwnode_provider(struct fwnode_handle *(*fn)(struct device *));
bool pci_pr3_present(struct pci_dev *pdev);
#else
static inline struct irq_domain *
pci_host_bridge_acpi_msi_domain(struct pci_bus *bus) { return NULL; }
static inline bool pci_pr3_present(struct pci_dev *pdev) { return false; }
#endif

#if defined(CONFIG_X86) && defined(CONFIG_ACPI)
bool arch_pci_dev_is_removable(struct pci_dev *pdev);
#else
static inline bool arch_pci_dev_is_removable(struct pci_dev *pdev) { return false; }
#endif

#ifdef CONFIG_EEH
static inline struct eeh_dev *pci_dev_to_eeh_dev(struct pci_dev *pdev)
{
        return pdev->dev.archdata.edev;
}
#endif

void pci_add_dma_alias(struct pci_dev *dev, u8 devfn_from, unsigned nr_devfns);
bool pci_devs_are_dma_aliases(struct pci_dev *dev1, struct pci_dev *dev2);
int pci_for_each_dma_alias(struct pci_dev *pdev,
                           int (*fn)(struct pci_dev *pdev,
                                     u16 alias, void *data), void *data);

/* Helper functions for operation of device flag */
static inline void pci_set_dev_assigned(struct pci_dev *pdev)
{
        pdev->dev_flags |= PCI_DEV_FLAGS_ASSIGNED;
}
static inline void pci_clear_dev_assigned(struct pci_dev *pdev)
{
        pdev->dev_flags &= ~PCI_DEV_FLAGS_ASSIGNED;
}
static inline bool pci_is_dev_assigned(struct pci_dev *pdev)
{
        return (pdev->dev_flags & PCI_DEV_FLAGS_ASSIGNED) == PCI_DEV_FLAGS_ASSIGNED;
}

/**
 * pci_ari_enabled - query ARI forwarding status
 * @bus: the PCI bus
 *
 * Returns true if ARI forwarding is enabled.
 */
static inline bool pci_ari_enabled(struct pci_bus *bus)
{
        return bus->self && bus->self->ari_enabled;
}

/**
 * pci_is_thunderbolt_attached - whether device is on a Thunderbolt daisy chain
 * @pdev: PCI device to check
 *
 * Walk upwards from @pdev and check for each encountered bridge if it's part
 * of a Thunderbolt controller.  Reaching the host bridge means @pdev is not
 * Thunderbolt-attached.  (But rather soldered to the mainboard usually.)
 */
static inline bool pci_is_thunderbolt_attached(struct pci_dev *pdev)
{
        struct pci_dev *parent = pdev;

        if (pdev->is_thunderbolt)
                return true;

        while ((parent = pci_upstream_bridge(parent)))
                if (parent->is_thunderbolt)
                        return true;

        return false;
}

#if defined(CONFIG_PCIEPORTBUS) || defined(CONFIG_EEH)
void pci_uevent_ers(struct pci_dev *pdev, enum  pci_ers_result err_type);
#endif

/* Provide the legacy pci_dma_* API */
#include <linux/pci-dma-compat.h>

#define pci_printk(level, pdev, fmt, arg...) \
        dev_printk(level, &(pdev)->dev, fmt, ##arg)

#define pci_emerg(pdev, fmt, arg...)        dev_emerg(&(pdev)->dev, fmt, ##arg)
#define pci_alert(pdev, fmt, arg...)        dev_alert(&(pdev)->dev, fmt, ##arg)
#define pci_crit(pdev, fmt, arg...)        dev_crit(&(pdev)->dev, fmt, ##arg)
#define pci_err(pdev, fmt, arg...)        dev_err(&(pdev)->dev, fmt, ##arg)
#define pci_warn(pdev, fmt, arg...)        dev_warn(&(pdev)->dev, fmt, ##arg)
#define pci_notice(pdev, fmt, arg...)        dev_notice(&(pdev)->dev, fmt, ##arg)
#define pci_info(pdev, fmt, arg...)        dev_info(&(pdev)->dev, fmt, ##arg)
#define pci_dbg(pdev, fmt, arg...)        dev_dbg(&(pdev)->dev, fmt, ##arg)

#define pci_notice_ratelimited(pdev, fmt, arg...) \
        dev_notice_ratelimited(&(pdev)->dev, fmt, ##arg)

#define pci_info_ratelimited(pdev, fmt, arg...) \
        dev_info_ratelimited(&(pdev)->dev, fmt, ##arg)

#define pci_WARN(pdev, condition, fmt, arg...) \
        WARN(condition, "%s %s: " fmt, \
             dev_driver_string(&(pdev)->dev), pci_name(pdev), ##arg)

#define pci_WARN_ONCE(pdev, condition, fmt, arg...) \
        WARN_ONCE(condition, "%s %s: " fmt, \
                  dev_driver_string(&(pdev)->dev), pci_name(pdev), ##arg)

#endif /* LINUX_PCI_H */



































































































































































































































































































































































    2 





























































































































    6 



    1 










    7 









































































































    7 























    7 





   11 


   11 






   11 
   11 

   11 







   11 

   11 









    7 
    7 








    7 







    7 





    7 

    7 


















    7 




    7 








    1 

    5 



    6 

    7 






   11 


   11 

   11 


















   25 







   25 











   24 
    8 







   25 
   25 















   15 
   15 

















   15 





   15 


    1 

   14 




    3 








   13 










   14 
   14 

















   14 






    1 

   13 





   10 








    1 
    9 










   16 









    1 


   15 



    3 









































   14 



   14 
   14 

















   14 




   14 













   15 







    1 


   14 



   10 






































   11 




   11 
   11 






   11 








   31 
















   31 


   15 


   16 
   31 

   25 





    2 


   13 






    3 


    3 




































































    1 



























































































































































































































































































































































































































































































































































































    2 


    2 




   49 

















   24 
















    4 








   21 
















   24 











   24 







































































  979 


  979 












   49 






























































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
// SPDX-License-Identifier: GPL-2.0-only
/*
 * GENEVE: Generic Network Virtualization Encapsulation
 *
 * Copyright (c) 2015 Red Hat, Inc.
 */

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#include <linux/ethtool.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/etherdevice.h>
#include <linux/hash.h>
#include <net/ipv6_stubs.h>
#include <net/dst_metadata.h>
#include <net/gro_cells.h>
#include <net/rtnetlink.h>
#include <net/geneve.h>
#include <net/protocol.h>

#define GENEVE_NETDEV_VER        "0.6"

#define GENEVE_N_VID                (1u << 24)
#define GENEVE_VID_MASK                (GENEVE_N_VID - 1)

#define VNI_HASH_BITS                10
#define VNI_HASH_SIZE                (1<<VNI_HASH_BITS)

static bool log_ecn_error = true;
module_param(log_ecn_error, bool, 0644);
MODULE_PARM_DESC(log_ecn_error, "Log packets received with corrupted ECN");

#define GENEVE_VER 0
#define GENEVE_BASE_HLEN (sizeof(struct udphdr) + sizeof(struct genevehdr))
#define GENEVE_IPV4_HLEN (ETH_HLEN + sizeof(struct iphdr) + GENEVE_BASE_HLEN)
#define GENEVE_IPV6_HLEN (ETH_HLEN + sizeof(struct ipv6hdr) + GENEVE_BASE_HLEN)

/* per-network namespace private data for this module */
struct geneve_net {
        struct list_head        geneve_list;
        struct list_head        sock_list;
};

static unsigned int geneve_net_id;

struct geneve_dev_node {
        struct hlist_node hlist;
        struct geneve_dev *geneve;
};

struct geneve_config {
        struct ip_tunnel_info        info;
        bool                        collect_md;
        bool                        use_udp6_rx_checksums;
        bool                        ttl_inherit;
        enum ifla_geneve_df        df;
        bool                        inner_proto_inherit;
};

/* Pseudo network device */
struct geneve_dev {
        struct geneve_dev_node hlist4;        /* vni hash table for IPv4 socket */
#if IS_ENABLED(CONFIG_IPV6)
        struct geneve_dev_node hlist6;        /* vni hash table for IPv6 socket */
#endif
        struct net           *net;        /* netns for packet i/o */
        struct net_device  *dev;        /* netdev for geneve tunnel */
        struct geneve_sock __rcu *sock4;        /* IPv4 socket used for geneve tunnel */
#if IS_ENABLED(CONFIG_IPV6)
        struct geneve_sock __rcu *sock6;        /* IPv6 socket used for geneve tunnel */
#endif
        struct list_head   next;        /* geneve's per namespace list */
        struct gro_cells   gro_cells;
        struct geneve_config cfg;
};

struct geneve_sock {
        bool                        collect_md;
        struct list_head        list;
        struct socket                *sock;
        struct rcu_head                rcu;
        int                        refcnt;
        struct hlist_head        vni_list[VNI_HASH_SIZE];
};

static inline __u32 geneve_net_vni_hash(u8 vni[3])
{
        __u32 vnid;

        vnid = (vni[0] << 16) | (vni[1] << 8) | vni[2];
        return hash_32(vnid, VNI_HASH_BITS);
}

static __be64 vni_to_tunnel_id(const __u8 *vni)
{
#ifdef __BIG_ENDIAN
        return (vni[0] << 16) | (vni[1] << 8) | vni[2];
#else
        return (__force __be64)(((__force u64)vni[0] << 40) |
                                ((__force u64)vni[1] << 48) |
                                ((__force u64)vni[2] << 56));
#endif
}

/* Convert 64 bit tunnel ID to 24 bit VNI. */
static void tunnel_id_to_vni(__be64 tun_id, __u8 *vni)
{
#ifdef __BIG_ENDIAN
        vni[0] = (__force __u8)(tun_id >> 16);
        vni[1] = (__force __u8)(tun_id >> 8);
        vni[2] = (__force __u8)tun_id;
#else
        vni[0] = (__force __u8)((__force u64)tun_id >> 40);
        vni[1] = (__force __u8)((__force u64)tun_id >> 48);
        vni[2] = (__force __u8)((__force u64)tun_id >> 56);
#endif
}

static bool eq_tun_id_and_vni(u8 *tun_id, u8 *vni)
{
        return !memcmp(vni, &tun_id[5], 3);
}

static sa_family_t geneve_get_sk_family(struct geneve_sock *gs)
{
        return gs->sock->sk->sk_family;
}

static struct geneve_dev *geneve_lookup(struct geneve_sock *gs,
                                        __be32 addr, u8 vni[])
{
        struct hlist_head *vni_list_head;
        struct geneve_dev_node *node;
        __u32 hash;

        /* Find the device for this VNI */
        hash = geneve_net_vni_hash(vni);
        vni_list_head = &gs->vni_list[hash];
        hlist_for_each_entry_rcu(node, vni_list_head, hlist) {
                if (eq_tun_id_and_vni((u8 *)&node->geneve->cfg.info.key.tun_id, vni) &&
                    addr == node->geneve->cfg.info.key.u.ipv4.dst)
                        return node->geneve;
        }
        return NULL;
}

#if IS_ENABLED(CONFIG_IPV6)
static struct geneve_dev *geneve6_lookup(struct geneve_sock *gs,
                                         struct in6_addr addr6, u8 vni[])
{
        struct hlist_head *vni_list_head;
        struct geneve_dev_node *node;
        __u32 hash;

        /* Find the device for this VNI */
        hash = geneve_net_vni_hash(vni);
        vni_list_head = &gs->vni_list[hash];
        hlist_for_each_entry_rcu(node, vni_list_head, hlist) {
                if (eq_tun_id_and_vni((u8 *)&node->geneve->cfg.info.key.tun_id, vni) &&
                    ipv6_addr_equal(&addr6, &node->geneve->cfg.info.key.u.ipv6.dst))
                        return node->geneve;
        }
        return NULL;
}
#endif

static inline struct genevehdr *geneve_hdr(const struct sk_buff *skb)
{
        return (struct genevehdr *)(udp_hdr(skb) + 1);
}

static struct geneve_dev *geneve_lookup_skb(struct geneve_sock *gs,
                                            struct sk_buff *skb)
{
        static u8 zero_vni[3];
        u8 *vni;

        if (geneve_get_sk_family(gs) == AF_INET) {
                struct iphdr *iph;
                __be32 addr;

                iph = ip_hdr(skb); /* outer IP header... */

                if (gs->collect_md) {
                        vni = zero_vni;
                        addr = 0;
                } else {
                        vni = geneve_hdr(skb)->vni;
                        addr = iph->saddr;
                }

                return geneve_lookup(gs, addr, vni);
#if IS_ENABLED(CONFIG_IPV6)
        } else if (geneve_get_sk_family(gs) == AF_INET6) {
                static struct in6_addr zero_addr6;
                struct ipv6hdr *ip6h;
                struct in6_addr addr6;

                ip6h = ipv6_hdr(skb); /* outer IPv6 header... */

                if (gs->collect_md) {
                        vni = zero_vni;
                        addr6 = zero_addr6;
                } else {
                        vni = geneve_hdr(skb)->vni;
                        addr6 = ip6h->saddr;
                }

                return geneve6_lookup(gs, addr6, vni);
#endif
        }
        return NULL;
}

/* geneve receive/decap routine */
static void geneve_rx(struct geneve_dev *geneve, struct geneve_sock *gs,
                      struct sk_buff *skb)
{
        struct genevehdr *gnvh = geneve_hdr(skb);
        struct metadata_dst *tun_dst = NULL;
        unsigned int len;
        int nh, err = 0;
        void *oiph;

        if (ip_tunnel_collect_metadata() || gs->collect_md) {
                __be16 flags;

                flags = TUNNEL_KEY | (gnvh->oam ? TUNNEL_OAM : 0) |
                        (gnvh->critical ? TUNNEL_CRIT_OPT : 0);

                tun_dst = udp_tun_rx_dst(skb, geneve_get_sk_family(gs), flags,
                                         vni_to_tunnel_id(gnvh->vni),
                                         gnvh->opt_len * 4);
                if (!tun_dst) {
                        geneve->dev->stats.rx_dropped++;
                        goto drop;
                }
                /* Update tunnel dst according to Geneve options. */
                ip_tunnel_info_opts_set(&tun_dst->u.tun_info,
                                        gnvh->options, gnvh->opt_len * 4,
                                        TUNNEL_GENEVE_OPT);
        } else {
                /* Drop packets w/ critical options,
                 * since we don't support any...
                 */
                if (gnvh->critical) {
                        geneve->dev->stats.rx_frame_errors++;
                        geneve->dev->stats.rx_errors++;
                        goto drop;
                }
        }

        if (tun_dst)
                skb_dst_set(skb, &tun_dst->dst);

        if (gnvh->proto_type == htons(ETH_P_TEB)) {
                skb_reset_mac_header(skb);
                skb->protocol = eth_type_trans(skb, geneve->dev);
                skb_postpull_rcsum(skb, eth_hdr(skb), ETH_HLEN);

                /* Ignore packet loops (and multicast echo) */
                if (ether_addr_equal(eth_hdr(skb)->h_source,
                                     geneve->dev->dev_addr)) {
                        geneve->dev->stats.rx_errors++;
                        goto drop;
                }
        } else {
                skb_reset_mac_header(skb);
                skb->dev = geneve->dev;
                skb->pkt_type = PACKET_HOST;
        }

        /* Save offset of outer header relative to skb->head,
         * because we are going to reset the network header to the inner header
         * and might change skb->head.
         */
        nh = skb_network_header(skb) - skb->head;

        skb_reset_network_header(skb);

        if (!pskb_inet_may_pull(skb)) {
                DEV_STATS_INC(geneve->dev, rx_length_errors);
                DEV_STATS_INC(geneve->dev, rx_errors);
                goto drop;
        }

        /* Get the outer header. */
        oiph = skb->head + nh;

        if (geneve_get_sk_family(gs) == AF_INET)
                err = IP_ECN_decapsulate(oiph, skb);
#if IS_ENABLED(CONFIG_IPV6)
        else
                err = IP6_ECN_decapsulate(oiph, skb);
#endif

        if (unlikely(err)) {
                if (log_ecn_error) {
                        if (geneve_get_sk_family(gs) == AF_INET)
                                net_info_ratelimited("non-ECT from %pI4 "
                                                     "with TOS=%#x\n",
                                                     &((struct iphdr *)oiph)->saddr,
                                                     ((struct iphdr *)oiph)->tos);
#if IS_ENABLED(CONFIG_IPV6)
                        else
                                net_info_ratelimited("non-ECT from %pI6\n",
                                                     &((struct ipv6hdr *)oiph)->saddr);
#endif
                }
                if (err > 1) {
                        ++geneve->dev->stats.rx_frame_errors;
                        ++geneve->dev->stats.rx_errors;
                        goto drop;
                }
        }

        len = skb->len;
        err = gro_cells_receive(&geneve->gro_cells, skb);
        if (likely(err == NET_RX_SUCCESS))
                dev_sw_netstats_rx_add(geneve->dev, len);

        return;
drop:
        /* Consume bad packet */
        kfree_skb(skb);
}

/* Setup stats when device is created */
static int geneve_init(struct net_device *dev)
{
        struct geneve_dev *geneve = netdev_priv(dev);
        int err;

        dev->tstats = netdev_alloc_pcpu_stats(struct pcpu_sw_netstats);
        if (!dev->tstats)
                return -ENOMEM;

        err = gro_cells_init(&geneve->gro_cells, dev);
        if (err) {
                free_percpu(dev->tstats);
                return err;
        }

        err = dst_cache_init(&geneve->cfg.info.dst_cache, GFP_KERNEL);
        if (err) {
                free_percpu(dev->tstats);
                gro_cells_destroy(&geneve->gro_cells);
                return err;
        }
        return 0;
}

static void geneve_uninit(struct net_device *dev)
{
        struct geneve_dev *geneve = netdev_priv(dev);

        dst_cache_destroy(&geneve->cfg.info.dst_cache);
        gro_cells_destroy(&geneve->gro_cells);
        free_percpu(dev->tstats);
}

/* Callback from net/ipv4/udp.c to receive packets */
static int geneve_udp_encap_recv(struct sock *sk, struct sk_buff *skb)
{
        struct genevehdr *geneveh;
        struct geneve_dev *geneve;
        struct geneve_sock *gs;
        __be16 inner_proto;
        int opts_len;

        /* Need UDP and Geneve header to be present */
        if (unlikely(!pskb_may_pull(skb, GENEVE_BASE_HLEN)))
                goto drop;

        /* Return packets with reserved bits set */
        geneveh = geneve_hdr(skb);
        if (unlikely(geneveh->ver != GENEVE_VER))
                goto drop;

        inner_proto = geneveh->proto_type;

        if (unlikely((inner_proto != htons(ETH_P_TEB) &&
                      inner_proto != htons(ETH_P_IP) &&
                      inner_proto != htons(ETH_P_IPV6))))
                goto drop;

        gs = rcu_dereference_sk_user_data(sk);
        if (!gs)
                goto drop;

        geneve = geneve_lookup_skb(gs, skb);
        if (!geneve)
                goto drop;

        if (unlikely((!geneve->cfg.inner_proto_inherit &&
                      inner_proto != htons(ETH_P_TEB)))) {
                geneve->dev->stats.rx_dropped++;
                goto drop;
        }

        opts_len = geneveh->opt_len * 4;
        if (iptunnel_pull_header(skb, GENEVE_BASE_HLEN + opts_len, inner_proto,
                                 !net_eq(geneve->net, dev_net(geneve->dev)))) {
                geneve->dev->stats.rx_dropped++;
                goto drop;
        }

        geneve_rx(geneve, gs, skb);
        return 0;

drop:
        /* Consume bad packet */
        kfree_skb(skb);
        return 0;
}

/* Callback from net/ipv{4,6}/udp.c to check that we have a tunnel for errors */
static int geneve_udp_encap_err_lookup(struct sock *sk, struct sk_buff *skb)
{
        struct genevehdr *geneveh;
        struct geneve_sock *gs;
        u8 zero_vni[3] = { 0 };
        u8 *vni = zero_vni;

        if (!pskb_may_pull(skb, skb_transport_offset(skb) + GENEVE_BASE_HLEN))
                return -EINVAL;

        geneveh = geneve_hdr(skb);
        if (geneveh->ver != GENEVE_VER)
                return -EINVAL;

        if (geneveh->proto_type != htons(ETH_P_TEB))
                return -EINVAL;

        gs = rcu_dereference_sk_user_data(sk);
        if (!gs)
                return -ENOENT;

        if (geneve_get_sk_family(gs) == AF_INET) {
                struct iphdr *iph = ip_hdr(skb);
                __be32 addr4 = 0;

                if (!gs->collect_md) {
                        vni = geneve_hdr(skb)->vni;
                        addr4 = iph->daddr;
                }

                return geneve_lookup(gs, addr4, vni) ? 0 : -ENOENT;
        }

#if IS_ENABLED(CONFIG_IPV6)
        if (geneve_get_sk_family(gs) == AF_INET6) {
                struct ipv6hdr *ip6h = ipv6_hdr(skb);
                struct in6_addr addr6;

                memset(&addr6, 0, sizeof(struct in6_addr));

                if (!gs->collect_md) {
                        vni = geneve_hdr(skb)->vni;
                        addr6 = ip6h->daddr;
                }

                return geneve6_lookup(gs, addr6, vni) ? 0 : -ENOENT;
        }
#endif

        return -EPFNOSUPPORT;
}

static struct socket *geneve_create_sock(struct net *net, bool ipv6,
                                         __be16 port, bool ipv6_rx_csum)
{
        struct socket *sock;
        struct udp_port_cfg udp_conf;
        int err;

        memset(&udp_conf, 0, sizeof(udp_conf));

        if (ipv6) {
                udp_conf.family = AF_INET6;
                udp_conf.ipv6_v6only = 1;
                udp_conf.use_udp6_rx_checksums = ipv6_rx_csum;
        } else {
                udp_conf.family = AF_INET;
                udp_conf.local_ip.s_addr = htonl(INADDR_ANY);
        }

        udp_conf.local_udp_port = port;

        /* Open UDP socket */
        err = udp_sock_create(net, &udp_conf, &sock);
        if (err < 0)
                return ERR_PTR(err);

        udp_allow_gso(sock->sk);
        return sock;
}

static int geneve_hlen(struct genevehdr *gh)
{
        return sizeof(*gh) + gh->opt_len * 4;
}

static struct sk_buff *geneve_gro_receive(struct sock *sk,
                                          struct list_head *head,
                                          struct sk_buff *skb)
{
        struct sk_buff *pp = NULL;
        struct sk_buff *p;
        struct genevehdr *gh, *gh2;
        unsigned int hlen, gh_len, off_gnv;
        const struct packet_offload *ptype;
        __be16 type;
        int flush = 1;

        off_gnv = skb_gro_offset(skb);
        hlen = off_gnv + sizeof(*gh);
        gh = skb_gro_header_fast(skb, off_gnv);
        if (skb_gro_header_hard(skb, hlen)) {
                gh = skb_gro_header_slow(skb, hlen, off_gnv);
                if (unlikely(!gh))
                        goto out;
        }

        if (gh->ver != GENEVE_VER || gh->oam)
                goto out;
        gh_len = geneve_hlen(gh);

        hlen = off_gnv + gh_len;
        if (skb_gro_header_hard(skb, hlen)) {
                gh = skb_gro_header_slow(skb, hlen, off_gnv);
                if (unlikely(!gh))
                        goto out;
        }

        list_for_each_entry(p, head, list) {
                if (!NAPI_GRO_CB(p)->same_flow)
                        continue;

                gh2 = (struct genevehdr *)(p->data + off_gnv);
                if (gh->opt_len != gh2->opt_len ||
                    memcmp(gh, gh2, gh_len)) {
                        NAPI_GRO_CB(p)->same_flow = 0;
                        continue;
                }
        }

        type = gh->proto_type;

        ptype = gro_find_receive_by_type(type);
        if (!ptype)
                goto out;

        skb_gro_pull(skb, gh_len);
        skb_gro_postpull_rcsum(skb, gh, gh_len);
        pp = call_gro_receive(ptype->callbacks.gro_receive, head, skb);
        flush = 0;

out:
        skb_gro_flush_final(skb, pp, flush);

        return pp;
}

static int geneve_gro_complete(struct sock *sk, struct sk_buff *skb,
                               int nhoff)
{
        struct genevehdr *gh;
        struct packet_offload *ptype;
        __be16 type;
        int gh_len;
        int err = -ENOSYS;

        gh = (struct genevehdr *)(skb->data + nhoff);
        gh_len = geneve_hlen(gh);
        type = gh->proto_type;

        ptype = gro_find_complete_by_type(type);
        if (ptype)
                err = ptype->callbacks.gro_complete(skb, nhoff + gh_len);

        skb_set_inner_mac_header(skb, nhoff + gh_len);

        return err;
}

/* Create new listen socket if needed */
static struct geneve_sock *geneve_socket_create(struct net *net, __be16 port,
                                                bool ipv6, bool ipv6_rx_csum)
{
        struct geneve_net *gn = net_generic(net, geneve_net_id);
        struct geneve_sock *gs;
        struct socket *sock;
        struct udp_tunnel_sock_cfg tunnel_cfg;
        int h;

        gs = kzalloc(sizeof(*gs), GFP_KERNEL);
        if (!gs)
                return ERR_PTR(-ENOMEM);

        sock = geneve_create_sock(net, ipv6, port, ipv6_rx_csum);
        if (IS_ERR(sock)) {
                kfree(gs);
                return ERR_CAST(sock);
        }

        gs->sock = sock;
        gs->refcnt = 1;
        for (h = 0; h < VNI_HASH_SIZE; ++h)
                INIT_HLIST_HEAD(&gs->vni_list[h]);

        /* Initialize the geneve udp offloads structure */
        udp_tunnel_notify_add_rx_port(gs->sock, UDP_TUNNEL_TYPE_GENEVE);

        /* Mark socket as an encapsulation socket */
        memset(&tunnel_cfg, 0, sizeof(tunnel_cfg));
        tunnel_cfg.sk_user_data = gs;
        tunnel_cfg.encap_type = 1;
        tunnel_cfg.gro_receive = geneve_gro_receive;
        tunnel_cfg.gro_complete = geneve_gro_complete;
        tunnel_cfg.encap_rcv = geneve_udp_encap_recv;
        tunnel_cfg.encap_err_lookup = geneve_udp_encap_err_lookup;
        tunnel_cfg.encap_destroy = NULL;
        setup_udp_tunnel_sock(net, sock, &tunnel_cfg);
        list_add(&gs->list, &gn->sock_list);
        return gs;
}

static void __geneve_sock_release(struct geneve_sock *gs)
{
        if (!gs || --gs->refcnt)
                return;

        list_del(&gs->list);
        udp_tunnel_notify_del_rx_port(gs->sock, UDP_TUNNEL_TYPE_GENEVE);
        udp_tunnel_sock_release(gs->sock);
        kfree_rcu(gs, rcu);
}

static void geneve_sock_release(struct geneve_dev *geneve)
{
        struct geneve_sock *gs4 = rtnl_dereference(geneve->sock4);
#if IS_ENABLED(CONFIG_IPV6)
        struct geneve_sock *gs6 = rtnl_dereference(geneve->sock6);

        rcu_assign_pointer(geneve->sock6, NULL);
#endif

        rcu_assign_pointer(geneve->sock4, NULL);
        synchronize_net();

        __geneve_sock_release(gs4);
#if IS_ENABLED(CONFIG_IPV6)
        __geneve_sock_release(gs6);
#endif
}

static struct geneve_sock *geneve_find_sock(struct geneve_net *gn,
                                            sa_family_t family,
                                            __be16 dst_port)
{
        struct geneve_sock *gs;

        list_for_each_entry(gs, &gn->sock_list, list) {
                if (inet_sk(gs->sock->sk)->inet_sport == dst_port &&
                    geneve_get_sk_family(gs) == family) {
                        return gs;
                }
        }
        return NULL;
}

static int geneve_sock_add(struct geneve_dev *geneve, bool ipv6)
{
        struct net *net = geneve->net;
        struct geneve_net *gn = net_generic(net, geneve_net_id);
        struct geneve_dev_node *node;
        struct geneve_sock *gs;
        __u8 vni[3];
        __u32 hash;

        gs = geneve_find_sock(gn, ipv6 ? AF_INET6 : AF_INET, geneve->cfg.info.key.tp_dst);
        if (gs) {
                gs->refcnt++;
                goto out;
        }

        gs = geneve_socket_create(net, geneve->cfg.info.key.tp_dst, ipv6,
                                  geneve->cfg.use_udp6_rx_checksums);
        if (IS_ERR(gs))
                return PTR_ERR(gs);

out:
        gs->collect_md = geneve->cfg.collect_md;
#if IS_ENABLED(CONFIG_IPV6)
        if (ipv6) {
                rcu_assign_pointer(geneve->sock6, gs);
                node = &geneve->hlist6;
        } else
#endif
        {
                rcu_assign_pointer(geneve->sock4, gs);
                node = &geneve->hlist4;
        }
        node->geneve = geneve;

        tunnel_id_to_vni(geneve->cfg.info.key.tun_id, vni);
        hash = geneve_net_vni_hash(vni);
        hlist_add_head_rcu(&node->hlist, &gs->vni_list[hash]);
        return 0;
}

static int geneve_open(struct net_device *dev)
{
        struct geneve_dev *geneve = netdev_priv(dev);
        bool metadata = geneve->cfg.collect_md;
        bool ipv4, ipv6;
        int ret = 0;

        ipv6 = geneve->cfg.info.mode & IP_TUNNEL_INFO_IPV6 || metadata;
        ipv4 = !ipv6 || metadata;
#if IS_ENABLED(CONFIG_IPV6)
        if (ipv6) {
                ret = geneve_sock_add(geneve, true);
                if (ret < 0 && ret != -EAFNOSUPPORT)
                        ipv4 = false;
        }
#endif
        if (ipv4)
                ret = geneve_sock_add(geneve, false);
        if (ret < 0)
                geneve_sock_release(geneve);

        return ret;
}

static int geneve_stop(struct net_device *dev)
{
        struct geneve_dev *geneve = netdev_priv(dev);

        hlist_del_init_rcu(&geneve->hlist4.hlist);
#if IS_ENABLED(CONFIG_IPV6)
        hlist_del_init_rcu(&geneve->hlist6.hlist);
#endif
        geneve_sock_release(geneve);
        return 0;
}

static void geneve_build_header(struct genevehdr *geneveh,
                                const struct ip_tunnel_info *info,
                                __be16 inner_proto)
{
        geneveh->ver = GENEVE_VER;
        geneveh->opt_len = info->options_len / 4;
        geneveh->oam = !!(info->key.tun_flags & TUNNEL_OAM);
        geneveh->critical = !!(info->key.tun_flags & TUNNEL_CRIT_OPT);
        geneveh->rsvd1 = 0;
        tunnel_id_to_vni(info->key.tun_id, geneveh->vni);
        geneveh->proto_type = inner_proto;
        geneveh->rsvd2 = 0;

        if (info->key.tun_flags & TUNNEL_GENEVE_OPT)
                ip_tunnel_info_opts_get(geneveh->options, info);
}

static int geneve_build_skb(struct dst_entry *dst, struct sk_buff *skb,
                            const struct ip_tunnel_info *info,
                            bool xnet, int ip_hdr_len,
                            bool inner_proto_inherit)
{
        bool udp_sum = !!(info->key.tun_flags & TUNNEL_CSUM);
        struct genevehdr *gnvh;
        __be16 inner_proto;
        int min_headroom;
        int err;

        skb_reset_mac_header(skb);
        skb_scrub_packet(skb, xnet);

        min_headroom = LL_RESERVED_SPACE(dst->dev) + dst->header_len +
                       GENEVE_BASE_HLEN + info->options_len + ip_hdr_len;
        err = skb_cow_head(skb, min_headroom);
        if (unlikely(err))
                goto free_dst;

        err = udp_tunnel_handle_offloads(skb, udp_sum);
        if (err)
                goto free_dst;

        gnvh = __skb_push(skb, sizeof(*gnvh) + info->options_len);
        inner_proto = inner_proto_inherit ? skb->protocol : htons(ETH_P_TEB);
        geneve_build_header(gnvh, info, inner_proto);
        skb_set_inner_protocol(skb, inner_proto);
        return 0;

free_dst:
        dst_release(dst);
        return err;
}

static struct rtable *geneve_get_v4_rt(struct sk_buff *skb,
                                       struct net_device *dev,
                                       struct geneve_sock *gs4,
                                       struct flowi4 *fl4,
                                       const struct ip_tunnel_info *info,
                                       __be16 dport, __be16 sport,
                                       __u8 *full_tos)
{
        bool use_cache = ip_tunnel_dst_cache_usable(skb, info);
        struct geneve_dev *geneve = netdev_priv(dev);
        struct dst_cache *dst_cache;
        struct rtable *rt = NULL;
        __u8 tos;

        if (!gs4)
                return ERR_PTR(-EIO);

        memset(fl4, 0, sizeof(*fl4));
        fl4->flowi4_mark = skb->mark;
        fl4->flowi4_proto = IPPROTO_UDP;
        fl4->daddr = info->key.u.ipv4.dst;
        fl4->saddr = info->key.u.ipv4.src;
        fl4->fl4_dport = dport;
        fl4->fl4_sport = sport;

        tos = info->key.tos;
        if ((tos == 1) && !geneve->cfg.collect_md) {
                tos = ip_tunnel_get_dsfield(ip_hdr(skb), skb);
                use_cache = false;
        }
        fl4->flowi4_tos = RT_TOS(tos);
        if (full_tos)
                *full_tos = tos;

        dst_cache = (struct dst_cache *)&info->dst_cache;
        if (use_cache) {
                rt = dst_cache_get_ip4(dst_cache, &fl4->saddr);
                if (rt)
                        return rt;
        }
        rt = ip_route_output_key(geneve->net, fl4);
        if (IS_ERR(rt)) {
                netdev_dbg(dev, "no route to %pI4\n", &fl4->daddr);
                return ERR_PTR(-ENETUNREACH);
        }
        if (rt->dst.dev == dev) { /* is this necessary? */
                netdev_dbg(dev, "circular route to %pI4\n", &fl4->daddr);
                ip_rt_put(rt);
                return ERR_PTR(-ELOOP);
        }
        if (use_cache)
                dst_cache_set_ip4(dst_cache, &rt->dst, fl4->saddr);
        return rt;
}

#if IS_ENABLED(CONFIG_IPV6)
static struct dst_entry *geneve_get_v6_dst(struct sk_buff *skb,
                                           struct net_device *dev,
                                           struct geneve_sock *gs6,
                                           struct flowi6 *fl6,
                                           const struct ip_tunnel_info *info,
                                           __be16 dport, __be16 sport)
{
        bool use_cache = ip_tunnel_dst_cache_usable(skb, info);
        struct geneve_dev *geneve = netdev_priv(dev);
        struct dst_entry *dst = NULL;
        struct dst_cache *dst_cache;
        __u8 prio;

        if (!gs6)
                return ERR_PTR(-EIO);

        memset(fl6, 0, sizeof(*fl6));
        fl6->flowi6_mark = skb->mark;
        fl6->flowi6_proto = IPPROTO_UDP;
        fl6->daddr = info->key.u.ipv6.dst;
        fl6->saddr = info->key.u.ipv6.src;
        fl6->fl6_dport = dport;
        fl6->fl6_sport = sport;

        prio = info->key.tos;
        if ((prio == 1) && !geneve->cfg.collect_md) {
                prio = ip_tunnel_get_dsfield(ip_hdr(skb), skb);
                use_cache = false;
        }

        fl6->flowlabel = ip6_make_flowinfo(prio, info->key.label);
        dst_cache = (struct dst_cache *)&info->dst_cache;
        if (use_cache) {
                dst = dst_cache_get_ip6(dst_cache, &fl6->saddr);
                if (dst)
                        return dst;
        }
        dst = ipv6_stub->ipv6_dst_lookup_flow(geneve->net, gs6->sock->sk, fl6,
                                              NULL);
        if (IS_ERR(dst)) {
                netdev_dbg(dev, "no route to %pI6\n", &fl6->daddr);
                return ERR_PTR(-ENETUNREACH);
        }
        if (dst->dev == dev) { /* is this necessary? */
                netdev_dbg(dev, "circular route to %pI6\n", &fl6->daddr);
                dst_release(dst);
                return ERR_PTR(-ELOOP);
        }

        if (use_cache)
                dst_cache_set_ip6(dst_cache, dst, &fl6->saddr);
        return dst;
}
#endif

static int geneve_xmit_skb(struct sk_buff *skb, struct net_device *dev,
                           struct geneve_dev *geneve,
                           const struct ip_tunnel_info *info)
{
        bool inner_proto_inherit = geneve->cfg.inner_proto_inherit;
        bool xnet = !net_eq(geneve->net, dev_net(geneve->dev));
        struct geneve_sock *gs4 = rcu_dereference(geneve->sock4);
        const struct ip_tunnel_key *key = &info->key;
        struct rtable *rt;
        struct flowi4 fl4;
        __u8 full_tos;
        __u8 tos, ttl;
        __be16 df = 0;
        __be16 sport;
        int err;

        if (!skb_vlan_inet_prepare(skb, inner_proto_inherit))
                return -EINVAL;

        sport = udp_flow_src_port(geneve->net, skb, 1, USHRT_MAX, true);
        rt = geneve_get_v4_rt(skb, dev, gs4, &fl4, info,
                              geneve->cfg.info.key.tp_dst, sport, &full_tos);
        if (IS_ERR(rt))
                return PTR_ERR(rt);

        err = skb_tunnel_check_pmtu(skb, &rt->dst,
                                    GENEVE_IPV4_HLEN + info->options_len,
                                    netif_is_any_bridge_port(dev));
        if (err < 0) {
                dst_release(&rt->dst);
                return err;
        } else if (err) {
                struct ip_tunnel_info *info;

                info = skb_tunnel_info(skb);
                if (info) {
                        struct ip_tunnel_info *unclone;

                        unclone = skb_tunnel_info_unclone(skb);
                        if (unlikely(!unclone)) {
                                dst_release(&rt->dst);
                                return -ENOMEM;
                        }

                        unclone->key.u.ipv4.dst = fl4.saddr;
                        unclone->key.u.ipv4.src = fl4.daddr;
                }

                if (!pskb_may_pull(skb, ETH_HLEN)) {
                        dst_release(&rt->dst);
                        return -EINVAL;
                }

                skb->protocol = eth_type_trans(skb, geneve->dev);
                netif_rx(skb);
                dst_release(&rt->dst);
                return -EMSGSIZE;
        }

        if (geneve->cfg.collect_md) {
                tos = ip_tunnel_ecn_encap(key->tos, ip_hdr(skb), skb);
                ttl = key->ttl;

                df = key->tun_flags & TUNNEL_DONT_FRAGMENT ? htons(IP_DF) : 0;
        } else {
                tos = ip_tunnel_ecn_encap(full_tos, ip_hdr(skb), skb);
                if (geneve->cfg.ttl_inherit)
                        ttl = ip_tunnel_get_ttl(ip_hdr(skb), skb);
                else
                        ttl = key->ttl;
                ttl = ttl ? : ip4_dst_hoplimit(&rt->dst);

                if (geneve->cfg.df == GENEVE_DF_SET) {
                        df = htons(IP_DF);
                } else if (geneve->cfg.df == GENEVE_DF_INHERIT) {
                        struct ethhdr *eth = skb_eth_hdr(skb);

                        if (ntohs(eth->h_proto) == ETH_P_IPV6) {
                                df = htons(IP_DF);
                        } else if (ntohs(eth->h_proto) == ETH_P_IP) {
                                struct iphdr *iph = ip_hdr(skb);

                                if (iph->frag_off & htons(IP_DF))
                                        df = htons(IP_DF);
                        }
                }
        }

        err = geneve_build_skb(&rt->dst, skb, info, xnet, sizeof(struct iphdr),
                               inner_proto_inherit);
        if (unlikely(err))
                return err;

        udp_tunnel_xmit_skb(rt, gs4->sock->sk, skb, fl4.saddr, fl4.daddr,
                            tos, ttl, df, sport, geneve->cfg.info.key.tp_dst,
                            !net_eq(geneve->net, dev_net(geneve->dev)),
                            !(info->key.tun_flags & TUNNEL_CSUM));
        return 0;
}

#if IS_ENABLED(CONFIG_IPV6)
static int geneve6_xmit_skb(struct sk_buff *skb, struct net_device *dev,
                            struct geneve_dev *geneve,
                            const struct ip_tunnel_info *info)
{
        bool inner_proto_inherit = geneve->cfg.inner_proto_inherit;
        bool xnet = !net_eq(geneve->net, dev_net(geneve->dev));
        struct geneve_sock *gs6 = rcu_dereference(geneve->sock6);
        const struct ip_tunnel_key *key = &info->key;
        struct dst_entry *dst = NULL;
        struct flowi6 fl6;
        __u8 prio, ttl;
        __be16 sport;
        int err;

        if (!skb_vlan_inet_prepare(skb, inner_proto_inherit))
                return -EINVAL;

        sport = udp_flow_src_port(geneve->net, skb, 1, USHRT_MAX, true);
        dst = geneve_get_v6_dst(skb, dev, gs6, &fl6, info,
                                geneve->cfg.info.key.tp_dst, sport);
        if (IS_ERR(dst))
                return PTR_ERR(dst);

        err = skb_tunnel_check_pmtu(skb, dst,
                                    GENEVE_IPV6_HLEN + info->options_len,
                                    netif_is_any_bridge_port(dev));
        if (err < 0) {
                dst_release(dst);
                return err;
        } else if (err) {
                struct ip_tunnel_info *info = skb_tunnel_info(skb);

                if (info) {
                        struct ip_tunnel_info *unclone;

                        unclone = skb_tunnel_info_unclone(skb);
                        if (unlikely(!unclone)) {
                                dst_release(dst);
                                return -ENOMEM;
                        }

                        unclone->key.u.ipv6.dst = fl6.saddr;
                        unclone->key.u.ipv6.src = fl6.daddr;
                }

                if (!pskb_may_pull(skb, ETH_HLEN)) {
                        dst_release(dst);
                        return -EINVAL;
                }

                skb->protocol = eth_type_trans(skb, geneve->dev);
                netif_rx(skb);
                dst_release(dst);
                return -EMSGSIZE;
        }

        if (geneve->cfg.collect_md) {
                prio = ip_tunnel_ecn_encap(key->tos, ip_hdr(skb), skb);
                ttl = key->ttl;
        } else {
                prio = ip_tunnel_ecn_encap(ip6_tclass(fl6.flowlabel),
                                           ip_hdr(skb), skb);
                if (geneve->cfg.ttl_inherit)
                        ttl = ip_tunnel_get_ttl(ip_hdr(skb), skb);
                else
                        ttl = key->ttl;
                ttl = ttl ? : ip6_dst_hoplimit(dst);
        }
        err = geneve_build_skb(dst, skb, info, xnet, sizeof(struct ipv6hdr),
                               inner_proto_inherit);
        if (unlikely(err))
                return err;

        udp_tunnel6_xmit_skb(dst, gs6->sock->sk, skb, dev,
                             &fl6.saddr, &fl6.daddr, prio, ttl,
                             info->key.label, sport, geneve->cfg.info.key.tp_dst,
                             !(info->key.tun_flags & TUNNEL_CSUM));
        return 0;
}
#endif

static netdev_tx_t geneve_xmit(struct sk_buff *skb, struct net_device *dev)
{
        struct geneve_dev *geneve = netdev_priv(dev);
        struct ip_tunnel_info *info = NULL;
        int err;

        if (geneve->cfg.collect_md) {
                info = skb_tunnel_info(skb);
                if (unlikely(!info || !(info->mode & IP_TUNNEL_INFO_TX))) {
                        netdev_dbg(dev, "no tunnel metadata\n");
                        dev_kfree_skb(skb);
                        dev->stats.tx_dropped++;
                        return NETDEV_TX_OK;
                }
        } else {
                info = &geneve->cfg.info;
        }

        rcu_read_lock();
#if IS_ENABLED(CONFIG_IPV6)
        if (info->mode & IP_TUNNEL_INFO_IPV6)
                err = geneve6_xmit_skb(skb, dev, geneve, info);
        else
#endif
                err = geneve_xmit_skb(skb, dev, geneve, info);
        rcu_read_unlock();

        if (likely(!err))
                return NETDEV_TX_OK;

        if (err != -EMSGSIZE)
                dev_kfree_skb(skb);

        if (err == -ELOOP)
                dev->stats.collisions++;
        else if (err == -ENETUNREACH)
                dev->stats.tx_carrier_errors++;

        dev->stats.tx_errors++;
        return NETDEV_TX_OK;
}

static int geneve_change_mtu(struct net_device *dev, int new_mtu)
{
        if (new_mtu > dev->max_mtu)
                new_mtu = dev->max_mtu;
        else if (new_mtu < dev->min_mtu)
                new_mtu = dev->min_mtu;

        dev->mtu = new_mtu;
        return 0;
}

static int geneve_fill_metadata_dst(struct net_device *dev, struct sk_buff *skb)
{
        struct ip_tunnel_info *info = skb_tunnel_info(skb);
        struct geneve_dev *geneve = netdev_priv(dev);
        __be16 sport;

        if (ip_tunnel_info_af(info) == AF_INET) {
                struct rtable *rt;
                struct flowi4 fl4;

                struct geneve_sock *gs4 = rcu_dereference(geneve->sock4);
                sport = udp_flow_src_port(geneve->net, skb,
                                          1, USHRT_MAX, true);

                rt = geneve_get_v4_rt(skb, dev, gs4, &fl4, info,
                                      geneve->cfg.info.key.tp_dst, sport, NULL);
                if (IS_ERR(rt))
                        return PTR_ERR(rt);

                ip_rt_put(rt);
                info->key.u.ipv4.src = fl4.saddr;
#if IS_ENABLED(CONFIG_IPV6)
        } else if (ip_tunnel_info_af(info) == AF_INET6) {
                struct dst_entry *dst;
                struct flowi6 fl6;

                struct geneve_sock *gs6 = rcu_dereference(geneve->sock6);
                sport = udp_flow_src_port(geneve->net, skb,
                                          1, USHRT_MAX, true);

                dst = geneve_get_v6_dst(skb, dev, gs6, &fl6, info,
                                        geneve->cfg.info.key.tp_dst, sport);
                if (IS_ERR(dst))
                        return PTR_ERR(dst);

                dst_release(dst);
                info->key.u.ipv6.src = fl6.saddr;
#endif
        } else {
                return -EINVAL;
        }

        info->key.tp_src = sport;
        info->key.tp_dst = geneve->cfg.info.key.tp_dst;
        return 0;
}

static const struct net_device_ops geneve_netdev_ops = {
        .ndo_init                = geneve_init,
        .ndo_uninit                = geneve_uninit,
        .ndo_open                = geneve_open,
        .ndo_stop                = geneve_stop,
        .ndo_start_xmit                = geneve_xmit,
        .ndo_get_stats64        = dev_get_tstats64,
        .ndo_change_mtu                = geneve_change_mtu,
        .ndo_validate_addr        = eth_validate_addr,
        .ndo_set_mac_address        = eth_mac_addr,
        .ndo_fill_metadata_dst        = geneve_fill_metadata_dst,
};

static void geneve_get_drvinfo(struct net_device *dev,
                               struct ethtool_drvinfo *drvinfo)
{
        strlcpy(drvinfo->version, GENEVE_NETDEV_VER, sizeof(drvinfo->version));
        strlcpy(drvinfo->driver, "geneve", sizeof(drvinfo->driver));
}

static const struct ethtool_ops geneve_ethtool_ops = {
        .get_drvinfo        = geneve_get_drvinfo,
        .get_link        = ethtool_op_get_link,
};

/* Info for udev, that this is a virtual tunnel endpoint */
static struct device_type geneve_type = {
        .name = "geneve",
};

/* Calls the ndo_udp_tunnel_add of the caller in order to
 * supply the listening GENEVE udp ports. Callers are expected
 * to implement the ndo_udp_tunnel_add.
 */
static void geneve_offload_rx_ports(struct net_device *dev, bool push)
{
        struct net *net = dev_net(dev);
        struct geneve_net *gn = net_generic(net, geneve_net_id);
        struct geneve_sock *gs;

        rcu_read_lock();
        list_for_each_entry_rcu(gs, &gn->sock_list, list) {
                if (push) {
                        udp_tunnel_push_rx_port(dev, gs->sock,
                                                UDP_TUNNEL_TYPE_GENEVE);
                } else {
                        udp_tunnel_drop_rx_port(dev, gs->sock,
                                                UDP_TUNNEL_TYPE_GENEVE);
                }
        }
        rcu_read_unlock();
}

/* Initialize the device structure. */
static void geneve_setup(struct net_device *dev)
{
        ether_setup(dev);

        dev->netdev_ops = &geneve_netdev_ops;
        dev->ethtool_ops = &geneve_ethtool_ops;
        dev->needs_free_netdev = true;

        SET_NETDEV_DEVTYPE(dev, &geneve_type);

        dev->features    |= NETIF_F_LLTX;
        dev->features    |= NETIF_F_SG | NETIF_F_HW_CSUM | NETIF_F_FRAGLIST;
        dev->features    |= NETIF_F_RXCSUM;
        dev->features    |= NETIF_F_GSO_SOFTWARE;

        dev->hw_features |= NETIF_F_SG | NETIF_F_HW_CSUM | NETIF_F_FRAGLIST;
        dev->hw_features |= NETIF_F_RXCSUM;
        dev->hw_features |= NETIF_F_GSO_SOFTWARE;

        /* MTU range: 68 - (something less than 65535) */
        dev->min_mtu = ETH_MIN_MTU;
        /* The max_mtu calculation does not take account of GENEVE
         * options, to avoid excluding potentially valid
         * configurations. This will be further reduced by IPvX hdr size.
         */
        dev->max_mtu = IP_MAX_MTU - GENEVE_BASE_HLEN - dev->hard_header_len;

        netif_keep_dst(dev);
        dev->priv_flags &= ~IFF_TX_SKB_SHARING;
        dev->priv_flags |= IFF_LIVE_ADDR_CHANGE | IFF_NO_QUEUE;
        eth_hw_addr_random(dev);
}

static const struct nla_policy geneve_policy[IFLA_GENEVE_MAX + 1] = {
        [IFLA_GENEVE_UNSPEC]                = { .strict_start_type = IFLA_GENEVE_INNER_PROTO_INHERIT },
        [IFLA_GENEVE_ID]                = { .type = NLA_U32 },
        [IFLA_GENEVE_REMOTE]                = { .len = sizeof_field(struct iphdr, daddr) },
        [IFLA_GENEVE_REMOTE6]                = { .len = sizeof(struct in6_addr) },
        [IFLA_GENEVE_TTL]                = { .type = NLA_U8 },
        [IFLA_GENEVE_TOS]                = { .type = NLA_U8 },
        [IFLA_GENEVE_LABEL]                = { .type = NLA_U32 },
        [IFLA_GENEVE_PORT]                = { .type = NLA_U16 },
        [IFLA_GENEVE_COLLECT_METADATA]        = { .type = NLA_FLAG },
        [IFLA_GENEVE_UDP_CSUM]                = { .type = NLA_U8 },
        [IFLA_GENEVE_UDP_ZERO_CSUM6_TX]        = { .type = NLA_U8 },
        [IFLA_GENEVE_UDP_ZERO_CSUM6_RX]        = { .type = NLA_U8 },
        [IFLA_GENEVE_TTL_INHERIT]        = { .type = NLA_U8 },
        [IFLA_GENEVE_DF]                = { .type = NLA_U8 },
        [IFLA_GENEVE_INNER_PROTO_INHERIT]        = { .type = NLA_FLAG },
};

static int geneve_validate(struct nlattr *tb[], struct nlattr *data[],
                           struct netlink_ext_ack *extack)
{
        if (tb[IFLA_ADDRESS]) {
                if (nla_len(tb[IFLA_ADDRESS]) != ETH_ALEN) {
                        NL_SET_ERR_MSG_ATTR(extack, tb[IFLA_ADDRESS],
                                            "Provided link layer address is not Ethernet");
                        return -EINVAL;
                }

                if (!is_valid_ether_addr(nla_data(tb[IFLA_ADDRESS]))) {
                        NL_SET_ERR_MSG_ATTR(extack, tb[IFLA_ADDRESS],
                                            "Provided Ethernet address is not unicast");
                        return -EADDRNOTAVAIL;
                }
        }

        if (!data) {
                NL_SET_ERR_MSG(extack,
                               "Not enough attributes provided to perform the operation");
                return -EINVAL;
        }

        if (data[IFLA_GENEVE_ID]) {
                __u32 vni =  nla_get_u32(data[IFLA_GENEVE_ID]);

                if (vni >= GENEVE_N_VID) {
                        NL_SET_ERR_MSG_ATTR(extack, data[IFLA_GENEVE_ID],
                                            "Geneve ID must be lower than 16777216");
                        return -ERANGE;
                }
        }

        if (data[IFLA_GENEVE_DF]) {
                enum ifla_geneve_df df = nla_get_u8(data[IFLA_GENEVE_DF]);

                if (df < 0 || df > GENEVE_DF_MAX) {
                        NL_SET_ERR_MSG_ATTR(extack, data[IFLA_GENEVE_DF],
                                            "Invalid DF attribute");
                        return -EINVAL;
                }
        }

        return 0;
}

static struct geneve_dev *geneve_find_dev(struct geneve_net *gn,
                                          const struct ip_tunnel_info *info,
                                          bool *tun_on_same_port,
                                          bool *tun_collect_md)
{
        struct geneve_dev *geneve, *t = NULL;

        *tun_on_same_port = false;
        *tun_collect_md = false;
        list_for_each_entry(geneve, &gn->geneve_list, next) {
                if (info->key.tp_dst == geneve->cfg.info.key.tp_dst) {
                        *tun_collect_md = geneve->cfg.collect_md;
                        *tun_on_same_port = true;
                }
                if (info->key.tun_id == geneve->cfg.info.key.tun_id &&
                    info->key.tp_dst == geneve->cfg.info.key.tp_dst &&
                    !memcmp(&info->key.u, &geneve->cfg.info.key.u, sizeof(info->key.u)))
                        t = geneve;
        }
        return t;
}

static bool is_tnl_info_zero(const struct ip_tunnel_info *info)
{
        return !(info->key.tun_id || info->key.tun_flags || info->key.tos ||
                 info->key.ttl || info->key.label || info->key.tp_src ||
                 memchr_inv(&info->key.u, 0, sizeof(info->key.u)));
}

static bool geneve_dst_addr_equal(struct ip_tunnel_info *a,
                                  struct ip_tunnel_info *b)
{
        if (ip_tunnel_info_af(a) == AF_INET)
                return a->key.u.ipv4.dst == b->key.u.ipv4.dst;
        else
                return ipv6_addr_equal(&a->key.u.ipv6.dst, &b->key.u.ipv6.dst);
}

static int geneve_configure(struct net *net, struct net_device *dev,
                            struct netlink_ext_ack *extack,
                            const struct geneve_config *cfg)
{
        struct geneve_net *gn = net_generic(net, geneve_net_id);
        struct geneve_dev *t, *geneve = netdev_priv(dev);
        const struct ip_tunnel_info *info = &cfg->info;
        bool tun_collect_md, tun_on_same_port;
        int err, encap_len;

        if (cfg->collect_md && !is_tnl_info_zero(info)) {
                NL_SET_ERR_MSG(extack,
                               "Device is externally controlled, so attributes (VNI, Port, and so on) must not be specified");
                return -EINVAL;
        }

        geneve->net = net;
        geneve->dev = dev;

        t = geneve_find_dev(gn, info, &tun_on_same_port, &tun_collect_md);
        if (t)
                return -EBUSY;

        /* make enough headroom for basic scenario */
        encap_len = GENEVE_BASE_HLEN + ETH_HLEN;
        if (!cfg->collect_md && ip_tunnel_info_af(info) == AF_INET) {
                encap_len += sizeof(struct iphdr);
                dev->max_mtu -= sizeof(struct iphdr);
        } else {
                encap_len += sizeof(struct ipv6hdr);
                dev->max_mtu -= sizeof(struct ipv6hdr);
        }
        dev->needed_headroom = encap_len + ETH_HLEN;

        if (cfg->collect_md) {
                if (tun_on_same_port) {
                        NL_SET_ERR_MSG(extack,
                                       "There can be only one externally controlled device on a destination port");
                        return -EPERM;
                }
        } else {
                if (tun_collect_md) {
                        NL_SET_ERR_MSG(extack,
                                       "There already exists an externally controlled device on this destination port");
                        return -EPERM;
                }
        }

        dst_cache_reset(&geneve->cfg.info.dst_cache);
        memcpy(&geneve->cfg, cfg, sizeof(*cfg));

        if (geneve->cfg.inner_proto_inherit) {
                dev->header_ops = NULL;
                dev->type = ARPHRD_NONE;
                dev->hard_header_len = 0;
                dev->addr_len = 0;
                dev->flags = IFF_NOARP;
        }

        err = register_netdevice(dev);
        if (err)
                return err;

        list_add(&geneve->next, &gn->geneve_list);
        return 0;
}

static void init_tnl_info(struct ip_tunnel_info *info, __u16 dst_port)
{
        memset(info, 0, sizeof(*info));
        info->key.tp_dst = htons(dst_port);
}

static int geneve_nl2info(struct nlattr *tb[], struct nlattr *data[],
                          struct netlink_ext_ack *extack,
                          struct geneve_config *cfg, bool changelink)
{
        struct ip_tunnel_info *info = &cfg->info;
        int attrtype;

        if (data[IFLA_GENEVE_REMOTE] && data[IFLA_GENEVE_REMOTE6]) {
                NL_SET_ERR_MSG(extack,
                               "Cannot specify both IPv4 and IPv6 Remote addresses");
                return -EINVAL;
        }

        if (data[IFLA_GENEVE_REMOTE]) {
                if (changelink && (ip_tunnel_info_af(info) == AF_INET6)) {
                        attrtype = IFLA_GENEVE_REMOTE;
                        goto change_notsup;
                }

                info->key.u.ipv4.dst =
                        nla_get_in_addr(data[IFLA_GENEVE_REMOTE]);

                if (ipv4_is_multicast(info->key.u.ipv4.dst)) {
                        NL_SET_ERR_MSG_ATTR(extack, data[IFLA_GENEVE_REMOTE],
                                            "Remote IPv4 address cannot be Multicast");
                        return -EINVAL;
                }
        }

        if (data[IFLA_GENEVE_REMOTE6]) {
#if IS_ENABLED(CONFIG_IPV6)
                if (changelink && (ip_tunnel_info_af(info) == AF_INET)) {
                        attrtype = IFLA_GENEVE_REMOTE6;
                        goto change_notsup;
                }

                info->mode = IP_TUNNEL_INFO_IPV6;
                info->key.u.ipv6.dst =
                        nla_get_in6_addr(data[IFLA_GENEVE_REMOTE6]);

                if (ipv6_addr_type(&info->key.u.ipv6.dst) &
                    IPV6_ADDR_LINKLOCAL) {
                        NL_SET_ERR_MSG_ATTR(extack, data[IFLA_GENEVE_REMOTE6],
                                            "Remote IPv6 address cannot be link-local");
                        return -EINVAL;
                }
                if (ipv6_addr_is_multicast(&info->key.u.ipv6.dst)) {
                        NL_SET_ERR_MSG_ATTR(extack, data[IFLA_GENEVE_REMOTE6],
                                            "Remote IPv6 address cannot be Multicast");
                        return -EINVAL;
                }
                info->key.tun_flags |= TUNNEL_CSUM;
                cfg->use_udp6_rx_checksums = true;
#else
                NL_SET_ERR_MSG_ATTR(extack, data[IFLA_GENEVE_REMOTE6],
                                    "IPv6 support not enabled in the kernel");
                return -EPFNOSUPPORT;
#endif
        }

        if (data[IFLA_GENEVE_ID]) {
                __u32 vni;
                __u8 tvni[3];
                __be64 tunid;

                vni = nla_get_u32(data[IFLA_GENEVE_ID]);
                tvni[0] = (vni & 0x00ff0000) >> 16;
                tvni[1] = (vni & 0x0000ff00) >> 8;
                tvni[2] =  vni & 0x000000ff;

                tunid = vni_to_tunnel_id(tvni);
                if (changelink && (tunid != info->key.tun_id)) {
                        attrtype = IFLA_GENEVE_ID;
                        goto change_notsup;
                }
                info->key.tun_id = tunid;
        }

        if (data[IFLA_GENEVE_TTL_INHERIT]) {
                if (nla_get_u8(data[IFLA_GENEVE_TTL_INHERIT]))
                        cfg->ttl_inherit = true;
                else
                        cfg->ttl_inherit = false;
        } else if (data[IFLA_GENEVE_TTL]) {
                info->key.ttl = nla_get_u8(data[IFLA_GENEVE_TTL]);
                cfg->ttl_inherit = false;
        }

        if (data[IFLA_GENEVE_TOS])
                info->key.tos = nla_get_u8(data[IFLA_GENEVE_TOS]);

        if (data[IFLA_GENEVE_DF])
                cfg->df = nla_get_u8(data[IFLA_GENEVE_DF]);

        if (data[IFLA_GENEVE_LABEL]) {
                info->key.label = nla_get_be32(data[IFLA_GENEVE_LABEL]) &
                                  IPV6_FLOWLABEL_MASK;
                if (info->key.label && (!(info->mode & IP_TUNNEL_INFO_IPV6))) {
                        NL_SET_ERR_MSG_ATTR(extack, data[IFLA_GENEVE_LABEL],
                                            "Label attribute only applies for IPv6 Geneve devices");
                        return -EINVAL;
                }
        }

        if (data[IFLA_GENEVE_PORT]) {
                if (changelink) {
                        attrtype = IFLA_GENEVE_PORT;
                        goto change_notsup;
                }
                info->key.tp_dst = nla_get_be16(data[IFLA_GENEVE_PORT]);
        }

        if (data[IFLA_GENEVE_COLLECT_METADATA]) {
                if (changelink) {
                        attrtype = IFLA_GENEVE_COLLECT_METADATA;
                        goto change_notsup;
                }
                cfg->collect_md = true;
        }

        if (data[IFLA_GENEVE_UDP_CSUM]) {
                if (changelink) {
                        attrtype = IFLA_GENEVE_UDP_CSUM;
                        goto change_notsup;
                }
                if (nla_get_u8(data[IFLA_GENEVE_UDP_CSUM]))
                        info->key.tun_flags |= TUNNEL_CSUM;
        }

        if (data[IFLA_GENEVE_UDP_ZERO_CSUM6_TX]) {
#if IS_ENABLED(CONFIG_IPV6)
                if (changelink) {
                        attrtype = IFLA_GENEVE_UDP_ZERO_CSUM6_TX;
                        goto change_notsup;
                }
                if (nla_get_u8(data[IFLA_GENEVE_UDP_ZERO_CSUM6_TX]))
                        info->key.tun_flags &= ~TUNNEL_CSUM;
#else
                NL_SET_ERR_MSG_ATTR(extack, data[IFLA_GENEVE_UDP_ZERO_CSUM6_TX],
                                    "IPv6 support not enabled in the kernel");
                return -EPFNOSUPPORT;
#endif
        }

        if (data[IFLA_GENEVE_UDP_ZERO_CSUM6_RX]) {
#if IS_ENABLED(CONFIG_IPV6)
                if (changelink) {
                        attrtype = IFLA_GENEVE_UDP_ZERO_CSUM6_RX;
                        goto change_notsup;
                }
                if (nla_get_u8(data[IFLA_GENEVE_UDP_ZERO_CSUM6_RX]))
                        cfg->use_udp6_rx_checksums = false;
#else
                NL_SET_ERR_MSG_ATTR(extack, data[IFLA_GENEVE_UDP_ZERO_CSUM6_RX],
                                    "IPv6 support not enabled in the kernel");
                return -EPFNOSUPPORT;
#endif
        }

        if (data[IFLA_GENEVE_INNER_PROTO_INHERIT]) {
                if (changelink) {
                        attrtype = IFLA_GENEVE_INNER_PROTO_INHERIT;
                        goto change_notsup;
                }
                cfg->inner_proto_inherit = true;
        }

        return 0;
change_notsup:
        NL_SET_ERR_MSG_ATTR(extack, data[attrtype],
                            "Changing VNI, Port, endpoint IP address family, external, inner_proto_inherit, and UDP checksum attributes are not supported");
        return -EOPNOTSUPP;
}

static void geneve_link_config(struct net_device *dev,
                               struct ip_tunnel_info *info, struct nlattr *tb[])
{
        struct geneve_dev *geneve = netdev_priv(dev);
        int ldev_mtu = 0;

        if (tb[IFLA_MTU]) {
                geneve_change_mtu(dev, nla_get_u32(tb[IFLA_MTU]));
                return;
        }

        switch (ip_tunnel_info_af(info)) {
        case AF_INET: {
                struct flowi4 fl4 = { .daddr = info->key.u.ipv4.dst };
                struct rtable *rt = ip_route_output_key(geneve->net, &fl4);

                if (!IS_ERR(rt) && rt->dst.dev) {
                        ldev_mtu = rt->dst.dev->mtu - GENEVE_IPV4_HLEN;
                        ip_rt_put(rt);
                }
                break;
        }
#if IS_ENABLED(CONFIG_IPV6)
        case AF_INET6: {
                struct rt6_info *rt;

                if (!__in6_dev_get(dev))
                        break;

                rt = rt6_lookup(geneve->net, &info->key.u.ipv6.dst, NULL, 0,
                                NULL, 0);

                if (rt && rt->dst.dev)
                        ldev_mtu = rt->dst.dev->mtu - GENEVE_IPV6_HLEN;
                ip6_rt_put(rt);
                break;
        }
#endif
        }

        if (ldev_mtu <= 0)
                return;

        geneve_change_mtu(dev, ldev_mtu - info->options_len);
}

static int geneve_newlink(struct net *net, struct net_device *dev,
                          struct nlattr *tb[], struct nlattr *data[],
                          struct netlink_ext_ack *extack)
{
        struct geneve_config cfg = {
                .df = GENEVE_DF_UNSET,
                .use_udp6_rx_checksums = false,
                .ttl_inherit = false,
                .collect_md = false,
        };
        int err;

        init_tnl_info(&cfg.info, GENEVE_UDP_PORT);
        err = geneve_nl2info(tb, data, extack, &cfg, false);
        if (err)
                return err;

        err = geneve_configure(net, dev, extack, &cfg);
        if (err)
                return err;

        geneve_link_config(dev, &cfg.info, tb);

        return 0;
}

/* Quiesces the geneve device data path for both TX and RX.
 *
 * On transmit geneve checks for non-NULL geneve_sock before it proceeds.
 * So, if we set that socket to NULL under RCU and wait for synchronize_net()
 * to complete for the existing set of in-flight packets to be transmitted,
 * then we would have quiesced the transmit data path. All the future packets
 * will get dropped until we unquiesce the data path.
 *
 * On receive geneve dereference the geneve_sock stashed in the socket. So,
 * if we set that to NULL under RCU and wait for synchronize_net() to
 * complete, then we would have quiesced the receive data path.
 */
static void geneve_quiesce(struct geneve_dev *geneve, struct geneve_sock **gs4,
                           struct geneve_sock **gs6)
{
        *gs4 = rtnl_dereference(geneve->sock4);
        rcu_assign_pointer(geneve->sock4, NULL);
        if (*gs4)
                rcu_assign_sk_user_data((*gs4)->sock->sk, NULL);
#if IS_ENABLED(CONFIG_IPV6)
        *gs6 = rtnl_dereference(geneve->sock6);
        rcu_assign_pointer(geneve->sock6, NULL);
        if (*gs6)
                rcu_assign_sk_user_data((*gs6)->sock->sk, NULL);
#else
        *gs6 = NULL;
#endif
        synchronize_net();
}

/* Resumes the geneve device data path for both TX and RX. */
static void geneve_unquiesce(struct geneve_dev *geneve, struct geneve_sock *gs4,
                             struct geneve_sock __maybe_unused *gs6)
{
        rcu_assign_pointer(geneve->sock4, gs4);
        if (gs4)
                rcu_assign_sk_user_data(gs4->sock->sk, gs4);
#if IS_ENABLED(CONFIG_IPV6)
        rcu_assign_pointer(geneve->sock6, gs6);
        if (gs6)
                rcu_assign_sk_user_data(gs6->sock->sk, gs6);
#endif
        synchronize_net();
}

static int geneve_changelink(struct net_device *dev, struct nlattr *tb[],
                             struct nlattr *data[],
                             struct netlink_ext_ack *extack)
{
        struct geneve_dev *geneve = netdev_priv(dev);
        struct geneve_sock *gs4, *gs6;
        struct geneve_config cfg;
        int err;

        /* If the geneve device is configured for metadata (or externally
         * controlled, for example, OVS), then nothing can be changed.
         */
        if (geneve->cfg.collect_md)
                return -EOPNOTSUPP;

        /* Start with the existing info. */
        memcpy(&cfg, &geneve->cfg, sizeof(cfg));
        err = geneve_nl2info(tb, data, extack, &cfg, true);
        if (err)
                return err;

        if (!geneve_dst_addr_equal(&geneve->cfg.info, &cfg.info)) {
                dst_cache_reset(&cfg.info.dst_cache);
                geneve_link_config(dev, &cfg.info, tb);
        }

        geneve_quiesce(geneve, &gs4, &gs6);
        memcpy(&geneve->cfg, &cfg, sizeof(cfg));
        geneve_unquiesce(geneve, gs4, gs6);

        return 0;
}

static void geneve_dellink(struct net_device *dev, struct list_head *head)
{
        struct geneve_dev *geneve = netdev_priv(dev);

        list_del(&geneve->next);
        unregister_netdevice_queue(dev, head);
}

static size_t geneve_get_size(const struct net_device *dev)
{
        return nla_total_size(sizeof(__u32)) +        /* IFLA_GENEVE_ID */
                nla_total_size(sizeof(struct in6_addr)) + /* IFLA_GENEVE_REMOTE{6} */
                nla_total_size(sizeof(__u8)) +  /* IFLA_GENEVE_TTL */
                nla_total_size(sizeof(__u8)) +  /* IFLA_GENEVE_TOS */
                nla_total_size(sizeof(__u8)) +        /* IFLA_GENEVE_DF */
                nla_total_size(sizeof(__be32)) +  /* IFLA_GENEVE_LABEL */
                nla_total_size(sizeof(__be16)) +  /* IFLA_GENEVE_PORT */
                nla_total_size(0) +         /* IFLA_GENEVE_COLLECT_METADATA */
                nla_total_size(sizeof(__u8)) + /* IFLA_GENEVE_UDP_CSUM */
                nla_total_size(sizeof(__u8)) + /* IFLA_GENEVE_UDP_ZERO_CSUM6_TX */
                nla_total_size(sizeof(__u8)) + /* IFLA_GENEVE_UDP_ZERO_CSUM6_RX */
                nla_total_size(sizeof(__u8)) + /* IFLA_GENEVE_TTL_INHERIT */
                nla_total_size(0) +         /* IFLA_GENEVE_INNER_PROTO_INHERIT */
                0;
}

static int geneve_fill_info(struct sk_buff *skb, const struct net_device *dev)
{
        struct geneve_dev *geneve = netdev_priv(dev);
        struct ip_tunnel_info *info = &geneve->cfg.info;
        bool ttl_inherit = geneve->cfg.ttl_inherit;
        bool metadata = geneve->cfg.collect_md;
        __u8 tmp_vni[3];
        __u32 vni;

        tunnel_id_to_vni(info->key.tun_id, tmp_vni);
        vni = (tmp_vni[0] << 16) | (tmp_vni[1] << 8) | tmp_vni[2];
        if (nla_put_u32(skb, IFLA_GENEVE_ID, vni))
                goto nla_put_failure;

        if (!metadata && ip_tunnel_info_af(info) == AF_INET) {
                if (nla_put_in_addr(skb, IFLA_GENEVE_REMOTE,
                                    info->key.u.ipv4.dst))
                        goto nla_put_failure;
                if (nla_put_u8(skb, IFLA_GENEVE_UDP_CSUM,
                               !!(info->key.tun_flags & TUNNEL_CSUM)))
                        goto nla_put_failure;

#if IS_ENABLED(CONFIG_IPV6)
        } else if (!metadata) {
                if (nla_put_in6_addr(skb, IFLA_GENEVE_REMOTE6,
                                     &info->key.u.ipv6.dst))
                        goto nla_put_failure;
                if (nla_put_u8(skb, IFLA_GENEVE_UDP_ZERO_CSUM6_TX,
                               !(info->key.tun_flags & TUNNEL_CSUM)))
                        goto nla_put_failure;
#endif
        }

        if (nla_put_u8(skb, IFLA_GENEVE_TTL, info->key.ttl) ||
            nla_put_u8(skb, IFLA_GENEVE_TOS, info->key.tos) ||
            nla_put_be32(skb, IFLA_GENEVE_LABEL, info->key.label))
                goto nla_put_failure;

        if (nla_put_u8(skb, IFLA_GENEVE_DF, geneve->cfg.df))
                goto nla_put_failure;

        if (nla_put_be16(skb, IFLA_GENEVE_PORT, info->key.tp_dst))
                goto nla_put_failure;

        if (metadata && nla_put_flag(skb, IFLA_GENEVE_COLLECT_METADATA))
                goto nla_put_failure;

#if IS_ENABLED(CONFIG_IPV6)
        if (nla_put_u8(skb, IFLA_GENEVE_UDP_ZERO_CSUM6_RX,
                       !geneve->cfg.use_udp6_rx_checksums))
                goto nla_put_failure;
#endif

        if (nla_put_u8(skb, IFLA_GENEVE_TTL_INHERIT, ttl_inherit))
                goto nla_put_failure;

        if (geneve->cfg.inner_proto_inherit &&
            nla_put_flag(skb, IFLA_GENEVE_INNER_PROTO_INHERIT))
                goto nla_put_failure;

        return 0;

nla_put_failure:
        return -EMSGSIZE;
}

static struct rtnl_link_ops geneve_link_ops __read_mostly = {
        .kind                = "geneve",
        .maxtype        = IFLA_GENEVE_MAX,
        .policy                = geneve_policy,
        .priv_size        = sizeof(struct geneve_dev),
        .setup                = geneve_setup,
        .validate        = geneve_validate,
        .newlink        = geneve_newlink,
        .changelink        = geneve_changelink,
        .dellink        = geneve_dellink,
        .get_size        = geneve_get_size,
        .fill_info        = geneve_fill_info,
};

struct net_device *geneve_dev_create_fb(struct net *net, const char *name,
                                        u8 name_assign_type, u16 dst_port)
{
        struct nlattr *tb[IFLA_MAX + 1];
        struct net_device *dev;
        LIST_HEAD(list_kill);
        int err;
        struct geneve_config cfg = {
                .df = GENEVE_DF_UNSET,
                .use_udp6_rx_checksums = true,
                .ttl_inherit = false,
                .collect_md = true,
        };

        memset(tb, 0, sizeof(tb));
        dev = rtnl_create_link(net, name, name_assign_type,
                               &geneve_link_ops, tb, NULL);
        if (IS_ERR(dev))
                return dev;

        init_tnl_info(&cfg.info, dst_port);
        err = geneve_configure(net, dev, NULL, &cfg);
        if (err) {
                free_netdev(dev);
                return ERR_PTR(err);
        }

        /* openvswitch users expect packet sizes to be unrestricted,
         * so set the largest MTU we can.
         */
        err = geneve_change_mtu(dev, IP_MAX_MTU);
        if (err)
                goto err;

        err = rtnl_configure_link(dev, NULL);
        if (err < 0)
                goto err;

        return dev;
err:
        geneve_dellink(dev, &list_kill);
        unregister_netdevice_many(&list_kill);
        return ERR_PTR(err);
}
EXPORT_SYMBOL_GPL(geneve_dev_create_fb);

static int geneve_netdevice_event(struct notifier_block *unused,
                                  unsigned long event, void *ptr)
{
        struct net_device *dev = netdev_notifier_info_to_dev(ptr);

        if (event == NETDEV_UDP_TUNNEL_PUSH_INFO)
                geneve_offload_rx_ports(dev, true);
        else if (event == NETDEV_UDP_TUNNEL_DROP_INFO)
                geneve_offload_rx_ports(dev, false);

        return NOTIFY_DONE;
}

static struct notifier_block geneve_notifier_block __read_mostly = {
        .notifier_call = geneve_netdevice_event,
};

static __net_init int geneve_init_net(struct net *net)
{
        struct geneve_net *gn = net_generic(net, geneve_net_id);

        INIT_LIST_HEAD(&gn->geneve_list);
        INIT_LIST_HEAD(&gn->sock_list);
        return 0;
}

static void geneve_destroy_tunnels(struct net *net, struct list_head *head)
{
        struct geneve_net *gn = net_generic(net, geneve_net_id);
        struct geneve_dev *geneve, *next;
        struct net_device *dev, *aux;

        /* gather any geneve devices that were moved into this ns */
        for_each_netdev_safe(net, dev, aux)
                if (dev->rtnl_link_ops == &geneve_link_ops)
                        unregister_netdevice_queue(dev, head);

        /* now gather any other geneve devices that were created in this ns */
        list_for_each_entry_safe(geneve, next, &gn->geneve_list, next) {
                /* If geneve->dev is in the same netns, it was already added
                 * to the list by the previous loop.
                 */
                if (!net_eq(dev_net(geneve->dev), net))
                        unregister_netdevice_queue(geneve->dev, head);
        }
}

static void __net_exit geneve_exit_batch_net(struct list_head *net_list)
{
        struct net *net;
        LIST_HEAD(list);

        rtnl_lock();
        list_for_each_entry(net, net_list, exit_list)
                geneve_destroy_tunnels(net, &list);

        /* unregister the devices gathered above */
        unregister_netdevice_many(&list);
        rtnl_unlock();

        list_for_each_entry(net, net_list, exit_list) {
                const struct geneve_net *gn = net_generic(net, geneve_net_id);

                WARN_ON_ONCE(!list_empty(&gn->sock_list));
        }
}

static struct pernet_operations geneve_net_ops = {
        .init = geneve_init_net,
        .exit_batch = geneve_exit_batch_net,
        .id   = &geneve_net_id,
        .size = sizeof(struct geneve_net),
};

static int __init geneve_init_module(void)
{
        int rc;

        rc = register_pernet_subsys(&geneve_net_ops);
        if (rc)
                goto out1;

        rc = register_netdevice_notifier(&geneve_notifier_block);
        if (rc)
                goto out2;

        rc = rtnl_link_register(&geneve_link_ops);
        if (rc)
                goto out3;

        return 0;
out3:
        unregister_netdevice_notifier(&geneve_notifier_block);
out2:
        unregister_pernet_subsys(&geneve_net_ops);
out1:
        return rc;
}
late_initcall(geneve_init_module);

static void __exit geneve_cleanup_module(void)
{
        rtnl_link_unregister(&geneve_link_ops);
        unregister_netdevice_notifier(&geneve_notifier_block);
        unregister_pernet_subsys(&geneve_net_ops);
}
module_exit(geneve_cleanup_module);

MODULE_LICENSE("GPL");
MODULE_VERSION(GENEVE_NETDEV_VER);
MODULE_AUTHOR("John W. Linville <linville@tuxdriver.com>");
MODULE_DESCRIPTION("Interface driver for GENEVE encapsulated traffic");
MODULE_ALIAS_RTNL_LINK("geneve");







































































































































































































































































































































































































































































































































































   49 













   49 









   49 























































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * IPVS:        Locality-Based Least-Connection scheduling module
 *
 * Authors:     Wensong Zhang <wensong@gnuchina.org>
 *
 * Changes:
 *     Martin Hamilton         :    fixed the terrible locking bugs
 *                                   *lock(tbl->lock) ==> *lock(&tbl->lock)
 *     Wensong Zhang           :    fixed the uninitialized tbl->lock bug
 *     Wensong Zhang           :    added doing full expiration check to
 *                                   collect stale entries of 24+ hours when
 *                                   no partial expire check in a half hour
 *     Julian Anastasov        :    replaced del_timer call with del_timer_sync
 *                                   to avoid the possible race between timer
 *                                   handler and del_timer thread in SMP
 */

/*
 * The lblc algorithm is as follows (pseudo code):
 *
 *       if cachenode[dest_ip] is null then
 *               n, cachenode[dest_ip] <- {weighted least-conn node};
 *       else
 *               n <- cachenode[dest_ip];
 *               if (n is dead) OR
 *                  (n.conns>n.weight AND
 *                   there is a node m with m.conns<m.weight/2) then
 *                 n, cachenode[dest_ip] <- {weighted least-conn node};
 *
 *       return n;
 *
 * Thanks must go to Wenzhuo Zhang for talking WCCP to me and pushing
 * me to write this module.
 */

#define KMSG_COMPONENT "IPVS"
#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt

#include <linux/ip.h>
#include <linux/slab.h>
#include <linux/module.h>
#include <linux/kernel.h>
#include <linux/skbuff.h>
#include <linux/jiffies.h>
#include <linux/hash.h>

/* for sysctl */
#include <linux/fs.h>
#include <linux/sysctl.h>

#include <net/ip_vs.h>


/*
 *    It is for garbage collection of stale IPVS lblc entries,
 *    when the table is full.
 */
#define CHECK_EXPIRE_INTERVAL   (60*HZ)
#define ENTRY_TIMEOUT           (6*60*HZ)

#define DEFAULT_EXPIRATION        (24*60*60*HZ)

/*
 *    It is for full expiration check.
 *    When there is no partial expiration check (garbage collection)
 *    in a half hour, do a full expiration check to collect stale
 *    entries that haven't been touched for a day.
 */
#define COUNT_FOR_FULL_EXPIRATION   30


/*
 *     for IPVS lblc entry hash table
 */
#ifndef CONFIG_IP_VS_LBLC_TAB_BITS
#define CONFIG_IP_VS_LBLC_TAB_BITS      10
#endif
#define IP_VS_LBLC_TAB_BITS     CONFIG_IP_VS_LBLC_TAB_BITS
#define IP_VS_LBLC_TAB_SIZE     (1 << IP_VS_LBLC_TAB_BITS)
#define IP_VS_LBLC_TAB_MASK     (IP_VS_LBLC_TAB_SIZE - 1)


/*
 *      IPVS lblc entry represents an association between destination
 *      IP address and its destination server
 */
struct ip_vs_lblc_entry {
        struct hlist_node        list;
        int                        af;                /* address family */
        union nf_inet_addr      addr;           /* destination IP address */
        struct ip_vs_dest        *dest;          /* real server (cache) */
        unsigned long           lastuse;        /* last used time */
        struct rcu_head                rcu_head;
};


/*
 *      IPVS lblc hash table
 */
struct ip_vs_lblc_table {
        struct rcu_head                rcu_head;
        struct hlist_head        bucket[IP_VS_LBLC_TAB_SIZE];  /* hash bucket */
        struct timer_list       periodic_timer; /* collect stale entries */
        struct ip_vs_service        *svc;                /* pointer back to service */
        atomic_t                entries;        /* number of entries */
        int                     max_size;       /* maximum size of entries */
        int                     rover;          /* rover for expire check */
        int                     counter;        /* counter for no expire */
        bool                        dead;
};


/*
 *      IPVS LBLC sysctl table
 */
#ifdef CONFIG_SYSCTL
static struct ctl_table vs_vars_table[] = {
        {
                .procname        = "lblc_expiration",
                .data                = NULL,
                .maxlen                = sizeof(int),
                .mode                = 0644,
                .proc_handler        = proc_dointvec_jiffies,
        },
        { }
};
#endif

static void ip_vs_lblc_rcu_free(struct rcu_head *head)
{
        struct ip_vs_lblc_entry *en = container_of(head,
                                                   struct ip_vs_lblc_entry,
                                                   rcu_head);

        ip_vs_dest_put_and_free(en->dest);
        kfree(en);
}

static inline void ip_vs_lblc_del(struct ip_vs_lblc_entry *en)
{
        hlist_del_rcu(&en->list);
        call_rcu(&en->rcu_head, ip_vs_lblc_rcu_free);
}

/*
 *        Returns hash value for IPVS LBLC entry
 */
static inline unsigned int
ip_vs_lblc_hashkey(int af, const union nf_inet_addr *addr)
{
        __be32 addr_fold = addr->ip;

#ifdef CONFIG_IP_VS_IPV6
        if (af == AF_INET6)
                addr_fold = addr->ip6[0]^addr->ip6[1]^
                            addr->ip6[2]^addr->ip6[3];
#endif
        return hash_32(ntohl(addr_fold), IP_VS_LBLC_TAB_BITS);
}


/*
 *        Hash an entry in the ip_vs_lblc_table.
 *        returns bool success.
 */
static void
ip_vs_lblc_hash(struct ip_vs_lblc_table *tbl, struct ip_vs_lblc_entry *en)
{
        unsigned int hash = ip_vs_lblc_hashkey(en->af, &en->addr);

        hlist_add_head_rcu(&en->list, &tbl->bucket[hash]);
        atomic_inc(&tbl->entries);
}


/* Get ip_vs_lblc_entry associated with supplied parameters. */
static inline struct ip_vs_lblc_entry *
ip_vs_lblc_get(int af, struct ip_vs_lblc_table *tbl,
               const union nf_inet_addr *addr)
{
        unsigned int hash = ip_vs_lblc_hashkey(af, addr);
        struct ip_vs_lblc_entry *en;

        hlist_for_each_entry_rcu(en, &tbl->bucket[hash], list)
                if (ip_vs_addr_equal(af, &en->addr, addr))
                        return en;

        return NULL;
}


/*
 * Create or update an ip_vs_lblc_entry, which is a mapping of a destination IP
 * address to a server. Called under spin lock.
 */
static inline struct ip_vs_lblc_entry *
ip_vs_lblc_new(struct ip_vs_lblc_table *tbl, const union nf_inet_addr *daddr,
               u16 af, struct ip_vs_dest *dest)
{
        struct ip_vs_lblc_entry *en;

        en = ip_vs_lblc_get(af, tbl, daddr);
        if (en) {
                if (en->dest == dest)
                        return en;
                ip_vs_lblc_del(en);
        }
        en = kmalloc(sizeof(*en), GFP_ATOMIC);
        if (!en)
                return NULL;

        en->af = af;
        ip_vs_addr_copy(af, &en->addr, daddr);
        en->lastuse = jiffies;

        ip_vs_dest_hold(dest);
        en->dest = dest;

        ip_vs_lblc_hash(tbl, en);

        return en;
}


/*
 *      Flush all the entries of the specified table.
 */
static void ip_vs_lblc_flush(struct ip_vs_service *svc)
{
        struct ip_vs_lblc_table *tbl = svc->sched_data;
        struct ip_vs_lblc_entry *en;
        struct hlist_node *next;
        int i;

        spin_lock_bh(&svc->sched_lock);
        tbl->dead = true;
        for (i = 0; i < IP_VS_LBLC_TAB_SIZE; i++) {
                hlist_for_each_entry_safe(en, next, &tbl->bucket[i], list) {
                        ip_vs_lblc_del(en);
                        atomic_dec(&tbl->entries);
                }
        }
        spin_unlock_bh(&svc->sched_lock);
}

static int sysctl_lblc_expiration(struct ip_vs_service *svc)
{
#ifdef CONFIG_SYSCTL
        return svc->ipvs->sysctl_lblc_expiration;
#else
        return DEFAULT_EXPIRATION;
#endif
}

static inline void ip_vs_lblc_full_check(struct ip_vs_service *svc)
{
        struct ip_vs_lblc_table *tbl = svc->sched_data;
        struct ip_vs_lblc_entry *en;
        struct hlist_node *next;
        unsigned long now = jiffies;
        int i, j;

        for (i = 0, j = tbl->rover; i < IP_VS_LBLC_TAB_SIZE; i++) {
                j = (j + 1) & IP_VS_LBLC_TAB_MASK;

                spin_lock(&svc->sched_lock);
                hlist_for_each_entry_safe(en, next, &tbl->bucket[j], list) {
                        if (time_before(now,
                                        en->lastuse +
                                        sysctl_lblc_expiration(svc)))
                                continue;

                        ip_vs_lblc_del(en);
                        atomic_dec(&tbl->entries);
                }
                spin_unlock(&svc->sched_lock);
        }
        tbl->rover = j;
}


/*
 *      Periodical timer handler for IPVS lblc table
 *      It is used to collect stale entries when the number of entries
 *      exceeds the maximum size of the table.
 *
 *      Fixme: we probably need more complicated algorithm to collect
 *             entries that have not been used for a long time even
 *             if the number of entries doesn't exceed the maximum size
 *             of the table.
 *      The full expiration check is for this purpose now.
 */
static void ip_vs_lblc_check_expire(struct timer_list *t)
{
        struct ip_vs_lblc_table *tbl = from_timer(tbl, t, periodic_timer);
        struct ip_vs_service *svc = tbl->svc;
        unsigned long now = jiffies;
        int goal;
        int i, j;
        struct ip_vs_lblc_entry *en;
        struct hlist_node *next;

        if ((tbl->counter % COUNT_FOR_FULL_EXPIRATION) == 0) {
                /* do full expiration check */
                ip_vs_lblc_full_check(svc);
                tbl->counter = 1;
                goto out;
        }

        if (atomic_read(&tbl->entries) <= tbl->max_size) {
                tbl->counter++;
                goto out;
        }

        goal = (atomic_read(&tbl->entries) - tbl->max_size)*4/3;
        if (goal > tbl->max_size/2)
                goal = tbl->max_size/2;

        for (i = 0, j = tbl->rover; i < IP_VS_LBLC_TAB_SIZE; i++) {
                j = (j + 1) & IP_VS_LBLC_TAB_MASK;

                spin_lock(&svc->sched_lock);
                hlist_for_each_entry_safe(en, next, &tbl->bucket[j], list) {
                        if (time_before(now, en->lastuse + ENTRY_TIMEOUT))
                                continue;

                        ip_vs_lblc_del(en);
                        atomic_dec(&tbl->entries);
                        goal--;
                }
                spin_unlock(&svc->sched_lock);
                if (goal <= 0)
                        break;
        }
        tbl->rover = j;

  out:
        mod_timer(&tbl->periodic_timer, jiffies + CHECK_EXPIRE_INTERVAL);
}


static int ip_vs_lblc_init_svc(struct ip_vs_service *svc)
{
        int i;
        struct ip_vs_lblc_table *tbl;

        /*
         *    Allocate the ip_vs_lblc_table for this service
         */
        tbl = kmalloc(sizeof(*tbl), GFP_KERNEL);
        if (tbl == NULL)
                return -ENOMEM;

        svc->sched_data = tbl;
        IP_VS_DBG(6, "LBLC hash table (memory=%zdbytes) allocated for "
                  "current service\n", sizeof(*tbl));

        /*
         *    Initialize the hash buckets
         */
        for (i = 0; i < IP_VS_LBLC_TAB_SIZE; i++) {
                INIT_HLIST_HEAD(&tbl->bucket[i]);
        }
        tbl->max_size = IP_VS_LBLC_TAB_SIZE*16;
        tbl->rover = 0;
        tbl->counter = 1;
        tbl->dead = false;
        tbl->svc = svc;
        atomic_set(&tbl->entries, 0);

        /*
         *    Hook periodic timer for garbage collection
         */
        timer_setup(&tbl->periodic_timer, ip_vs_lblc_check_expire, 0);
        mod_timer(&tbl->periodic_timer, jiffies + CHECK_EXPIRE_INTERVAL);

        return 0;
}


static void ip_vs_lblc_done_svc(struct ip_vs_service *svc)
{
        struct ip_vs_lblc_table *tbl = svc->sched_data;

        /* remove periodic timer */
        del_timer_sync(&tbl->periodic_timer);

        /* got to clean up table entries here */
        ip_vs_lblc_flush(svc);

        /* release the table itself */
        kfree_rcu(tbl, rcu_head);
        IP_VS_DBG(6, "LBLC hash table (memory=%zdbytes) released\n",
                  sizeof(*tbl));
}


static inline struct ip_vs_dest *
__ip_vs_lblc_schedule(struct ip_vs_service *svc)
{
        struct ip_vs_dest *dest, *least;
        int loh, doh;

        /*
         * We use the following formula to estimate the load:
         *                (dest overhead) / dest->weight
         *
         * Remember -- no floats in kernel mode!!!
         * The comparison of h1*w2 > h2*w1 is equivalent to that of
         *                h1/w1 > h2/w2
         * if every weight is larger than zero.
         *
         * The server with weight=0 is quiesced and will not receive any
         * new connection.
         */
        list_for_each_entry_rcu(dest, &svc->destinations, n_list) {
                if (dest->flags & IP_VS_DEST_F_OVERLOAD)
                        continue;
                if (atomic_read(&dest->weight) > 0) {
                        least = dest;
                        loh = ip_vs_dest_conn_overhead(least);
                        goto nextstage;
                }
        }
        return NULL;

        /*
         *    Find the destination with the least load.
         */
  nextstage:
        list_for_each_entry_continue_rcu(dest, &svc->destinations, n_list) {
                if (dest->flags & IP_VS_DEST_F_OVERLOAD)
                        continue;

                doh = ip_vs_dest_conn_overhead(dest);
                if ((__s64)loh * atomic_read(&dest->weight) >
                    (__s64)doh * atomic_read(&least->weight)) {
                        least = dest;
                        loh = doh;
                }
        }

        IP_VS_DBG_BUF(6, "LBLC: server %s:%d "
                      "activeconns %d refcnt %d weight %d overhead %d\n",
                      IP_VS_DBG_ADDR(least->af, &least->addr),
                      ntohs(least->port),
                      atomic_read(&least->activeconns),
                      refcount_read(&least->refcnt),
                      atomic_read(&least->weight), loh);

        return least;
}


/*
 *   If this destination server is overloaded and there is a less loaded
 *   server, then return true.
 */
static inline int
is_overloaded(struct ip_vs_dest *dest, struct ip_vs_service *svc)
{
        if (atomic_read(&dest->activeconns) > atomic_read(&dest->weight)) {
                struct ip_vs_dest *d;

                list_for_each_entry_rcu(d, &svc->destinations, n_list) {
                        if (atomic_read(&d->activeconns)*2
                            < atomic_read(&d->weight)) {
                                return 1;
                        }
                }
        }
        return 0;
}


/*
 *    Locality-Based (weighted) Least-Connection scheduling
 */
static struct ip_vs_dest *
ip_vs_lblc_schedule(struct ip_vs_service *svc, const struct sk_buff *skb,
                    struct ip_vs_iphdr *iph)
{
        struct ip_vs_lblc_table *tbl = svc->sched_data;
        struct ip_vs_dest *dest = NULL;
        struct ip_vs_lblc_entry *en;

        IP_VS_DBG(6, "%s(): Scheduling...\n", __func__);

        /* First look in our cache */
        en = ip_vs_lblc_get(svc->af, tbl, &iph->daddr);
        if (en) {
                /* We only hold a read lock, but this is atomic */
                en->lastuse = jiffies;

                /*
                 * If the destination is not available, i.e. it's in the trash,
                 * we must ignore it, as it may be removed from under our feet,
                 * if someone drops our reference count. Our caller only makes
                 * sure that destinations, that are not in the trash, are not
                 * moved to the trash, while we are scheduling. But anyone can
                 * free up entries from the trash at any time.
                 */

                dest = en->dest;
                if ((dest->flags & IP_VS_DEST_F_AVAILABLE) &&
                    atomic_read(&dest->weight) > 0 && !is_overloaded(dest, svc))
                        goto out;
        }

        /* No cache entry or it is invalid, time to schedule */
        dest = __ip_vs_lblc_schedule(svc);
        if (!dest) {
                ip_vs_scheduler_err(svc, "no destination available");
                return NULL;
        }

        /* If we fail to create a cache entry, we'll just use the valid dest */
        spin_lock_bh(&svc->sched_lock);
        if (!tbl->dead)
                ip_vs_lblc_new(tbl, &iph->daddr, svc->af, dest);
        spin_unlock_bh(&svc->sched_lock);

out:
        IP_VS_DBG_BUF(6, "LBLC: destination IP address %s --> server %s:%d\n",
                      IP_VS_DBG_ADDR(svc->af, &iph->daddr),
                      IP_VS_DBG_ADDR(dest->af, &dest->addr), ntohs(dest->port));

        return dest;
}


/*
 *      IPVS LBLC Scheduler structure
 */
static struct ip_vs_scheduler ip_vs_lblc_scheduler = {
        .name =                        "lblc",
        .refcnt =                ATOMIC_INIT(0),
        .module =                THIS_MODULE,
        .n_list =                LIST_HEAD_INIT(ip_vs_lblc_scheduler.n_list),
        .init_service =                ip_vs_lblc_init_svc,
        .done_service =                ip_vs_lblc_done_svc,
        .schedule =                ip_vs_lblc_schedule,
};

/*
 *  per netns init.
 */
#ifdef CONFIG_SYSCTL
static int __net_init __ip_vs_lblc_init(struct net *net)
{
        struct netns_ipvs *ipvs = net_ipvs(net);

        if (!ipvs)
                return -ENOENT;

        if (!net_eq(net, &init_net)) {
                ipvs->lblc_ctl_table = kmemdup(vs_vars_table,
                                                sizeof(vs_vars_table),
                                                GFP_KERNEL);
                if (ipvs->lblc_ctl_table == NULL)
                        return -ENOMEM;

                /* Don't export sysctls to unprivileged users */
                if (net->user_ns != &init_user_ns)
                        ipvs->lblc_ctl_table[0].procname = NULL;

        } else
                ipvs->lblc_ctl_table = vs_vars_table;
        ipvs->sysctl_lblc_expiration = DEFAULT_EXPIRATION;
        ipvs->lblc_ctl_table[0].data = &ipvs->sysctl_lblc_expiration;

        ipvs->lblc_ctl_header =
                register_net_sysctl(net, "net/ipv4/vs", ipvs->lblc_ctl_table);
        if (!ipvs->lblc_ctl_header) {
                if (!net_eq(net, &init_net))
                        kfree(ipvs->lblc_ctl_table);
                return -ENOMEM;
        }

        return 0;
}

static void __net_exit __ip_vs_lblc_exit(struct net *net)
{
        struct netns_ipvs *ipvs = net_ipvs(net);

        unregister_net_sysctl_table(ipvs->lblc_ctl_header);

        if (!net_eq(net, &init_net))
                kfree(ipvs->lblc_ctl_table);
}

#else

static int __net_init __ip_vs_lblc_init(struct net *net) { return 0; }
static void __net_exit __ip_vs_lblc_exit(struct net *net) { }

#endif

static struct pernet_operations ip_vs_lblc_ops = {
        .init = __ip_vs_lblc_init,
        .exit = __ip_vs_lblc_exit,
};

static int __init ip_vs_lblc_init(void)
{
        int ret;

        ret = register_pernet_subsys(&ip_vs_lblc_ops);
        if (ret)
                return ret;

        ret = register_ip_vs_scheduler(&ip_vs_lblc_scheduler);
        if (ret)
                unregister_pernet_subsys(&ip_vs_lblc_ops);
        return ret;
}

static void __exit ip_vs_lblc_cleanup(void)
{
        unregister_ip_vs_scheduler(&ip_vs_lblc_scheduler);
        unregister_pernet_subsys(&ip_vs_lblc_ops);
        rcu_barrier();
}


module_init(ip_vs_lblc_init);
module_exit(ip_vs_lblc_cleanup);
MODULE_LICENSE("GPL");

























































































































    6 







































   21 






1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
/* SPDX-License-Identifier: GPL-2.0 */
/* Copyright (C) B.A.T.M.A.N. contributors:
 *
 * Marek Lindner, Simon Wunderlich
 */

#ifndef _NET_BATMAN_ADV_ORIGINATOR_H_
#define _NET_BATMAN_ADV_ORIGINATOR_H_

#include "main.h"

#include <linux/compiler.h>
#include <linux/if_ether.h>
#include <linux/jhash.h>
#include <linux/kref.h>
#include <linux/netlink.h>
#include <linux/skbuff.h>
#include <linux/types.h>

bool batadv_compare_orig(const struct hlist_node *node, const void *data2);
int batadv_originator_init(struct batadv_priv *bat_priv);
void batadv_originator_free(struct batadv_priv *bat_priv);
void batadv_purge_orig_ref(struct batadv_priv *bat_priv);
void batadv_orig_node_release(struct kref *ref);
struct batadv_orig_node *batadv_orig_node_new(struct batadv_priv *bat_priv,
                                              const u8 *addr);
struct batadv_hardif_neigh_node *
batadv_hardif_neigh_get(const struct batadv_hard_iface *hard_iface,
                        const u8 *neigh_addr);
void batadv_hardif_neigh_release(struct kref *ref);
struct batadv_neigh_node *
batadv_neigh_node_get_or_create(struct batadv_orig_node *orig_node,
                                struct batadv_hard_iface *hard_iface,
                                const u8 *neigh_addr);
void batadv_neigh_node_release(struct kref *ref);
struct batadv_neigh_node *
batadv_orig_router_get(struct batadv_orig_node *orig_node,
                       const struct batadv_hard_iface *if_outgoing);
struct batadv_neigh_ifinfo *
batadv_neigh_ifinfo_new(struct batadv_neigh_node *neigh,
                        struct batadv_hard_iface *if_outgoing);
struct batadv_neigh_ifinfo *
batadv_neigh_ifinfo_get(struct batadv_neigh_node *neigh,
                        struct batadv_hard_iface *if_outgoing);
void batadv_neigh_ifinfo_release(struct kref *ref);

int batadv_hardif_neigh_dump(struct sk_buff *msg, struct netlink_callback *cb);

struct batadv_orig_ifinfo *
batadv_orig_ifinfo_get(struct batadv_orig_node *orig_node,
                       struct batadv_hard_iface *if_outgoing);
struct batadv_orig_ifinfo *
batadv_orig_ifinfo_new(struct batadv_orig_node *orig_node,
                       struct batadv_hard_iface *if_outgoing);
void batadv_orig_ifinfo_release(struct kref *ref);

int batadv_orig_dump(struct sk_buff *msg, struct netlink_callback *cb);
struct batadv_orig_node_vlan *
batadv_orig_node_vlan_new(struct batadv_orig_node *orig_node,
                          unsigned short vid);
struct batadv_orig_node_vlan *
batadv_orig_node_vlan_get(struct batadv_orig_node *orig_node,
                          unsigned short vid);
void batadv_orig_node_vlan_release(struct kref *ref);

/**
 * batadv_choose_orig() - Return the index of the orig entry in the hash table
 * @data: mac address of the originator node
 * @size: the size of the hash table
 *
 * Return: the hash index where the object represented by @data should be
 * stored at.
 */
static inline u32 batadv_choose_orig(const void *data, u32 size)
{
        u32 hash = 0;

        hash = jhash(data, ETH_ALEN, hash);
        return hash % size;
}

struct batadv_orig_node *
batadv_orig_hash_find(struct batadv_priv *bat_priv, const void *data);

/**
 * batadv_orig_node_vlan_put() - decrement the refcounter and possibly release
 *  the originator-vlan object
 * @orig_vlan: the originator-vlan object to release
 */
static inline void
batadv_orig_node_vlan_put(struct batadv_orig_node_vlan *orig_vlan)
{
        if (!orig_vlan)
                return;

        kref_put(&orig_vlan->refcount, batadv_orig_node_vlan_release);
}

/**
 * batadv_neigh_ifinfo_put() - decrement the refcounter and possibly release
 *  the neigh_ifinfo
 * @neigh_ifinfo: the neigh_ifinfo object to release
 */
static inline void
batadv_neigh_ifinfo_put(struct batadv_neigh_ifinfo *neigh_ifinfo)
{
        if (!neigh_ifinfo)
                return;

        kref_put(&neigh_ifinfo->refcount, batadv_neigh_ifinfo_release);
}

/**
 * batadv_hardif_neigh_put() - decrement the hardif neighbors refcounter
 *  and possibly release it
 * @hardif_neigh: hardif neigh neighbor to free
 */
static inline void
batadv_hardif_neigh_put(struct batadv_hardif_neigh_node *hardif_neigh)
{
        if (!hardif_neigh)
                return;

        kref_put(&hardif_neigh->refcount, batadv_hardif_neigh_release);
}

/**
 * batadv_neigh_node_put() - decrement the neighbors refcounter and possibly
 *  release it
 * @neigh_node: neigh neighbor to free
 */
static inline void batadv_neigh_node_put(struct batadv_neigh_node *neigh_node)
{
        if (!neigh_node)
                return;

        kref_put(&neigh_node->refcount, batadv_neigh_node_release);
}

/**
 * batadv_orig_ifinfo_put() - decrement the refcounter and possibly release
 *  the orig_ifinfo
 * @orig_ifinfo: the orig_ifinfo object to release
 */
static inline void
batadv_orig_ifinfo_put(struct batadv_orig_ifinfo *orig_ifinfo)
{
        if (!orig_ifinfo)
                return;

        kref_put(&orig_ifinfo->refcount, batadv_orig_ifinfo_release);
}

/**
 * batadv_orig_node_put() - decrement the orig node refcounter and possibly
 *  release it
 * @orig_node: the orig node to free
 */
static inline void batadv_orig_node_put(struct batadv_orig_node *orig_node)
{
        if (!orig_node)
                return;

        kref_put(&orig_node->refcount, batadv_orig_node_release);
}

#endif /* _NET_BATMAN_ADV_ORIGINATOR_H_ */













































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































  147 
  149 



  149 




    3 



   10 

  136 

  131 

  146 







   35 











































































   46 
   46 



   46 












   46 

    3 






   38 
   35 


    6 




   29 




































   43 







   43 
   42 

   43 


   43 



  446 
  446 


  446 

































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965
2966
2967
2968
2969
2970
2971
2972
2973
2974
2975
2976
2977
2978
2979
2980
2981
2982
2983
2984
2985
2986
2987
2988
2989
2990
2991
2992
2993
2994
2995
2996
2997
/*
 * Performance events x86 architecture code
 *
 *  Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
 *  Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar
 *  Copyright (C) 2009 Jaswinder Singh Rajput
 *  Copyright (C) 2009 Advanced Micro Devices, Inc., Robert Richter
 *  Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra
 *  Copyright (C) 2009 Intel Corporation, <markus.t.metzger@intel.com>
 *  Copyright (C) 2009 Google, Inc., Stephane Eranian
 *
 *  For licencing details see kernel-base/COPYING
 */

#include <linux/perf_event.h>
#include <linux/capability.h>
#include <linux/notifier.h>
#include <linux/hardirq.h>
#include <linux/kprobes.h>
#include <linux/export.h>
#include <linux/init.h>
#include <linux/kdebug.h>
#include <linux/sched/mm.h>
#include <linux/sched/clock.h>
#include <linux/uaccess.h>
#include <linux/slab.h>
#include <linux/cpu.h>
#include <linux/bitops.h>
#include <linux/device.h>
#include <linux/nospec.h>
#include <linux/static_call.h>

#include <asm/apic.h>
#include <asm/stacktrace.h>
#include <asm/nmi.h>
#include <asm/smp.h>
#include <asm/alternative.h>
#include <asm/mmu_context.h>
#include <asm/tlbflush.h>
#include <asm/timer.h>
#include <asm/desc.h>
#include <asm/ldt.h>
#include <asm/unwind.h>

#include "perf_event.h"

struct x86_pmu x86_pmu __read_mostly;
static struct pmu pmu;

DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events) = {
        .enabled = 1,
        .pmu = &pmu,
};

DEFINE_STATIC_KEY_FALSE(rdpmc_never_available_key);
DEFINE_STATIC_KEY_FALSE(rdpmc_always_available_key);
DEFINE_STATIC_KEY_FALSE(perf_is_hybrid);

/*
 * This here uses DEFINE_STATIC_CALL_NULL() to get a static_call defined
 * from just a typename, as opposed to an actual function.
 */
DEFINE_STATIC_CALL_NULL(x86_pmu_handle_irq,  *x86_pmu.handle_irq);
DEFINE_STATIC_CALL_NULL(x86_pmu_disable_all, *x86_pmu.disable_all);
DEFINE_STATIC_CALL_NULL(x86_pmu_enable_all,  *x86_pmu.enable_all);
DEFINE_STATIC_CALL_NULL(x86_pmu_enable,             *x86_pmu.enable);
DEFINE_STATIC_CALL_NULL(x86_pmu_disable,     *x86_pmu.disable);

DEFINE_STATIC_CALL_NULL(x86_pmu_add,  *x86_pmu.add);
DEFINE_STATIC_CALL_NULL(x86_pmu_del,  *x86_pmu.del);
DEFINE_STATIC_CALL_NULL(x86_pmu_read, *x86_pmu.read);

DEFINE_STATIC_CALL_NULL(x86_pmu_schedule_events,       *x86_pmu.schedule_events);
DEFINE_STATIC_CALL_NULL(x86_pmu_get_event_constraints, *x86_pmu.get_event_constraints);
DEFINE_STATIC_CALL_NULL(x86_pmu_put_event_constraints, *x86_pmu.put_event_constraints);

DEFINE_STATIC_CALL_NULL(x86_pmu_start_scheduling,  *x86_pmu.start_scheduling);
DEFINE_STATIC_CALL_NULL(x86_pmu_commit_scheduling, *x86_pmu.commit_scheduling);
DEFINE_STATIC_CALL_NULL(x86_pmu_stop_scheduling,   *x86_pmu.stop_scheduling);

DEFINE_STATIC_CALL_NULL(x86_pmu_sched_task,    *x86_pmu.sched_task);
DEFINE_STATIC_CALL_NULL(x86_pmu_swap_task_ctx, *x86_pmu.swap_task_ctx);

DEFINE_STATIC_CALL_NULL(x86_pmu_drain_pebs,   *x86_pmu.drain_pebs);
DEFINE_STATIC_CALL_NULL(x86_pmu_pebs_aliases, *x86_pmu.pebs_aliases);

/*
 * This one is magic, it will get called even when PMU init fails (because
 * there is no PMU), in which case it should simply return NULL.
 */
DEFINE_STATIC_CALL_RET0(x86_pmu_guest_get_msrs, *x86_pmu.guest_get_msrs);

u64 __read_mostly hw_cache_event_ids
                                [PERF_COUNT_HW_CACHE_MAX]
                                [PERF_COUNT_HW_CACHE_OP_MAX]
                                [PERF_COUNT_HW_CACHE_RESULT_MAX];
u64 __read_mostly hw_cache_extra_regs
                                [PERF_COUNT_HW_CACHE_MAX]
                                [PERF_COUNT_HW_CACHE_OP_MAX]
                                [PERF_COUNT_HW_CACHE_RESULT_MAX];

/*
 * Propagate event elapsed time into the generic event.
 * Can only be executed on the CPU where the event is active.
 * Returns the delta events processed.
 */
u64 x86_perf_event_update(struct perf_event *event)
{
        struct hw_perf_event *hwc = &event->hw;
        int shift = 64 - x86_pmu.cntval_bits;
        u64 prev_raw_count, new_raw_count;
        u64 delta;

        if (unlikely(!hwc->event_base))
                return 0;

        if (unlikely(is_topdown_count(event)) && x86_pmu.update_topdown_event)
                return x86_pmu.update_topdown_event(event);

        /*
         * Careful: an NMI might modify the previous event value.
         *
         * Our tactic to handle this is to first atomically read and
         * exchange a new raw count - then add that new-prev delta
         * count to the generic event atomically:
         */
again:
        prev_raw_count = local64_read(&hwc->prev_count);
        rdpmcl(hwc->event_base_rdpmc, new_raw_count);

        if (local64_cmpxchg(&hwc->prev_count, prev_raw_count,
                                        new_raw_count) != prev_raw_count)
                goto again;

        /*
         * Now we have the new raw value and have updated the prev
         * timestamp already. We can now calculate the elapsed delta
         * (event-)time and add that to the generic event.
         *
         * Careful, not all hw sign-extends above the physical width
         * of the count.
         */
        delta = (new_raw_count << shift) - (prev_raw_count << shift);
        delta >>= shift;

        local64_add(delta, &event->count);
        local64_sub(delta, &hwc->period_left);

        return new_raw_count;
}

/*
 * Find and validate any extra registers to set up.
 */
static int x86_pmu_extra_regs(u64 config, struct perf_event *event)
{
        struct extra_reg *extra_regs = hybrid(event->pmu, extra_regs);
        struct hw_perf_event_extra *reg;
        struct extra_reg *er;

        reg = &event->hw.extra_reg;

        if (!extra_regs)
                return 0;

        for (er = extra_regs; er->msr; er++) {
                if (er->event != (config & er->config_mask))
                        continue;
                if (event->attr.config1 & ~er->valid_mask)
                        return -EINVAL;
                /* Check if the extra msrs can be safely accessed*/
                if (!er->extra_msr_access)
                        return -ENXIO;

                reg->idx = er->idx;
                reg->config = event->attr.config1;
                reg->reg = er->msr;
                break;
        }
        return 0;
}

static atomic_t active_events;
static atomic_t pmc_refcount;
static DEFINE_MUTEX(pmc_reserve_mutex);

#ifdef CONFIG_X86_LOCAL_APIC

static inline int get_possible_num_counters(void)
{
        int i, num_counters = x86_pmu.num_counters;

        if (!is_hybrid())
                return num_counters;

        for (i = 0; i < x86_pmu.num_hybrid_pmus; i++)
                num_counters = max_t(int, num_counters, x86_pmu.hybrid_pmu[i].num_counters);

        return num_counters;
}

static bool reserve_pmc_hardware(void)
{
        int i, num_counters = get_possible_num_counters();

        for (i = 0; i < num_counters; i++) {
                if (!reserve_perfctr_nmi(x86_pmu_event_addr(i)))
                        goto perfctr_fail;
        }

        for (i = 0; i < num_counters; i++) {
                if (!reserve_evntsel_nmi(x86_pmu_config_addr(i)))
                        goto eventsel_fail;
        }

        return true;

eventsel_fail:
        for (i--; i >= 0; i--)
                release_evntsel_nmi(x86_pmu_config_addr(i));

        i = num_counters;

perfctr_fail:
        for (i--; i >= 0; i--)
                release_perfctr_nmi(x86_pmu_event_addr(i));

        return false;
}

static void release_pmc_hardware(void)
{
        int i, num_counters = get_possible_num_counters();

        for (i = 0; i < num_counters; i++) {
                release_perfctr_nmi(x86_pmu_event_addr(i));
                release_evntsel_nmi(x86_pmu_config_addr(i));
        }
}

#else

static bool reserve_pmc_hardware(void) { return true; }
static void release_pmc_hardware(void) {}

#endif

bool check_hw_exists(struct pmu *pmu, int num_counters, int num_counters_fixed)
{
        u64 val, val_fail = -1, val_new= ~0;
        int i, reg, reg_fail = -1, ret = 0;
        int bios_fail = 0;
        int reg_safe = -1;

        /*
         * Check to see if the BIOS enabled any of the counters, if so
         * complain and bail.
         */
        for (i = 0; i < num_counters; i++) {
                reg = x86_pmu_config_addr(i);
                ret = rdmsrl_safe(reg, &val);
                if (ret)
                        goto msr_fail;
                if (val & ARCH_PERFMON_EVENTSEL_ENABLE) {
                        bios_fail = 1;
                        val_fail = val;
                        reg_fail = reg;
                } else {
                        reg_safe = i;
                }
        }

        if (num_counters_fixed) {
                reg = MSR_ARCH_PERFMON_FIXED_CTR_CTRL;
                ret = rdmsrl_safe(reg, &val);
                if (ret)
                        goto msr_fail;
                for (i = 0; i < num_counters_fixed; i++) {
                        if (fixed_counter_disabled(i, pmu))
                                continue;
                        if (val & (0x03ULL << i*4)) {
                                bios_fail = 1;
                                val_fail = val;
                                reg_fail = reg;
                        }
                }
        }

        /*
         * If all the counters are enabled, the below test will always
         * fail.  The tools will also become useless in this scenario.
         * Just fail and disable the hardware counters.
         */

        if (reg_safe == -1) {
                reg = reg_safe;
                goto msr_fail;
        }

        /*
         * Read the current value, change it and read it back to see if it
         * matches, this is needed to detect certain hardware emulators
         * (qemu/kvm) that don't trap on the MSR access and always return 0s.
         */
        reg = x86_pmu_event_addr(reg_safe);
        if (rdmsrl_safe(reg, &val))
                goto msr_fail;
        val ^= 0xffffUL;
        ret = wrmsrl_safe(reg, val);
        ret |= rdmsrl_safe(reg, &val_new);
        if (ret || val != val_new)
                goto msr_fail;

        /*
         * We still allow the PMU driver to operate:
         */
        if (bios_fail) {
                pr_cont("Broken BIOS detected, complain to your hardware vendor.\n");
                pr_err(FW_BUG "the BIOS has corrupted hw-PMU resources (MSR %x is %Lx)\n",
                              reg_fail, val_fail);
        }

        return true;

msr_fail:
        if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) {
                pr_cont("PMU not available due to virtualization, using software events only.\n");
        } else {
                pr_cont("Broken PMU hardware detected, using software events only.\n");
                pr_err("Failed to access perfctr msr (MSR %x is %Lx)\n",
                       reg, val_new);
        }

        return false;
}

static void hw_perf_event_destroy(struct perf_event *event)
{
        x86_release_hardware();
        atomic_dec(&active_events);
}

void hw_perf_lbr_event_destroy(struct perf_event *event)
{
        hw_perf_event_destroy(event);

        /* undo the lbr/bts event accounting */
        x86_del_exclusive(x86_lbr_exclusive_lbr);
}

static inline int x86_pmu_initialized(void)
{
        return x86_pmu.handle_irq != NULL;
}

static inline int
set_ext_hw_attr(struct hw_perf_event *hwc, struct perf_event *event)
{
        struct perf_event_attr *attr = &event->attr;
        unsigned int cache_type, cache_op, cache_result;
        u64 config, val;

        config = attr->config;

        cache_type = (config >> 0) & 0xff;
        if (cache_type >= PERF_COUNT_HW_CACHE_MAX)
                return -EINVAL;
        cache_type = array_index_nospec(cache_type, PERF_COUNT_HW_CACHE_MAX);

        cache_op = (config >>  8) & 0xff;
        if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX)
                return -EINVAL;
        cache_op = array_index_nospec(cache_op, PERF_COUNT_HW_CACHE_OP_MAX);

        cache_result = (config >> 16) & 0xff;
        if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX)
                return -EINVAL;
        cache_result = array_index_nospec(cache_result, PERF_COUNT_HW_CACHE_RESULT_MAX);

        val = hybrid_var(event->pmu, hw_cache_event_ids)[cache_type][cache_op][cache_result];
        if (val == 0)
                return -ENOENT;

        if (val == -1)
                return -EINVAL;

        hwc->config |= val;
        attr->config1 = hybrid_var(event->pmu, hw_cache_extra_regs)[cache_type][cache_op][cache_result];
        return x86_pmu_extra_regs(val, event);
}

int x86_reserve_hardware(void)
{
        int err = 0;

        if (!atomic_inc_not_zero(&pmc_refcount)) {
                mutex_lock(&pmc_reserve_mutex);
                if (atomic_read(&pmc_refcount) == 0) {
                        if (!reserve_pmc_hardware()) {
                                err = -EBUSY;
                        } else {
                                reserve_ds_buffers();
                                reserve_lbr_buffers();
                        }
                }
                if (!err)
                        atomic_inc(&pmc_refcount);
                mutex_unlock(&pmc_reserve_mutex);
        }

        return err;
}

void x86_release_hardware(void)
{
        if (atomic_dec_and_mutex_lock(&pmc_refcount, &pmc_reserve_mutex)) {
                release_pmc_hardware();
                release_ds_buffers();
                release_lbr_buffers();
                mutex_unlock(&pmc_reserve_mutex);
        }
}

/*
 * Check if we can create event of a certain type (that no conflicting events
 * are present).
 */
int x86_add_exclusive(unsigned int what)
{
        int i;

        /*
         * When lbr_pt_coexist we allow PT to coexist with either LBR or BTS.
         * LBR and BTS are still mutually exclusive.
         */
        if (x86_pmu.lbr_pt_coexist && what == x86_lbr_exclusive_pt)
                goto out;

        if (!atomic_inc_not_zero(&x86_pmu.lbr_exclusive[what])) {
                mutex_lock(&pmc_reserve_mutex);
                for (i = 0; i < ARRAY_SIZE(x86_pmu.lbr_exclusive); i++) {
                        if (i != what && atomic_read(&x86_pmu.lbr_exclusive[i]))
                                goto fail_unlock;
                }
                atomic_inc(&x86_pmu.lbr_exclusive[what]);
                mutex_unlock(&pmc_reserve_mutex);
        }

out:
        atomic_inc(&active_events);
        return 0;

fail_unlock:
        mutex_unlock(&pmc_reserve_mutex);
        return -EBUSY;
}

void x86_del_exclusive(unsigned int what)
{
        atomic_dec(&active_events);

        /*
         * See the comment in x86_add_exclusive().
         */
        if (x86_pmu.lbr_pt_coexist && what == x86_lbr_exclusive_pt)
                return;

        atomic_dec(&x86_pmu.lbr_exclusive[what]);
}

int x86_setup_perfctr(struct perf_event *event)
{
        struct perf_event_attr *attr = &event->attr;
        struct hw_perf_event *hwc = &event->hw;
        u64 config;

        if (!is_sampling_event(event)) {
                hwc->sample_period = x86_pmu.max_period;
                hwc->last_period = hwc->sample_period;
                local64_set(&hwc->period_left, hwc->sample_period);
        }

        if (attr->type == event->pmu->type)
                return x86_pmu_extra_regs(event->attr.config, event);

        if (attr->type == PERF_TYPE_HW_CACHE)
                return set_ext_hw_attr(hwc, event);

        if (attr->config >= x86_pmu.max_events)
                return -EINVAL;

        attr->config = array_index_nospec((unsigned long)attr->config, x86_pmu.max_events);

        /*
         * The generic map:
         */
        config = x86_pmu.event_map(attr->config);

        if (config == 0)
                return -ENOENT;

        if (config == -1LL)
                return -EINVAL;

        hwc->config |= config;

        return 0;
}

/*
 * check that branch_sample_type is compatible with
 * settings needed for precise_ip > 1 which implies
 * using the LBR to capture ALL taken branches at the
 * priv levels of the measurement
 */
static inline int precise_br_compat(struct perf_event *event)
{
        u64 m = event->attr.branch_sample_type;
        u64 b = 0;

        /* must capture all branches */
        if (!(m & PERF_SAMPLE_BRANCH_ANY))
                return 0;

        m &= PERF_SAMPLE_BRANCH_KERNEL | PERF_SAMPLE_BRANCH_USER;

        if (!event->attr.exclude_user)
                b |= PERF_SAMPLE_BRANCH_USER;

        if (!event->attr.exclude_kernel)
                b |= PERF_SAMPLE_BRANCH_KERNEL;

        /*
         * ignore PERF_SAMPLE_BRANCH_HV, not supported on x86
         */

        return m == b;
}

int x86_pmu_max_precise(void)
{
        int precise = 0;

        /* Support for constant skid */
        if (x86_pmu.pebs_active && !x86_pmu.pebs_broken) {
                precise++;

                /* Support for IP fixup */
                if (x86_pmu.lbr_nr || x86_pmu.intel_cap.pebs_format >= 2)
                        precise++;

                if (x86_pmu.pebs_prec_dist)
                        precise++;
        }
        return precise;
}

int x86_pmu_hw_config(struct perf_event *event)
{
        if (event->attr.precise_ip) {
                int precise = x86_pmu_max_precise();

                if (event->attr.precise_ip > precise)
                        return -EOPNOTSUPP;

                /* There's no sense in having PEBS for non sampling events: */
                if (!is_sampling_event(event))
                        return -EINVAL;
        }
        /*
         * check that PEBS LBR correction does not conflict with
         * whatever the user is asking with attr->branch_sample_type
         */
        if (event->attr.precise_ip > 1 && x86_pmu.intel_cap.pebs_format < 2) {
                u64 *br_type = &event->attr.branch_sample_type;

                if (has_branch_stack(event)) {
                        if (!precise_br_compat(event))
                                return -EOPNOTSUPP;

                        /* branch_sample_type is compatible */

                } else {
                        /*
                         * user did not specify  branch_sample_type
                         *
                         * For PEBS fixups, we capture all
                         * the branches at the priv level of the
                         * event.
                         */
                        *br_type = PERF_SAMPLE_BRANCH_ANY;

                        if (!event->attr.exclude_user)
                                *br_type |= PERF_SAMPLE_BRANCH_USER;

                        if (!event->attr.exclude_kernel)
                                *br_type |= PERF_SAMPLE_BRANCH_KERNEL;
                }
        }

        if (event->attr.branch_sample_type & PERF_SAMPLE_BRANCH_CALL_STACK)
                event->attach_state |= PERF_ATTACH_TASK_DATA;

        /*
         * Generate PMC IRQs:
         * (keep 'enabled' bit clear for now)
         */
        event->hw.config = ARCH_PERFMON_EVENTSEL_INT;

        /*
         * Count user and OS events unless requested not to
         */
        if (!event->attr.exclude_user)
                event->hw.config |= ARCH_PERFMON_EVENTSEL_USR;
        if (!event->attr.exclude_kernel)
                event->hw.config |= ARCH_PERFMON_EVENTSEL_OS;

        if (event->attr.type == event->pmu->type)
                event->hw.config |= event->attr.config & X86_RAW_EVENT_MASK;

        if (event->attr.sample_period && x86_pmu.limit_period) {
                if (x86_pmu.limit_period(event, event->attr.sample_period) >
                                event->attr.sample_period)
                        return -EINVAL;
        }

        /* sample_regs_user never support XMM registers */
        if (unlikely(event->attr.sample_regs_user & PERF_REG_EXTENDED_MASK))
                return -EINVAL;
        /*
         * Besides the general purpose registers, XMM registers may
         * be collected in PEBS on some platforms, e.g. Icelake
         */
        if (unlikely(event->attr.sample_regs_intr & PERF_REG_EXTENDED_MASK)) {
                if (!(event->pmu->capabilities & PERF_PMU_CAP_EXTENDED_REGS))
                        return -EINVAL;

                if (!event->attr.precise_ip)
                        return -EINVAL;
        }

        return x86_setup_perfctr(event);
}

/*
 * Setup the hardware configuration for a given attr_type
 */
static int __x86_pmu_event_init(struct perf_event *event)
{
        int err;

        if (!x86_pmu_initialized())
                return -ENODEV;

        err = x86_reserve_hardware();
        if (err)
                return err;

        atomic_inc(&active_events);
        event->destroy = hw_perf_event_destroy;

        event->hw.idx = -1;
        event->hw.last_cpu = -1;
        event->hw.last_tag = ~0ULL;

        /* mark unused */
        event->hw.extra_reg.idx = EXTRA_REG_NONE;
        event->hw.branch_reg.idx = EXTRA_REG_NONE;

        return x86_pmu.hw_config(event);
}

void x86_pmu_disable_all(void)
{
        struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
        int idx;

        for (idx = 0; idx < x86_pmu.num_counters; idx++) {
                struct hw_perf_event *hwc = &cpuc->events[idx]->hw;
                u64 val;

                if (!test_bit(idx, cpuc->active_mask))
                        continue;
                rdmsrl(x86_pmu_config_addr(idx), val);
                if (!(val & ARCH_PERFMON_EVENTSEL_ENABLE))
                        continue;
                val &= ~ARCH_PERFMON_EVENTSEL_ENABLE;
                wrmsrl(x86_pmu_config_addr(idx), val);
                if (is_counter_pair(hwc))
                        wrmsrl(x86_pmu_config_addr(idx + 1), 0);
        }
}

struct perf_guest_switch_msr *perf_guest_get_msrs(int *nr)
{
        return static_call(x86_pmu_guest_get_msrs)(nr);
}
EXPORT_SYMBOL_GPL(perf_guest_get_msrs);

/*
 * There may be PMI landing after enabled=0. The PMI hitting could be before or
 * after disable_all.
 *
 * If PMI hits before disable_all, the PMU will be disabled in the NMI handler.
 * It will not be re-enabled in the NMI handler again, because enabled=0. After
 * handling the NMI, disable_all will be called, which will not change the
 * state either. If PMI hits after disable_all, the PMU is already disabled
 * before entering NMI handler. The NMI handler will not change the state
 * either.
 *
 * So either situation is harmless.
 */
static void x86_pmu_disable(struct pmu *pmu)
{
        struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);

        if (!x86_pmu_initialized())
                return;

        if (!cpuc->enabled)
                return;

        cpuc->n_added = 0;
        cpuc->enabled = 0;
        barrier();

        static_call(x86_pmu_disable_all)();
}

void x86_pmu_enable_all(int added)
{
        struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
        int idx;

        for (idx = 0; idx < x86_pmu.num_counters; idx++) {
                struct hw_perf_event *hwc = &cpuc->events[idx]->hw;

                if (!test_bit(idx, cpuc->active_mask))
                        continue;

                __x86_pmu_enable_event(hwc, ARCH_PERFMON_EVENTSEL_ENABLE);
        }
}

static inline int is_x86_event(struct perf_event *event)
{
        int i;

        if (!is_hybrid())
                return event->pmu == &pmu;

        for (i = 0; i < x86_pmu.num_hybrid_pmus; i++) {
                if (event->pmu == &x86_pmu.hybrid_pmu[i].pmu)
                        return true;
        }

        return false;
}

struct pmu *x86_get_pmu(unsigned int cpu)
{
        struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu);

        /*
         * All CPUs of the hybrid type have been offline.
         * The x86_get_pmu() should not be invoked.
         */
        if (WARN_ON_ONCE(!cpuc->pmu))
                return &pmu;

        return cpuc->pmu;
}
/*
 * Event scheduler state:
 *
 * Assign events iterating over all events and counters, beginning
 * with events with least weights first. Keep the current iterator
 * state in struct sched_state.
 */
struct sched_state {
        int        weight;
        int        event;                /* event index */
        int        counter;        /* counter index */
        int        unassigned;        /* number of events to be assigned left */
        int        nr_gp;                /* number of GP counters used */
        u64        used;
};

/* Total max is X86_PMC_IDX_MAX, but we are O(n!) limited */
#define        SCHED_STATES_MAX        2

struct perf_sched {
        int                        max_weight;
        int                        max_events;
        int                        max_gp;
        int                        saved_states;
        struct event_constraint        **constraints;
        struct sched_state        state;
        struct sched_state        saved[SCHED_STATES_MAX];
};

/*
 * Initialize iterator that runs through all events and counters.
 */
static void perf_sched_init(struct perf_sched *sched, struct event_constraint **constraints,
                            int num, int wmin, int wmax, int gpmax)
{
        int idx;

        memset(sched, 0, sizeof(*sched));
        sched->max_events        = num;
        sched->max_weight        = wmax;
        sched->max_gp                = gpmax;
        sched->constraints        = constraints;

        for (idx = 0; idx < num; idx++) {
                if (constraints[idx]->weight == wmin)
                        break;
        }

        sched->state.event        = idx;                /* start with min weight */
        sched->state.weight        = wmin;
        sched->state.unassigned        = num;
}

static void perf_sched_save_state(struct perf_sched *sched)
{
        if (WARN_ON_ONCE(sched->saved_states >= SCHED_STATES_MAX))
                return;

        sched->saved[sched->saved_states] = sched->state;
        sched->saved_states++;
}

static bool perf_sched_restore_state(struct perf_sched *sched)
{
        if (!sched->saved_states)
                return false;

        sched->saved_states--;
        sched->state = sched->saved[sched->saved_states];

        /* this assignment didn't work out */
        /* XXX broken vs EVENT_PAIR */
        sched->state.used &= ~BIT_ULL(sched->state.counter);

        /* try the next one */
        sched->state.counter++;

        return true;
}

/*
 * Select a counter for the current event to schedule. Return true on
 * success.
 */
static bool __perf_sched_find_counter(struct perf_sched *sched)
{
        struct event_constraint *c;
        int idx;

        if (!sched->state.unassigned)
                return false;

        if (sched->state.event >= sched->max_events)
                return false;

        c = sched->constraints[sched->state.event];
        /* Prefer fixed purpose counters */
        if (c->idxmsk64 & (~0ULL << INTEL_PMC_IDX_FIXED)) {
                idx = INTEL_PMC_IDX_FIXED;
                for_each_set_bit_from(idx, c->idxmsk, X86_PMC_IDX_MAX) {
                        u64 mask = BIT_ULL(idx);

                        if (sched->state.used & mask)
                                continue;

                        sched->state.used |= mask;
                        goto done;
                }
        }

        /* Grab the first unused counter starting with idx */
        idx = sched->state.counter;
        for_each_set_bit_from(idx, c->idxmsk, INTEL_PMC_IDX_FIXED) {
                u64 mask = BIT_ULL(idx);

                if (c->flags & PERF_X86_EVENT_PAIR)
                        mask |= mask << 1;

                if (sched->state.used & mask)
                        continue;

                if (sched->state.nr_gp++ >= sched->max_gp)
                        return false;

                sched->state.used |= mask;
                goto done;
        }

        return false;

done:
        sched->state.counter = idx;

        if (c->overlap)
                perf_sched_save_state(sched);

        return true;
}

static bool perf_sched_find_counter(struct perf_sched *sched)
{
        while (!__perf_sched_find_counter(sched)) {
                if (!perf_sched_restore_state(sched))
                        return false;
        }

        return true;
}

/*
 * Go through all unassigned events and find the next one to schedule.
 * Take events with the least weight first. Return true on success.
 */
static bool perf_sched_next_event(struct perf_sched *sched)
{
        struct event_constraint *c;

        if (!sched->state.unassigned || !--sched->state.unassigned)
                return false;

        do {
                /* next event */
                sched->state.event++;
                if (sched->state.event >= sched->max_events) {
                        /* next weight */
                        sched->state.event = 0;
                        sched->state.weight++;
                        if (sched->state.weight > sched->max_weight)
                                return false;
                }
                c = sched->constraints[sched->state.event];
        } while (c->weight != sched->state.weight);

        sched->state.counter = 0;        /* start with first counter */

        return true;
}

/*
 * Assign a counter for each event.
 */
int perf_assign_events(struct event_constraint **constraints, int n,
                        int wmin, int wmax, int gpmax, int *assign)
{
        struct perf_sched sched;

        perf_sched_init(&sched, constraints, n, wmin, wmax, gpmax);

        do {
                if (!perf_sched_find_counter(&sched))
                        break;        /* failed */
                if (assign)
                        assign[sched.state.event] = sched.state.counter;
        } while (perf_sched_next_event(&sched));

        return sched.state.unassigned;
}
EXPORT_SYMBOL_GPL(perf_assign_events);

int x86_schedule_events(struct cpu_hw_events *cpuc, int n, int *assign)
{
        int num_counters = hybrid(cpuc->pmu, num_counters);
        struct event_constraint *c;
        struct perf_event *e;
        int n0, i, wmin, wmax, unsched = 0;
        struct hw_perf_event *hwc;
        u64 used_mask = 0;

        /*
         * Compute the number of events already present; see x86_pmu_add(),
         * validate_group() and x86_pmu_commit_txn(). For the former two
         * cpuc->n_events hasn't been updated yet, while for the latter
         * cpuc->n_txn contains the number of events added in the current
         * transaction.
         */
        n0 = cpuc->n_events;
        if (cpuc->txn_flags & PERF_PMU_TXN_ADD)
                n0 -= cpuc->n_txn;

        static_call_cond(x86_pmu_start_scheduling)(cpuc);

        for (i = 0, wmin = X86_PMC_IDX_MAX, wmax = 0; i < n; i++) {
                c = cpuc->event_constraint[i];

                /*
                 * Previously scheduled events should have a cached constraint,
                 * while new events should not have one.
                 */
                WARN_ON_ONCE((c && i >= n0) || (!c && i < n0));

                /*
                 * Request constraints for new events; or for those events that
                 * have a dynamic constraint -- for those the constraint can
                 * change due to external factors (sibling state, allow_tfa).
                 */
                if (!c || (c->flags & PERF_X86_EVENT_DYNAMIC)) {
                        c = static_call(x86_pmu_get_event_constraints)(cpuc, i, cpuc->event_list[i]);
                        cpuc->event_constraint[i] = c;
                }

                wmin = min(wmin, c->weight);
                wmax = max(wmax, c->weight);
        }

        /*
         * fastpath, try to reuse previous register
         */
        for (i = 0; i < n; i++) {
                u64 mask;

                hwc = &cpuc->event_list[i]->hw;
                c = cpuc->event_constraint[i];

                /* never assigned */
                if (hwc->idx == -1)
                        break;

                /* constraint still honored */
                if (!test_bit(hwc->idx, c->idxmsk))
                        break;

                mask = BIT_ULL(hwc->idx);
                if (is_counter_pair(hwc))
                        mask |= mask << 1;

                /* not already used */
                if (used_mask & mask)
                        break;

                used_mask |= mask;

                if (assign)
                        assign[i] = hwc->idx;
        }

        /* slow path */
        if (i != n) {
                int gpmax = num_counters;

                /*
                 * Do not allow scheduling of more than half the available
                 * generic counters.
                 *
                 * This helps avoid counter starvation of sibling thread by
                 * ensuring at most half the counters cannot be in exclusive
                 * mode. There is no designated counters for the limits. Any
                 * N/2 counters can be used. This helps with events with
                 * specific counter constraints.
                 */
                if (is_ht_workaround_enabled() && !cpuc->is_fake &&
                    READ_ONCE(cpuc->excl_cntrs->exclusive_present))
                        gpmax /= 2;

                /*
                 * Reduce the amount of available counters to allow fitting
                 * the extra Merge events needed by large increment events.
                 */
                if (x86_pmu.flags & PMU_FL_PAIR) {
                        gpmax = num_counters - cpuc->n_pair;
                        WARN_ON(gpmax <= 0);
                }

                unsched = perf_assign_events(cpuc->event_constraint, n, wmin,
                                             wmax, gpmax, assign);
        }

        /*
         * In case of success (unsched = 0), mark events as committed,
         * so we do not put_constraint() in case new events are added
         * and fail to be scheduled
         *
         * We invoke the lower level commit callback to lock the resource
         *
         * We do not need to do all of this in case we are called to
         * validate an event group (assign == NULL)
         */
        if (!unsched && assign) {
                for (i = 0; i < n; i++)
                        static_call_cond(x86_pmu_commit_scheduling)(cpuc, i, assign[i]);
        } else {
                for (i = n0; i < n; i++) {
                        e = cpuc->event_list[i];

                        /*
                         * release events that failed scheduling
                         */
                        static_call_cond(x86_pmu_put_event_constraints)(cpuc, e);

                        cpuc->event_constraint[i] = NULL;
                }
        }

        static_call_cond(x86_pmu_stop_scheduling)(cpuc);

        return unsched ? -EINVAL : 0;
}

static int add_nr_metric_event(struct cpu_hw_events *cpuc,
                               struct perf_event *event)
{
        if (is_metric_event(event)) {
                if (cpuc->n_metric == INTEL_TD_METRIC_NUM)
                        return -EINVAL;
                cpuc->n_metric++;
                cpuc->n_txn_metric++;
        }

        return 0;
}

static void del_nr_metric_event(struct cpu_hw_events *cpuc,
                                struct perf_event *event)
{
        if (is_metric_event(event))
                cpuc->n_metric--;
}

static int collect_event(struct cpu_hw_events *cpuc, struct perf_event *event,
                         int max_count, int n)
{
        union perf_capabilities intel_cap = hybrid(cpuc->pmu, intel_cap);

        if (intel_cap.perf_metrics && add_nr_metric_event(cpuc, event))
                return -EINVAL;

        if (n >= max_count + cpuc->n_metric)
                return -EINVAL;

        cpuc->event_list[n] = event;
        if (is_counter_pair(&event->hw)) {
                cpuc->n_pair++;
                cpuc->n_txn_pair++;
        }

        return 0;
}

/*
 * dogrp: true if must collect siblings events (group)
 * returns total number of events and error code
 */
static int collect_events(struct cpu_hw_events *cpuc, struct perf_event *leader, bool dogrp)
{
        int num_counters = hybrid(cpuc->pmu, num_counters);
        int num_counters_fixed = hybrid(cpuc->pmu, num_counters_fixed);
        struct perf_event *event;
        int n, max_count;

        max_count = num_counters + num_counters_fixed;

        /* current number of events already accepted */
        n = cpuc->n_events;
        if (!cpuc->n_events)
                cpuc->pebs_output = 0;

        if (!cpuc->is_fake && leader->attr.precise_ip) {
                /*
                 * For PEBS->PT, if !aux_event, the group leader (PT) went
                 * away, the group was broken down and this singleton event
                 * can't schedule any more.
                 */
                if (is_pebs_pt(leader) && !leader->aux_event)
                        return -EINVAL;

                /*
                 * pebs_output: 0: no PEBS so far, 1: PT, 2: DS
                 */
                if (cpuc->pebs_output &&
                    cpuc->pebs_output != is_pebs_pt(leader) + 1)
                        return -EINVAL;

                cpuc->pebs_output = is_pebs_pt(leader) + 1;
        }

        if (is_x86_event(leader)) {
                if (collect_event(cpuc, leader, max_count, n))
                        return -EINVAL;
                n++;
        }

        if (!dogrp)
                return n;

        for_each_sibling_event(event, leader) {
                if (!is_x86_event(event) || event->state <= PERF_EVENT_STATE_OFF)
                        continue;

                if (collect_event(cpuc, event, max_count, n))
                        return -EINVAL;

                n++;
        }
        return n;
}

static inline void x86_assign_hw_event(struct perf_event *event,
                                struct cpu_hw_events *cpuc, int i)
{
        struct hw_perf_event *hwc = &event->hw;
        int idx;

        idx = hwc->idx = cpuc->assign[i];
        hwc->last_cpu = smp_processor_id();
        hwc->last_tag = ++cpuc->tags[i];

        switch (hwc->idx) {
        case INTEL_PMC_IDX_FIXED_BTS:
        case INTEL_PMC_IDX_FIXED_VLBR:
                hwc->config_base = 0;
                hwc->event_base        = 0;
                break;

        case INTEL_PMC_IDX_METRIC_BASE ... INTEL_PMC_IDX_METRIC_END:
                /* All the metric events are mapped onto the fixed counter 3. */
                idx = INTEL_PMC_IDX_FIXED_SLOTS;
                fallthrough;
        case INTEL_PMC_IDX_FIXED ... INTEL_PMC_IDX_FIXED_BTS-1:
                hwc->config_base = MSR_ARCH_PERFMON_FIXED_CTR_CTRL;
                hwc->event_base = MSR_ARCH_PERFMON_FIXED_CTR0 +
                                (idx - INTEL_PMC_IDX_FIXED);
                hwc->event_base_rdpmc = (idx - INTEL_PMC_IDX_FIXED) |
                                        INTEL_PMC_FIXED_RDPMC_BASE;
                break;

        default:
                hwc->config_base = x86_pmu_config_addr(hwc->idx);
                hwc->event_base  = x86_pmu_event_addr(hwc->idx);
                hwc->event_base_rdpmc = x86_pmu_rdpmc_index(hwc->idx);
                break;
        }
}

/**
 * x86_perf_rdpmc_index - Return PMC counter used for event
 * @event: the perf_event to which the PMC counter was assigned
 *
 * The counter assigned to this performance event may change if interrupts
 * are enabled. This counter should thus never be used while interrupts are
 * enabled. Before this function is used to obtain the assigned counter the
 * event should be checked for validity using, for example,
 * perf_event_read_local(), within the same interrupt disabled section in
 * which this counter is planned to be used.
 *
 * Return: The index of the performance monitoring counter assigned to
 * @perf_event.
 */
int x86_perf_rdpmc_index(struct perf_event *event)
{
        lockdep_assert_irqs_disabled();

        return event->hw.event_base_rdpmc;
}

static inline int match_prev_assignment(struct hw_perf_event *hwc,
                                        struct cpu_hw_events *cpuc,
                                        int i)
{
        return hwc->idx == cpuc->assign[i] &&
                hwc->last_cpu == smp_processor_id() &&
                hwc->last_tag == cpuc->tags[i];
}

static void x86_pmu_start(struct perf_event *event, int flags);

static void x86_pmu_enable(struct pmu *pmu)
{
        struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
        struct perf_event *event;
        struct hw_perf_event *hwc;
        int i, added = cpuc->n_added;

        if (!x86_pmu_initialized())
                return;

        if (cpuc->enabled)
                return;

        if (cpuc->n_added) {
                int n_running = cpuc->n_events - cpuc->n_added;
                /*
                 * apply assignment obtained either from
                 * hw_perf_group_sched_in() or x86_pmu_enable()
                 *
                 * step1: save events moving to new counters
                 */
                for (i = 0; i < n_running; i++) {
                        event = cpuc->event_list[i];
                        hwc = &event->hw;

                        /*
                         * we can avoid reprogramming counter if:
                         * - assigned same counter as last time
                         * - running on same CPU as last time
                         * - no other event has used the counter since
                         */
                        if (hwc->idx == -1 ||
                            match_prev_assignment(hwc, cpuc, i))
                                continue;

                        /*
                         * Ensure we don't accidentally enable a stopped
                         * counter simply because we rescheduled.
                         */
                        if (hwc->state & PERF_HES_STOPPED)
                                hwc->state |= PERF_HES_ARCH;

                        x86_pmu_stop(event, PERF_EF_UPDATE);
                }

                /*
                 * step2: reprogram moved events into new counters
                 */
                for (i = 0; i < cpuc->n_events; i++) {
                        event = cpuc->event_list[i];
                        hwc = &event->hw;

                        if (!match_prev_assignment(hwc, cpuc, i))
                                x86_assign_hw_event(event, cpuc, i);
                        else if (i < n_running)
                                continue;

                        if (hwc->state & PERF_HES_ARCH)
                                continue;

                        x86_pmu_start(event, PERF_EF_RELOAD);
                }
                cpuc->n_added = 0;
                perf_events_lapic_init();
        }

        cpuc->enabled = 1;
        barrier();

        static_call(x86_pmu_enable_all)(added);
}

static DEFINE_PER_CPU(u64 [X86_PMC_IDX_MAX], pmc_prev_left);

/*
 * Set the next IRQ period, based on the hwc->period_left value.
 * To be called with the event disabled in hw:
 */
int x86_perf_event_set_period(struct perf_event *event)
{
        struct hw_perf_event *hwc = &event->hw;
        s64 left = local64_read(&hwc->period_left);
        s64 period = hwc->sample_period;
        int ret = 0, idx = hwc->idx;

        if (unlikely(!hwc->event_base))
                return 0;

        if (unlikely(is_topdown_count(event)) &&
            x86_pmu.set_topdown_event_period)
                return x86_pmu.set_topdown_event_period(event);

        /*
         * If we are way outside a reasonable range then just skip forward:
         */
        if (unlikely(left <= -period)) {
                left = period;
                local64_set(&hwc->period_left, left);
                hwc->last_period = period;
                ret = 1;
        }

        if (unlikely(left <= 0)) {
                left += period;
                local64_set(&hwc->period_left, left);
                hwc->last_period = period;
                ret = 1;
        }
        /*
         * Quirk: certain CPUs dont like it if just 1 hw_event is left:
         */
        if (unlikely(left < 2))
                left = 2;

        if (left > x86_pmu.max_period)
                left = x86_pmu.max_period;

        if (x86_pmu.limit_period)
                left = x86_pmu.limit_period(event, left);

        per_cpu(pmc_prev_left[idx], smp_processor_id()) = left;

        /*
         * The hw event starts counting from this event offset,
         * mark it to be able to extra future deltas:
         */
        local64_set(&hwc->prev_count, (u64)-left);

        wrmsrl(hwc->event_base, (u64)(-left) & x86_pmu.cntval_mask);

        /*
         * Sign extend the Merge event counter's upper 16 bits since
         * we currently declare a 48-bit counter width
         */
        if (is_counter_pair(hwc))
                wrmsrl(x86_pmu_event_addr(idx + 1), 0xffff);

        /*
         * Due to erratum on certan cpu we need
         * a second write to be sure the register
         * is updated properly
         */
        if (x86_pmu.perfctr_second_write) {
                wrmsrl(hwc->event_base,
                        (u64)(-left) & x86_pmu.cntval_mask);
        }

        perf_event_update_userpage(event);

        return ret;
}

void x86_pmu_enable_event(struct perf_event *event)
{
        if (__this_cpu_read(cpu_hw_events.enabled))
                __x86_pmu_enable_event(&event->hw,
                                       ARCH_PERFMON_EVENTSEL_ENABLE);
}

/*
 * Add a single event to the PMU.
 *
 * The event is added to the group of enabled events
 * but only if it can be scheduled with existing events.
 */
static int x86_pmu_add(struct perf_event *event, int flags)
{
        struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
        struct hw_perf_event *hwc;
        int assign[X86_PMC_IDX_MAX];
        int n, n0, ret;

        hwc = &event->hw;

        n0 = cpuc->n_events;
        ret = n = collect_events(cpuc, event, false);
        if (ret < 0)
                goto out;

        hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED;
        if (!(flags & PERF_EF_START))
                hwc->state |= PERF_HES_ARCH;

        /*
         * If group events scheduling transaction was started,
         * skip the schedulability test here, it will be performed
         * at commit time (->commit_txn) as a whole.
         *
         * If commit fails, we'll call ->del() on all events
         * for which ->add() was called.
         */
        if (cpuc->txn_flags & PERF_PMU_TXN_ADD)
                goto done_collect;

        ret = static_call(x86_pmu_schedule_events)(cpuc, n, assign);
        if (ret)
                goto out;
        /*
         * copy new assignment, now we know it is possible
         * will be used by hw_perf_enable()
         */
        memcpy(cpuc->assign, assign, n*sizeof(int));

done_collect:
        /*
         * Commit the collect_events() state. See x86_pmu_del() and
         * x86_pmu_*_txn().
         */
        cpuc->n_events = n;
        cpuc->n_added += n - n0;
        cpuc->n_txn += n - n0;

        /*
         * This is before x86_pmu_enable() will call x86_pmu_start(),
         * so we enable LBRs before an event needs them etc..
         */
        static_call_cond(x86_pmu_add)(event);

        ret = 0;
out:
        return ret;
}

static void x86_pmu_start(struct perf_event *event, int flags)
{
        struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
        int idx = event->hw.idx;

        if (WARN_ON_ONCE(!(event->hw.state & PERF_HES_STOPPED)))
                return;

        if (WARN_ON_ONCE(idx == -1))
                return;

        if (flags & PERF_EF_RELOAD) {
                WARN_ON_ONCE(!(event->hw.state & PERF_HES_UPTODATE));
                x86_perf_event_set_period(event);
        }

        event->hw.state = 0;

        cpuc->events[idx] = event;
        __set_bit(idx, cpuc->active_mask);
        static_call(x86_pmu_enable)(event);
        perf_event_update_userpage(event);
}

void perf_event_print_debug(void)
{
        u64 ctrl, status, overflow, pmc_ctrl, pmc_count, prev_left, fixed;
        u64 pebs, debugctl;
        int cpu = smp_processor_id();
        struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu);
        int num_counters = hybrid(cpuc->pmu, num_counters);
        int num_counters_fixed = hybrid(cpuc->pmu, num_counters_fixed);
        struct event_constraint *pebs_constraints = hybrid(cpuc->pmu, pebs_constraints);
        unsigned long flags;
        int idx;

        if (!num_counters)
                return;

        local_irq_save(flags);

        if (x86_pmu.version >= 2) {
                rdmsrl(MSR_CORE_PERF_GLOBAL_CTRL, ctrl);
                rdmsrl(MSR_CORE_PERF_GLOBAL_STATUS, status);
                rdmsrl(MSR_CORE_PERF_GLOBAL_OVF_CTRL, overflow);
                rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR_CTRL, fixed);

                pr_info("\n");
                pr_info("CPU#%d: ctrl:       %016llx\n", cpu, ctrl);
                pr_info("CPU#%d: status:     %016llx\n", cpu, status);
                pr_info("CPU#%d: overflow:   %016llx\n", cpu, overflow);
                pr_info("CPU#%d: fixed:      %016llx\n", cpu, fixed);
                if (pebs_constraints) {
                        rdmsrl(MSR_IA32_PEBS_ENABLE, pebs);
                        pr_info("CPU#%d: pebs:       %016llx\n", cpu, pebs);
                }
                if (x86_pmu.lbr_nr) {
                        rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);
                        pr_info("CPU#%d: debugctl:   %016llx\n", cpu, debugctl);
                }
        }
        pr_info("CPU#%d: active:     %016llx\n", cpu, *(u64 *)cpuc->active_mask);

        for (idx = 0; idx < num_counters; idx++) {
                rdmsrl(x86_pmu_config_addr(idx), pmc_ctrl);
                rdmsrl(x86_pmu_event_addr(idx), pmc_count);

                prev_left = per_cpu(pmc_prev_left[idx], cpu);

                pr_info("CPU#%d:   gen-PMC%d ctrl:  %016llx\n",
                        cpu, idx, pmc_ctrl);
                pr_info("CPU#%d:   gen-PMC%d count: %016llx\n",
                        cpu, idx, pmc_count);
                pr_info("CPU#%d:   gen-PMC%d left:  %016llx\n",
                        cpu, idx, prev_left);
        }
        for (idx = 0; idx < num_counters_fixed; idx++) {
                if (fixed_counter_disabled(idx, cpuc->pmu))
                        continue;
                rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR0 + idx, pmc_count);

                pr_info("CPU#%d: fixed-PMC%d count: %016llx\n",
                        cpu, idx, pmc_count);
        }
        local_irq_restore(flags);
}

void x86_pmu_stop(struct perf_event *event, int flags)
{
        struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
        struct hw_perf_event *hwc = &event->hw;

        if (test_bit(hwc->idx, cpuc->active_mask)) {
                static_call(x86_pmu_disable)(event);
                __clear_bit(hwc->idx, cpuc->active_mask);
                cpuc->events[hwc->idx] = NULL;
                WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED);
                hwc->state |= PERF_HES_STOPPED;
        }

        if ((flags & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE)) {
                /*
                 * Drain the remaining delta count out of a event
                 * that we are disabling:
                 */
                x86_perf_event_update(event);
                hwc->state |= PERF_HES_UPTODATE;
        }
}

static void x86_pmu_del(struct perf_event *event, int flags)
{
        struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
        union perf_capabilities intel_cap = hybrid(cpuc->pmu, intel_cap);
        int i;

        /*
         * If we're called during a txn, we only need to undo x86_pmu.add.
         * The events never got scheduled and ->cancel_txn will truncate
         * the event_list.
         *
         * XXX assumes any ->del() called during a TXN will only be on
         * an event added during that same TXN.
         */
        if (cpuc->txn_flags & PERF_PMU_TXN_ADD)
                goto do_del;

        __set_bit(event->hw.idx, cpuc->dirty);

        /*
         * Not a TXN, therefore cleanup properly.
         */
        x86_pmu_stop(event, PERF_EF_UPDATE);

        for (i = 0; i < cpuc->n_events; i++) {
                if (event == cpuc->event_list[i])
                        break;
        }

        if (WARN_ON_ONCE(i == cpuc->n_events)) /* called ->del() without ->add() ? */
                return;

        /* If we have a newly added event; make sure to decrease n_added. */
        if (i >= cpuc->n_events - cpuc->n_added)
                --cpuc->n_added;

        static_call_cond(x86_pmu_put_event_constraints)(cpuc, event);

        /* Delete the array entry. */
        while (++i < cpuc->n_events) {
                cpuc->event_list[i-1] = cpuc->event_list[i];
                cpuc->event_constraint[i-1] = cpuc->event_constraint[i];
                cpuc->assign[i-1] = cpuc->assign[i];
        }
        cpuc->event_constraint[i-1] = NULL;
        --cpuc->n_events;
        if (intel_cap.perf_metrics)
                del_nr_metric_event(cpuc, event);

        perf_event_update_userpage(event);

do_del:

        /*
         * This is after x86_pmu_stop(); so we disable LBRs after any
         * event can need them etc..
         */
        static_call_cond(x86_pmu_del)(event);
}

int x86_pmu_handle_irq(struct pt_regs *regs)
{
        struct perf_sample_data data;
        struct cpu_hw_events *cpuc;
        struct perf_event *event;
        int idx, handled = 0;
        u64 val;

        cpuc = this_cpu_ptr(&cpu_hw_events);

        /*
         * Some chipsets need to unmask the LVTPC in a particular spot
         * inside the nmi handler.  As a result, the unmasking was pushed
         * into all the nmi handlers.
         *
         * This generic handler doesn't seem to have any issues where the
         * unmasking occurs so it was left at the top.
         */
        apic_write(APIC_LVTPC, APIC_DM_NMI);

        for (idx = 0; idx < x86_pmu.num_counters; idx++) {
                if (!test_bit(idx, cpuc->active_mask))
                        continue;

                event = cpuc->events[idx];

                val = x86_perf_event_update(event);
                if (val & (1ULL << (x86_pmu.cntval_bits - 1)))
                        continue;

                /*
                 * event overflow
                 */
                handled++;
                perf_sample_data_init(&data, 0, event->hw.last_period);

                if (!x86_perf_event_set_period(event))
                        continue;

                if (perf_event_overflow(event, &data, regs))
                        x86_pmu_stop(event, 0);
        }

        if (handled)
                inc_irq_stat(apic_perf_irqs);

        return handled;
}

void perf_events_lapic_init(void)
{
        if (!x86_pmu.apic || !x86_pmu_initialized())
                return;

        /*
         * Always use NMI for PMU
         */
        apic_write(APIC_LVTPC, APIC_DM_NMI);
}

static int
perf_event_nmi_handler(unsigned int cmd, struct pt_regs *regs)
{
        u64 start_clock;
        u64 finish_clock;
        int ret;

        /*
         * All PMUs/events that share this PMI handler should make sure to
         * increment active_events for their events.
         */
        if (!atomic_read(&active_events))
                return NMI_DONE;

        start_clock = sched_clock();
        ret = static_call(x86_pmu_handle_irq)(regs);
        finish_clock = sched_clock();

        perf_sample_event_took(finish_clock - start_clock);

        return ret;
}
NOKPROBE_SYMBOL(perf_event_nmi_handler);

struct event_constraint emptyconstraint;
struct event_constraint unconstrained;

static int x86_pmu_prepare_cpu(unsigned int cpu)
{
        struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu);
        int i;

        for (i = 0 ; i < X86_PERF_KFREE_MAX; i++)
                cpuc->kfree_on_online[i] = NULL;
        if (x86_pmu.cpu_prepare)
                return x86_pmu.cpu_prepare(cpu);
        return 0;
}

static int x86_pmu_dead_cpu(unsigned int cpu)
{
        if (x86_pmu.cpu_dead)
                x86_pmu.cpu_dead(cpu);
        return 0;
}

static int x86_pmu_online_cpu(unsigned int cpu)
{
        struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu);
        int i;

        for (i = 0 ; i < X86_PERF_KFREE_MAX; i++) {
                kfree(cpuc->kfree_on_online[i]);
                cpuc->kfree_on_online[i] = NULL;
        }
        return 0;
}

static int x86_pmu_starting_cpu(unsigned int cpu)
{
        if (x86_pmu.cpu_starting)
                x86_pmu.cpu_starting(cpu);
        return 0;
}

static int x86_pmu_dying_cpu(unsigned int cpu)
{
        if (x86_pmu.cpu_dying)
                x86_pmu.cpu_dying(cpu);
        return 0;
}

static void __init pmu_check_apic(void)
{
        if (boot_cpu_has(X86_FEATURE_APIC))
                return;

        x86_pmu.apic = 0;
        pr_info("no APIC, boot with the \"lapic\" boot parameter to force-enable it.\n");
        pr_info("no hardware sampling interrupt available.\n");

        /*
         * If we have a PMU initialized but no APIC
         * interrupts, we cannot sample hardware
         * events (user-space has to fall back and
         * sample via a hrtimer based software event):
         */
        pmu.capabilities |= PERF_PMU_CAP_NO_INTERRUPT;

}

static struct attribute_group x86_pmu_format_group __ro_after_init = {
        .name = "format",
        .attrs = NULL,
};

ssize_t events_sysfs_show(struct device *dev, struct device_attribute *attr, char *page)
{
        struct perf_pmu_events_attr *pmu_attr =
                container_of(attr, struct perf_pmu_events_attr, attr);
        u64 config = 0;

        if (pmu_attr->id < x86_pmu.max_events)
                config = x86_pmu.event_map(pmu_attr->id);

        /* string trumps id */
        if (pmu_attr->event_str)
                return sprintf(page, "%s", pmu_attr->event_str);

        return x86_pmu.events_sysfs_show(page, config);
}
EXPORT_SYMBOL_GPL(events_sysfs_show);

ssize_t events_ht_sysfs_show(struct device *dev, struct device_attribute *attr,
                          char *page)
{
        struct perf_pmu_events_ht_attr *pmu_attr =
                container_of(attr, struct perf_pmu_events_ht_attr, attr);

        /*
         * Report conditional events depending on Hyper-Threading.
         *
         * This is overly conservative as usually the HT special
         * handling is not needed if the other CPU thread is idle.
         *
         * Note this does not (and cannot) handle the case when thread
         * siblings are invisible, for example with virtualization
         * if they are owned by some other guest.  The user tool
         * has to re-read when a thread sibling gets onlined later.
         */
        return sprintf(page, "%s",
                        topology_max_smt_threads() > 1 ?
                        pmu_attr->event_str_ht :
                        pmu_attr->event_str_noht);
}

ssize_t events_hybrid_sysfs_show(struct device *dev,
                                 struct device_attribute *attr,
                                 char *page)
{
        struct perf_pmu_events_hybrid_attr *pmu_attr =
                container_of(attr, struct perf_pmu_events_hybrid_attr, attr);
        struct x86_hybrid_pmu *pmu;
        const char *str, *next_str;
        int i;

        if (hweight64(pmu_attr->pmu_type) == 1)
                return sprintf(page, "%s", pmu_attr->event_str);

        /*
         * Hybrid PMUs may support the same event name, but with different
         * event encoding, e.g., the mem-loads event on an Atom PMU has
         * different event encoding from a Core PMU.
         *
         * The event_str includes all event encodings. Each event encoding
         * is divided by ";". The order of the event encodings must follow
         * the order of the hybrid PMU index.
         */
        pmu = container_of(dev_get_drvdata(dev), struct x86_hybrid_pmu, pmu);

        str = pmu_attr->event_str;
        for (i = 0; i < x86_pmu.num_hybrid_pmus; i++) {
                if (!(x86_pmu.hybrid_pmu[i].cpu_type & pmu_attr->pmu_type))
                        continue;
                if (x86_pmu.hybrid_pmu[i].cpu_type & pmu->cpu_type) {
                        next_str = strchr(str, ';');
                        if (next_str)
                                return snprintf(page, next_str - str + 1, "%s", str);
                        else
                                return sprintf(page, "%s", str);
                }
                str = strchr(str, ';');
                str++;
        }

        return 0;
}
EXPORT_SYMBOL_GPL(events_hybrid_sysfs_show);

EVENT_ATTR(cpu-cycles,                        CPU_CYCLES                );
EVENT_ATTR(instructions,                INSTRUCTIONS                );
EVENT_ATTR(cache-references,                CACHE_REFERENCES        );
EVENT_ATTR(cache-misses,                 CACHE_MISSES                );
EVENT_ATTR(branch-instructions,                BRANCH_INSTRUCTIONS        );
EVENT_ATTR(branch-misses,                BRANCH_MISSES                );
EVENT_ATTR(bus-cycles,                        BUS_CYCLES                );
EVENT_ATTR(stalled-cycles-frontend,        STALLED_CYCLES_FRONTEND        );
EVENT_ATTR(stalled-cycles-backend,        STALLED_CYCLES_BACKEND        );
EVENT_ATTR(ref-cycles,                        REF_CPU_CYCLES                );

static struct attribute *empty_attrs;

static struct attribute *events_attr[] = {
        EVENT_PTR(CPU_CYCLES),
        EVENT_PTR(INSTRUCTIONS),
        EVENT_PTR(CACHE_REFERENCES),
        EVENT_PTR(CACHE_MISSES),
        EVENT_PTR(BRANCH_INSTRUCTIONS),
        EVENT_PTR(BRANCH_MISSES),
        EVENT_PTR(BUS_CYCLES),
        EVENT_PTR(STALLED_CYCLES_FRONTEND),
        EVENT_PTR(STALLED_CYCLES_BACKEND),
        EVENT_PTR(REF_CPU_CYCLES),
        NULL,
};

/*
 * Remove all undefined events (x86_pmu.event_map(id) == 0)
 * out of events_attr attributes.
 */
static umode_t
is_visible(struct kobject *kobj, struct attribute *attr, int idx)
{
        struct perf_pmu_events_attr *pmu_attr;

        if (idx >= x86_pmu.max_events)
                return 0;

        pmu_attr = container_of(attr, struct perf_pmu_events_attr, attr.attr);
        /* str trumps id */
        return pmu_attr->event_str || x86_pmu.event_map(idx) ? attr->mode : 0;
}

static struct attribute_group x86_pmu_events_group __ro_after_init = {
        .name = "events",
        .attrs = events_attr,
        .is_visible = is_visible,
};

ssize_t x86_event_sysfs_show(char *page, u64 config, u64 event)
{
        u64 umask  = (config & ARCH_PERFMON_EVENTSEL_UMASK) >> 8;
        u64 cmask  = (config & ARCH_PERFMON_EVENTSEL_CMASK) >> 24;
        bool edge  = (config & ARCH_PERFMON_EVENTSEL_EDGE);
        bool pc    = (config & ARCH_PERFMON_EVENTSEL_PIN_CONTROL);
        bool any   = (config & ARCH_PERFMON_EVENTSEL_ANY);
        bool inv   = (config & ARCH_PERFMON_EVENTSEL_INV);
        ssize_t ret;

        /*
        * We have whole page size to spend and just little data
        * to write, so we can safely use sprintf.
        */
        ret = sprintf(page, "event=0x%02llx", event);

        if (umask)
                ret += sprintf(page + ret, ",umask=0x%02llx", umask);

        if (edge)
                ret += sprintf(page + ret, ",edge");

        if (pc)
                ret += sprintf(page + ret, ",pc");

        if (any)
                ret += sprintf(page + ret, ",any");

        if (inv)
                ret += sprintf(page + ret, ",inv");

        if (cmask)
                ret += sprintf(page + ret, ",cmask=0x%02llx", cmask);

        ret += sprintf(page + ret, "\n");

        return ret;
}

static struct attribute_group x86_pmu_attr_group;
static struct attribute_group x86_pmu_caps_group;

static void x86_pmu_static_call_update(void)
{
        static_call_update(x86_pmu_handle_irq, x86_pmu.handle_irq);
        static_call_update(x86_pmu_disable_all, x86_pmu.disable_all);
        static_call_update(x86_pmu_enable_all, x86_pmu.enable_all);
        static_call_update(x86_pmu_enable, x86_pmu.enable);
        static_call_update(x86_pmu_disable, x86_pmu.disable);

        static_call_update(x86_pmu_add, x86_pmu.add);
        static_call_update(x86_pmu_del, x86_pmu.del);
        static_call_update(x86_pmu_read, x86_pmu.read);

        static_call_update(x86_pmu_schedule_events, x86_pmu.schedule_events);
        static_call_update(x86_pmu_get_event_constraints, x86_pmu.get_event_constraints);
        static_call_update(x86_pmu_put_event_constraints, x86_pmu.put_event_constraints);

        static_call_update(x86_pmu_start_scheduling, x86_pmu.start_scheduling);
        static_call_update(x86_pmu_commit_scheduling, x86_pmu.commit_scheduling);
        static_call_update(x86_pmu_stop_scheduling, x86_pmu.stop_scheduling);

        static_call_update(x86_pmu_sched_task, x86_pmu.sched_task);
        static_call_update(x86_pmu_swap_task_ctx, x86_pmu.swap_task_ctx);

        static_call_update(x86_pmu_drain_pebs, x86_pmu.drain_pebs);
        static_call_update(x86_pmu_pebs_aliases, x86_pmu.pebs_aliases);

        static_call_update(x86_pmu_guest_get_msrs, x86_pmu.guest_get_msrs);
}

static void _x86_pmu_read(struct perf_event *event)
{
        x86_perf_event_update(event);
}

void x86_pmu_show_pmu_cap(int num_counters, int num_counters_fixed,
                          u64 intel_ctrl)
{
        pr_info("... version:                %d\n",     x86_pmu.version);
        pr_info("... bit width:              %d\n",     x86_pmu.cntval_bits);
        pr_info("... generic registers:      %d\n",     num_counters);
        pr_info("... value mask:             %016Lx\n", x86_pmu.cntval_mask);
        pr_info("... max period:             %016Lx\n", x86_pmu.max_period);
        pr_info("... fixed-purpose events:   %lu\n",
                        hweight64((((1ULL << num_counters_fixed) - 1)
                                        << INTEL_PMC_IDX_FIXED) & intel_ctrl));
        pr_info("... event mask:             %016Lx\n", intel_ctrl);
}

/*
 * The generic code is not hybrid friendly. The hybrid_pmu->pmu
 * of the first registered PMU is unconditionally assigned to
 * each possible cpuctx->ctx.pmu.
 * Update the correct hybrid PMU to the cpuctx->ctx.pmu.
 */
void x86_pmu_update_cpu_context(struct pmu *pmu, int cpu)
{
        struct perf_cpu_context *cpuctx;

        if (!pmu->pmu_cpu_context)
                return;

        cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);
        cpuctx->ctx.pmu = pmu;
}

static int __init init_hw_perf_events(void)
{
        struct x86_pmu_quirk *quirk;
        int err;

        pr_info("Performance Events: ");

        switch (boot_cpu_data.x86_vendor) {
        case X86_VENDOR_INTEL:
                err = intel_pmu_init();
                break;
        case X86_VENDOR_AMD:
                err = amd_pmu_init();
                break;
        case X86_VENDOR_HYGON:
                err = amd_pmu_init();
                x86_pmu.name = "HYGON";
                break;
        case X86_VENDOR_ZHAOXIN:
        case X86_VENDOR_CENTAUR:
                err = zhaoxin_pmu_init();
                break;
        default:
                err = -ENOTSUPP;
        }
        if (err != 0) {
                pr_cont("no PMU driver, software events only.\n");
                return 0;
        }

        pmu_check_apic();

        /* sanity check that the hardware exists or is emulated */
        if (!check_hw_exists(&pmu, x86_pmu.num_counters, x86_pmu.num_counters_fixed))
                return 0;

        pr_cont("%s PMU driver.\n", x86_pmu.name);

        x86_pmu.attr_rdpmc = 1; /* enable userspace RDPMC usage by default */

        for (quirk = x86_pmu.quirks; quirk; quirk = quirk->next)
                quirk->func();

        if (!x86_pmu.intel_ctrl)
                x86_pmu.intel_ctrl = (1 << x86_pmu.num_counters) - 1;

        perf_events_lapic_init();
        register_nmi_handler(NMI_LOCAL, perf_event_nmi_handler, 0, "PMI");

        unconstrained = (struct event_constraint)
                __EVENT_CONSTRAINT(0, (1ULL << x86_pmu.num_counters) - 1,
                                   0, x86_pmu.num_counters, 0, 0);

        x86_pmu_format_group.attrs = x86_pmu.format_attrs;

        if (!x86_pmu.events_sysfs_show)
                x86_pmu_events_group.attrs = &empty_attrs;

        pmu.attr_update = x86_pmu.attr_update;

        if (!is_hybrid()) {
                x86_pmu_show_pmu_cap(x86_pmu.num_counters,
                                     x86_pmu.num_counters_fixed,
                                     x86_pmu.intel_ctrl);
        }

        if (!x86_pmu.read)
                x86_pmu.read = _x86_pmu_read;

        if (!x86_pmu.guest_get_msrs)
                x86_pmu.guest_get_msrs = (void *)&__static_call_return0;

        x86_pmu_static_call_update();

        /*
         * Install callbacks. Core will call them for each online
         * cpu.
         */
        err = cpuhp_setup_state(CPUHP_PERF_X86_PREPARE, "perf/x86:prepare",
                                x86_pmu_prepare_cpu, x86_pmu_dead_cpu);
        if (err)
                return err;

        err = cpuhp_setup_state(CPUHP_AP_PERF_X86_STARTING,
                                "perf/x86:starting", x86_pmu_starting_cpu,
                                x86_pmu_dying_cpu);
        if (err)
                goto out;

        err = cpuhp_setup_state(CPUHP_AP_PERF_X86_ONLINE, "perf/x86:online",
                                x86_pmu_online_cpu, NULL);
        if (err)
                goto out1;

        if (!is_hybrid()) {
                err = perf_pmu_register(&pmu, "cpu", PERF_TYPE_RAW);
                if (err)
                        goto out2;
        } else {
                u8 cpu_type = get_this_hybrid_cpu_type();
                struct x86_hybrid_pmu *hybrid_pmu;
                int i, j;

                if (!cpu_type && x86_pmu.get_hybrid_cpu_type)
                        cpu_type = x86_pmu.get_hybrid_cpu_type();

                for (i = 0; i < x86_pmu.num_hybrid_pmus; i++) {
                        hybrid_pmu = &x86_pmu.hybrid_pmu[i];

                        hybrid_pmu->pmu = pmu;
                        hybrid_pmu->pmu.type = -1;
                        hybrid_pmu->pmu.attr_update = x86_pmu.attr_update;
                        hybrid_pmu->pmu.capabilities |= PERF_PMU_CAP_HETEROGENEOUS_CPUS;
                        hybrid_pmu->pmu.capabilities |= PERF_PMU_CAP_EXTENDED_HW_TYPE;

                        err = perf_pmu_register(&hybrid_pmu->pmu, hybrid_pmu->name,
                                                (hybrid_pmu->cpu_type == hybrid_big) ? PERF_TYPE_RAW : -1);
                        if (err)
                                break;

                        if (cpu_type == hybrid_pmu->cpu_type)
                                x86_pmu_update_cpu_context(&hybrid_pmu->pmu, raw_smp_processor_id());
                }

                if (i < x86_pmu.num_hybrid_pmus) {
                        for (j = 0; j < i; j++)
                                perf_pmu_unregister(&x86_pmu.hybrid_pmu[j].pmu);
                        pr_warn("Failed to register hybrid PMUs\n");
                        kfree(x86_pmu.hybrid_pmu);
                        x86_pmu.hybrid_pmu = NULL;
                        x86_pmu.num_hybrid_pmus = 0;
                        goto out2;
                }
        }

        return 0;

out2:
        cpuhp_remove_state(CPUHP_AP_PERF_X86_ONLINE);
out1:
        cpuhp_remove_state(CPUHP_AP_PERF_X86_STARTING);
out:
        cpuhp_remove_state(CPUHP_PERF_X86_PREPARE);
        return err;
}
early_initcall(init_hw_perf_events);

static void x86_pmu_read(struct perf_event *event)
{
        static_call(x86_pmu_read)(event);
}

/*
 * Start group events scheduling transaction
 * Set the flag to make pmu::enable() not perform the
 * schedulability test, it will be performed at commit time
 *
 * We only support PERF_PMU_TXN_ADD transactions. Save the
 * transaction flags but otherwise ignore non-PERF_PMU_TXN_ADD
 * transactions.
 */
static void x86_pmu_start_txn(struct pmu *pmu, unsigned int txn_flags)
{
        struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);

        WARN_ON_ONCE(cpuc->txn_flags);                /* txn already in flight */

        cpuc->txn_flags = txn_flags;
        if (txn_flags & ~PERF_PMU_TXN_ADD)
                return;

        perf_pmu_disable(pmu);
        __this_cpu_write(cpu_hw_events.n_txn, 0);
        __this_cpu_write(cpu_hw_events.n_txn_pair, 0);
        __this_cpu_write(cpu_hw_events.n_txn_metric, 0);
}

/*
 * Stop group events scheduling transaction
 * Clear the flag and pmu::enable() will perform the
 * schedulability test.
 */
static void x86_pmu_cancel_txn(struct pmu *pmu)
{
        unsigned int txn_flags;
        struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);

        WARN_ON_ONCE(!cpuc->txn_flags);        /* no txn in flight */

        txn_flags = cpuc->txn_flags;
        cpuc->txn_flags = 0;
        if (txn_flags & ~PERF_PMU_TXN_ADD)
                return;

        /*
         * Truncate collected array by the number of events added in this
         * transaction. See x86_pmu_add() and x86_pmu_*_txn().
         */
        __this_cpu_sub(cpu_hw_events.n_added, __this_cpu_read(cpu_hw_events.n_txn));
        __this_cpu_sub(cpu_hw_events.n_events, __this_cpu_read(cpu_hw_events.n_txn));
        __this_cpu_sub(cpu_hw_events.n_pair, __this_cpu_read(cpu_hw_events.n_txn_pair));
        __this_cpu_sub(cpu_hw_events.n_metric, __this_cpu_read(cpu_hw_events.n_txn_metric));
        perf_pmu_enable(pmu);
}

/*
 * Commit group events scheduling transaction
 * Perform the group schedulability test as a whole
 * Return 0 if success
 *
 * Does not cancel the transaction on failure; expects the caller to do this.
 */
static int x86_pmu_commit_txn(struct pmu *pmu)
{
        struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
        int assign[X86_PMC_IDX_MAX];
        int n, ret;

        WARN_ON_ONCE(!cpuc->txn_flags);        /* no txn in flight */

        if (cpuc->txn_flags & ~PERF_PMU_TXN_ADD) {
                cpuc->txn_flags = 0;
                return 0;
        }

        n = cpuc->n_events;

        if (!x86_pmu_initialized())
                return -EAGAIN;

        ret = static_call(x86_pmu_schedule_events)(cpuc, n, assign);
        if (ret)
                return ret;

        /*
         * copy new assignment, now we know it is possible
         * will be used by hw_perf_enable()
         */
        memcpy(cpuc->assign, assign, n*sizeof(int));

        cpuc->txn_flags = 0;
        perf_pmu_enable(pmu);
        return 0;
}
/*
 * a fake_cpuc is used to validate event groups. Due to
 * the extra reg logic, we need to also allocate a fake
 * per_core and per_cpu structure. Otherwise, group events
 * using extra reg may conflict without the kernel being
 * able to catch this when the last event gets added to
 * the group.
 */
static void free_fake_cpuc(struct cpu_hw_events *cpuc)
{
        intel_cpuc_finish(cpuc);
        kfree(cpuc);
}

static struct cpu_hw_events *allocate_fake_cpuc(struct pmu *event_pmu)
{
        struct cpu_hw_events *cpuc;
        int cpu;

        cpuc = kzalloc(sizeof(*cpuc), GFP_KERNEL);
        if (!cpuc)
                return ERR_PTR(-ENOMEM);
        cpuc->is_fake = 1;

        if (is_hybrid()) {
                struct x86_hybrid_pmu *h_pmu;

                h_pmu = hybrid_pmu(event_pmu);
                if (cpumask_empty(&h_pmu->supported_cpus))
                        goto error;
                cpu = cpumask_first(&h_pmu->supported_cpus);
        } else
                cpu = raw_smp_processor_id();
        cpuc->pmu = event_pmu;

        if (intel_cpuc_prepare(cpuc, cpu))
                goto error;

        return cpuc;
error:
        free_fake_cpuc(cpuc);
        return ERR_PTR(-ENOMEM);
}

/*
 * validate that we can schedule this event
 */
static int validate_event(struct perf_event *event)
{
        struct cpu_hw_events *fake_cpuc;
        struct event_constraint *c;
        int ret = 0;

        fake_cpuc = allocate_fake_cpuc(event->pmu);
        if (IS_ERR(fake_cpuc))
                return PTR_ERR(fake_cpuc);

        c = x86_pmu.get_event_constraints(fake_cpuc, 0, event);

        if (!c || !c->weight)
                ret = -EINVAL;

        if (x86_pmu.put_event_constraints)
                x86_pmu.put_event_constraints(fake_cpuc, event);

        free_fake_cpuc(fake_cpuc);

        return ret;
}

/*
 * validate a single event group
 *
 * validation include:
 *        - check events are compatible which each other
 *        - events do not compete for the same counter
 *        - number of events <= number of counters
 *
 * validation ensures the group can be loaded onto the
 * PMU if it was the only group available.
 */
static int validate_group(struct perf_event *event)
{
        struct perf_event *leader = event->group_leader;
        struct cpu_hw_events *fake_cpuc;
        int ret = -EINVAL, n;

        /*
         * Reject events from different hybrid PMUs.
         */
        if (is_hybrid()) {
                struct perf_event *sibling;
                struct pmu *pmu = NULL;

                if (is_x86_event(leader))
                        pmu = leader->pmu;

                for_each_sibling_event(sibling, leader) {
                        if (!is_x86_event(sibling))
                                continue;
                        if (!pmu)
                                pmu = sibling->pmu;
                        else if (pmu != sibling->pmu)
                                return ret;
                }
        }

        fake_cpuc = allocate_fake_cpuc(event->pmu);
        if (IS_ERR(fake_cpuc))
                return PTR_ERR(fake_cpuc);
        /*
         * the event is not yet connected with its
         * siblings therefore we must first collect
         * existing siblings, then add the new event
         * before we can simulate the scheduling
         */
        n = collect_events(fake_cpuc, leader, true);
        if (n < 0)
                goto out;

        fake_cpuc->n_events = n;
        n = collect_events(fake_cpuc, event, false);
        if (n < 0)
                goto out;

        fake_cpuc->n_events = 0;
        ret = x86_pmu.schedule_events(fake_cpuc, n, NULL);

out:
        free_fake_cpuc(fake_cpuc);
        return ret;
}

static int x86_pmu_event_init(struct perf_event *event)
{
        struct x86_hybrid_pmu *pmu = NULL;
        int err;

        if ((event->attr.type != event->pmu->type) &&
            (event->attr.type != PERF_TYPE_HARDWARE) &&
            (event->attr.type != PERF_TYPE_HW_CACHE))
                return -ENOENT;

        if (is_hybrid() && (event->cpu != -1)) {
                pmu = hybrid_pmu(event->pmu);
                if (!cpumask_test_cpu(event->cpu, &pmu->supported_cpus))
                        return -ENOENT;
        }

        err = __x86_pmu_event_init(event);
        if (!err) {
                if (event->group_leader != event)
                        err = validate_group(event);
                else
                        err = validate_event(event);
        }
        if (err) {
                if (event->destroy)
                        event->destroy(event);
                event->destroy = NULL;
        }

        if (READ_ONCE(x86_pmu.attr_rdpmc) &&
            !(event->hw.flags & PERF_X86_EVENT_LARGE_PEBS))
                event->hw.flags |= PERF_X86_EVENT_RDPMC_ALLOWED;

        return err;
}

void perf_clear_dirty_counters(void)
{
        struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
        int i;

         /* Don't need to clear the assigned counter. */
        for (i = 0; i < cpuc->n_events; i++)
                __clear_bit(cpuc->assign[i], cpuc->dirty);

        if (bitmap_empty(cpuc->dirty, X86_PMC_IDX_MAX))
                return;

        for_each_set_bit(i, cpuc->dirty, X86_PMC_IDX_MAX) {
                if (i >= INTEL_PMC_IDX_FIXED) {
                        /* Metrics and fake events don't have corresponding HW counters. */
                        if ((i - INTEL_PMC_IDX_FIXED) >= hybrid(cpuc->pmu, num_counters_fixed))
                                continue;

                        wrmsrl(MSR_ARCH_PERFMON_FIXED_CTR0 + (i - INTEL_PMC_IDX_FIXED), 0);
                } else {
                        wrmsrl(x86_pmu_event_addr(i), 0);
                }
        }

        bitmap_zero(cpuc->dirty, X86_PMC_IDX_MAX);
}

static void x86_pmu_event_mapped(struct perf_event *event, struct mm_struct *mm)
{
        if (!(event->hw.flags & PERF_X86_EVENT_RDPMC_ALLOWED))
                return;

        /*
         * This function relies on not being called concurrently in two
         * tasks in the same mm.  Otherwise one task could observe
         * perf_rdpmc_allowed > 1 and return all the way back to
         * userspace with CR4.PCE clear while another task is still
         * doing on_each_cpu_mask() to propagate CR4.PCE.
         *
         * For now, this can't happen because all callers hold mmap_lock
         * for write.  If this changes, we'll need a different solution.
         */
        mmap_assert_write_locked(mm);

        if (atomic_inc_return(&mm->context.perf_rdpmc_allowed) == 1)
                on_each_cpu_mask(mm_cpumask(mm), cr4_update_pce, NULL, 1);
}

static void x86_pmu_event_unmapped(struct perf_event *event, struct mm_struct *mm)
{
        if (!(event->hw.flags & PERF_X86_EVENT_RDPMC_ALLOWED))
                return;

        if (atomic_dec_and_test(&mm->context.perf_rdpmc_allowed))
                on_each_cpu_mask(mm_cpumask(mm), cr4_update_pce, NULL, 1);
}

static int x86_pmu_event_idx(struct perf_event *event)
{
        struct hw_perf_event *hwc = &event->hw;

        if (!(hwc->flags & PERF_X86_EVENT_RDPMC_ALLOWED))
                return 0;

        if (is_metric_idx(hwc->idx))
                return INTEL_PMC_FIXED_RDPMC_METRICS + 1;
        else
                return hwc->event_base_rdpmc + 1;
}

static ssize_t get_attr_rdpmc(struct device *cdev,
                              struct device_attribute *attr,
                              char *buf)
{
        return snprintf(buf, 40, "%d\n", x86_pmu.attr_rdpmc);
}

static ssize_t set_attr_rdpmc(struct device *cdev,
                              struct device_attribute *attr,
                              const char *buf, size_t count)
{
        unsigned long val;
        ssize_t ret;

        ret = kstrtoul(buf, 0, &val);
        if (ret)
                return ret;

        if (val > 2)
                return -EINVAL;

        if (x86_pmu.attr_rdpmc_broken)
                return -ENOTSUPP;

        if (val != x86_pmu.attr_rdpmc) {
                /*
                 * Changing into or out of never available or always available,
                 * aka perf-event-bypassing mode. This path is extremely slow,
                 * but only root can trigger it, so it's okay.
                 */
                if (val == 0)
                        static_branch_inc(&rdpmc_never_available_key);
                else if (x86_pmu.attr_rdpmc == 0)
                        static_branch_dec(&rdpmc_never_available_key);

                if (val == 2)
                        static_branch_inc(&rdpmc_always_available_key);
                else if (x86_pmu.attr_rdpmc == 2)
                        static_branch_dec(&rdpmc_always_available_key);

                on_each_cpu(cr4_update_pce, NULL, 1);
                x86_pmu.attr_rdpmc = val;
        }

        return count;
}

static DEVICE_ATTR(rdpmc, S_IRUSR | S_IWUSR, get_attr_rdpmc, set_attr_rdpmc);

static struct attribute *x86_pmu_attrs[] = {
        &dev_attr_rdpmc.attr,
        NULL,
};

static struct attribute_group x86_pmu_attr_group __ro_after_init = {
        .attrs = x86_pmu_attrs,
};

static ssize_t max_precise_show(struct device *cdev,
                                  struct device_attribute *attr,
                                  char *buf)
{
        return snprintf(buf, PAGE_SIZE, "%d\n", x86_pmu_max_precise());
}

static DEVICE_ATTR_RO(max_precise);

static struct attribute *x86_pmu_caps_attrs[] = {
        &dev_attr_max_precise.attr,
        NULL
};

static struct attribute_group x86_pmu_caps_group __ro_after_init = {
        .name = "caps",
        .attrs = x86_pmu_caps_attrs,
};

static const struct attribute_group *x86_pmu_attr_groups[] = {
        &x86_pmu_attr_group,
        &x86_pmu_format_group,
        &x86_pmu_events_group,
        &x86_pmu_caps_group,
        NULL,
};

static void x86_pmu_sched_task(struct perf_event_context *ctx, bool sched_in)
{
        static_call_cond(x86_pmu_sched_task)(ctx, sched_in);
}

static void x86_pmu_swap_task_ctx(struct perf_event_context *prev,
                                  struct perf_event_context *next)
{
        static_call_cond(x86_pmu_swap_task_ctx)(prev, next);
}

void perf_check_microcode(void)
{
        if (x86_pmu.check_microcode)
                x86_pmu.check_microcode();
}

static int x86_pmu_check_period(struct perf_event *event, u64 value)
{
        if (x86_pmu.check_period && x86_pmu.check_period(event, value))
                return -EINVAL;

        if (value && x86_pmu.limit_period) {
                if (x86_pmu.limit_period(event, value) > value)
                        return -EINVAL;
        }

        return 0;
}

static int x86_pmu_aux_output_match(struct perf_event *event)
{
        if (!(pmu.capabilities & PERF_PMU_CAP_AUX_OUTPUT))
                return 0;

        if (x86_pmu.aux_output_match)
                return x86_pmu.aux_output_match(event);

        return 0;
}

static int x86_pmu_filter_match(struct perf_event *event)
{
        if (x86_pmu.filter_match)
                return x86_pmu.filter_match(event);

        return 1;
}

static struct pmu pmu = {
        .pmu_enable                = x86_pmu_enable,
        .pmu_disable                = x86_pmu_disable,

        .attr_groups                = x86_pmu_attr_groups,

        .event_init                = x86_pmu_event_init,

        .event_mapped                = x86_pmu_event_mapped,
        .event_unmapped                = x86_pmu_event_unmapped,

        .add                        = x86_pmu_add,
        .del                        = x86_pmu_del,
        .start                        = x86_pmu_start,
        .stop                        = x86_pmu_stop,
        .read                        = x86_pmu_read,

        .start_txn                = x86_pmu_start_txn,
        .cancel_txn                = x86_pmu_cancel_txn,
        .commit_txn                = x86_pmu_commit_txn,

        .event_idx                = x86_pmu_event_idx,
        .sched_task                = x86_pmu_sched_task,
        .swap_task_ctx                = x86_pmu_swap_task_ctx,
        .check_period                = x86_pmu_check_period,

        .aux_output_match        = x86_pmu_aux_output_match,

        .filter_match                = x86_pmu_filter_match,
};

void arch_perf_update_userpage(struct perf_event *event,
                               struct perf_event_mmap_page *userpg, u64 now)
{
        struct cyc2ns_data data;
        u64 offset;

        userpg->cap_user_time = 0;
        userpg->cap_user_time_zero = 0;
        userpg->cap_user_rdpmc =
                !!(event->hw.flags & PERF_X86_EVENT_RDPMC_ALLOWED);
        userpg->pmc_width = x86_pmu.cntval_bits;

        if (!using_native_sched_clock() || !sched_clock_stable())
                return;

        cyc2ns_read_begin(&data);

        offset = data.cyc2ns_offset + __sched_clock_offset;

        /*
         * Internal timekeeping for enabled/running/stopped times
         * is always in the local_clock domain.
         */
        userpg->cap_user_time = 1;
        userpg->time_mult = data.cyc2ns_mul;
        userpg->time_shift = data.cyc2ns_shift;
        userpg->time_offset = offset - now;

        /*
         * cap_user_time_zero doesn't make sense when we're using a different
         * time base for the records.
         */
        if (!event->attr.use_clockid) {
                userpg->cap_user_time_zero = 1;
                userpg->time_zero = offset;
        }

        cyc2ns_read_end();
}

/*
 * Determine whether the regs were taken from an irq/exception handler rather
 * than from perf_arch_fetch_caller_regs().
 */
static bool perf_hw_regs(struct pt_regs *regs)
{
        return regs->flags & X86_EFLAGS_FIXED;
}

void
perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs)
{
        struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs();
        struct unwind_state state;
        unsigned long addr;

        if (guest_cbs && guest_cbs->is_in_guest()) {
                /* TODO: We don't support guest os callchain now */
                return;
        }

        if (perf_callchain_store(entry, regs->ip))
                return;

        if (perf_hw_regs(regs))
                unwind_start(&state, current, regs, NULL);
        else
                unwind_start(&state, current, NULL, (void *)regs->sp);

        for (; !unwind_done(&state); unwind_next_frame(&state)) {
                addr = unwind_get_return_address(&state);
                if (!addr || perf_callchain_store(entry, addr))
                        return;
        }
}

static inline int
valid_user_frame(const void __user *fp, unsigned long size)
{
        return (__range_not_ok(fp, size, TASK_SIZE) == 0);
}

static unsigned long get_segment_base(unsigned int segment)
{
        struct desc_struct *desc;
        unsigned int idx = segment >> 3;

        if ((segment & SEGMENT_TI_MASK) == SEGMENT_LDT) {
#ifdef CONFIG_MODIFY_LDT_SYSCALL
                struct ldt_struct *ldt;

                /* IRQs are off, so this synchronizes with smp_store_release */
                ldt = READ_ONCE(current->active_mm->context.ldt);
                if (!ldt || idx >= ldt->nr_entries)
                        return 0;

                desc = &ldt->entries[idx];
#else
                return 0;
#endif
        } else {
                if (idx >= GDT_ENTRIES)
                        return 0;

                desc = raw_cpu_ptr(gdt_page.gdt) + idx;
        }

        return get_desc_base(desc);
}

#ifdef CONFIG_IA32_EMULATION

#include <linux/compat.h>

static inline int
perf_callchain_user32(struct pt_regs *regs, struct perf_callchain_entry_ctx *entry)
{
        /* 32-bit process in 64-bit kernel. */
        unsigned long ss_base, cs_base;
        struct stack_frame_ia32 frame;
        const struct stack_frame_ia32 __user *fp;

        if (user_64bit_mode(regs))
                return 0;

        cs_base = get_segment_base(regs->cs);
        ss_base = get_segment_base(regs->ss);

        fp = compat_ptr(ss_base + regs->bp);
        pagefault_disable();
        while (entry->nr < entry->max_stack) {
                if (!valid_user_frame(fp, sizeof(frame)))
                        break;

                if (__get_user(frame.next_frame, &fp->next_frame))
                        break;
                if (__get_user(frame.return_address, &fp->return_address))
                        break;

                perf_callchain_store(entry, cs_base + frame.return_address);
                fp = compat_ptr(ss_base + frame.next_frame);
        }
        pagefault_enable();
        return 1;
}
#else
static inline int
perf_callchain_user32(struct pt_regs *regs, struct perf_callchain_entry_ctx *entry)
{
    return 0;
}
#endif

void
perf_callchain_user(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs)
{
        struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs();
        struct stack_frame frame;
        const struct stack_frame __user *fp;

        if (guest_cbs && guest_cbs->is_in_guest()) {
                /* TODO: We don't support guest os callchain now */
                return;
        }

        /*
         * We don't know what to do with VM86 stacks.. ignore them for now.
         */
        if (regs->flags & (X86_VM_MASK | PERF_EFLAGS_VM))
                return;

        fp = (void __user *)regs->bp;

        perf_callchain_store(entry, regs->ip);

        if (!nmi_uaccess_okay())
                return;

        if (perf_callchain_user32(regs, entry))
                return;

        pagefault_disable();
        while (entry->nr < entry->max_stack) {
                if (!valid_user_frame(fp, sizeof(frame)))
                        break;

                if (__get_user(frame.next_frame, &fp->next_frame))
                        break;
                if (__get_user(frame.return_address, &fp->return_address))
                        break;

                perf_callchain_store(entry, frame.return_address);
                fp = (void __user *)frame.next_frame;
        }
        pagefault_enable();
}

/*
 * Deal with code segment offsets for the various execution modes:
 *
 *   VM86 - the good olde 16 bit days, where the linear address is
 *          20 bits and we use regs->ip + 0x10 * regs->cs.
 *
 *   IA32 - Where we need to look at GDT/LDT segment descriptor tables
 *          to figure out what the 32bit base address is.
 *
 *    X32 - has TIF_X32 set, but is running in x86_64
 *
 * X86_64 - CS,DS,SS,ES are all zero based.
 */
static unsigned long code_segment_base(struct pt_regs *regs)
{
        /*
         * For IA32 we look at the GDT/LDT segment base to convert the
         * effective IP to a linear address.
         */

#ifdef CONFIG_X86_32
        /*
         * If we are in VM86 mode, add the segment offset to convert to a
         * linear address.
         */
        if (regs->flags & X86_VM_MASK)
                return 0x10 * regs->cs;

        if (user_mode(regs) && regs->cs != __USER_CS)
                return get_segment_base(regs->cs);
#else
        if (user_mode(regs) && !user_64bit_mode(regs) &&
            regs->cs != __USER32_CS)
                return get_segment_base(regs->cs);
#endif
        return 0;
}

unsigned long perf_instruction_pointer(struct pt_regs *regs)
{
        struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs();

        if (guest_cbs && guest_cbs->is_in_guest())
                return guest_cbs->get_guest_ip();

        return regs->ip + code_segment_base(regs);
}

unsigned long perf_misc_flags(struct pt_regs *regs)
{
        struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs();
        int misc = 0;

        if (guest_cbs && guest_cbs->is_in_guest()) {
                if (guest_cbs->is_user_mode())
                        misc |= PERF_RECORD_MISC_GUEST_USER;
                else
                        misc |= PERF_RECORD_MISC_GUEST_KERNEL;
        } else {
                if (user_mode(regs))
                        misc |= PERF_RECORD_MISC_USER;
                else
                        misc |= PERF_RECORD_MISC_KERNEL;
        }

        if (regs->flags & PERF_EFLAGS_EXACT)
                misc |= PERF_RECORD_MISC_EXACT_IP;

        return misc;
}

void perf_get_x86_pmu_capability(struct x86_pmu_capability *cap)
{
        cap->version                = x86_pmu.version;
        /*
         * KVM doesn't support the hybrid PMU yet.
         * Return the common value in global x86_pmu,
         * which available for all cores.
         */
        cap->num_counters_gp        = x86_pmu.num_counters;
        cap->num_counters_fixed        = x86_pmu.num_counters_fixed;
        cap->bit_width_gp        = x86_pmu.cntval_bits;
        cap->bit_width_fixed        = x86_pmu.cntval_bits;
        cap->events_mask        = (unsigned int)x86_pmu.events_maskl;
        cap->events_mask_len        = x86_pmu.events_mask_len;
}
EXPORT_SYMBOL_GPL(perf_get_x86_pmu_capability);














































   16 













    8 


   15 

















    1 






























    3 


























    3 























    3 
    3 

    3 




    3 




    3 




    3 





    3 




    3 


    3 



    3 
















    3 













   29 






    3 

   15 




   12 
    6 
    9 








































































































    3 




    4 


    5 










    6 
    3 






















    1 





    1 
    1 
















    2 






   13 














    1 





    1 

   19 




    2 




    4 





    4 



    4 







    4 


    3 
    2 






    5 



    5 








   10 




    5 
    4 


    5 







    4 




    2 



    4 

    1 














    2 






















   23 



   24 



































































































    9 





    4 






















   23 



    1 






    5 














   21 
    3 




























    3 


    1 


    1 



    4 




























   13 


   28 






























































    2 



























   22 






    1 




    1 











    8 









    7 








    5 




   13 









































    1 






























    1 









    2 



    2 



    1 
    1 







    1 


    1 





    1 


    1 


    1 












    1 
    2 



    2 




    2 




    1 
    2 



    2 
    1 















    1 


























    1 


    3 




    3 

    5 

    1 

    3 

    1 






    1 




















































































    1 






    3 

    5 





    8 

    8 

    8 


    3 



    1 
    7 





    1 



    7 



















































    1 








    1 








    1 




































































    9 


    5 





















    2 
























    9 


    6 
























    3 


    1 






















    1 


















































   30 
    2 
   16 
    1 
    3 
   11 
    2 

   15 
   11 
    3 
    9 


    1 

    1 
   12 
    3 

    5 
    1 

    3 
    1 
   18 
   32 
    1 









































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
// SPDX-License-Identifier: GPL-2.0
/*
 * cfg80211 - wext compat code
 *
 * This is temporary code until all wireless functionality is migrated
 * into cfg80211, when that happens all the exports here go away and
 * we directly assign the wireless handlers of wireless interfaces.
 *
 * Copyright 2008-2009        Johannes Berg <johannes@sipsolutions.net>
 * Copyright (C) 2019-2021 Intel Corporation
 */

#include <linux/export.h>
#include <linux/wireless.h>
#include <linux/nl80211.h>
#include <linux/if_arp.h>
#include <linux/etherdevice.h>
#include <linux/slab.h>
#include <net/iw_handler.h>
#include <net/cfg80211.h>
#include <net/cfg80211-wext.h>
#include "wext-compat.h"
#include "core.h"
#include "rdev-ops.h"

int cfg80211_wext_giwname(struct net_device *dev,
                          struct iw_request_info *info,
                          char *name, char *extra)
{
        strcpy(name, "IEEE 802.11");
        return 0;
}
EXPORT_WEXT_HANDLER(cfg80211_wext_giwname);

int cfg80211_wext_siwmode(struct net_device *dev, struct iw_request_info *info,
                          u32 *mode, char *extra)
{
        struct wireless_dev *wdev = dev->ieee80211_ptr;
        struct cfg80211_registered_device *rdev;
        struct vif_params vifparams;
        enum nl80211_iftype type;
        int ret;

        rdev = wiphy_to_rdev(wdev->wiphy);

        switch (*mode) {
        case IW_MODE_INFRA:
                type = NL80211_IFTYPE_STATION;
                break;
        case IW_MODE_ADHOC:
                type = NL80211_IFTYPE_ADHOC;
                break;
        case IW_MODE_MONITOR:
                type = NL80211_IFTYPE_MONITOR;
                break;
        default:
                return -EINVAL;
        }

        if (type == wdev->iftype)
                return 0;

        memset(&vifparams, 0, sizeof(vifparams));

        wiphy_lock(wdev->wiphy);
        ret = cfg80211_change_iface(rdev, dev, type, &vifparams);
        wiphy_unlock(wdev->wiphy);

        return ret;
}
EXPORT_WEXT_HANDLER(cfg80211_wext_siwmode);

int cfg80211_wext_giwmode(struct net_device *dev, struct iw_request_info *info,
                          u32 *mode, char *extra)
{
        struct wireless_dev *wdev = dev->ieee80211_ptr;

        if (!wdev)
                return -EOPNOTSUPP;

        switch (wdev->iftype) {
        case NL80211_IFTYPE_AP:
                *mode = IW_MODE_MASTER;
                break;
        case NL80211_IFTYPE_STATION:
                *mode = IW_MODE_INFRA;
                break;
        case NL80211_IFTYPE_ADHOC:
                *mode = IW_MODE_ADHOC;
                break;
        case NL80211_IFTYPE_MONITOR:
                *mode = IW_MODE_MONITOR;
                break;
        case NL80211_IFTYPE_WDS:
                *mode = IW_MODE_REPEAT;
                break;
        case NL80211_IFTYPE_AP_VLAN:
                *mode = IW_MODE_SECOND;                /* FIXME */
                break;
        default:
                *mode = IW_MODE_AUTO;
                break;
        }
        return 0;
}
EXPORT_WEXT_HANDLER(cfg80211_wext_giwmode);


int cfg80211_wext_giwrange(struct net_device *dev,
                           struct iw_request_info *info,
                           struct iw_point *data, char *extra)
{
        struct wireless_dev *wdev = dev->ieee80211_ptr;
        struct iw_range *range = (struct iw_range *) extra;
        enum nl80211_band band;
        int i, c = 0;

        if (!wdev)
                return -EOPNOTSUPP;

        data->length = sizeof(struct iw_range);
        memset(range, 0, sizeof(struct iw_range));

        range->we_version_compiled = WIRELESS_EXT;
        range->we_version_source = 21;
        range->retry_capa = IW_RETRY_LIMIT;
        range->retry_flags = IW_RETRY_LIMIT;
        range->min_retry = 0;
        range->max_retry = 255;
        range->min_rts = 0;
        range->max_rts = 2347;
        range->min_frag = 256;
        range->max_frag = 2346;

        range->max_encoding_tokens = 4;

        range->max_qual.updated = IW_QUAL_NOISE_INVALID;

        switch (wdev->wiphy->signal_type) {
        case CFG80211_SIGNAL_TYPE_NONE:
                break;
        case CFG80211_SIGNAL_TYPE_MBM:
                range->max_qual.level = (u8)-110;
                range->max_qual.qual = 70;
                range->avg_qual.qual = 35;
                range->max_qual.updated |= IW_QUAL_DBM;
                range->max_qual.updated |= IW_QUAL_QUAL_UPDATED;
                range->max_qual.updated |= IW_QUAL_LEVEL_UPDATED;
                break;
        case CFG80211_SIGNAL_TYPE_UNSPEC:
                range->max_qual.level = 100;
                range->max_qual.qual = 100;
                range->avg_qual.qual = 50;
                range->max_qual.updated |= IW_QUAL_QUAL_UPDATED;
                range->max_qual.updated |= IW_QUAL_LEVEL_UPDATED;
                break;
        }

        range->avg_qual.level = range->max_qual.level / 2;
        range->avg_qual.noise = range->max_qual.noise / 2;
        range->avg_qual.updated = range->max_qual.updated;

        for (i = 0; i < wdev->wiphy->n_cipher_suites; i++) {
                switch (wdev->wiphy->cipher_suites[i]) {
                case WLAN_CIPHER_SUITE_TKIP:
                        range->enc_capa |= (IW_ENC_CAPA_CIPHER_TKIP |
                                            IW_ENC_CAPA_WPA);
                        break;

                case WLAN_CIPHER_SUITE_CCMP:
                        range->enc_capa |= (IW_ENC_CAPA_CIPHER_CCMP |
                                            IW_ENC_CAPA_WPA2);
                        break;

                case WLAN_CIPHER_SUITE_WEP40:
                        range->encoding_size[range->num_encoding_sizes++] =
                                WLAN_KEY_LEN_WEP40;
                        break;

                case WLAN_CIPHER_SUITE_WEP104:
                        range->encoding_size[range->num_encoding_sizes++] =
                                WLAN_KEY_LEN_WEP104;
                        break;
                }
        }

        for (band = 0; band < NUM_NL80211_BANDS; band ++) {
                struct ieee80211_supported_band *sband;

                sband = wdev->wiphy->bands[band];

                if (!sband)
                        continue;

                for (i = 0; i < sband->n_channels && c < IW_MAX_FREQUENCIES; i++) {
                        struct ieee80211_channel *chan = &sband->channels[i];

                        if (!(chan->flags & IEEE80211_CHAN_DISABLED)) {
                                range->freq[c].i =
                                        ieee80211_frequency_to_channel(
                                                chan->center_freq);
                                range->freq[c].m = chan->center_freq;
                                range->freq[c].e = 6;
                                c++;
                        }
                }
        }
        range->num_channels = c;
        range->num_frequency = c;

        IW_EVENT_CAPA_SET_KERNEL(range->event_capa);
        IW_EVENT_CAPA_SET(range->event_capa, SIOCGIWAP);
        IW_EVENT_CAPA_SET(range->event_capa, SIOCGIWSCAN);

        if (wdev->wiphy->max_scan_ssids > 0)
                range->scan_capa |= IW_SCAN_CAPA_ESSID;

        return 0;
}
EXPORT_WEXT_HANDLER(cfg80211_wext_giwrange);


/**
 * cfg80211_wext_freq - get wext frequency for non-"auto"
 * @freq: the wext freq encoding
 *
 * Returns a frequency, or a negative error code, or 0 for auto.
 */
int cfg80211_wext_freq(struct iw_freq *freq)
{
        /*
         * Parse frequency - return 0 for auto and
         * -EINVAL for impossible things.
         */
        if (freq->e == 0) {
                enum nl80211_band band = NL80211_BAND_2GHZ;
                if (freq->m < 0)
                        return 0;
                if (freq->m > 14)
                        band = NL80211_BAND_5GHZ;
                return ieee80211_channel_to_frequency(freq->m, band);
        } else {
                int i, div = 1000000;
                for (i = 0; i < freq->e; i++)
                        div /= 10;
                if (div <= 0)
                        return -EINVAL;
                return freq->m / div;
        }
}

int cfg80211_wext_siwrts(struct net_device *dev,
                         struct iw_request_info *info,
                         struct iw_param *rts, char *extra)
{
        struct wireless_dev *wdev = dev->ieee80211_ptr;
        struct cfg80211_registered_device *rdev = wiphy_to_rdev(wdev->wiphy);
        u32 orts = wdev->wiphy->rts_threshold;
        int err;

        wiphy_lock(&rdev->wiphy);
        if (rts->disabled || !rts->fixed) {
                wdev->wiphy->rts_threshold = (u32) -1;
        } else if (rts->value < 0) {
                err = -EINVAL;
                goto out;
        } else {
                wdev->wiphy->rts_threshold = rts->value;
        }

        err = rdev_set_wiphy_params(rdev, WIPHY_PARAM_RTS_THRESHOLD);

        if (err)
                wdev->wiphy->rts_threshold = orts;

out:
        wiphy_unlock(&rdev->wiphy);
        return err;
}
EXPORT_WEXT_HANDLER(cfg80211_wext_siwrts);

int cfg80211_wext_giwrts(struct net_device *dev,
                         struct iw_request_info *info,
                         struct iw_param *rts, char *extra)
{
        struct wireless_dev *wdev = dev->ieee80211_ptr;

        rts->value = wdev->wiphy->rts_threshold;
        rts->disabled = rts->value == (u32) -1;
        rts->fixed = 1;

        return 0;
}
EXPORT_WEXT_HANDLER(cfg80211_wext_giwrts);

int cfg80211_wext_siwfrag(struct net_device *dev,
                          struct iw_request_info *info,
                          struct iw_param *frag, char *extra)
{
        struct wireless_dev *wdev = dev->ieee80211_ptr;
        struct cfg80211_registered_device *rdev = wiphy_to_rdev(wdev->wiphy);
        u32 ofrag = wdev->wiphy->frag_threshold;
        int err;

        wiphy_lock(&rdev->wiphy);
        if (frag->disabled || !frag->fixed) {
                wdev->wiphy->frag_threshold = (u32) -1;
        } else if (frag->value < 256) {
                err = -EINVAL;
                goto out;
        } else {
                /* Fragment length must be even, so strip LSB. */
                wdev->wiphy->frag_threshold = frag->value & ~0x1;
        }

        err = rdev_set_wiphy_params(rdev, WIPHY_PARAM_FRAG_THRESHOLD);
        if (err)
                wdev->wiphy->frag_threshold = ofrag;
out:
        wiphy_unlock(&rdev->wiphy);

        return err;
}
EXPORT_WEXT_HANDLER(cfg80211_wext_siwfrag);

int cfg80211_wext_giwfrag(struct net_device *dev,
                          struct iw_request_info *info,
                          struct iw_param *frag, char *extra)
{
        struct wireless_dev *wdev = dev->ieee80211_ptr;

        frag->value = wdev->wiphy->frag_threshold;
        frag->disabled = frag->value == (u32) -1;
        frag->fixed = 1;

        return 0;
}
EXPORT_WEXT_HANDLER(cfg80211_wext_giwfrag);

static int cfg80211_wext_siwretry(struct net_device *dev,
                                  struct iw_request_info *info,
                                  struct iw_param *retry, char *extra)
{
        struct wireless_dev *wdev = dev->ieee80211_ptr;
        struct cfg80211_registered_device *rdev = wiphy_to_rdev(wdev->wiphy);
        u32 changed = 0;
        u8 olong = wdev->wiphy->retry_long;
        u8 oshort = wdev->wiphy->retry_short;
        int err;

        if (retry->disabled || retry->value < 1 || retry->value > 255 ||
            (retry->flags & IW_RETRY_TYPE) != IW_RETRY_LIMIT)
                return -EINVAL;

        wiphy_lock(&rdev->wiphy);
        if (retry->flags & IW_RETRY_LONG) {
                wdev->wiphy->retry_long = retry->value;
                changed |= WIPHY_PARAM_RETRY_LONG;
        } else if (retry->flags & IW_RETRY_SHORT) {
                wdev->wiphy->retry_short = retry->value;
                changed |= WIPHY_PARAM_RETRY_SHORT;
        } else {
                wdev->wiphy->retry_short = retry->value;
                wdev->wiphy->retry_long = retry->value;
                changed |= WIPHY_PARAM_RETRY_LONG;
                changed |= WIPHY_PARAM_RETRY_SHORT;
        }

        err = rdev_set_wiphy_params(rdev, changed);
        if (err) {
                wdev->wiphy->retry_short = oshort;
                wdev->wiphy->retry_long = olong;
        }
        wiphy_unlock(&rdev->wiphy);

        return err;
}

int cfg80211_wext_giwretry(struct net_device *dev,
                           struct iw_request_info *info,
                           struct iw_param *retry, char *extra)
{
        struct wireless_dev *wdev = dev->ieee80211_ptr;

        retry->disabled = 0;

        if (retry->flags == 0 || (retry->flags & IW_RETRY_SHORT)) {
                /*
                 * First return short value, iwconfig will ask long value
                 * later if needed
                 */
                retry->flags |= IW_RETRY_LIMIT | IW_RETRY_SHORT;
                retry->value = wdev->wiphy->retry_short;
                if (wdev->wiphy->retry_long == wdev->wiphy->retry_short)
                        retry->flags |= IW_RETRY_LONG;

                return 0;
        }

        if (retry->flags & IW_RETRY_LONG) {
                retry->flags = IW_RETRY_LIMIT | IW_RETRY_LONG;
                retry->value = wdev->wiphy->retry_long;
        }

        return 0;
}
EXPORT_WEXT_HANDLER(cfg80211_wext_giwretry);

static int __cfg80211_set_encryption(struct cfg80211_registered_device *rdev,
                                     struct net_device *dev, bool pairwise,
                                     const u8 *addr, bool remove, bool tx_key,
                                     int idx, struct key_params *params)
{
        struct wireless_dev *wdev = dev->ieee80211_ptr;
        int err, i;
        bool rejoin = false;

        if (pairwise && !addr)
                return -EINVAL;

        /*
         * In many cases we won't actually need this, but it's better
         * to do it first in case the allocation fails. Don't use wext.
         */
        if (!wdev->wext.keys) {
                wdev->wext.keys = kzalloc(sizeof(*wdev->wext.keys),
                                          GFP_KERNEL);
                if (!wdev->wext.keys)
                        return -ENOMEM;
                for (i = 0; i < CFG80211_MAX_WEP_KEYS; i++)
                        wdev->wext.keys->params[i].key =
                                wdev->wext.keys->data[i];
        }

        if (wdev->iftype != NL80211_IFTYPE_ADHOC &&
            wdev->iftype != NL80211_IFTYPE_STATION)
                return -EOPNOTSUPP;

        if (params->cipher == WLAN_CIPHER_SUITE_AES_CMAC) {
                if (!wdev->current_bss)
                        return -ENOLINK;

                if (!rdev->ops->set_default_mgmt_key)
                        return -EOPNOTSUPP;

                if (idx < 4 || idx > 5)
                        return -EINVAL;
        } else if (idx < 0 || idx > 3)
                return -EINVAL;

        if (remove) {
                err = 0;
                if (wdev->current_bss) {
                        /*
                         * If removing the current TX key, we will need to
                         * join a new IBSS without the privacy bit clear.
                         */
                        if (idx == wdev->wext.default_key &&
                            wdev->iftype == NL80211_IFTYPE_ADHOC) {
                                __cfg80211_leave_ibss(rdev, wdev->netdev, true);
                                rejoin = true;
                        }

                        if (!pairwise && addr &&
                            !(rdev->wiphy.flags & WIPHY_FLAG_IBSS_RSN))
                                err = -ENOENT;
                        else
                                err = rdev_del_key(rdev, dev, idx, pairwise,
                                                   addr);
                }
                wdev->wext.connect.privacy = false;
                /*
                 * Applications using wireless extensions expect to be
                 * able to delete keys that don't exist, so allow that.
                 */
                if (err == -ENOENT)
                        err = 0;
                if (!err) {
                        if (!addr && idx < 4) {
                                memset(wdev->wext.keys->data[idx], 0,
                                       sizeof(wdev->wext.keys->data[idx]));
                                wdev->wext.keys->params[idx].key_len = 0;
                                wdev->wext.keys->params[idx].cipher = 0;
                        }
                        if (idx == wdev->wext.default_key)
                                wdev->wext.default_key = -1;
                        else if (idx == wdev->wext.default_mgmt_key)
                                wdev->wext.default_mgmt_key = -1;
                }

                if (!err && rejoin)
                        err = cfg80211_ibss_wext_join(rdev, wdev);

                return err;
        }

        if (addr)
                tx_key = false;

        if (cfg80211_validate_key_settings(rdev, params, idx, pairwise, addr))
                return -EINVAL;

        err = 0;
        if (wdev->current_bss)
                err = rdev_add_key(rdev, dev, idx, pairwise, addr, params);
        else if (params->cipher != WLAN_CIPHER_SUITE_WEP40 &&
                 params->cipher != WLAN_CIPHER_SUITE_WEP104)
                return -EINVAL;
        if (err)
                return err;

        /*
         * We only need to store WEP keys, since they're the only keys that
         * can be set before a connection is established and persist after
         * disconnecting.
         */
        if (!addr && (params->cipher == WLAN_CIPHER_SUITE_WEP40 ||
                      params->cipher == WLAN_CIPHER_SUITE_WEP104)) {
                wdev->wext.keys->params[idx] = *params;
                memcpy(wdev->wext.keys->data[idx],
                        params->key, params->key_len);
                wdev->wext.keys->params[idx].key =
                        wdev->wext.keys->data[idx];
        }

        if ((params->cipher == WLAN_CIPHER_SUITE_WEP40 ||
             params->cipher == WLAN_CIPHER_SUITE_WEP104) &&
            (tx_key || (!addr && wdev->wext.default_key == -1))) {
                if (wdev->current_bss) {
                        /*
                         * If we are getting a new TX key from not having
                         * had one before we need to join a new IBSS with
                         * the privacy bit set.
                         */
                        if (wdev->iftype == NL80211_IFTYPE_ADHOC &&
                            wdev->wext.default_key == -1) {
                                __cfg80211_leave_ibss(rdev, wdev->netdev, true);
                                rejoin = true;
                        }
                        err = rdev_set_default_key(rdev, dev, idx, true, true);
                }
                if (!err) {
                        wdev->wext.default_key = idx;
                        if (rejoin)
                                err = cfg80211_ibss_wext_join(rdev, wdev);
                }
                return err;
        }

        if (params->cipher == WLAN_CIPHER_SUITE_AES_CMAC &&
            (tx_key || (!addr && wdev->wext.default_mgmt_key == -1))) {
                if (wdev->current_bss)
                        err = rdev_set_default_mgmt_key(rdev, dev, idx);
                if (!err)
                        wdev->wext.default_mgmt_key = idx;
                return err;
        }

        return 0;
}

static int cfg80211_set_encryption(struct cfg80211_registered_device *rdev,
                                   struct net_device *dev, bool pairwise,
                                   const u8 *addr, bool remove, bool tx_key,
                                   int idx, struct key_params *params)
{
        int err;

        wdev_lock(dev->ieee80211_ptr);
        err = __cfg80211_set_encryption(rdev, dev, pairwise, addr,
                                        remove, tx_key, idx, params);
        wdev_unlock(dev->ieee80211_ptr);

        return err;
}

static int cfg80211_wext_siwencode(struct net_device *dev,
                                   struct iw_request_info *info,
                                   struct iw_point *erq, char *keybuf)
{
        struct wireless_dev *wdev = dev->ieee80211_ptr;
        struct cfg80211_registered_device *rdev = wiphy_to_rdev(wdev->wiphy);
        int idx, err;
        bool remove = false;
        struct key_params params;

        if (wdev->iftype != NL80211_IFTYPE_STATION &&
            wdev->iftype != NL80211_IFTYPE_ADHOC)
                return -EOPNOTSUPP;

        /* no use -- only MFP (set_default_mgmt_key) is optional */
        if (!rdev->ops->del_key ||
            !rdev->ops->add_key ||
            !rdev->ops->set_default_key)
                return -EOPNOTSUPP;

        wiphy_lock(&rdev->wiphy);
        idx = erq->flags & IW_ENCODE_INDEX;
        if (idx == 0) {
                idx = wdev->wext.default_key;
                if (idx < 0)
                        idx = 0;
        } else if (idx < 1 || idx > 4) {
                err = -EINVAL;
                goto out;
        } else {
                idx--;
        }

        if (erq->flags & IW_ENCODE_DISABLED)
                remove = true;
        else if (erq->length == 0) {
                /* No key data - just set the default TX key index */
                err = 0;
                wdev_lock(wdev);
                if (wdev->current_bss)
                        err = rdev_set_default_key(rdev, dev, idx, true,
                                                   true);
                if (!err)
                        wdev->wext.default_key = idx;
                wdev_unlock(wdev);
                goto out;
        }

        memset(&params, 0, sizeof(params));
        params.key = keybuf;
        params.key_len = erq->length;
        if (erq->length == 5) {
                params.cipher = WLAN_CIPHER_SUITE_WEP40;
        } else if (erq->length == 13) {
                params.cipher = WLAN_CIPHER_SUITE_WEP104;
        } else if (!remove) {
                err = -EINVAL;
                goto out;
        }

        err = cfg80211_set_encryption(rdev, dev, false, NULL, remove,
                                      wdev->wext.default_key == -1,
                                      idx, &params);
out:
        wiphy_unlock(&rdev->wiphy);

        return err;
}

static int cfg80211_wext_siwencodeext(struct net_device *dev,
                                      struct iw_request_info *info,
                                      struct iw_point *erq, char *extra)
{
        struct wireless_dev *wdev = dev->ieee80211_ptr;
        struct cfg80211_registered_device *rdev = wiphy_to_rdev(wdev->wiphy);
        struct iw_encode_ext *ext = (struct iw_encode_ext *) extra;
        const u8 *addr;
        int idx;
        bool remove = false;
        struct key_params params;
        u32 cipher;
        int ret;

        if (wdev->iftype != NL80211_IFTYPE_STATION &&
            wdev->iftype != NL80211_IFTYPE_ADHOC)
                return -EOPNOTSUPP;

        /* no use -- only MFP (set_default_mgmt_key) is optional */
        if (!rdev->ops->del_key ||
            !rdev->ops->add_key ||
            !rdev->ops->set_default_key)
                return -EOPNOTSUPP;

        switch (ext->alg) {
        case IW_ENCODE_ALG_NONE:
                remove = true;
                cipher = 0;
                break;
        case IW_ENCODE_ALG_WEP:
                if (ext->key_len == 5)
                        cipher = WLAN_CIPHER_SUITE_WEP40;
                else if (ext->key_len == 13)
                        cipher = WLAN_CIPHER_SUITE_WEP104;
                else
                        return -EINVAL;
                break;
        case IW_ENCODE_ALG_TKIP:
                cipher = WLAN_CIPHER_SUITE_TKIP;
                break;
        case IW_ENCODE_ALG_CCMP:
                cipher = WLAN_CIPHER_SUITE_CCMP;
                break;
        case IW_ENCODE_ALG_AES_CMAC:
                cipher = WLAN_CIPHER_SUITE_AES_CMAC;
                break;
        default:
                return -EOPNOTSUPP;
        }

        if (erq->flags & IW_ENCODE_DISABLED)
                remove = true;

        idx = erq->flags & IW_ENCODE_INDEX;
        if (cipher == WLAN_CIPHER_SUITE_AES_CMAC) {
                if (idx < 4 || idx > 5) {
                        idx = wdev->wext.default_mgmt_key;
                        if (idx < 0)
                                return -EINVAL;
                } else
                        idx--;
        } else {
                if (idx < 1 || idx > 4) {
                        idx = wdev->wext.default_key;
                        if (idx < 0)
                                return -EINVAL;
                } else
                        idx--;
        }

        addr = ext->addr.sa_data;
        if (is_broadcast_ether_addr(addr))
                addr = NULL;

        memset(&params, 0, sizeof(params));
        params.key = ext->key;
        params.key_len = ext->key_len;
        params.cipher = cipher;

        if (ext->ext_flags & IW_ENCODE_EXT_RX_SEQ_VALID) {
                params.seq = ext->rx_seq;
                params.seq_len = 6;
        }

        wiphy_lock(wdev->wiphy);
        ret = cfg80211_set_encryption(
                        rdev, dev,
                        !(ext->ext_flags & IW_ENCODE_EXT_GROUP_KEY),
                        addr, remove,
                        ext->ext_flags & IW_ENCODE_EXT_SET_TX_KEY,
                        idx, &params);
        wiphy_unlock(wdev->wiphy);

        return ret;
}

static int cfg80211_wext_giwencode(struct net_device *dev,
                                   struct iw_request_info *info,
                                   struct iw_point *erq, char *keybuf)
{
        struct wireless_dev *wdev = dev->ieee80211_ptr;
        int idx;

        if (wdev->iftype != NL80211_IFTYPE_STATION &&
            wdev->iftype != NL80211_IFTYPE_ADHOC)
                return -EOPNOTSUPP;

        idx = erq->flags & IW_ENCODE_INDEX;
        if (idx == 0) {
                idx = wdev->wext.default_key;
                if (idx < 0)
                        idx = 0;
        } else if (idx < 1 || idx > 4)
                return -EINVAL;
        else
                idx--;

        erq->flags = idx + 1;

        if (!wdev->wext.keys || !wdev->wext.keys->params[idx].cipher) {
                erq->flags |= IW_ENCODE_DISABLED;
                erq->length = 0;
                return 0;
        }

        erq->length = min_t(size_t, erq->length,
                            wdev->wext.keys->params[idx].key_len);
        memcpy(keybuf, wdev->wext.keys->params[idx].key, erq->length);
        erq->flags |= IW_ENCODE_ENABLED;

        return 0;
}

static int cfg80211_wext_siwfreq(struct net_device *dev,
                                 struct iw_request_info *info,
                                 struct iw_freq *wextfreq, char *extra)
{
        struct wireless_dev *wdev = dev->ieee80211_ptr;
        struct cfg80211_registered_device *rdev = wiphy_to_rdev(wdev->wiphy);
        struct cfg80211_chan_def chandef = {
                .width = NL80211_CHAN_WIDTH_20_NOHT,
        };
        int freq, ret;

        wiphy_lock(&rdev->wiphy);

        switch (wdev->iftype) {
        case NL80211_IFTYPE_STATION:
                ret = cfg80211_mgd_wext_siwfreq(dev, info, wextfreq, extra);
                break;
        case NL80211_IFTYPE_ADHOC:
                ret = cfg80211_ibss_wext_siwfreq(dev, info, wextfreq, extra);
                break;
        case NL80211_IFTYPE_MONITOR:
                freq = cfg80211_wext_freq(wextfreq);
                if (freq < 0) {
                        ret = freq;
                        break;
                }
                if (freq == 0) {
                        ret = -EINVAL;
                        break;
                }
                chandef.center_freq1 = freq;
                chandef.chan = ieee80211_get_channel(&rdev->wiphy, freq);
                if (!chandef.chan) {
                        ret = -EINVAL;
                        break;
                }
                ret = cfg80211_set_monitor_channel(rdev, &chandef);
                break;
        case NL80211_IFTYPE_MESH_POINT:
                freq = cfg80211_wext_freq(wextfreq);
                if (freq < 0) {
                        ret = freq;
                        break;
                }
                if (freq == 0) {
                        ret = -EINVAL;
                        break;
                }
                chandef.center_freq1 = freq;
                chandef.chan = ieee80211_get_channel(&rdev->wiphy, freq);
                if (!chandef.chan) {
                        ret = -EINVAL;
                        break;
                }
                ret = cfg80211_set_mesh_channel(rdev, wdev, &chandef);
                break;
        default:
                ret = -EOPNOTSUPP;
                break;
        }

        wiphy_unlock(&rdev->wiphy);

        return ret;
}

static int cfg80211_wext_giwfreq(struct net_device *dev,
                                 struct iw_request_info *info,
                                 struct iw_freq *freq, char *extra)
{
        struct wireless_dev *wdev = dev->ieee80211_ptr;
        struct cfg80211_registered_device *rdev = wiphy_to_rdev(wdev->wiphy);
        struct cfg80211_chan_def chandef = {};
        int ret;

        wiphy_lock(&rdev->wiphy);
        switch (wdev->iftype) {
        case NL80211_IFTYPE_STATION:
                ret = cfg80211_mgd_wext_giwfreq(dev, info, freq, extra);
                break;
        case NL80211_IFTYPE_ADHOC:
                ret = cfg80211_ibss_wext_giwfreq(dev, info, freq, extra);
                break;
        case NL80211_IFTYPE_MONITOR:
                if (!rdev->ops->get_channel) {
                        ret = -EINVAL;
                        break;
                }

                ret = rdev_get_channel(rdev, wdev, &chandef);
                if (ret)
                        break;
                freq->m = chandef.chan->center_freq;
                freq->e = 6;
                ret = 0;
                break;
        default:
                ret = -EINVAL;
                break;
        }

        wiphy_unlock(&rdev->wiphy);

        return ret;
}

static int cfg80211_wext_siwtxpower(struct net_device *dev,
                                    struct iw_request_info *info,
                                    union iwreq_data *data, char *extra)
{
        struct wireless_dev *wdev = dev->ieee80211_ptr;
        struct cfg80211_registered_device *rdev = wiphy_to_rdev(wdev->wiphy);
        enum nl80211_tx_power_setting type;
        int dbm = 0;
        int ret;

        if ((data->txpower.flags & IW_TXPOW_TYPE) != IW_TXPOW_DBM)
                return -EINVAL;
        if (data->txpower.flags & IW_TXPOW_RANGE)
                return -EINVAL;

        if (!rdev->ops->set_tx_power)
                return -EOPNOTSUPP;

        /* only change when not disabling */
        if (!data->txpower.disabled) {
                rfkill_set_sw_state(rdev->wiphy.rfkill, false);

                if (data->txpower.fixed) {
                        /*
                         * wext doesn't support negative values, see
                         * below where it's for automatic
                         */
                        if (data->txpower.value < 0)
                                return -EINVAL;
                        dbm = data->txpower.value;
                        type = NL80211_TX_POWER_FIXED;
                        /* TODO: do regulatory check! */
                } else {
                        /*
                         * Automatic power level setting, max being the value
                         * passed in from userland.
                         */
                        if (data->txpower.value < 0) {
                                type = NL80211_TX_POWER_AUTOMATIC;
                        } else {
                                dbm = data->txpower.value;
                                type = NL80211_TX_POWER_LIMITED;
                        }
                }
        } else {
                if (rfkill_set_sw_state(rdev->wiphy.rfkill, true))
                        schedule_work(&rdev->rfkill_block);
                return 0;
        }

        wiphy_lock(&rdev->wiphy);
        ret = rdev_set_tx_power(rdev, wdev, type, DBM_TO_MBM(dbm));
        wiphy_unlock(&rdev->wiphy);

        return ret;
}

static int cfg80211_wext_giwtxpower(struct net_device *dev,
                                    struct iw_request_info *info,
                                    union iwreq_data *data, char *extra)
{
        struct wireless_dev *wdev = dev->ieee80211_ptr;
        struct cfg80211_registered_device *rdev = wiphy_to_rdev(wdev->wiphy);
        int err, val;

        if ((data->txpower.flags & IW_TXPOW_TYPE) != IW_TXPOW_DBM)
                return -EINVAL;
        if (data->txpower.flags & IW_TXPOW_RANGE)
                return -EINVAL;

        if (!rdev->ops->get_tx_power)
                return -EOPNOTSUPP;

        wiphy_lock(&rdev->wiphy);
        err = rdev_get_tx_power(rdev, wdev, &val);
        wiphy_unlock(&rdev->wiphy);
        if (err)
                return err;

        /* well... oh well */
        data->txpower.fixed = 1;
        data->txpower.disabled = rfkill_blocked(rdev->wiphy.rfkill);
        data->txpower.value = val;
        data->txpower.flags = IW_TXPOW_DBM;

        return 0;
}

static int cfg80211_set_auth_alg(struct wireless_dev *wdev,
                                 s32 auth_alg)
{
        int nr_alg = 0;

        if (!auth_alg)
                return -EINVAL;

        if (auth_alg & ~(IW_AUTH_ALG_OPEN_SYSTEM |
                         IW_AUTH_ALG_SHARED_KEY |
                         IW_AUTH_ALG_LEAP))
                return -EINVAL;

        if (auth_alg & IW_AUTH_ALG_OPEN_SYSTEM) {
                nr_alg++;
                wdev->wext.connect.auth_type = NL80211_AUTHTYPE_OPEN_SYSTEM;
        }

        if (auth_alg & IW_AUTH_ALG_SHARED_KEY) {
                nr_alg++;
                wdev->wext.connect.auth_type = NL80211_AUTHTYPE_SHARED_KEY;
        }

        if (auth_alg & IW_AUTH_ALG_LEAP) {
                nr_alg++;
                wdev->wext.connect.auth_type = NL80211_AUTHTYPE_NETWORK_EAP;
        }

        if (nr_alg > 1)
                wdev->wext.connect.auth_type = NL80211_AUTHTYPE_AUTOMATIC;

        return 0;
}

static int cfg80211_set_wpa_version(struct wireless_dev *wdev, u32 wpa_versions)
{
        if (wpa_versions & ~(IW_AUTH_WPA_VERSION_WPA |
                             IW_AUTH_WPA_VERSION_WPA2|
                             IW_AUTH_WPA_VERSION_DISABLED))
                return -EINVAL;

        if ((wpa_versions & IW_AUTH_WPA_VERSION_DISABLED) &&
            (wpa_versions & (IW_AUTH_WPA_VERSION_WPA|
                             IW_AUTH_WPA_VERSION_WPA2)))
                return -EINVAL;

        if (wpa_versions & IW_AUTH_WPA_VERSION_DISABLED)
                wdev->wext.connect.crypto.wpa_versions &=
                        ~(NL80211_WPA_VERSION_1|NL80211_WPA_VERSION_2);

        if (wpa_versions & IW_AUTH_WPA_VERSION_WPA)
                wdev->wext.connect.crypto.wpa_versions |=
                        NL80211_WPA_VERSION_1;

        if (wpa_versions & IW_AUTH_WPA_VERSION_WPA2)
                wdev->wext.connect.crypto.wpa_versions |=
                        NL80211_WPA_VERSION_2;

        return 0;
}

static int cfg80211_set_cipher_group(struct wireless_dev *wdev, u32 cipher)
{
        if (cipher & IW_AUTH_CIPHER_WEP40)
                wdev->wext.connect.crypto.cipher_group =
                        WLAN_CIPHER_SUITE_WEP40;
        else if (cipher & IW_AUTH_CIPHER_WEP104)
                wdev->wext.connect.crypto.cipher_group =
                        WLAN_CIPHER_SUITE_WEP104;
        else if (cipher & IW_AUTH_CIPHER_TKIP)
                wdev->wext.connect.crypto.cipher_group =
                        WLAN_CIPHER_SUITE_TKIP;
        else if (cipher & IW_AUTH_CIPHER_CCMP)
                wdev->wext.connect.crypto.cipher_group =
                        WLAN_CIPHER_SUITE_CCMP;
        else if (cipher & IW_AUTH_CIPHER_AES_CMAC)
                wdev->wext.connect.crypto.cipher_group =
                        WLAN_CIPHER_SUITE_AES_CMAC;
        else if (cipher & IW_AUTH_CIPHER_NONE)
                wdev->wext.connect.crypto.cipher_group = 0;
        else
                return -EINVAL;

        return 0;
}

static int cfg80211_set_cipher_pairwise(struct wireless_dev *wdev, u32 cipher)
{
        int nr_ciphers = 0;
        u32 *ciphers_pairwise = wdev->wext.connect.crypto.ciphers_pairwise;

        if (cipher & IW_AUTH_CIPHER_WEP40) {
                ciphers_pairwise[nr_ciphers] = WLAN_CIPHER_SUITE_WEP40;
                nr_ciphers++;
        }

        if (cipher & IW_AUTH_CIPHER_WEP104) {
                ciphers_pairwise[nr_ciphers] = WLAN_CIPHER_SUITE_WEP104;
                nr_ciphers++;
        }

        if (cipher & IW_AUTH_CIPHER_TKIP) {
                ciphers_pairwise[nr_ciphers] = WLAN_CIPHER_SUITE_TKIP;
                nr_ciphers++;
        }

        if (cipher & IW_AUTH_CIPHER_CCMP) {
                ciphers_pairwise[nr_ciphers] = WLAN_CIPHER_SUITE_CCMP;
                nr_ciphers++;
        }

        if (cipher & IW_AUTH_CIPHER_AES_CMAC) {
                ciphers_pairwise[nr_ciphers] = WLAN_CIPHER_SUITE_AES_CMAC;
                nr_ciphers++;
        }

        BUILD_BUG_ON(NL80211_MAX_NR_CIPHER_SUITES < 5);

        wdev->wext.connect.crypto.n_ciphers_pairwise = nr_ciphers;

        return 0;
}


static int cfg80211_set_key_mgt(struct wireless_dev *wdev, u32 key_mgt)
{
        int nr_akm_suites = 0;

        if (key_mgt & ~(IW_AUTH_KEY_MGMT_802_1X |
                        IW_AUTH_KEY_MGMT_PSK))
                return -EINVAL;

        if (key_mgt & IW_AUTH_KEY_MGMT_802_1X) {
                wdev->wext.connect.crypto.akm_suites[nr_akm_suites] =
                        WLAN_AKM_SUITE_8021X;
                nr_akm_suites++;
        }

        if (key_mgt & IW_AUTH_KEY_MGMT_PSK) {
                wdev->wext.connect.crypto.akm_suites[nr_akm_suites] =
                        WLAN_AKM_SUITE_PSK;
                nr_akm_suites++;
        }

        wdev->wext.connect.crypto.n_akm_suites = nr_akm_suites;

        return 0;
}

static int cfg80211_wext_siwauth(struct net_device *dev,
                                 struct iw_request_info *info,
                                 struct iw_param *data, char *extra)
{
        struct wireless_dev *wdev = dev->ieee80211_ptr;

        if (wdev->iftype != NL80211_IFTYPE_STATION)
                return -EOPNOTSUPP;

        switch (data->flags & IW_AUTH_INDEX) {
        case IW_AUTH_PRIVACY_INVOKED:
                wdev->wext.connect.privacy = data->value;
                return 0;
        case IW_AUTH_WPA_VERSION:
                return cfg80211_set_wpa_version(wdev, data->value);
        case IW_AUTH_CIPHER_GROUP:
                return cfg80211_set_cipher_group(wdev, data->value);
        case IW_AUTH_KEY_MGMT:
                return cfg80211_set_key_mgt(wdev, data->value);
        case IW_AUTH_CIPHER_PAIRWISE:
                return cfg80211_set_cipher_pairwise(wdev, data->value);
        case IW_AUTH_80211_AUTH_ALG:
                return cfg80211_set_auth_alg(wdev, data->value);
        case IW_AUTH_WPA_ENABLED:
        case IW_AUTH_RX_UNENCRYPTED_EAPOL:
        case IW_AUTH_DROP_UNENCRYPTED:
        case IW_AUTH_MFP:
                return 0;
        default:
                return -EOPNOTSUPP;
        }
}

static int cfg80211_wext_giwauth(struct net_device *dev,
                                 struct iw_request_info *info,
                                 struct iw_param *data, char *extra)
{
        /* XXX: what do we need? */

        return -EOPNOTSUPP;
}

static int cfg80211_wext_siwpower(struct net_device *dev,
                                  struct iw_request_info *info,
                                  struct iw_param *wrq, char *extra)
{
        struct wireless_dev *wdev = dev->ieee80211_ptr;
        struct cfg80211_registered_device *rdev = wiphy_to_rdev(wdev->wiphy);
        bool ps;
        int timeout = wdev->ps_timeout;
        int err;

        if (wdev->iftype != NL80211_IFTYPE_STATION)
                return -EINVAL;

        if (!rdev->ops->set_power_mgmt)
                return -EOPNOTSUPP;

        if (wrq->disabled) {
                ps = false;
        } else {
                switch (wrq->flags & IW_POWER_MODE) {
                case IW_POWER_ON:       /* If not specified */
                case IW_POWER_MODE:     /* If set all mask */
                case IW_POWER_ALL_R:    /* If explicitely state all */
                        ps = true;
                        break;
                default:                /* Otherwise we ignore */
                        return -EINVAL;
                }

                if (wrq->flags & ~(IW_POWER_MODE | IW_POWER_TIMEOUT))
                        return -EINVAL;

                if (wrq->flags & IW_POWER_TIMEOUT)
                        timeout = wrq->value / 1000;
        }

        wiphy_lock(&rdev->wiphy);
        err = rdev_set_power_mgmt(rdev, dev, ps, timeout);
        wiphy_unlock(&rdev->wiphy);
        if (err)
                return err;

        wdev->ps = ps;
        wdev->ps_timeout = timeout;

        return 0;

}

static int cfg80211_wext_giwpower(struct net_device *dev,
                                  struct iw_request_info *info,
                                  struct iw_param *wrq, char *extra)
{
        struct wireless_dev *wdev = dev->ieee80211_ptr;

        wrq->disabled = !wdev->ps;

        return 0;
}

static int cfg80211_wext_siwrate(struct net_device *dev,
                                 struct iw_request_info *info,
                                 struct iw_param *rate, char *extra)
{
        struct wireless_dev *wdev = dev->ieee80211_ptr;
        struct cfg80211_registered_device *rdev = wiphy_to_rdev(wdev->wiphy);
        struct cfg80211_bitrate_mask mask;
        u32 fixed, maxrate;
        struct ieee80211_supported_band *sband;
        int band, ridx, ret;
        bool match = false;

        if (!rdev->ops->set_bitrate_mask)
                return -EOPNOTSUPP;

        memset(&mask, 0, sizeof(mask));
        fixed = 0;
        maxrate = (u32)-1;

        if (rate->value < 0) {
                /* nothing */
        } else if (rate->fixed) {
                fixed = rate->value / 100000;
        } else {
                maxrate = rate->value / 100000;
        }

        for (band = 0; band < NUM_NL80211_BANDS; band++) {
                sband = wdev->wiphy->bands[band];
                if (sband == NULL)
                        continue;
                for (ridx = 0; ridx < sband->n_bitrates; ridx++) {
                        struct ieee80211_rate *srate = &sband->bitrates[ridx];
                        if (fixed == srate->bitrate) {
                                mask.control[band].legacy = 1 << ridx;
                                match = true;
                                break;
                        }
                        if (srate->bitrate <= maxrate) {
                                mask.control[band].legacy |= 1 << ridx;
                                match = true;
                        }
                }
        }

        if (!match)
                return -EINVAL;

        wiphy_lock(&rdev->wiphy);
        ret = rdev_set_bitrate_mask(rdev, dev, NULL, &mask);
        wiphy_unlock(&rdev->wiphy);

        return ret;
}

static int cfg80211_wext_giwrate(struct net_device *dev,
                                 struct iw_request_info *info,
                                 struct iw_param *rate, char *extra)
{
        struct wireless_dev *wdev = dev->ieee80211_ptr;
        struct cfg80211_registered_device *rdev = wiphy_to_rdev(wdev->wiphy);
        struct station_info sinfo = {};
        u8 addr[ETH_ALEN];
        int err;

        if (wdev->iftype != NL80211_IFTYPE_STATION)
                return -EOPNOTSUPP;

        if (!rdev->ops->get_station)
                return -EOPNOTSUPP;

        err = 0;
        wdev_lock(wdev);
        if (wdev->current_bss)
                memcpy(addr, wdev->current_bss->pub.bssid, ETH_ALEN);
        else
                err = -EOPNOTSUPP;
        wdev_unlock(wdev);
        if (err)
                return err;

        wiphy_lock(&rdev->wiphy);
        err = rdev_get_station(rdev, dev, addr, &sinfo);
        wiphy_unlock(&rdev->wiphy);
        if (err)
                return err;

        if (!(sinfo.filled & BIT_ULL(NL80211_STA_INFO_TX_BITRATE))) {
                err = -EOPNOTSUPP;
                goto free;
        }

        rate->value = 100000 * cfg80211_calculate_bitrate(&sinfo.txrate);

free:
        cfg80211_sinfo_release_content(&sinfo);
        return err;
}

/* Get wireless statistics.  Called by /proc/net/wireless and by SIOCGIWSTATS */
static struct iw_statistics *cfg80211_wireless_stats(struct net_device *dev)
{
        struct wireless_dev *wdev = dev->ieee80211_ptr;
        struct cfg80211_registered_device *rdev = wiphy_to_rdev(wdev->wiphy);
        /* we are under RTNL - globally locked - so can use static structs */
        static struct iw_statistics wstats;
        static struct station_info sinfo = {};
        u8 bssid[ETH_ALEN];
        int ret;

        if (dev->ieee80211_ptr->iftype != NL80211_IFTYPE_STATION)
                return NULL;

        if (!rdev->ops->get_station)
                return NULL;

        /* Grab BSSID of current BSS, if any */
        wdev_lock(wdev);
        if (!wdev->current_bss) {
                wdev_unlock(wdev);
                return NULL;
        }
        memcpy(bssid, wdev->current_bss->pub.bssid, ETH_ALEN);
        wdev_unlock(wdev);

        memset(&sinfo, 0, sizeof(sinfo));

        wiphy_lock(&rdev->wiphy);
        ret = rdev_get_station(rdev, dev, bssid, &sinfo);
        wiphy_unlock(&rdev->wiphy);

        if (ret)
                return NULL;

        memset(&wstats, 0, sizeof(wstats));

        switch (rdev->wiphy.signal_type) {
        case CFG80211_SIGNAL_TYPE_MBM:
                if (sinfo.filled & BIT_ULL(NL80211_STA_INFO_SIGNAL)) {
                        int sig = sinfo.signal;
                        wstats.qual.updated |= IW_QUAL_LEVEL_UPDATED;
                        wstats.qual.updated |= IW_QUAL_QUAL_UPDATED;
                        wstats.qual.updated |= IW_QUAL_DBM;
                        wstats.qual.level = sig;
                        if (sig < -110)
                                sig = -110;
                        else if (sig > -40)
                                sig = -40;
                        wstats.qual.qual = sig + 110;
                        break;
                }
                fallthrough;
        case CFG80211_SIGNAL_TYPE_UNSPEC:
                if (sinfo.filled & BIT_ULL(NL80211_STA_INFO_SIGNAL)) {
                        wstats.qual.updated |= IW_QUAL_LEVEL_UPDATED;
                        wstats.qual.updated |= IW_QUAL_QUAL_UPDATED;
                        wstats.qual.level = sinfo.signal;
                        wstats.qual.qual = sinfo.signal;
                        break;
                }
                fallthrough;
        default:
                wstats.qual.updated |= IW_QUAL_LEVEL_INVALID;
                wstats.qual.updated |= IW_QUAL_QUAL_INVALID;
        }

        wstats.qual.updated |= IW_QUAL_NOISE_INVALID;
        if (sinfo.filled & BIT_ULL(NL80211_STA_INFO_RX_DROP_MISC))
                wstats.discard.misc = sinfo.rx_dropped_misc;
        if (sinfo.filled & BIT_ULL(NL80211_STA_INFO_TX_FAILED))
                wstats.discard.retries = sinfo.tx_failed;

        cfg80211_sinfo_release_content(&sinfo);

        return &wstats;
}

static int cfg80211_wext_siwap(struct net_device *dev,
                               struct iw_request_info *info,
                               struct sockaddr *ap_addr, char *extra)
{
        struct wireless_dev *wdev = dev->ieee80211_ptr;
        struct cfg80211_registered_device *rdev = wiphy_to_rdev(wdev->wiphy);
        int ret;

        wiphy_lock(&rdev->wiphy);
        switch (wdev->iftype) {
        case NL80211_IFTYPE_ADHOC:
                ret = cfg80211_ibss_wext_siwap(dev, info, ap_addr, extra);
                break;
        case NL80211_IFTYPE_STATION:
                ret = cfg80211_mgd_wext_siwap(dev, info, ap_addr, extra);
                break;
        default:
                ret = -EOPNOTSUPP;
                break;
        }
        wiphy_unlock(&rdev->wiphy);

        return ret;
}

static int cfg80211_wext_giwap(struct net_device *dev,
                               struct iw_request_info *info,
                               struct sockaddr *ap_addr, char *extra)
{
        struct wireless_dev *wdev = dev->ieee80211_ptr;
        struct cfg80211_registered_device *rdev = wiphy_to_rdev(wdev->wiphy);
        int ret;

        wiphy_lock(&rdev->wiphy);
        switch (wdev->iftype) {
        case NL80211_IFTYPE_ADHOC:
                ret = cfg80211_ibss_wext_giwap(dev, info, ap_addr, extra);
                break;
        case NL80211_IFTYPE_STATION:
                ret = cfg80211_mgd_wext_giwap(dev, info, ap_addr, extra);
                break;
        default:
                ret = -EOPNOTSUPP;
                break;
        }
        wiphy_unlock(&rdev->wiphy);

        return ret;
}

static int cfg80211_wext_siwessid(struct net_device *dev,
                                  struct iw_request_info *info,
                                  struct iw_point *data, char *ssid)
{
        struct wireless_dev *wdev = dev->ieee80211_ptr;
        struct cfg80211_registered_device *rdev = wiphy_to_rdev(wdev->wiphy);
        int ret;

        wiphy_lock(&rdev->wiphy);
        switch (wdev->iftype) {
        case NL80211_IFTYPE_ADHOC:
                ret = cfg80211_ibss_wext_siwessid(dev, info, data, ssid);
                break;
        case NL80211_IFTYPE_STATION:
                ret = cfg80211_mgd_wext_siwessid(dev, info, data, ssid);
                break;
        default:
                ret = -EOPNOTSUPP;
                break;
        }
        wiphy_unlock(&rdev->wiphy);

        return ret;
}

static int cfg80211_wext_giwessid(struct net_device *dev,
                                  struct iw_request_info *info,
                                  struct iw_point *data, char *ssid)
{
        struct wireless_dev *wdev = dev->ieee80211_ptr;
        struct cfg80211_registered_device *rdev = wiphy_to_rdev(wdev->wiphy);
        int ret;

        data->flags = 0;
        data->length = 0;

        wiphy_lock(&rdev->wiphy);
        switch (wdev->iftype) {
        case NL80211_IFTYPE_ADHOC:
                ret = cfg80211_ibss_wext_giwessid(dev, info, data, ssid);
                break;
        case NL80211_IFTYPE_STATION:
                ret = cfg80211_mgd_wext_giwessid(dev, info, data, ssid);
                break;
        default:
                ret = -EOPNOTSUPP;
                break;
        }
        wiphy_unlock(&rdev->wiphy);

        return ret;
}

static int cfg80211_wext_siwpmksa(struct net_device *dev,
                                  struct iw_request_info *info,
                                  struct iw_point *data, char *extra)
{
        struct wireless_dev *wdev = dev->ieee80211_ptr;
        struct cfg80211_registered_device *rdev = wiphy_to_rdev(wdev->wiphy);
        struct cfg80211_pmksa cfg_pmksa;
        struct iw_pmksa *pmksa = (struct iw_pmksa *)extra;
        int ret;

        memset(&cfg_pmksa, 0, sizeof(struct cfg80211_pmksa));

        if (wdev->iftype != NL80211_IFTYPE_STATION)
                return -EINVAL;

        cfg_pmksa.bssid = pmksa->bssid.sa_data;
        cfg_pmksa.pmkid = pmksa->pmkid;

        wiphy_lock(&rdev->wiphy);
        switch (pmksa->cmd) {
        case IW_PMKSA_ADD:
                if (!rdev->ops->set_pmksa) {
                        ret = -EOPNOTSUPP;
                        break;
                }

                ret = rdev_set_pmksa(rdev, dev, &cfg_pmksa);
                break;
        case IW_PMKSA_REMOVE:
                if (!rdev->ops->del_pmksa) {
                        ret = -EOPNOTSUPP;
                        break;
                }

                ret = rdev_del_pmksa(rdev, dev, &cfg_pmksa);
                break;
        case IW_PMKSA_FLUSH:
                if (!rdev->ops->flush_pmksa) {
                        ret = -EOPNOTSUPP;
                        break;
                }

                ret = rdev_flush_pmksa(rdev, dev);
                break;
        default:
                ret = -EOPNOTSUPP;
                break;
        }
        wiphy_unlock(&rdev->wiphy);

        return ret;
}

#define DEFINE_WEXT_COMPAT_STUB(func, type)                        \
        static int __ ## func(struct net_device *dev,                \
                              struct iw_request_info *info,        \
                              union iwreq_data *wrqu,                \
                              char *extra)                        \
        {                                                        \
                return func(dev, info, (type *)wrqu, extra);        \
        }

DEFINE_WEXT_COMPAT_STUB(cfg80211_wext_giwname, char)
DEFINE_WEXT_COMPAT_STUB(cfg80211_wext_siwfreq, struct iw_freq)
DEFINE_WEXT_COMPAT_STUB(cfg80211_wext_giwfreq, struct iw_freq)
DEFINE_WEXT_COMPAT_STUB(cfg80211_wext_siwmode, u32)
DEFINE_WEXT_COMPAT_STUB(cfg80211_wext_giwmode, u32)
DEFINE_WEXT_COMPAT_STUB(cfg80211_wext_giwrange, struct iw_point)
DEFINE_WEXT_COMPAT_STUB(cfg80211_wext_siwap, struct sockaddr)
DEFINE_WEXT_COMPAT_STUB(cfg80211_wext_giwap, struct sockaddr)
DEFINE_WEXT_COMPAT_STUB(cfg80211_wext_siwmlme, struct iw_point)
DEFINE_WEXT_COMPAT_STUB(cfg80211_wext_giwscan, struct iw_point)
DEFINE_WEXT_COMPAT_STUB(cfg80211_wext_siwessid, struct iw_point)
DEFINE_WEXT_COMPAT_STUB(cfg80211_wext_giwessid, struct iw_point)
DEFINE_WEXT_COMPAT_STUB(cfg80211_wext_siwrate, struct iw_param)
DEFINE_WEXT_COMPAT_STUB(cfg80211_wext_giwrate, struct iw_param)
DEFINE_WEXT_COMPAT_STUB(cfg80211_wext_siwrts, struct iw_param)
DEFINE_WEXT_COMPAT_STUB(cfg80211_wext_giwrts, struct iw_param)
DEFINE_WEXT_COMPAT_STUB(cfg80211_wext_siwfrag, struct iw_param)
DEFINE_WEXT_COMPAT_STUB(cfg80211_wext_giwfrag, struct iw_param)
DEFINE_WEXT_COMPAT_STUB(cfg80211_wext_siwretry, struct iw_param)
DEFINE_WEXT_COMPAT_STUB(cfg80211_wext_giwretry, struct iw_param)
DEFINE_WEXT_COMPAT_STUB(cfg80211_wext_siwencode, struct iw_point)
DEFINE_WEXT_COMPAT_STUB(cfg80211_wext_giwencode, struct iw_point)
DEFINE_WEXT_COMPAT_STUB(cfg80211_wext_giwpower, struct iw_param)
DEFINE_WEXT_COMPAT_STUB(cfg80211_wext_siwpower, struct iw_param)
DEFINE_WEXT_COMPAT_STUB(cfg80211_wext_siwgenie, struct iw_point)
DEFINE_WEXT_COMPAT_STUB(cfg80211_wext_giwauth, struct iw_param)
DEFINE_WEXT_COMPAT_STUB(cfg80211_wext_siwauth, struct iw_param)
DEFINE_WEXT_COMPAT_STUB(cfg80211_wext_siwencodeext, struct iw_point)
DEFINE_WEXT_COMPAT_STUB(cfg80211_wext_siwpmksa, struct iw_point)

static const iw_handler cfg80211_handlers[] = {
        [IW_IOCTL_IDX(SIOCGIWNAME)]        = __cfg80211_wext_giwname,
        [IW_IOCTL_IDX(SIOCSIWFREQ)]        = __cfg80211_wext_siwfreq,
        [IW_IOCTL_IDX(SIOCGIWFREQ)]        = __cfg80211_wext_giwfreq,
        [IW_IOCTL_IDX(SIOCSIWMODE)]        = __cfg80211_wext_siwmode,
        [IW_IOCTL_IDX(SIOCGIWMODE)]        = __cfg80211_wext_giwmode,
        [IW_IOCTL_IDX(SIOCGIWRANGE)]        = __cfg80211_wext_giwrange,
        [IW_IOCTL_IDX(SIOCSIWAP)]        = __cfg80211_wext_siwap,
        [IW_IOCTL_IDX(SIOCGIWAP)]        = __cfg80211_wext_giwap,
        [IW_IOCTL_IDX(SIOCSIWMLME)]        = __cfg80211_wext_siwmlme,
        [IW_IOCTL_IDX(SIOCSIWSCAN)]        = cfg80211_wext_siwscan,
        [IW_IOCTL_IDX(SIOCGIWSCAN)]        = __cfg80211_wext_giwscan,
        [IW_IOCTL_IDX(SIOCSIWESSID)]        = __cfg80211_wext_siwessid,
        [IW_IOCTL_IDX(SIOCGIWESSID)]        = __cfg80211_wext_giwessid,
        [IW_IOCTL_IDX(SIOCSIWRATE)]        = __cfg80211_wext_siwrate,
        [IW_IOCTL_IDX(SIOCGIWRATE)]        = __cfg80211_wext_giwrate,
        [IW_IOCTL_IDX(SIOCSIWRTS)]        = __cfg80211_wext_siwrts,
        [IW_IOCTL_IDX(SIOCGIWRTS)]        = __cfg80211_wext_giwrts,
        [IW_IOCTL_IDX(SIOCSIWFRAG)]        = __cfg80211_wext_siwfrag,
        [IW_IOCTL_IDX(SIOCGIWFRAG)]        = __cfg80211_wext_giwfrag,
        [IW_IOCTL_IDX(SIOCSIWTXPOW)]        = cfg80211_wext_siwtxpower,
        [IW_IOCTL_IDX(SIOCGIWTXPOW)]        = cfg80211_wext_giwtxpower,
        [IW_IOCTL_IDX(SIOCSIWRETRY)]        = __cfg80211_wext_siwretry,
        [IW_IOCTL_IDX(SIOCGIWRETRY)]        = __cfg80211_wext_giwretry,
        [IW_IOCTL_IDX(SIOCSIWENCODE)]        = __cfg80211_wext_siwencode,
        [IW_IOCTL_IDX(SIOCGIWENCODE)]        = __cfg80211_wext_giwencode,
        [IW_IOCTL_IDX(SIOCSIWPOWER)]        = __cfg80211_wext_siwpower,
        [IW_IOCTL_IDX(SIOCGIWPOWER)]        = __cfg80211_wext_giwpower,
        [IW_IOCTL_IDX(SIOCSIWGENIE)]        = __cfg80211_wext_siwgenie,
        [IW_IOCTL_IDX(SIOCSIWAUTH)]        = __cfg80211_wext_siwauth,
        [IW_IOCTL_IDX(SIOCGIWAUTH)]        = __cfg80211_wext_giwauth,
        [IW_IOCTL_IDX(SIOCSIWENCODEEXT)]= __cfg80211_wext_siwencodeext,
        [IW_IOCTL_IDX(SIOCSIWPMKSA)]        = __cfg80211_wext_siwpmksa,
};

const struct iw_handler_def cfg80211_wext_handler = {
        .num_standard                = ARRAY_SIZE(cfg80211_handlers),
        .standard                = cfg80211_handlers,
        .get_wireless_stats = cfg80211_wireless_stats,
};






























































  193 








  193 








  194 














   48 













   37 

































  126 


















 2473 




   18 

 2476 


















































 5211 



 2789 




 4850 



 3034 




 4518 



 3923 




  715 














  250 




  250 




  250 




  250 










 5121 
 5102 


  239 


































































































































































 6298 
















 6049 







 2954 




 6267 

  149 















 3469 

 3480 

 5124 







 2465 
 6292 
    6 
 1842 




 6303 
   15 
  562 



 6261 

    7 
  149 



    7 
  148 




    6 


 6308 
    5 
  789 




 6310 





 6328 
 2054 
 6291 



 6307 
    4 
    4 































 1135 
















 6321 


 6321 




 1135 

















 6320 

 6331 

 5256 
 4469 









































































 6321 



 6343 





   12 




























































































    1 
    1 


































   12 





   12 









    4 



    4 




























































































































    6 











    1 

    2 
    2 




    2 

    6 











































































































































    1 





























    1 
    1 
    1 

    1 
    1 


    1 
    1 
    1 






























































































    5 









    5 




















    5 



    5 




  345 





























  345 






  345 
  345 











  374 






















   11 
  363 
  303 




   11 









  303 


  357 





  304 
   69 



  303 




  290 

   24 



   67 

  145 



  362 

   11 









    7 





    1 
    6 









  381 


    7 
  376 



  384 





  332 




  334 






































































































  713 


































































































  233 























































































































































































































































































































































































































































































































































































































































































































  954 
  721 










    1 



























  233 

    4 


























    1 














    8 


























 6805 















    1 









 6830 





 6666 










    6 

  552 
    2 
  148 
 2453 












 6633 








 6657 







    1 







 6674 


 4460 












 6075 

 2905 



 6307 



  944 



   68 




  273 



  919 




























 1401 


 4461 

    3 










 4153 


















    1 

































 6644 


















 6659 








 5264 
 6487 












    1 






 2907 






 2918 



 2912 








 6302 



  940 

  946 





   66 

















 1392 


 4448 








    3 

















 2948 


 3810 







  623 
 6634 



























 2933 




 2979 






















 5769 























   47 




   47 







































 2427 



































































































































































































  406 































  406 





    3 


   25 
  292 



















    1 





    1 



    1 


















    8 








    4 




    3 
    4 










   19 






    2 

















































   32 
  374 












































  222 













  166 




  221 
   25 





  210 



























  159 

    8 





  159 























    6 





  156 













    8 






    8 

























































































  158 







  158 

  156 




  126 



   20 


























    2 






  125 

   18 




  145 















  221 










1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965
2966
2967
2968
2969
2970
2971
2972
2973
2974
2975
2976
2977
2978
2979
2980
2981
2982
2983
2984
2985
2986
2987
2988
2989
2990
2991
2992
2993
2994
2995
2996
2997
2998
2999
3000
3001
3002
3003
3004
3005
3006
3007
3008
3009
3010
3011
3012
3013
3014
3015
3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029
3030
3031
3032
3033
3034
3035
3036
3037
3038
3039
3040
3041
3042
3043
3044
3045
3046
3047
3048
3049
3050
3051
3052
3053
3054
3055
3056
3057
3058
3059
3060
3061
3062
3063
3064
3065
3066
3067
3068
3069
3070
3071
3072
3073
3074
3075
3076
3077
3078
3079
3080
3081
3082
3083
3084
3085
3086
3087
3088
3089
3090
3091
3092
3093
3094
3095
3096
3097
3098
3099
3100
3101
3102
3103
3104
3105
3106
3107
3108
3109
3110
3111
3112
3113
3114
3115
3116
3117
3118
3119
3120
3121
3122
3123
3124
3125
3126
3127
3128
3129
3130
3131
3132
3133
3134
3135
3136
3137
3138
3139
3140
3141
3142
3143
3144
3145
3146
3147
3148
3149
3150
3151
3152
3153
3154
3155
3156
3157
3158
3159
3160
3161
3162
3163
3164
3165
3166
3167
3168
3169
3170
3171
3172
3173
3174
3175
3176
3177
3178
3179
3180
3181
3182
3183
3184
3185
3186
3187
3188
3189
3190
3191
3192
3193
3194
3195
3196
3197
3198
3199
3200
3201
3202
3203
3204
3205
3206
3207
3208
3209
3210
3211
3212
3213
3214
3215
3216
3217
3218
3219
3220
3221
3222
3223
3224
3225
3226
3227
3228
3229
3230
3231
3232
3233
3234
3235
3236
3237
3238
3239
3240
3241
3242
3243
3244
3245
3246
3247
3248
3249
3250
3251
3252
3253
3254
3255
3256
3257
3258
3259
3260
3261
3262
3263
3264
3265
3266
3267
3268
3269
3270
3271
3272
3273
3274
3275
3276
3277
3278
3279
3280
3281
3282
3283
3284
3285
3286
3287
3288
3289
3290
3291
3292
3293
3294
3295
3296
3297
3298
3299
3300
3301
3302
3303
3304
3305
3306
3307
3308
3309
3310
3311
3312
3313
3314
3315
3316
3317
3318
3319
3320
3321
3322
3323
3324
3325
3326
3327
3328
3329
3330
3331
3332
3333
3334
3335
3336
3337
3338
3339
3340
3341
3342
3343
3344
3345
3346
3347
3348
3349
3350
3351
3352
3353
3354
3355
3356
3357
3358
3359
3360
3361
3362
3363
3364
3365
3366
3367
3368
3369
3370
3371
3372
3373
3374
3375
3376
3377
3378
3379
3380
3381
3382
3383
3384
3385
3386
3387
3388
3389
3390
3391
3392
3393
3394
3395
3396
3397
3398
3399
3400
3401
3402
3403
3404
3405
3406
3407
3408
3409
3410
3411
3412
3413
3414
3415
3416
3417
3418
3419
3420
3421
3422
3423
3424
3425
3426
3427
3428
3429
3430
3431
3432
3433
3434
3435
3436
3437
3438
3439
3440
3441
3442
3443
3444
3445
3446
3447
3448
3449
3450
3451
3452
3453
3454
3455
3456
3457
3458
3459
3460
3461
3462
3463
3464
3465
3466
3467
3468
3469
3470
3471
3472
3473
3474
3475
3476
3477
3478
3479
3480
3481
3482
3483
3484
3485
3486
3487
3488
3489
3490
3491
3492
3493
3494
3495
3496
3497
3498
3499
3500
3501
3502
3503
3504
3505
3506
3507
3508
3509
3510
3511
3512
3513
3514
3515
3516
3517
3518
3519
3520
3521
3522
3523
3524
3525
3526
3527
3528
3529
3530
3531
3532
3533
3534
3535
3536
3537
3538
3539
3540
3541
3542
3543
3544
3545
3546
3547
3548
3549
3550
3551
3552
3553
3554
3555
3556
3557
3558
3559
3560
3561
3562
3563
3564
3565
3566
3567
3568
3569
3570
3571
3572
3573
3574
3575
3576
3577
3578
3579
3580
3581
3582
3583
3584
3585
3586
3587
3588
3589
3590
3591
3592
3593
3594
3595
3596
3597
3598
3599
3600
3601
3602
3603
3604
3605
3606
3607
3608
3609
3610
3611
3612
3613
3614
3615
3616
3617
3618
3619
3620
3621
3622
3623
3624
3625
3626
3627
3628
3629
3630
3631
3632
3633
3634
3635
3636
3637
3638
3639
3640
3641
3642
3643
3644
3645
3646
3647
3648
3649
3650
3651
3652
3653
3654
3655
3656
3657
3658
3659
3660
3661
3662
3663
3664
3665
3666
3667
3668
3669
3670
3671
3672
3673
3674
3675
3676
3677
3678
3679
3680
3681
3682
3683
3684
3685
3686
3687
3688
3689
3690
3691
3692
3693
3694
3695
3696
3697
3698
3699
3700
3701
3702
3703
3704
3705
3706
3707
3708
3709
3710
3711
3712
3713
// SPDX-License-Identifier: GPL-2.0-only
/*
 *  linux/lib/vsprintf.c
 *
 *  Copyright (C) 1991, 1992  Linus Torvalds
 */

/* vsprintf.c -- Lars Wirzenius & Linus Torvalds. */
/*
 * Wirzenius wrote this portably, Torvalds fucked it up :-)
 */

/*
 * Fri Jul 13 2001 Crutcher Dunnavant <crutcher+kernel@datastacks.com>
 * - changed to provide snprintf and vsnprintf functions
 * So Feb  1 16:51:32 CET 2004 Juergen Quade <quade@hsnr.de>
 * - scnprintf and vscnprintf
 */

#include <linux/stdarg.h>
#include <linux/build_bug.h>
#include <linux/clk.h>
#include <linux/clk-provider.h>
#include <linux/errname.h>
#include <linux/module.h>        /* for KSYM_SYMBOL_LEN */
#include <linux/types.h>
#include <linux/string.h>
#include <linux/ctype.h>
#include <linux/kernel.h>
#include <linux/kallsyms.h>
#include <linux/math64.h>
#include <linux/uaccess.h>
#include <linux/ioport.h>
#include <linux/dcache.h>
#include <linux/cred.h>
#include <linux/rtc.h>
#include <linux/time.h>
#include <linux/uuid.h>
#include <linux/of.h>
#include <net/addrconf.h>
#include <linux/siphash.h>
#include <linux/compiler.h>
#include <linux/property.h>
#ifdef CONFIG_BLOCK
#include <linux/blkdev.h>
#endif

#include "../mm/internal.h"        /* For the trace_print_flags arrays */

#include <asm/page.h>                /* for PAGE_SIZE */
#include <asm/byteorder.h>        /* cpu_to_le16 */
#include <asm/unaligned.h>

#include <linux/string_helpers.h>
#include "kstrtox.h"

/* Disable pointer hashing if requested */
bool no_hash_pointers __ro_after_init;
EXPORT_SYMBOL_GPL(no_hash_pointers);

static noinline unsigned long long simple_strntoull(const char *startp, size_t max_chars, char **endp, unsigned int base)
{
        const char *cp;
        unsigned long long result = 0ULL;
        size_t prefix_chars;
        unsigned int rv;

        cp = _parse_integer_fixup_radix(startp, &base);
        prefix_chars = cp - startp;
        if (prefix_chars < max_chars) {
                rv = _parse_integer_limit(cp, base, &result, max_chars - prefix_chars);
                /* FIXME */
                cp += (rv & ~KSTRTOX_OVERFLOW);
        } else {
                /* Field too short for prefix + digit, skip over without converting */
                cp = startp + max_chars;
        }

        if (endp)
                *endp = (char *)cp;

        return result;
}

/**
 * simple_strtoull - convert a string to an unsigned long long
 * @cp: The start of the string
 * @endp: A pointer to the end of the parsed string will be placed here
 * @base: The number base to use
 *
 * This function has caveats. Please use kstrtoull instead.
 */
noinline
unsigned long long simple_strtoull(const char *cp, char **endp, unsigned int base)
{
        return simple_strntoull(cp, INT_MAX, endp, base);
}
EXPORT_SYMBOL(simple_strtoull);

/**
 * simple_strtoul - convert a string to an unsigned long
 * @cp: The start of the string
 * @endp: A pointer to the end of the parsed string will be placed here
 * @base: The number base to use
 *
 * This function has caveats. Please use kstrtoul instead.
 */
unsigned long simple_strtoul(const char *cp, char **endp, unsigned int base)
{
        return simple_strtoull(cp, endp, base);
}
EXPORT_SYMBOL(simple_strtoul);

/**
 * simple_strtol - convert a string to a signed long
 * @cp: The start of the string
 * @endp: A pointer to the end of the parsed string will be placed here
 * @base: The number base to use
 *
 * This function has caveats. Please use kstrtol instead.
 */
long simple_strtol(const char *cp, char **endp, unsigned int base)
{
        if (*cp == '-')
                return -simple_strtoul(cp + 1, endp, base);

        return simple_strtoul(cp, endp, base);
}
EXPORT_SYMBOL(simple_strtol);

static long long simple_strntoll(const char *cp, size_t max_chars, char **endp,
                                 unsigned int base)
{
        /*
         * simple_strntoull() safely handles receiving max_chars==0 in the
         * case cp[0] == '-' && max_chars == 1.
         * If max_chars == 0 we can drop through and pass it to simple_strntoull()
         * and the content of *cp is irrelevant.
         */
        if (*cp == '-' && max_chars > 0)
                return -simple_strntoull(cp + 1, max_chars - 1, endp, base);

        return simple_strntoull(cp, max_chars, endp, base);
}

/**
 * simple_strtoll - convert a string to a signed long long
 * @cp: The start of the string
 * @endp: A pointer to the end of the parsed string will be placed here
 * @base: The number base to use
 *
 * This function has caveats. Please use kstrtoll instead.
 */
long long simple_strtoll(const char *cp, char **endp, unsigned int base)
{
        return simple_strntoll(cp, INT_MAX, endp, base);
}
EXPORT_SYMBOL(simple_strtoll);

static noinline_for_stack
int skip_atoi(const char **s)
{
        int i = 0;

        do {
                i = i*10 + *((*s)++) - '0';
        } while (isdigit(**s));

        return i;
}

/*
 * Decimal conversion is by far the most typical, and is used for
 * /proc and /sys data. This directly impacts e.g. top performance
 * with many processes running. We optimize it for speed by emitting
 * two characters at a time, using a 200 byte lookup table. This
 * roughly halves the number of multiplications compared to computing
 * the digits one at a time. Implementation strongly inspired by the
 * previous version, which in turn used ideas described at
 * <http://www.cs.uiowa.edu/~jones/bcd/divide.html> (with permission
 * from the author, Douglas W. Jones).
 *
 * It turns out there is precisely one 26 bit fixed-point
 * approximation a of 64/100 for which x/100 == (x * (u64)a) >> 32
 * holds for all x in [0, 10^8-1], namely a = 0x28f5c29. The actual
 * range happens to be somewhat larger (x <= 1073741898), but that's
 * irrelevant for our purpose.
 *
 * For dividing a number in the range [10^4, 10^6-1] by 100, we still
 * need a 32x32->64 bit multiply, so we simply use the same constant.
 *
 * For dividing a number in the range [100, 10^4-1] by 100, there are
 * several options. The simplest is (x * 0x147b) >> 19, which is valid
 * for all x <= 43698.
 */

static const u16 decpair[100] = {
#define _(x) (__force u16) cpu_to_le16(((x % 10) | ((x / 10) << 8)) + 0x3030)
        _( 0), _( 1), _( 2), _( 3), _( 4), _( 5), _( 6), _( 7), _( 8), _( 9),
        _(10), _(11), _(12), _(13), _(14), _(15), _(16), _(17), _(18), _(19),
        _(20), _(21), _(22), _(23), _(24), _(25), _(26), _(27), _(28), _(29),
        _(30), _(31), _(32), _(33), _(34), _(35), _(36), _(37), _(38), _(39),
        _(40), _(41), _(42), _(43), _(44), _(45), _(46), _(47), _(48), _(49),
        _(50), _(51), _(52), _(53), _(54), _(55), _(56), _(57), _(58), _(59),
        _(60), _(61), _(62), _(63), _(64), _(65), _(66), _(67), _(68), _(69),
        _(70), _(71), _(72), _(73), _(74), _(75), _(76), _(77), _(78), _(79),
        _(80), _(81), _(82), _(83), _(84), _(85), _(86), _(87), _(88), _(89),
        _(90), _(91), _(92), _(93), _(94), _(95), _(96), _(97), _(98), _(99),
#undef _
};

/*
 * This will print a single '0' even if r == 0, since we would
 * immediately jump to out_r where two 0s would be written but only
 * one of them accounted for in buf. This is needed by ip4_string
 * below. All other callers pass a non-zero value of r.
*/
static noinline_for_stack
char *put_dec_trunc8(char *buf, unsigned r)
{
        unsigned q;

        /* 1 <= r < 10^8 */
        if (r < 100)
                goto out_r;

        /* 100 <= r < 10^8 */
        q = (r * (u64)0x28f5c29) >> 32;
        *((u16 *)buf) = decpair[r - 100*q];
        buf += 2;

        /* 1 <= q < 10^6 */
        if (q < 100)
                goto out_q;

        /*  100 <= q < 10^6 */
        r = (q * (u64)0x28f5c29) >> 32;
        *((u16 *)buf) = decpair[q - 100*r];
        buf += 2;

        /* 1 <= r < 10^4 */
        if (r < 100)
                goto out_r;

        /* 100 <= r < 10^4 */
        q = (r * 0x147b) >> 19;
        *((u16 *)buf) = decpair[r - 100*q];
        buf += 2;
out_q:
        /* 1 <= q < 100 */
        r = q;
out_r:
        /* 1 <= r < 100 */
        *((u16 *)buf) = decpair[r];
        buf += r < 10 ? 1 : 2;
        return buf;
}

#if BITS_PER_LONG == 64 && BITS_PER_LONG_LONG == 64
static noinline_for_stack
char *put_dec_full8(char *buf, unsigned r)
{
        unsigned q;

        /* 0 <= r < 10^8 */
        q = (r * (u64)0x28f5c29) >> 32;
        *((u16 *)buf) = decpair[r - 100*q];
        buf += 2;

        /* 0 <= q < 10^6 */
        r = (q * (u64)0x28f5c29) >> 32;
        *((u16 *)buf) = decpair[q - 100*r];
        buf += 2;

        /* 0 <= r < 10^4 */
        q = (r * 0x147b) >> 19;
        *((u16 *)buf) = decpair[r - 100*q];
        buf += 2;

        /* 0 <= q < 100 */
        *((u16 *)buf) = decpair[q];
        buf += 2;
        return buf;
}

static noinline_for_stack
char *put_dec(char *buf, unsigned long long n)
{
        if (n >= 100*1000*1000)
                buf = put_dec_full8(buf, do_div(n, 100*1000*1000));
        /* 1 <= n <= 1.6e11 */
        if (n >= 100*1000*1000)
                buf = put_dec_full8(buf, do_div(n, 100*1000*1000));
        /* 1 <= n < 1e8 */
        return put_dec_trunc8(buf, n);
}

#elif BITS_PER_LONG == 32 && BITS_PER_LONG_LONG == 64

static void
put_dec_full4(char *buf, unsigned r)
{
        unsigned q;

        /* 0 <= r < 10^4 */
        q = (r * 0x147b) >> 19;
        *((u16 *)buf) = decpair[r - 100*q];
        buf += 2;
        /* 0 <= q < 100 */
        *((u16 *)buf) = decpair[q];
}

/*
 * Call put_dec_full4 on x % 10000, return x / 10000.
 * The approximation x/10000 == (x * 0x346DC5D7) >> 43
 * holds for all x < 1,128,869,999.  The largest value this
 * helper will ever be asked to convert is 1,125,520,955.
 * (second call in the put_dec code, assuming n is all-ones).
 */
static noinline_for_stack
unsigned put_dec_helper4(char *buf, unsigned x)
{
        uint32_t q = (x * (uint64_t)0x346DC5D7) >> 43;

        put_dec_full4(buf, x - q * 10000);
        return q;
}

/* Based on code by Douglas W. Jones found at
 * <http://www.cs.uiowa.edu/~jones/bcd/decimal.html#sixtyfour>
 * (with permission from the author).
 * Performs no 64-bit division and hence should be fast on 32-bit machines.
 */
static
char *put_dec(char *buf, unsigned long long n)
{
        uint32_t d3, d2, d1, q, h;

        if (n < 100*1000*1000)
                return put_dec_trunc8(buf, n);

        d1  = ((uint32_t)n >> 16); /* implicit "& 0xffff" */
        h   = (n >> 32);
        d2  = (h      ) & 0xffff;
        d3  = (h >> 16); /* implicit "& 0xffff" */

        /* n = 2^48 d3 + 2^32 d2 + 2^16 d1 + d0
             = 281_4749_7671_0656 d3 + 42_9496_7296 d2 + 6_5536 d1 + d0 */
        q   = 656 * d3 + 7296 * d2 + 5536 * d1 + ((uint32_t)n & 0xffff);
        q = put_dec_helper4(buf, q);

        q += 7671 * d3 + 9496 * d2 + 6 * d1;
        q = put_dec_helper4(buf+4, q);

        q += 4749 * d3 + 42 * d2;
        q = put_dec_helper4(buf+8, q);

        q += 281 * d3;
        buf += 12;
        if (q)
                buf = put_dec_trunc8(buf, q);
        else while (buf[-1] == '0')
                --buf;

        return buf;
}

#endif

/*
 * Convert passed number to decimal string.
 * Returns the length of string.  On buffer overflow, returns 0.
 *
 * If speed is not important, use snprintf(). It's easy to read the code.
 */
int num_to_str(char *buf, int size, unsigned long long num, unsigned int width)
{
        /* put_dec requires 2-byte alignment of the buffer. */
        char tmp[sizeof(num) * 3] __aligned(2);
        int idx, len;

        /* put_dec() may work incorrectly for num = 0 (generate "", not "0") */
        if (num <= 9) {
                tmp[0] = '0' + num;
                len = 1;
        } else {
                len = put_dec(tmp, num) - tmp;
        }

        if (len > size || width > size)
                return 0;

        if (width > len) {
                width = width - len;
                for (idx = 0; idx < width; idx++)
                        buf[idx] = ' ';
        } else {
                width = 0;
        }

        for (idx = 0; idx < len; ++idx)
                buf[idx + width] = tmp[len - idx - 1];

        return len + width;
}

#define SIGN        1                /* unsigned/signed, must be 1 */
#define LEFT        2                /* left justified */
#define PLUS        4                /* show plus */
#define SPACE        8                /* space if plus */
#define ZEROPAD        16                /* pad with zero, must be 16 == '0' - ' ' */
#define SMALL        32                /* use lowercase in hex (must be 32 == 0x20) */
#define SPECIAL        64                /* prefix hex with "0x", octal with "0" */

static_assert(ZEROPAD == ('0' - ' '));
static_assert(SMALL == ' ');

enum format_type {
        FORMAT_TYPE_NONE, /* Just a string part */
        FORMAT_TYPE_WIDTH,
        FORMAT_TYPE_PRECISION,
        FORMAT_TYPE_CHAR,
        FORMAT_TYPE_STR,
        FORMAT_TYPE_PTR,
        FORMAT_TYPE_PERCENT_CHAR,
        FORMAT_TYPE_INVALID,
        FORMAT_TYPE_LONG_LONG,
        FORMAT_TYPE_ULONG,
        FORMAT_TYPE_LONG,
        FORMAT_TYPE_UBYTE,
        FORMAT_TYPE_BYTE,
        FORMAT_TYPE_USHORT,
        FORMAT_TYPE_SHORT,
        FORMAT_TYPE_UINT,
        FORMAT_TYPE_INT,
        FORMAT_TYPE_SIZE_T,
        FORMAT_TYPE_PTRDIFF
};

struct printf_spec {
        unsigned int        type:8;                /* format_type enum */
        signed int        field_width:24;        /* width of output field */
        unsigned int        flags:8;        /* flags to number() */
        unsigned int        base:8;                /* number base, 8, 10 or 16 only */
        signed int        precision:16;        /* # of digits/chars */
} __packed;
static_assert(sizeof(struct printf_spec) == 8);

#define FIELD_WIDTH_MAX ((1 << 23) - 1)
#define PRECISION_MAX ((1 << 15) - 1)

static noinline_for_stack
char *number(char *buf, char *end, unsigned long long num,
             struct printf_spec spec)
{
        /* put_dec requires 2-byte alignment of the buffer. */
        char tmp[3 * sizeof(num)] __aligned(2);
        char sign;
        char locase;
        int need_pfx = ((spec.flags & SPECIAL) && spec.base != 10);
        int i;
        bool is_zero = num == 0LL;
        int field_width = spec.field_width;
        int precision = spec.precision;

        /* locase = 0 or 0x20. ORing digits or letters with 'locase'
         * produces same digits or (maybe lowercased) letters */
        locase = (spec.flags & SMALL);
        if (spec.flags & LEFT)
                spec.flags &= ~ZEROPAD;
        sign = 0;
        if (spec.flags & SIGN) {
                if ((signed long long)num < 0) {
                        sign = '-';
                        num = -(signed long long)num;
                        field_width--;
                } else if (spec.flags & PLUS) {
                        sign = '+';
                        field_width--;
                } else if (spec.flags & SPACE) {
                        sign = ' ';
                        field_width--;
                }
        }
        if (need_pfx) {
                if (spec.base == 16)
                        field_width -= 2;
                else if (!is_zero)
                        field_width--;
        }

        /* generate full string in tmp[], in reverse order */
        i = 0;
        if (num < spec.base)
                tmp[i++] = hex_asc_upper[num] | locase;
        else if (spec.base != 10) { /* 8 or 16 */
                int mask = spec.base - 1;
                int shift = 3;

                if (spec.base == 16)
                        shift = 4;
                do {
                        tmp[i++] = (hex_asc_upper[((unsigned char)num) & mask] | locase);
                        num >>= shift;
                } while (num);
        } else { /* base 10 */
                i = put_dec(tmp, num) - tmp;
        }

        /* printing 100 using %2d gives "100", not "00" */
        if (i > precision)
                precision = i;
        /* leading space padding */
        field_width -= precision;
        if (!(spec.flags & (ZEROPAD | LEFT))) {
                while (--field_width >= 0) {
                        if (buf < end)
                                *buf = ' ';
                        ++buf;
                }
        }
        /* sign */
        if (sign) {
                if (buf < end)
                        *buf = sign;
                ++buf;
        }
        /* "0x" / "0" prefix */
        if (need_pfx) {
                if (spec.base == 16 || !is_zero) {
                        if (buf < end)
                                *buf = '0';
                        ++buf;
                }
                if (spec.base == 16) {
                        if (buf < end)
                                *buf = ('X' | locase);
                        ++buf;
                }
        }
        /* zero or space padding */
        if (!(spec.flags & LEFT)) {
                char c = ' ' + (spec.flags & ZEROPAD);

                while (--field_width >= 0) {
                        if (buf < end)
                                *buf = c;
                        ++buf;
                }
        }
        /* hmm even more zero padding? */
        while (i <= --precision) {
                if (buf < end)
                        *buf = '0';
                ++buf;
        }
        /* actual digits of result */
        while (--i >= 0) {
                if (buf < end)
                        *buf = tmp[i];
                ++buf;
        }
        /* trailing space padding */
        while (--field_width >= 0) {
                if (buf < end)
                        *buf = ' ';
                ++buf;
        }

        return buf;
}

static noinline_for_stack
char *special_hex_number(char *buf, char *end, unsigned long long num, int size)
{
        struct printf_spec spec;

        spec.type = FORMAT_TYPE_PTR;
        spec.field_width = 2 + 2 * size;        /* 0x + hex */
        spec.flags = SPECIAL | SMALL | ZEROPAD;
        spec.base = 16;
        spec.precision = -1;

        return number(buf, end, num, spec);
}

static void move_right(char *buf, char *end, unsigned len, unsigned spaces)
{
        size_t size;
        if (buf >= end)        /* nowhere to put anything */
                return;
        size = end - buf;
        if (size <= spaces) {
                memset(buf, ' ', size);
                return;
        }
        if (len) {
                if (len > size - spaces)
                        len = size - spaces;
                memmove(buf + spaces, buf, len);
        }
        memset(buf, ' ', spaces);
}

/*
 * Handle field width padding for a string.
 * @buf: current buffer position
 * @n: length of string
 * @end: end of output buffer
 * @spec: for field width and flags
 * Returns: new buffer position after padding.
 */
static noinline_for_stack
char *widen_string(char *buf, int n, char *end, struct printf_spec spec)
{
        unsigned spaces;

        if (likely(n >= spec.field_width))
                return buf;
        /* we want to pad the sucker */
        spaces = spec.field_width - n;
        if (!(spec.flags & LEFT)) {
                move_right(buf - n, end, n, spaces);
                return buf + spaces;
        }
        while (spaces--) {
                if (buf < end)
                        *buf = ' ';
                ++buf;
        }
        return buf;
}

/* Handle string from a well known address. */
static char *string_nocheck(char *buf, char *end, const char *s,
                            struct printf_spec spec)
{
        int len = 0;
        int lim = spec.precision;

        while (lim--) {
                char c = *s++;
                if (!c)
                        break;
                if (buf < end)
                        *buf = c;
                ++buf;
                ++len;
        }
        return widen_string(buf, len, end, spec);
}

static char *err_ptr(char *buf, char *end, void *ptr,
                     struct printf_spec spec)
{
        int err = PTR_ERR(ptr);
        const char *sym = errname(err);

        if (sym)
                return string_nocheck(buf, end, sym, spec);

        /*
         * Somebody passed ERR_PTR(-1234) or some other non-existing
         * Efoo - or perhaps CONFIG_SYMBOLIC_ERRNAME=n. Fall back to
         * printing it as its decimal representation.
         */
        spec.flags |= SIGN;
        spec.base = 10;
        return number(buf, end, err, spec);
}

/* Be careful: error messages must fit into the given buffer. */
static char *error_string(char *buf, char *end, const char *s,
                          struct printf_spec spec)
{
        /*
         * Hard limit to avoid a completely insane messages. It actually
         * works pretty well because most error messages are in
         * the many pointer format modifiers.
         */
        if (spec.precision == -1)
                spec.precision = 2 * sizeof(void *);

        return string_nocheck(buf, end, s, spec);
}

/*
 * Do not call any complex external code here. Nested printk()/vsprintf()
 * might cause infinite loops. Failures might break printk() and would
 * be hard to debug.
 */
static const char *check_pointer_msg(const void *ptr)
{
        if (!ptr)
                return "(null)";

        if ((unsigned long)ptr < PAGE_SIZE || IS_ERR_VALUE(ptr))
                return "(efault)";

        return NULL;
}

static int check_pointer(char **buf, char *end, const void *ptr,
                         struct printf_spec spec)
{
        const char *err_msg;

        err_msg = check_pointer_msg(ptr);
        if (err_msg) {
                *buf = error_string(*buf, end, err_msg, spec);
                return -EFAULT;
        }

        return 0;
}

static noinline_for_stack
char *string(char *buf, char *end, const char *s,
             struct printf_spec spec)
{
        if (check_pointer(&buf, end, s, spec))
                return buf;

        return string_nocheck(buf, end, s, spec);
}

static char *pointer_string(char *buf, char *end,
                            const void *ptr,
                            struct printf_spec spec)
{
        spec.base = 16;
        spec.flags |= SMALL;
        if (spec.field_width == -1) {
                spec.field_width = 2 * sizeof(ptr);
                spec.flags |= ZEROPAD;
        }

        return number(buf, end, (unsigned long int)ptr, spec);
}

/* Make pointers available for printing early in the boot sequence. */
static int debug_boot_weak_hash __ro_after_init;

static int __init debug_boot_weak_hash_enable(char *str)
{
        debug_boot_weak_hash = 1;
        pr_info("debug_boot_weak_hash enabled\n");
        return 0;
}
early_param("debug_boot_weak_hash", debug_boot_weak_hash_enable);

static DEFINE_STATIC_KEY_TRUE(not_filled_random_ptr_key);
static siphash_key_t ptr_key __read_mostly;

static void enable_ptr_key_workfn(struct work_struct *work)
{
        get_random_bytes(&ptr_key, sizeof(ptr_key));
        /* Needs to run from preemptible context */
        static_branch_disable(&not_filled_random_ptr_key);
}

static DECLARE_WORK(enable_ptr_key_work, enable_ptr_key_workfn);

static int fill_random_ptr_key(struct notifier_block *nb,
                               unsigned long action, void *data)
{
        /* This may be in an interrupt handler. */
        queue_work(system_unbound_wq, &enable_ptr_key_work);
        return 0;
}

static struct notifier_block random_ready = {
        .notifier_call = fill_random_ptr_key
};

static int __init initialize_ptr_random(void)
{
        int key_size = sizeof(ptr_key);
        int ret;

        /* Use hw RNG if available. */
        if (get_random_bytes_arch(&ptr_key, key_size) == key_size) {
                static_branch_disable(&not_filled_random_ptr_key);
                return 0;
        }

        ret = register_random_ready_notifier(&random_ready);
        if (!ret) {
                return 0;
        } else if (ret == -EALREADY) {
                /* This is in preemptible context */
                enable_ptr_key_workfn(&enable_ptr_key_work);
                return 0;
        }

        return ret;
}
early_initcall(initialize_ptr_random);

/* Maps a pointer to a 32 bit unique identifier. */
static inline int __ptr_to_hashval(const void *ptr, unsigned long *hashval_out)
{
        unsigned long hashval;

        if (static_branch_unlikely(&not_filled_random_ptr_key))
                return -EAGAIN;

#ifdef CONFIG_64BIT
        hashval = (unsigned long)siphash_1u64((u64)ptr, &ptr_key);
        /*
         * Mask off the first 32 bits, this makes explicit that we have
         * modified the address (and 32 bits is plenty for a unique ID).
         */
        hashval = hashval & 0xffffffff;
#else
        hashval = (unsigned long)siphash_1u32((u32)ptr, &ptr_key);
#endif
        *hashval_out = hashval;
        return 0;
}

int ptr_to_hashval(const void *ptr, unsigned long *hashval_out)
{
        return __ptr_to_hashval(ptr, hashval_out);
}

static char *ptr_to_id(char *buf, char *end, const void *ptr,
                       struct printf_spec spec)
{
        const char *str = sizeof(ptr) == 8 ? "(____ptrval____)" : "(ptrval)";
        unsigned long hashval;
        int ret;

        /*
         * Print the real pointer value for NULL and error pointers,
         * as they are not actual addresses.
         */
        if (IS_ERR_OR_NULL(ptr))
                return pointer_string(buf, end, ptr, spec);

        /* When debugging early boot use non-cryptographically secure hash. */
        if (unlikely(debug_boot_weak_hash)) {
                hashval = hash_long((unsigned long)ptr, 32);
                return pointer_string(buf, end, (const void *)hashval, spec);
        }

        ret = __ptr_to_hashval(ptr, &hashval);
        if (ret) {
                spec.field_width = 2 * sizeof(ptr);
                /* string length must be less than default_width */
                return error_string(buf, end, str, spec);
        }

        return pointer_string(buf, end, (const void *)hashval, spec);
}

static char *default_pointer(char *buf, char *end, const void *ptr,
                             struct printf_spec spec)
{
        /*
         * default is to _not_ leak addresses, so hash before printing,
         * unless no_hash_pointers is specified on the command line.
         */
        if (unlikely(no_hash_pointers))
                return pointer_string(buf, end, ptr, spec);

        return ptr_to_id(buf, end, ptr, spec);
}

int kptr_restrict __read_mostly;

static noinline_for_stack
char *restricted_pointer(char *buf, char *end, const void *ptr,
                         struct printf_spec spec)
{
        switch (kptr_restrict) {
        case 0:
                /* Handle as %p, hash and do _not_ leak addresses. */
                return default_pointer(buf, end, ptr, spec);
        case 1: {
                const struct cred *cred;

                /*
                 * kptr_restrict==1 cannot be used in IRQ context
                 * because its test for CAP_SYSLOG would be meaningless.
                 */
                if (in_irq() || in_serving_softirq() || in_nmi()) {
                        if (spec.field_width == -1)
                                spec.field_width = 2 * sizeof(ptr);
                        return error_string(buf, end, "pK-error", spec);
                }

                /*
                 * Only print the real pointer value if the current
                 * process has CAP_SYSLOG and is running with the
                 * same credentials it started with. This is because
                 * access to files is checked at open() time, but %pK
                 * checks permission at read() time. We don't want to
                 * leak pointer values if a binary opens a file using
                 * %pK and then elevates privileges before reading it.
                 */
                cred = current_cred();
                if (!has_capability_noaudit(current, CAP_SYSLOG) ||
                    !uid_eq(cred->euid, cred->uid) ||
                    !gid_eq(cred->egid, cred->gid))
                        ptr = NULL;
                break;
        }
        case 2:
        default:
                /* Always print 0's for %pK */
                ptr = NULL;
                break;
        }

        return pointer_string(buf, end, ptr, spec);
}

static noinline_for_stack
char *dentry_name(char *buf, char *end, const struct dentry *d, struct printf_spec spec,
                  const char *fmt)
{
        const char *array[4], *s;
        const struct dentry *p;
        int depth;
        int i, n;

        switch (fmt[1]) {
                case '2': case '3': case '4':
                        depth = fmt[1] - '0';
                        break;
                default:
                        depth = 1;
        }

        rcu_read_lock();
        for (i = 0; i < depth; i++, d = p) {
                if (check_pointer(&buf, end, d, spec)) {
                        rcu_read_unlock();
                        return buf;
                }

                p = READ_ONCE(d->d_parent);
                array[i] = READ_ONCE(d->d_name.name);
                if (p == d) {
                        if (i)
                                array[i] = "";
                        i++;
                        break;
                }
        }
        s = array[--i];
        for (n = 0; n != spec.precision; n++, buf++) {
                char c = *s++;
                if (!c) {
                        if (!i)
                                break;
                        c = '/';
                        s = array[--i];
                }
                if (buf < end)
                        *buf = c;
        }
        rcu_read_unlock();
        return widen_string(buf, n, end, spec);
}

static noinline_for_stack
char *file_dentry_name(char *buf, char *end, const struct file *f,
                        struct printf_spec spec, const char *fmt)
{
        if (check_pointer(&buf, end, f, spec))
                return buf;

        return dentry_name(buf, end, f->f_path.dentry, spec, fmt);
}
#ifdef CONFIG_BLOCK
static noinline_for_stack
char *bdev_name(char *buf, char *end, struct block_device *bdev,
                struct printf_spec spec, const char *fmt)
{
        struct gendisk *hd;

        if (check_pointer(&buf, end, bdev, spec))
                return buf;

        hd = bdev->bd_disk;
        buf = string(buf, end, hd->disk_name, spec);
        if (bdev->bd_partno) {
                if (isdigit(hd->disk_name[strlen(hd->disk_name)-1])) {
                        if (buf < end)
                                *buf = 'p';
                        buf++;
                }
                buf = number(buf, end, bdev->bd_partno, spec);
        }
        return buf;
}
#endif

static noinline_for_stack
char *symbol_string(char *buf, char *end, void *ptr,
                    struct printf_spec spec, const char *fmt)
{
        unsigned long value;
#ifdef CONFIG_KALLSYMS
        char sym[KSYM_SYMBOL_LEN];
#endif

        if (fmt[1] == 'R')
                ptr = __builtin_extract_return_addr(ptr);
        value = (unsigned long)ptr;

#ifdef CONFIG_KALLSYMS
        if (*fmt == 'B' && fmt[1] == 'b')
                sprint_backtrace_build_id(sym, value);
        else if (*fmt == 'B')
                sprint_backtrace(sym, value);
        else if (*fmt == 'S' && (fmt[1] == 'b' || (fmt[1] == 'R' && fmt[2] == 'b')))
                sprint_symbol_build_id(sym, value);
        else if (*fmt != 's')
                sprint_symbol(sym, value);
        else
                sprint_symbol_no_offset(sym, value);

        return string_nocheck(buf, end, sym, spec);
#else
        return special_hex_number(buf, end, value, sizeof(void *));
#endif
}

static const struct printf_spec default_str_spec = {
        .field_width = -1,
        .precision = -1,
};

static const struct printf_spec default_flag_spec = {
        .base = 16,
        .precision = -1,
        .flags = SPECIAL | SMALL,
};

static const struct printf_spec default_dec_spec = {
        .base = 10,
        .precision = -1,
};

static const struct printf_spec default_dec02_spec = {
        .base = 10,
        .field_width = 2,
        .precision = -1,
        .flags = ZEROPAD,
};

static const struct printf_spec default_dec04_spec = {
        .base = 10,
        .field_width = 4,
        .precision = -1,
        .flags = ZEROPAD,
};

static noinline_for_stack
char *resource_string(char *buf, char *end, struct resource *res,
                      struct printf_spec spec, const char *fmt)
{
#ifndef IO_RSRC_PRINTK_SIZE
#define IO_RSRC_PRINTK_SIZE        6
#endif

#ifndef MEM_RSRC_PRINTK_SIZE
#define MEM_RSRC_PRINTK_SIZE        10
#endif
        static const struct printf_spec io_spec = {
                .base = 16,
                .field_width = IO_RSRC_PRINTK_SIZE,
                .precision = -1,
                .flags = SPECIAL | SMALL | ZEROPAD,
        };
        static const struct printf_spec mem_spec = {
                .base = 16,
                .field_width = MEM_RSRC_PRINTK_SIZE,
                .precision = -1,
                .flags = SPECIAL | SMALL | ZEROPAD,
        };
        static const struct printf_spec bus_spec = {
                .base = 16,
                .field_width = 2,
                .precision = -1,
                .flags = SMALL | ZEROPAD,
        };
        static const struct printf_spec str_spec = {
                .field_width = -1,
                .precision = 10,
                .flags = LEFT,
        };

        /* 32-bit res (sizeof==4): 10 chars in dec, 10 in hex ("0x" + 8)
         * 64-bit res (sizeof==8): 20 chars in dec, 18 in hex ("0x" + 16) */
#define RSRC_BUF_SIZE                ((2 * sizeof(resource_size_t)) + 4)
#define FLAG_BUF_SIZE                (2 * sizeof(res->flags))
#define DECODED_BUF_SIZE        sizeof("[mem - 64bit pref window disabled]")
#define RAW_BUF_SIZE                sizeof("[mem - flags 0x]")
        char sym[max(2*RSRC_BUF_SIZE + DECODED_BUF_SIZE,
                     2*RSRC_BUF_SIZE + FLAG_BUF_SIZE + RAW_BUF_SIZE)];

        char *p = sym, *pend = sym + sizeof(sym);
        int decode = (fmt[0] == 'R') ? 1 : 0;
        const struct printf_spec *specp;

        if (check_pointer(&buf, end, res, spec))
                return buf;

        *p++ = '[';
        if (res->flags & IORESOURCE_IO) {
                p = string_nocheck(p, pend, "io  ", str_spec);
                specp = &io_spec;
        } else if (res->flags & IORESOURCE_MEM) {
                p = string_nocheck(p, pend, "mem ", str_spec);
                specp = &mem_spec;
        } else if (res->flags & IORESOURCE_IRQ) {
                p = string_nocheck(p, pend, "irq ", str_spec);
                specp = &default_dec_spec;
        } else if (res->flags & IORESOURCE_DMA) {
                p = string_nocheck(p, pend, "dma ", str_spec);
                specp = &default_dec_spec;
        } else if (res->flags & IORESOURCE_BUS) {
                p = string_nocheck(p, pend, "bus ", str_spec);
                specp = &bus_spec;
        } else {
                p = string_nocheck(p, pend, "??? ", str_spec);
                specp = &mem_spec;
                decode = 0;
        }
        if (decode && res->flags & IORESOURCE_UNSET) {
                p = string_nocheck(p, pend, "size ", str_spec);
                p = number(p, pend, resource_size(res), *specp);
        } else {
                p = number(p, pend, res->start, *specp);
                if (res->start != res->end) {
                        *p++ = '-';
                        p = number(p, pend, res->end, *specp);
                }
        }
        if (decode) {
                if (res->flags & IORESOURCE_MEM_64)
                        p = string_nocheck(p, pend, " 64bit", str_spec);
                if (res->flags & IORESOURCE_PREFETCH)
                        p = string_nocheck(p, pend, " pref", str_spec);
                if (res->flags & IORESOURCE_WINDOW)
                        p = string_nocheck(p, pend, " window", str_spec);
                if (res->flags & IORESOURCE_DISABLED)
                        p = string_nocheck(p, pend, " disabled", str_spec);
        } else {
                p = string_nocheck(p, pend, " flags ", str_spec);
                p = number(p, pend, res->flags, default_flag_spec);
        }
        *p++ = ']';
        *p = '\0';

        return string_nocheck(buf, end, sym, spec);
}

static noinline_for_stack
char *hex_string(char *buf, char *end, u8 *addr, struct printf_spec spec,
                 const char *fmt)
{
        int i, len = 1;                /* if we pass '%ph[CDN]', field width remains
                                   negative value, fallback to the default */
        char separator;

        if (spec.field_width == 0)
                /* nothing to print */
                return buf;

        if (check_pointer(&buf, end, addr, spec))
                return buf;

        switch (fmt[1]) {
        case 'C':
                separator = ':';
                break;
        case 'D':
                separator = '-';
                break;
        case 'N':
                separator = 0;
                break;
        default:
                separator = ' ';
                break;
        }

        if (spec.field_width > 0)
                len = min_t(int, spec.field_width, 64);

        for (i = 0; i < len; ++i) {
                if (buf < end)
                        *buf = hex_asc_hi(addr[i]);
                ++buf;
                if (buf < end)
                        *buf = hex_asc_lo(addr[i]);
                ++buf;

                if (separator && i != len - 1) {
                        if (buf < end)
                                *buf = separator;
                        ++buf;
                }
        }

        return buf;
}

static noinline_for_stack
char *bitmap_string(char *buf, char *end, unsigned long *bitmap,
                    struct printf_spec spec, const char *fmt)
{
        const int CHUNKSZ = 32;
        int nr_bits = max_t(int, spec.field_width, 0);
        int i, chunksz;
        bool first = true;

        if (check_pointer(&buf, end, bitmap, spec))
                return buf;

        /* reused to print numbers */
        spec = (struct printf_spec){ .flags = SMALL | ZEROPAD, .base = 16 };

        chunksz = nr_bits & (CHUNKSZ - 1);
        if (chunksz == 0)
                chunksz = CHUNKSZ;

        i = ALIGN(nr_bits, CHUNKSZ) - CHUNKSZ;
        for (; i >= 0; i -= CHUNKSZ) {
                u32 chunkmask, val;
                int word, bit;

                chunkmask = ((1ULL << chunksz) - 1);
                word = i / BITS_PER_LONG;
                bit = i % BITS_PER_LONG;
                val = (bitmap[word] >> bit) & chunkmask;

                if (!first) {
                        if (buf < end)
                                *buf = ',';
                        buf++;
                }
                first = false;

                spec.field_width = DIV_ROUND_UP(chunksz, 4);
                buf = number(buf, end, val, spec);

                chunksz = CHUNKSZ;
        }
        return buf;
}

static noinline_for_stack
char *bitmap_list_string(char *buf, char *end, unsigned long *bitmap,
                         struct printf_spec spec, const char *fmt)
{
        int nr_bits = max_t(int, spec.field_width, 0);
        /* current bit is 'cur', most recently seen range is [rbot, rtop] */
        int cur, rbot, rtop;
        bool first = true;

        if (check_pointer(&buf, end, bitmap, spec))
                return buf;

        rbot = cur = find_first_bit(bitmap, nr_bits);
        while (cur < nr_bits) {
                rtop = cur;
                cur = find_next_bit(bitmap, nr_bits, cur + 1);
                if (cur < nr_bits && cur <= rtop + 1)
                        continue;

                if (!first) {
                        if (buf < end)
                                *buf = ',';
                        buf++;
                }
                first = false;

                buf = number(buf, end, rbot, default_dec_spec);
                if (rbot < rtop) {
                        if (buf < end)
                                *buf = '-';
                        buf++;

                        buf = number(buf, end, rtop, default_dec_spec);
                }

                rbot = cur;
        }
        return buf;
}

static noinline_for_stack
char *mac_address_string(char *buf, char *end, u8 *addr,
                         struct printf_spec spec, const char *fmt)
{
        char mac_addr[sizeof("xx:xx:xx:xx:xx:xx")];
        char *p = mac_addr;
        int i;
        char separator;
        bool reversed = false;

        if (check_pointer(&buf, end, addr, spec))
                return buf;

        switch (fmt[1]) {
        case 'F':
                separator = '-';
                break;

        case 'R':
                reversed = true;
                fallthrough;

        default:
                separator = ':';
                break;
        }

        for (i = 0; i < 6; i++) {
                if (reversed)
                        p = hex_byte_pack(p, addr[5 - i]);
                else
                        p = hex_byte_pack(p, addr[i]);

                if (fmt[0] == 'M' && i != 5)
                        *p++ = separator;
        }
        *p = '\0';

        return string_nocheck(buf, end, mac_addr, spec);
}

static noinline_for_stack
char *ip4_string(char *p, const u8 *addr, const char *fmt)
{
        int i;
        bool leading_zeros = (fmt[0] == 'i');
        int index;
        int step;

        switch (fmt[2]) {
        case 'h':
#ifdef __BIG_ENDIAN
                index = 0;
                step = 1;
#else
                index = 3;
                step = -1;
#endif
                break;
        case 'l':
                index = 3;
                step = -1;
                break;
        case 'n':
        case 'b':
        default:
                index = 0;
                step = 1;
                break;
        }
        for (i = 0; i < 4; i++) {
                char temp[4] __aligned(2);        /* hold each IP quad in reverse order */
                int digits = put_dec_trunc8(temp, addr[index]) - temp;
                if (leading_zeros) {
                        if (digits < 3)
                                *p++ = '0';
                        if (digits < 2)
                                *p++ = '0';
                }
                /* reverse the digits in the quad */
                while (digits--)
                        *p++ = temp[digits];
                if (i < 3)
                        *p++ = '.';
                index += step;
        }
        *p = '\0';

        return p;
}

static noinline_for_stack
char *ip6_compressed_string(char *p, const char *addr)
{
        int i, j, range;
        unsigned char zerolength[8];
        int longest = 1;
        int colonpos = -1;
        u16 word;
        u8 hi, lo;
        bool needcolon = false;
        bool useIPv4;
        struct in6_addr in6;

        memcpy(&in6, addr, sizeof(struct in6_addr));

        useIPv4 = ipv6_addr_v4mapped(&in6) || ipv6_addr_is_isatap(&in6);

        memset(zerolength, 0, sizeof(zerolength));

        if (useIPv4)
                range = 6;
        else
                range = 8;

        /* find position of longest 0 run */
        for (i = 0; i < range; i++) {
                for (j = i; j < range; j++) {
                        if (in6.s6_addr16[j] != 0)
                                break;
                        zerolength[i]++;
                }
        }
        for (i = 0; i < range; i++) {
                if (zerolength[i] > longest) {
                        longest = zerolength[i];
                        colonpos = i;
                }
        }
        if (longest == 1)                /* don't compress a single 0 */
                colonpos = -1;

        /* emit address */
        for (i = 0; i < range; i++) {
                if (i == colonpos) {
                        if (needcolon || i == 0)
                                *p++ = ':';
                        *p++ = ':';
                        needcolon = false;
                        i += longest - 1;
                        continue;
                }
                if (needcolon) {
                        *p++ = ':';
                        needcolon = false;
                }
                /* hex u16 without leading 0s */
                word = ntohs(in6.s6_addr16[i]);
                hi = word >> 8;
                lo = word & 0xff;
                if (hi) {
                        if (hi > 0x0f)
                                p = hex_byte_pack(p, hi);
                        else
                                *p++ = hex_asc_lo(hi);
                        p = hex_byte_pack(p, lo);
                }
                else if (lo > 0x0f)
                        p = hex_byte_pack(p, lo);
                else
                        *p++ = hex_asc_lo(lo);
                needcolon = true;
        }

        if (useIPv4) {
                if (needcolon)
                        *p++ = ':';
                p = ip4_string(p, &in6.s6_addr[12], "I4");
        }
        *p = '\0';

        return p;
}

static noinline_for_stack
char *ip6_string(char *p, const char *addr, const char *fmt)
{
        int i;

        for (i = 0; i < 8; i++) {
                p = hex_byte_pack(p, *addr++);
                p = hex_byte_pack(p, *addr++);
                if (fmt[0] == 'I' && i != 7)
                        *p++ = ':';
        }
        *p = '\0';

        return p;
}

static noinline_for_stack
char *ip6_addr_string(char *buf, char *end, const u8 *addr,
                      struct printf_spec spec, const char *fmt)
{
        char ip6_addr[sizeof("xxxx:xxxx:xxxx:xxxx:xxxx:xxxx:255.255.255.255")];

        if (fmt[0] == 'I' && fmt[2] == 'c')
                ip6_compressed_string(ip6_addr, addr);
        else
                ip6_string(ip6_addr, addr, fmt);

        return string_nocheck(buf, end, ip6_addr, spec);
}

static noinline_for_stack
char *ip4_addr_string(char *buf, char *end, const u8 *addr,
                      struct printf_spec spec, const char *fmt)
{
        char ip4_addr[sizeof("255.255.255.255")];

        ip4_string(ip4_addr, addr, fmt);

        return string_nocheck(buf, end, ip4_addr, spec);
}

static noinline_for_stack
char *ip6_addr_string_sa(char *buf, char *end, const struct sockaddr_in6 *sa,
                         struct printf_spec spec, const char *fmt)
{
        bool have_p = false, have_s = false, have_f = false, have_c = false;
        char ip6_addr[sizeof("[xxxx:xxxx:xxxx:xxxx:xxxx:xxxx:255.255.255.255]") +
                      sizeof(":12345") + sizeof("/123456789") +
                      sizeof("%1234567890")];
        char *p = ip6_addr, *pend = ip6_addr + sizeof(ip6_addr);
        const u8 *addr = (const u8 *) &sa->sin6_addr;
        char fmt6[2] = { fmt[0], '6' };
        u8 off = 0;

        fmt++;
        while (isalpha(*++fmt)) {
                switch (*fmt) {
                case 'p':
                        have_p = true;
                        break;
                case 'f':
                        have_f = true;
                        break;
                case 's':
                        have_s = true;
                        break;
                case 'c':
                        have_c = true;
                        break;
                }
        }

        if (have_p || have_s || have_f) {
                *p = '[';
                off = 1;
        }

        if (fmt6[0] == 'I' && have_c)
                p = ip6_compressed_string(ip6_addr + off, addr);
        else
                p = ip6_string(ip6_addr + off, addr, fmt6);

        if (have_p || have_s || have_f)
                *p++ = ']';

        if (have_p) {
                *p++ = ':';
                p = number(p, pend, ntohs(sa->sin6_port), spec);
        }
        if (have_f) {
                *p++ = '/';
                p = number(p, pend, ntohl(sa->sin6_flowinfo &
                                          IPV6_FLOWINFO_MASK), spec);
        }
        if (have_s) {
                *p++ = '%';
                p = number(p, pend, sa->sin6_scope_id, spec);
        }
        *p = '\0';

        return string_nocheck(buf, end, ip6_addr, spec);
}

static noinline_for_stack
char *ip4_addr_string_sa(char *buf, char *end, const struct sockaddr_in *sa,
                         struct printf_spec spec, const char *fmt)
{
        bool have_p = false;
        char *p, ip4_addr[sizeof("255.255.255.255") + sizeof(":12345")];
        char *pend = ip4_addr + sizeof(ip4_addr);
        const u8 *addr = (const u8 *) &sa->sin_addr.s_addr;
        char fmt4[3] = { fmt[0], '4', 0 };

        fmt++;
        while (isalpha(*++fmt)) {
                switch (*fmt) {
                case 'p':
                        have_p = true;
                        break;
                case 'h':
                case 'l':
                case 'n':
                case 'b':
                        fmt4[2] = *fmt;
                        break;
                }
        }

        p = ip4_string(ip4_addr, addr, fmt4);
        if (have_p) {
                *p++ = ':';
                p = number(p, pend, ntohs(sa->sin_port), spec);
        }
        *p = '\0';

        return string_nocheck(buf, end, ip4_addr, spec);
}

static noinline_for_stack
char *ip_addr_string(char *buf, char *end, const void *ptr,
                     struct printf_spec spec, const char *fmt)
{
        char *err_fmt_msg;

        if (check_pointer(&buf, end, ptr, spec))
                return buf;

        switch (fmt[1]) {
        case '6':
                return ip6_addr_string(buf, end, ptr, spec, fmt);
        case '4':
                return ip4_addr_string(buf, end, ptr, spec, fmt);
        case 'S': {
                const union {
                        struct sockaddr                raw;
                        struct sockaddr_in        v4;
                        struct sockaddr_in6        v6;
                } *sa = ptr;

                switch (sa->raw.sa_family) {
                case AF_INET:
                        return ip4_addr_string_sa(buf, end, &sa->v4, spec, fmt);
                case AF_INET6:
                        return ip6_addr_string_sa(buf, end, &sa->v6, spec, fmt);
                default:
                        return error_string(buf, end, "(einval)", spec);
                }}
        }

        err_fmt_msg = fmt[0] == 'i' ? "(%pi?)" : "(%pI?)";
        return error_string(buf, end, err_fmt_msg, spec);
}

static noinline_for_stack
char *escaped_string(char *buf, char *end, u8 *addr, struct printf_spec spec,
                     const char *fmt)
{
        bool found = true;
        int count = 1;
        unsigned int flags = 0;
        int len;

        if (spec.field_width == 0)
                return buf;                                /* nothing to print */

        if (check_pointer(&buf, end, addr, spec))
                return buf;

        do {
                switch (fmt[count++]) {
                case 'a':
                        flags |= ESCAPE_ANY;
                        break;
                case 'c':
                        flags |= ESCAPE_SPECIAL;
                        break;
                case 'h':
                        flags |= ESCAPE_HEX;
                        break;
                case 'n':
                        flags |= ESCAPE_NULL;
                        break;
                case 'o':
                        flags |= ESCAPE_OCTAL;
                        break;
                case 'p':
                        flags |= ESCAPE_NP;
                        break;
                case 's':
                        flags |= ESCAPE_SPACE;
                        break;
                default:
                        found = false;
                        break;
                }
        } while (found);

        if (!flags)
                flags = ESCAPE_ANY_NP;

        len = spec.field_width < 0 ? 1 : spec.field_width;

        /*
         * string_escape_mem() writes as many characters as it can to
         * the given buffer, and returns the total size of the output
         * had the buffer been big enough.
         */
        buf += string_escape_mem(addr, len, buf, buf < end ? end - buf : 0, flags, NULL);

        return buf;
}

static char *va_format(char *buf, char *end, struct va_format *va_fmt,
                       struct printf_spec spec, const char *fmt)
{
        va_list va;

        if (check_pointer(&buf, end, va_fmt, spec))
                return buf;

        va_copy(va, *va_fmt->va);
        buf += vsnprintf(buf, end > buf ? end - buf : 0, va_fmt->fmt, va);
        va_end(va);

        return buf;
}

static noinline_for_stack
char *uuid_string(char *buf, char *end, const u8 *addr,
                  struct printf_spec spec, const char *fmt)
{
        char uuid[UUID_STRING_LEN + 1];
        char *p = uuid;
        int i;
        const u8 *index = uuid_index;
        bool uc = false;

        if (check_pointer(&buf, end, addr, spec))
                return buf;

        switch (*(++fmt)) {
        case 'L':
                uc = true;
                fallthrough;
        case 'l':
                index = guid_index;
                break;
        case 'B':
                uc = true;
                break;
        }

        for (i = 0; i < 16; i++) {
                if (uc)
                        p = hex_byte_pack_upper(p, addr[index[i]]);
                else
                        p = hex_byte_pack(p, addr[index[i]]);
                switch (i) {
                case 3:
                case 5:
                case 7:
                case 9:
                        *p++ = '-';
                        break;
                }
        }

        *p = 0;

        return string_nocheck(buf, end, uuid, spec);
}

static noinline_for_stack
char *netdev_bits(char *buf, char *end, const void *addr,
                  struct printf_spec spec,  const char *fmt)
{
        unsigned long long num;
        int size;

        if (check_pointer(&buf, end, addr, spec))
                return buf;

        switch (fmt[1]) {
        case 'F':
                num = *(const netdev_features_t *)addr;
                size = sizeof(netdev_features_t);
                break;
        default:
                return error_string(buf, end, "(%pN?)", spec);
        }

        return special_hex_number(buf, end, num, size);
}

static noinline_for_stack
char *fourcc_string(char *buf, char *end, const u32 *fourcc,
                    struct printf_spec spec, const char *fmt)
{
        char output[sizeof("0123 little-endian (0x01234567)")];
        char *p = output;
        unsigned int i;
        u32 orig, val;

        if (fmt[1] != 'c' || fmt[2] != 'c')
                return error_string(buf, end, "(%p4?)", spec);

        if (check_pointer(&buf, end, fourcc, spec))
                return buf;

        orig = get_unaligned(fourcc);
        val = orig & ~BIT(31);

        for (i = 0; i < sizeof(u32); i++) {
                unsigned char c = val >> (i * 8);

                /* Print non-control ASCII characters as-is, dot otherwise */
                *p++ = isascii(c) && isprint(c) ? c : '.';
        }

        strcpy(p, orig & BIT(31) ? " big-endian" : " little-endian");
        p += strlen(p);

        *p++ = ' ';
        *p++ = '(';
        p = special_hex_number(p, output + sizeof(output) - 2, orig, sizeof(u32));
        *p++ = ')';
        *p = '\0';

        return string(buf, end, output, spec);
}

static noinline_for_stack
char *address_val(char *buf, char *end, const void *addr,
                  struct printf_spec spec, const char *fmt)
{
        unsigned long long num;
        int size;

        if (check_pointer(&buf, end, addr, spec))
                return buf;

        switch (fmt[1]) {
        case 'd':
                num = *(const dma_addr_t *)addr;
                size = sizeof(dma_addr_t);
                break;
        case 'p':
        default:
                num = *(const phys_addr_t *)addr;
                size = sizeof(phys_addr_t);
                break;
        }

        return special_hex_number(buf, end, num, size);
}

static noinline_for_stack
char *date_str(char *buf, char *end, const struct rtc_time *tm, bool r)
{
        int year = tm->tm_year + (r ? 0 : 1900);
        int mon = tm->tm_mon + (r ? 0 : 1);

        buf = number(buf, end, year, default_dec04_spec);
        if (buf < end)
                *buf = '-';
        buf++;

        buf = number(buf, end, mon, default_dec02_spec);
        if (buf < end)
                *buf = '-';
        buf++;

        return number(buf, end, tm->tm_mday, default_dec02_spec);
}

static noinline_for_stack
char *time_str(char *buf, char *end, const struct rtc_time *tm, bool r)
{
        buf = number(buf, end, tm->tm_hour, default_dec02_spec);
        if (buf < end)
                *buf = ':';
        buf++;

        buf = number(buf, end, tm->tm_min, default_dec02_spec);
        if (buf < end)
                *buf = ':';
        buf++;

        return number(buf, end, tm->tm_sec, default_dec02_spec);
}

static noinline_for_stack
char *rtc_str(char *buf, char *end, const struct rtc_time *tm,
              struct printf_spec spec, const char *fmt)
{
        bool have_t = true, have_d = true;
        bool raw = false, iso8601_separator = true;
        bool found = true;
        int count = 2;

        if (check_pointer(&buf, end, tm, spec))
                return buf;

        switch (fmt[count]) {
        case 'd':
                have_t = false;
                count++;
                break;
        case 't':
                have_d = false;
                count++;
                break;
        }

        do {
                switch (fmt[count++]) {
                case 'r':
                        raw = true;
                        break;
                case 's':
                        iso8601_separator = false;
                        break;
                default:
                        found = false;
                        break;
                }
        } while (found);

        if (have_d)
                buf = date_str(buf, end, tm, raw);
        if (have_d && have_t) {
                if (buf < end)
                        *buf = iso8601_separator ? 'T' : ' ';
                buf++;
        }
        if (have_t)
                buf = time_str(buf, end, tm, raw);

        return buf;
}

static noinline_for_stack
char *time64_str(char *buf, char *end, const time64_t time,
                 struct printf_spec spec, const char *fmt)
{
        struct rtc_time rtc_time;
        struct tm tm;

        time64_to_tm(time, 0, &tm);

        rtc_time.tm_sec = tm.tm_sec;
        rtc_time.tm_min = tm.tm_min;
        rtc_time.tm_hour = tm.tm_hour;
        rtc_time.tm_mday = tm.tm_mday;
        rtc_time.tm_mon = tm.tm_mon;
        rtc_time.tm_year = tm.tm_year;
        rtc_time.tm_wday = tm.tm_wday;
        rtc_time.tm_yday = tm.tm_yday;

        rtc_time.tm_isdst = 0;

        return rtc_str(buf, end, &rtc_time, spec, fmt);
}

static noinline_for_stack
char *time_and_date(char *buf, char *end, void *ptr, struct printf_spec spec,
                    const char *fmt)
{
        switch (fmt[1]) {
        case 'R':
                return rtc_str(buf, end, (const struct rtc_time *)ptr, spec, fmt);
        case 'T':
                return time64_str(buf, end, *(const time64_t *)ptr, spec, fmt);
        default:
                return error_string(buf, end, "(%pt?)", spec);
        }
}

static noinline_for_stack
char *clock(char *buf, char *end, struct clk *clk, struct printf_spec spec,
            const char *fmt)
{
        if (!IS_ENABLED(CONFIG_HAVE_CLK))
                return error_string(buf, end, "(%pC?)", spec);

        if (check_pointer(&buf, end, clk, spec))
                return buf;

        switch (fmt[1]) {
        case 'n':
        default:
#ifdef CONFIG_COMMON_CLK
                return string(buf, end, __clk_get_name(clk), spec);
#else
                return ptr_to_id(buf, end, clk, spec);
#endif
        }
}

static
char *format_flags(char *buf, char *end, unsigned long flags,
                                        const struct trace_print_flags *names)
{
        unsigned long mask;

        for ( ; flags && names->name; names++) {
                mask = names->mask;
                if ((flags & mask) != mask)
                        continue;

                buf = string(buf, end, names->name, default_str_spec);

                flags &= ~mask;
                if (flags) {
                        if (buf < end)
                                *buf = '|';
                        buf++;
                }
        }

        if (flags)
                buf = number(buf, end, flags, default_flag_spec);

        return buf;
}

struct page_flags_fields {
        int width;
        int shift;
        int mask;
        const struct printf_spec *spec;
        const char *name;
};

static const struct page_flags_fields pff[] = {
        {SECTIONS_WIDTH, SECTIONS_PGSHIFT, SECTIONS_MASK,
         &default_dec_spec, "section"},
        {NODES_WIDTH, NODES_PGSHIFT, NODES_MASK,
         &default_dec_spec, "node"},
        {ZONES_WIDTH, ZONES_PGSHIFT, ZONES_MASK,
         &default_dec_spec, "zone"},
        {LAST_CPUPID_WIDTH, LAST_CPUPID_PGSHIFT, LAST_CPUPID_MASK,
         &default_flag_spec, "lastcpupid"},
        {KASAN_TAG_WIDTH, KASAN_TAG_PGSHIFT, KASAN_TAG_MASK,
         &default_flag_spec, "kasantag"},
};

static
char *format_page_flags(char *buf, char *end, unsigned long flags)
{
        unsigned long main_flags = flags & PAGEFLAGS_MASK;
        bool append = false;
        int i;

        /* Page flags from the main area. */
        if (main_flags) {
                buf = format_flags(buf, end, main_flags, pageflag_names);
                append = true;
        }

        /* Page flags from the fields area */
        for (i = 0; i < ARRAY_SIZE(pff); i++) {
                /* Skip undefined fields. */
                if (!pff[i].width)
                        continue;

                /* Format: Flag Name + '=' (equals sign) + Number + '|' (separator) */
                if (append) {
                        if (buf < end)
                                *buf = '|';
                        buf++;
                }

                buf = string(buf, end, pff[i].name, default_str_spec);
                if (buf < end)
                        *buf = '=';
                buf++;
                buf = number(buf, end, (flags >> pff[i].shift) & pff[i].mask,
                             *pff[i].spec);

                append = true;
        }

        return buf;
}

static noinline_for_stack
char *flags_string(char *buf, char *end, void *flags_ptr,
                   struct printf_spec spec, const char *fmt)
{
        unsigned long flags;
        const struct trace_print_flags *names;

        if (check_pointer(&buf, end, flags_ptr, spec))
                return buf;

        switch (fmt[1]) {
        case 'p':
                return format_page_flags(buf, end, *(unsigned long *)flags_ptr);
        case 'v':
                flags = *(unsigned long *)flags_ptr;
                names = vmaflag_names;
                break;
        case 'g':
                flags = (__force unsigned long)(*(gfp_t *)flags_ptr);
                names = gfpflag_names;
                break;
        default:
                return error_string(buf, end, "(%pG?)", spec);
        }

        return format_flags(buf, end, flags, names);
}

static noinline_for_stack
char *fwnode_full_name_string(struct fwnode_handle *fwnode, char *buf,
                              char *end)
{
        int depth;

        /* Loop starting from the root node to the current node. */
        for (depth = fwnode_count_parents(fwnode); depth >= 0; depth--) {
                /*
                 * Only get a reference for other nodes (i.e. parent nodes).
                 * fwnode refcount may be 0 here.
                 */
                struct fwnode_handle *__fwnode = depth ?
                        fwnode_get_nth_parent(fwnode, depth) : fwnode;

                buf = string(buf, end, fwnode_get_name_prefix(__fwnode),
                             default_str_spec);
                buf = string(buf, end, fwnode_get_name(__fwnode),
                             default_str_spec);

                if (depth)
                        fwnode_handle_put(__fwnode);
        }

        return buf;
}

static noinline_for_stack
char *device_node_string(char *buf, char *end, struct device_node *dn,
                         struct printf_spec spec, const char *fmt)
{
        char tbuf[sizeof("xxxx") + 1];
        const char *p;
        int ret;
        char *buf_start = buf;
        struct property *prop;
        bool has_mult, pass;

        struct printf_spec str_spec = spec;
        str_spec.field_width = -1;

        if (fmt[0] != 'F')
                return error_string(buf, end, "(%pO?)", spec);

        if (!IS_ENABLED(CONFIG_OF))
                return error_string(buf, end, "(%pOF?)", spec);

        if (check_pointer(&buf, end, dn, spec))
                return buf;

        /* simple case without anything any more format specifiers */
        fmt++;
        if (fmt[0] == '\0' || strcspn(fmt,"fnpPFcC") > 0)
                fmt = "f";

        for (pass = false; strspn(fmt,"fnpPFcC"); fmt++, pass = true) {
                int precision;
                if (pass) {
                        if (buf < end)
                                *buf = ':';
                        buf++;
                }

                switch (*fmt) {
                case 'f':        /* full_name */
                        buf = fwnode_full_name_string(of_fwnode_handle(dn), buf,
                                                      end);
                        break;
                case 'n':        /* name */
                        p = fwnode_get_name(of_fwnode_handle(dn));
                        precision = str_spec.precision;
                        str_spec.precision = strchrnul(p, '@') - p;
                        buf = string(buf, end, p, str_spec);
                        str_spec.precision = precision;
                        break;
                case 'p':        /* phandle */
                        buf = number(buf, end, (unsigned int)dn->phandle, default_dec_spec);
                        break;
                case 'P':        /* path-spec */
                        p = fwnode_get_name(of_fwnode_handle(dn));
                        if (!p[1])
                                p = "/";
                        buf = string(buf, end, p, str_spec);
                        break;
                case 'F':        /* flags */
                        tbuf[0] = of_node_check_flag(dn, OF_DYNAMIC) ? 'D' : '-';
                        tbuf[1] = of_node_check_flag(dn, OF_DETACHED) ? 'd' : '-';
                        tbuf[2] = of_node_check_flag(dn, OF_POPULATED) ? 'P' : '-';
                        tbuf[3] = of_node_check_flag(dn, OF_POPULATED_BUS) ? 'B' : '-';
                        tbuf[4] = 0;
                        buf = string_nocheck(buf, end, tbuf, str_spec);
                        break;
                case 'c':        /* major compatible string */
                        ret = of_property_read_string(dn, "compatible", &p);
                        if (!ret)
                                buf = string(buf, end, p, str_spec);
                        break;
                case 'C':        /* full compatible string */
                        has_mult = false;
                        of_property_for_each_string(dn, "compatible", prop, p) {
                                if (has_mult)
                                        buf = string_nocheck(buf, end, ",", str_spec);
                                buf = string_nocheck(buf, end, "\"", str_spec);
                                buf = string(buf, end, p, str_spec);
                                buf = string_nocheck(buf, end, "\"", str_spec);

                                has_mult = true;
                        }
                        break;
                default:
                        break;
                }
        }

        return widen_string(buf, buf - buf_start, end, spec);
}

static noinline_for_stack
char *fwnode_string(char *buf, char *end, struct fwnode_handle *fwnode,
                    struct printf_spec spec, const char *fmt)
{
        struct printf_spec str_spec = spec;
        char *buf_start = buf;

        str_spec.field_width = -1;

        if (*fmt != 'w')
                return error_string(buf, end, "(%pf?)", spec);

        if (check_pointer(&buf, end, fwnode, spec))
                return buf;

        fmt++;

        switch (*fmt) {
        case 'P':        /* name */
                buf = string(buf, end, fwnode_get_name(fwnode), str_spec);
                break;
        case 'f':        /* full_name */
        default:
                buf = fwnode_full_name_string(fwnode, buf, end);
                break;
        }

        return widen_string(buf, buf - buf_start, end, spec);
}

int __init no_hash_pointers_enable(char *str)
{
        if (no_hash_pointers)
                return 0;

        no_hash_pointers = true;

        pr_warn("**********************************************************\n");
        pr_warn("**   NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE   **\n");
        pr_warn("**                                                      **\n");
        pr_warn("** This system shows unhashed kernel memory addresses   **\n");
        pr_warn("** via the console, logs, and other interfaces. This    **\n");
        pr_warn("** might reduce the security of your system.            **\n");
        pr_warn("**                                                      **\n");
        pr_warn("** If you see this message and you are not debugging    **\n");
        pr_warn("** the kernel, report this immediately to your system   **\n");
        pr_warn("** administrator!                                       **\n");
        pr_warn("**                                                      **\n");
        pr_warn("**   NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE   **\n");
        pr_warn("**********************************************************\n");

        return 0;
}
early_param("no_hash_pointers", no_hash_pointers_enable);

/*
 * Show a '%p' thing.  A kernel extension is that the '%p' is followed
 * by an extra set of alphanumeric characters that are extended format
 * specifiers.
 *
 * Please update scripts/checkpatch.pl when adding/removing conversion
 * characters.  (Search for "check for vsprintf extension").
 *
 * Right now we handle:
 *
 * - 'S' For symbolic direct pointers (or function descriptors) with offset
 * - 's' For symbolic direct pointers (or function descriptors) without offset
 * - '[Ss]R' as above with __builtin_extract_return_addr() translation
 * - 'S[R]b' as above with module build ID (for use in backtraces)
 * - '[Ff]' %pf and %pF were obsoleted and later removed in favor of
 *            %ps and %pS. Be careful when re-using these specifiers.
 * - 'B' For backtraced symbolic direct pointers with offset
 * - 'Bb' as above with module build ID (for use in backtraces)
 * - 'R' For decoded struct resource, e.g., [mem 0x0-0x1f 64bit pref]
 * - 'r' For raw struct resource, e.g., [mem 0x0-0x1f flags 0x201]
 * - 'b[l]' For a bitmap, the number of bits is determined by the field
 *       width which must be explicitly specified either as part of the
 *       format string '%32b[l]' or through '%*b[l]', [l] selects
 *       range-list format instead of hex format
 * - 'M' For a 6-byte MAC address, it prints the address in the
 *       usual colon-separated hex notation
 * - 'm' For a 6-byte MAC address, it prints the hex address without colons
 * - 'MF' For a 6-byte MAC FDDI address, it prints the address
 *       with a dash-separated hex notation
 * - '[mM]R' For a 6-byte MAC address, Reverse order (Bluetooth)
 * - 'I' [46] for IPv4/IPv6 addresses printed in the usual way
 *       IPv4 uses dot-separated decimal without leading 0's (1.2.3.4)
 *       IPv6 uses colon separated network-order 16 bit hex with leading 0's
 *       [S][pfs]
 *       Generic IPv4/IPv6 address (struct sockaddr *) that falls back to
 *       [4] or [6] and is able to print port [p], flowinfo [f], scope [s]
 * - 'i' [46] for 'raw' IPv4/IPv6 addresses
 *       IPv6 omits the colons (01020304...0f)
 *       IPv4 uses dot-separated decimal with leading 0's (010.123.045.006)
 *       [S][pfs]
 *       Generic IPv4/IPv6 address (struct sockaddr *) that falls back to
 *       [4] or [6] and is able to print port [p], flowinfo [f], scope [s]
 * - '[Ii][4S][hnbl]' IPv4 addresses in host, network, big or little endian order
 * - 'I[6S]c' for IPv6 addresses printed as specified by
 *       https://tools.ietf.org/html/rfc5952
 * - 'E[achnops]' For an escaped buffer, where rules are defined by combination
 *                of the following flags (see string_escape_mem() for the
 *                details):
 *                  a - ESCAPE_ANY
 *                  c - ESCAPE_SPECIAL
 *                  h - ESCAPE_HEX
 *                  n - ESCAPE_NULL
 *                  o - ESCAPE_OCTAL
 *                  p - ESCAPE_NP
 *                  s - ESCAPE_SPACE
 *                By default ESCAPE_ANY_NP is used.
 * - 'U' For a 16 byte UUID/GUID, it prints the UUID/GUID in the form
 *       "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"
 *       Options for %pU are:
 *         b big endian lower case hex (default)
 *         B big endian UPPER case hex
 *         l little endian lower case hex
 *         L little endian UPPER case hex
 *           big endian output byte order is:
 *             [0][1][2][3]-[4][5]-[6][7]-[8][9]-[10][11][12][13][14][15]
 *           little endian output byte order is:
 *             [3][2][1][0]-[5][4]-[7][6]-[8][9]-[10][11][12][13][14][15]
 * - 'V' For a struct va_format which contains a format string * and va_list *,
 *       call vsnprintf(->format, *->va_list).
 *       Implements a "recursive vsnprintf".
 *       Do not use this feature without some mechanism to verify the
 *       correctness of the format string and va_list arguments.
 * - 'K' For a kernel pointer that should be hidden from unprivileged users.
 *       Use only for procfs, sysfs and similar files, not printk(); please
 *       read the documentation (path below) first.
 * - 'NF' For a netdev_features_t
 * - '4cc' V4L2 or DRM FourCC code, with endianness and raw numerical value.
 * - 'h[CDN]' For a variable-length buffer, it prints it as a hex string with
 *            a certain separator (' ' by default):
 *              C colon
 *              D dash
 *              N no separator
 *            The maximum supported length is 64 bytes of the input. Consider
 *            to use print_hex_dump() for the larger input.
 * - 'a[pd]' For address types [p] phys_addr_t, [d] dma_addr_t and derivatives
 *           (default assumed to be phys_addr_t, passed by reference)
 * - 'd[234]' For a dentry name (optionally 2-4 last components)
 * - 'D[234]' Same as 'd' but for a struct file
 * - 'g' For block_device name (gendisk + partition number)
 * - 't[RT][dt][r][s]' For time and date as represented by:
 *      R    struct rtc_time
 *      T    time64_t
 * - 'C' For a clock, it prints the name (Common Clock Framework) or address
 *       (legacy clock framework) of the clock
 * - 'Cn' For a clock, it prints the name (Common Clock Framework) or address
 *        (legacy clock framework) of the clock
 * - 'G' For flags to be printed as a collection of symbolic strings that would
 *       construct the specific value. Supported flags given by option:
 *       p page flags (see struct page) given as pointer to unsigned long
 *       g gfp flags (GFP_* and __GFP_*) given as pointer to gfp_t
 *       v vma flags (VM_*) given as pointer to unsigned long
 * - 'OF[fnpPcCF]'  For a device tree object
 *                  Without any optional arguments prints the full_name
 *                  f device node full_name
 *                  n device node name
 *                  p device node phandle
 *                  P device node path spec (name + @unit)
 *                  F device node flags
 *                  c major compatible string
 *                  C full compatible string
 * - 'fw[fP]'        For a firmware node (struct fwnode_handle) pointer
 *                Without an option prints the full name of the node
 *                f full name
 *                P node name, including a possible unit address
 * - 'x' For printing the address unmodified. Equivalent to "%lx".
 *       Please read the documentation (path below) before using!
 * - '[ku]s' For a BPF/tracing related format specifier, e.g. used out of
 *           bpf_trace_printk() where [ku] prefix specifies either kernel (k)
 *           or user (u) memory to probe, and:
 *              s a string, equivalent to "%s" on direct vsnprintf() use
 *
 * ** When making changes please also update:
 *        Documentation/core-api/printk-formats.rst
 *
 * Note: The default behaviour (unadorned %p) is to hash the address,
 * rendering it useful as a unique identifier.
 */
static noinline_for_stack
char *pointer(const char *fmt, char *buf, char *end, void *ptr,
              struct printf_spec spec)
{
        switch (*fmt) {
        case 'S':
        case 's':
                ptr = dereference_symbol_descriptor(ptr);
                fallthrough;
        case 'B':
                return symbol_string(buf, end, ptr, spec, fmt);
        case 'R':
        case 'r':
                return resource_string(buf, end, ptr, spec, fmt);
        case 'h':
                return hex_string(buf, end, ptr, spec, fmt);
        case 'b':
                switch (fmt[1]) {
                case 'l':
                        return bitmap_list_string(buf, end, ptr, spec, fmt);
                default:
                        return bitmap_string(buf, end, ptr, spec, fmt);
                }
        case 'M':                        /* Colon separated: 00:01:02:03:04:05 */
        case 'm':                        /* Contiguous: 000102030405 */
                                        /* [mM]F (FDDI) */
                                        /* [mM]R (Reverse order; Bluetooth) */
                return mac_address_string(buf, end, ptr, spec, fmt);
        case 'I':                        /* Formatted IP supported
                                         * 4:        1.2.3.4
                                         * 6:        0001:0203:...:0708
                                         * 6c:        1::708 or 1::1.2.3.4
                                         */
        case 'i':                        /* Contiguous:
                                         * 4:        001.002.003.004
                                         * 6:   000102...0f
                                         */
                return ip_addr_string(buf, end, ptr, spec, fmt);
        case 'E':
                return escaped_string(buf, end, ptr, spec, fmt);
        case 'U':
                return uuid_string(buf, end, ptr, spec, fmt);
        case 'V':
                return va_format(buf, end, ptr, spec, fmt);
        case 'K':
                return restricted_pointer(buf, end, ptr, spec);
        case 'N':
                return netdev_bits(buf, end, ptr, spec, fmt);
        case '4':
                return fourcc_string(buf, end, ptr, spec, fmt);
        case 'a':
                return address_val(buf, end, ptr, spec, fmt);
        case 'd':
                return dentry_name(buf, end, ptr, spec, fmt);
        case 't':
                return time_and_date(buf, end, ptr, spec, fmt);
        case 'C':
                return clock(buf, end, ptr, spec, fmt);
        case 'D':
                return file_dentry_name(buf, end, ptr, spec, fmt);
#ifdef CONFIG_BLOCK
        case 'g':
                return bdev_name(buf, end, ptr, spec, fmt);
#endif

        case 'G':
                return flags_string(buf, end, ptr, spec, fmt);
        case 'O':
                return device_node_string(buf, end, ptr, spec, fmt + 1);
        case 'f':
                return fwnode_string(buf, end, ptr, spec, fmt + 1);
        case 'x':
                return pointer_string(buf, end, ptr, spec);
        case 'e':
                /* %pe with a non-ERR_PTR gets treated as plain %p */
                if (!IS_ERR(ptr))
                        return default_pointer(buf, end, ptr, spec);
                return err_ptr(buf, end, ptr, spec);
        case 'u':
        case 'k':
                switch (fmt[1]) {
                case 's':
                        return string(buf, end, ptr, spec);
                default:
                        return error_string(buf, end, "(einval)", spec);
                }
        default:
                return default_pointer(buf, end, ptr, spec);
        }
}

/*
 * Helper function to decode printf style format.
 * Each call decode a token from the format and return the
 * number of characters read (or likely the delta where it wants
 * to go on the next call).
 * The decoded token is returned through the parameters
 *
 * 'h', 'l', or 'L' for integer fields
 * 'z' support added 23/7/1999 S.H.
 * 'z' changed to 'Z' --davidm 1/25/99
 * 'Z' changed to 'z' --adobriyan 2017-01-25
 * 't' added for ptrdiff_t
 *
 * @fmt: the format string
 * @type of the token returned
 * @flags: various flags such as +, -, # tokens..
 * @field_width: overwritten width
 * @base: base of the number (octal, hex, ...)
 * @precision: precision of a number
 * @qualifier: qualifier of a number (long, size_t, ...)
 */
static noinline_for_stack
int format_decode(const char *fmt, struct printf_spec *spec)
{
        const char *start = fmt;
        char qualifier;

        /* we finished early by reading the field width */
        if (spec->type == FORMAT_TYPE_WIDTH) {
                if (spec->field_width < 0) {
                        spec->field_width = -spec->field_width;
                        spec->flags |= LEFT;
                }
                spec->type = FORMAT_TYPE_NONE;
                goto precision;
        }

        /* we finished early by reading the precision */
        if (spec->type == FORMAT_TYPE_PRECISION) {
                if (spec->precision < 0)
                        spec->precision = 0;

                spec->type = FORMAT_TYPE_NONE;
                goto qualifier;
        }

        /* By default */
        spec->type = FORMAT_TYPE_NONE;

        for (; *fmt ; ++fmt) {
                if (*fmt == '%')
                        break;
        }

        /* Return the current non-format string */
        if (fmt != start || !*fmt)
                return fmt - start;

        /* Process flags */
        spec->flags = 0;

        while (1) { /* this also skips first '%' */
                bool found = true;

                ++fmt;

                switch (*fmt) {
                case '-': spec->flags |= LEFT;    break;
                case '+': spec->flags |= PLUS;    break;
                case ' ': spec->flags |= SPACE;   break;
                case '#': spec->flags |= SPECIAL; break;
                case '0': spec->flags |= ZEROPAD; break;
                default:  found = false;
                }

                if (!found)
                        break;
        }

        /* get field width */
        spec->field_width = -1;

        if (isdigit(*fmt))
                spec->field_width = skip_atoi(&fmt);
        else if (*fmt == '*') {
                /* it's the next argument */
                spec->type = FORMAT_TYPE_WIDTH;
                return ++fmt - start;
        }

precision:
        /* get the precision */
        spec->precision = -1;
        if (*fmt == '.') {
                ++fmt;
                if (isdigit(*fmt)) {
                        spec->precision = skip_atoi(&fmt);
                        if (spec->precision < 0)
                                spec->precision = 0;
                } else if (*fmt == '*') {
                        /* it's the next argument */
                        spec->type = FORMAT_TYPE_PRECISION;
                        return ++fmt - start;
                }
        }

qualifier:
        /* get the conversion qualifier */
        qualifier = 0;
        if (*fmt == 'h' || _tolower(*fmt) == 'l' ||
            *fmt == 'z' || *fmt == 't') {
                qualifier = *fmt++;
                if (unlikely(qualifier == *fmt)) {
                        if (qualifier == 'l') {
                                qualifier = 'L';
                                ++fmt;
                        } else if (qualifier == 'h') {
                                qualifier = 'H';
                                ++fmt;
                        }
                }
        }

        /* default base */
        spec->base = 10;
        switch (*fmt) {
        case 'c':
                spec->type = FORMAT_TYPE_CHAR;
                return ++fmt - start;

        case 's':
                spec->type = FORMAT_TYPE_STR;
                return ++fmt - start;

        case 'p':
                spec->type = FORMAT_TYPE_PTR;
                return ++fmt - start;

        case '%':
                spec->type = FORMAT_TYPE_PERCENT_CHAR;
                return ++fmt - start;

        /* integer number formats - set up the flags and "break" */
        case 'o':
                spec->base = 8;
                break;

        case 'x':
                spec->flags |= SMALL;
                fallthrough;

        case 'X':
                spec->base = 16;
                break;

        case 'd':
        case 'i':
                spec->flags |= SIGN;
                break;
        case 'u':
                break;

        case 'n':
                /*
                 * Since %n poses a greater security risk than
                 * utility, treat it as any other invalid or
                 * unsupported format specifier.
                 */
                fallthrough;

        default:
                WARN_ONCE(1, "Please remove unsupported %%%c in format string\n", *fmt);
                spec->type = FORMAT_TYPE_INVALID;
                return fmt - start;
        }

        if (qualifier == 'L')
                spec->type = FORMAT_TYPE_LONG_LONG;
        else if (qualifier == 'l') {
                BUILD_BUG_ON(FORMAT_TYPE_ULONG + SIGN != FORMAT_TYPE_LONG);
                spec->type = FORMAT_TYPE_ULONG + (spec->flags & SIGN);
        } else if (qualifier == 'z') {
                spec->type = FORMAT_TYPE_SIZE_T;
        } else if (qualifier == 't') {
                spec->type = FORMAT_TYPE_PTRDIFF;
        } else if (qualifier == 'H') {
                BUILD_BUG_ON(FORMAT_TYPE_UBYTE + SIGN != FORMAT_TYPE_BYTE);
                spec->type = FORMAT_TYPE_UBYTE + (spec->flags & SIGN);
        } else if (qualifier == 'h') {
                BUILD_BUG_ON(FORMAT_TYPE_USHORT + SIGN != FORMAT_TYPE_SHORT);
                spec->type = FORMAT_TYPE_USHORT + (spec->flags & SIGN);
        } else {
                BUILD_BUG_ON(FORMAT_TYPE_UINT + SIGN != FORMAT_TYPE_INT);
                spec->type = FORMAT_TYPE_UINT + (spec->flags & SIGN);
        }

        return ++fmt - start;
}

static void
set_field_width(struct printf_spec *spec, int width)
{
        spec->field_width = width;
        if (WARN_ONCE(spec->field_width != width, "field width %d too large", width)) {
                spec->field_width = clamp(width, -FIELD_WIDTH_MAX, FIELD_WIDTH_MAX);
        }
}

static void
set_precision(struct printf_spec *spec, int prec)
{
        spec->precision = prec;
        if (WARN_ONCE(spec->precision != prec, "precision %d too large", prec)) {
                spec->precision = clamp(prec, 0, PRECISION_MAX);
        }
}

/**
 * vsnprintf - Format a string and place it in a buffer
 * @buf: The buffer to place the result into
 * @size: The size of the buffer, including the trailing null space
 * @fmt: The format string to use
 * @args: Arguments for the format string
 *
 * This function generally follows C99 vsnprintf, but has some
 * extensions and a few limitations:
 *
 *  - ``%n`` is unsupported
 *  - ``%p*`` is handled by pointer()
 *
 * See pointer() or Documentation/core-api/printk-formats.rst for more
 * extensive description.
 *
 * **Please update the documentation in both places when making changes**
 *
 * The return value is the number of characters which would
 * be generated for the given input, excluding the trailing
 * '\0', as per ISO C99. If you want to have the exact
 * number of characters written into @buf as return value
 * (not including the trailing '\0'), use vscnprintf(). If the
 * return is greater than or equal to @size, the resulting
 * string is truncated.
 *
 * If you're not already dealing with a va_list consider using snprintf().
 */
int vsnprintf(char *buf, size_t size, const char *fmt, va_list args)
{
        unsigned long long num;
        char *str, *end;
        struct printf_spec spec = {0};

        /* Reject out-of-range values early.  Large positive sizes are
           used for unknown buffer sizes. */
        if (WARN_ON_ONCE(size > INT_MAX))
                return 0;

        str = buf;
        end = buf + size;

        /* Make sure end is always >= buf */
        if (end < buf) {
                end = ((void *)-1);
                size = end - buf;
        }

        while (*fmt) {
                const char *old_fmt = fmt;
                int read = format_decode(fmt, &spec);

                fmt += read;

                switch (spec.type) {
                case FORMAT_TYPE_NONE: {
                        int copy = read;
                        if (str < end) {
                                if (copy > end - str)
                                        copy = end - str;
                                memcpy(str, old_fmt, copy);
                        }
                        str += read;
                        break;
                }

                case FORMAT_TYPE_WIDTH:
                        set_field_width(&spec, va_arg(args, int));
                        break;

                case FORMAT_TYPE_PRECISION:
                        set_precision(&spec, va_arg(args, int));
                        break;

                case FORMAT_TYPE_CHAR: {
                        char c;

                        if (!(spec.flags & LEFT)) {
                                while (--spec.field_width > 0) {
                                        if (str < end)
                                                *str = ' ';
                                        ++str;

                                }
                        }
                        c = (unsigned char) va_arg(args, int);
                        if (str < end)
                                *str = c;
                        ++str;
                        while (--spec.field_width > 0) {
                                if (str < end)
                                        *str = ' ';
                                ++str;
                        }
                        break;
                }

                case FORMAT_TYPE_STR:
                        str = string(str, end, va_arg(args, char *), spec);
                        break;

                case FORMAT_TYPE_PTR:
                        str = pointer(fmt, str, end, va_arg(args, void *),
                                      spec);
                        while (isalnum(*fmt))
                                fmt++;
                        break;

                case FORMAT_TYPE_PERCENT_CHAR:
                        if (str < end)
                                *str = '%';
                        ++str;
                        break;

                case FORMAT_TYPE_INVALID:
                        /*
                         * Presumably the arguments passed gcc's type
                         * checking, but there is no safe or sane way
                         * for us to continue parsing the format and
                         * fetching from the va_list; the remaining
                         * specifiers and arguments would be out of
                         * sync.
                         */
                        goto out;

                default:
                        switch (spec.type) {
                        case FORMAT_TYPE_LONG_LONG:
                                num = va_arg(args, long long);
                                break;
                        case FORMAT_TYPE_ULONG:
                                num = va_arg(args, unsigned long);
                                break;
                        case FORMAT_TYPE_LONG:
                                num = va_arg(args, long);
                                break;
                        case FORMAT_TYPE_SIZE_T:
                                if (spec.flags & SIGN)
                                        num = va_arg(args, ssize_t);
                                else
                                        num = va_arg(args, size_t);
                                break;
                        case FORMAT_TYPE_PTRDIFF:
                                num = va_arg(args, ptrdiff_t);
                                break;
                        case FORMAT_TYPE_UBYTE:
                                num = (unsigned char) va_arg(args, int);
                                break;
                        case FORMAT_TYPE_BYTE:
                                num = (signed char) va_arg(args, int);
                                break;
                        case FORMAT_TYPE_USHORT:
                                num = (unsigned short) va_arg(args, int);
                                break;
                        case FORMAT_TYPE_SHORT:
                                num = (short) va_arg(args, int);
                                break;
                        case FORMAT_TYPE_INT:
                                num = (int) va_arg(args, int);
                                break;
                        default:
                                num = va_arg(args, unsigned int);
                        }

                        str = number(str, end, num, spec);
                }
        }

out:
        if (size > 0) {
                if (str < end)
                        *str = '\0';
                else
                        end[-1] = '\0';
        }

        /* the trailing null byte doesn't count towards the total */
        return str-buf;

}
EXPORT_SYMBOL(vsnprintf);

/**
 * vscnprintf - Format a string and place it in a buffer
 * @buf: The buffer to place the result into
 * @size: The size of the buffer, including the trailing null space
 * @fmt: The format string to use
 * @args: Arguments for the format string
 *
 * The return value is the number of characters which have been written into
 * the @buf not including the trailing '\0'. If @size is == 0 the function
 * returns 0.
 *
 * If you're not already dealing with a va_list consider using scnprintf().
 *
 * See the vsnprintf() documentation for format string extensions over C99.
 */
int vscnprintf(char *buf, size_t size, const char *fmt, va_list args)
{
        int i;

        i = vsnprintf(buf, size, fmt, args);

        if (likely(i < size))
                return i;
        if (size != 0)
                return size - 1;
        return 0;
}
EXPORT_SYMBOL(vscnprintf);

/**
 * snprintf - Format a string and place it in a buffer
 * @buf: The buffer to place the result into
 * @size: The size of the buffer, including the trailing null space
 * @fmt: The format string to use
 * @...: Arguments for the format string
 *
 * The return value is the number of characters which would be
 * generated for the given input, excluding the trailing null,
 * as per ISO C99.  If the return is greater than or equal to
 * @size, the resulting string is truncated.
 *
 * See the vsnprintf() documentation for format string extensions over C99.
 */
int snprintf(char *buf, size_t size, const char *fmt, ...)
{
        va_list args;
        int i;

        va_start(args, fmt);
        i = vsnprintf(buf, size, fmt, args);
        va_end(args);

        return i;
}
EXPORT_SYMBOL(snprintf);

/**
 * scnprintf - Format a string and place it in a buffer
 * @buf: The buffer to place the result into
 * @size: The size of the buffer, including the trailing null space
 * @fmt: The format string to use
 * @...: Arguments for the format string
 *
 * The return value is the number of characters written into @buf not including
 * the trailing '\0'. If @size is == 0 the function returns 0.
 */

int scnprintf(char *buf, size_t size, const char *fmt, ...)
{
        va_list args;
        int i;

        va_start(args, fmt);
        i = vscnprintf(buf, size, fmt, args);
        va_end(args);

        return i;
}
EXPORT_SYMBOL(scnprintf);

/**
 * vsprintf - Format a string and place it in a buffer
 * @buf: The buffer to place the result into
 * @fmt: The format string to use
 * @args: Arguments for the format string
 *
 * The function returns the number of characters written
 * into @buf. Use vsnprintf() or vscnprintf() in order to avoid
 * buffer overflows.
 *
 * If you're not already dealing with a va_list consider using sprintf().
 *
 * See the vsnprintf() documentation for format string extensions over C99.
 */
int vsprintf(char *buf, const char *fmt, va_list args)
{
        return vsnprintf(buf, INT_MAX, fmt, args);
}
EXPORT_SYMBOL(vsprintf);

/**
 * sprintf - Format a string and place it in a buffer
 * @buf: The buffer to place the result into
 * @fmt: The format string to use
 * @...: Arguments for the format string
 *
 * The function returns the number of characters written
 * into @buf. Use snprintf() or scnprintf() in order to avoid
 * buffer overflows.
 *
 * See the vsnprintf() documentation for format string extensions over C99.
 */
int sprintf(char *buf, const char *fmt, ...)
{
        va_list args;
        int i;

        va_start(args, fmt);
        i = vsnprintf(buf, INT_MAX, fmt, args);
        va_end(args);

        return i;
}
EXPORT_SYMBOL(sprintf);

#ifdef CONFIG_BINARY_PRINTF
/*
 * bprintf service:
 * vbin_printf() - VA arguments to binary data
 * bstr_printf() - Binary data to text string
 */

/**
 * vbin_printf - Parse a format string and place args' binary value in a buffer
 * @bin_buf: The buffer to place args' binary value
 * @size: The size of the buffer(by words(32bits), not characters)
 * @fmt: The format string to use
 * @args: Arguments for the format string
 *
 * The format follows C99 vsnprintf, except %n is ignored, and its argument
 * is skipped.
 *
 * The return value is the number of words(32bits) which would be generated for
 * the given input.
 *
 * NOTE:
 * If the return value is greater than @size, the resulting bin_buf is NOT
 * valid for bstr_printf().
 */
int vbin_printf(u32 *bin_buf, size_t size, const char *fmt, va_list args)
{
        struct printf_spec spec = {0};
        char *str, *end;
        int width;

        str = (char *)bin_buf;
        end = (char *)(bin_buf + size);

#define save_arg(type)                                                        \
({                                                                        \
        unsigned long long value;                                        \
        if (sizeof(type) == 8) {                                        \
                unsigned long long val8;                                \
                str = PTR_ALIGN(str, sizeof(u32));                        \
                val8 = va_arg(args, unsigned long long);                \
                if (str + sizeof(type) <= end) {                        \
                        *(u32 *)str = *(u32 *)&val8;                        \
                        *(u32 *)(str + 4) = *((u32 *)&val8 + 1);        \
                }                                                        \
                value = val8;                                                \
        } else {                                                        \
                unsigned int val4;                                        \
                str = PTR_ALIGN(str, sizeof(type));                        \
                val4 = va_arg(args, int);                                \
                if (str + sizeof(type) <= end)                                \
                        *(typeof(type) *)str = (type)(long)val4;        \
                value = (unsigned long long)val4;                        \
        }                                                                \
        str += sizeof(type);                                                \
        value;                                                                \
})

        while (*fmt) {
                int read = format_decode(fmt, &spec);

                fmt += read;

                switch (spec.type) {
                case FORMAT_TYPE_NONE:
                case FORMAT_TYPE_PERCENT_CHAR:
                        break;
                case FORMAT_TYPE_INVALID:
                        goto out;

                case FORMAT_TYPE_WIDTH:
                case FORMAT_TYPE_PRECISION:
                        width = (int)save_arg(int);
                        /* Pointers may require the width */
                        if (*fmt == 'p')
                                set_field_width(&spec, width);
                        break;

                case FORMAT_TYPE_CHAR:
                        save_arg(char);
                        break;

                case FORMAT_TYPE_STR: {
                        const char *save_str = va_arg(args, char *);
                        const char *err_msg;
                        size_t len;

                        err_msg = check_pointer_msg(save_str);
                        if (err_msg)
                                save_str = err_msg;

                        len = strlen(save_str) + 1;
                        if (str + len < end)
                                memcpy(str, save_str, len);
                        str += len;
                        break;
                }

                case FORMAT_TYPE_PTR:
                        /* Dereferenced pointers must be done now */
                        switch (*fmt) {
                        /* Dereference of functions is still OK */
                        case 'S':
                        case 's':
                        case 'x':
                        case 'K':
                        case 'e':
                                save_arg(void *);
                                break;
                        default:
                                if (!isalnum(*fmt)) {
                                        save_arg(void *);
                                        break;
                                }
                                str = pointer(fmt, str, end, va_arg(args, void *),
                                              spec);
                                if (str + 1 < end)
                                        *str++ = '\0';
                                else
                                        end[-1] = '\0'; /* Must be nul terminated */
                        }
                        /* skip all alphanumeric pointer suffixes */
                        while (isalnum(*fmt))
                                fmt++;
                        break;

                default:
                        switch (spec.type) {

                        case FORMAT_TYPE_LONG_LONG:
                                save_arg(long long);
                                break;
                        case FORMAT_TYPE_ULONG:
                        case FORMAT_TYPE_LONG:
                                save_arg(unsigned long);
                                break;
                        case FORMAT_TYPE_SIZE_T:
                                save_arg(size_t);
                                break;
                        case FORMAT_TYPE_PTRDIFF:
                                save_arg(ptrdiff_t);
                                break;
                        case FORMAT_TYPE_UBYTE:
                        case FORMAT_TYPE_BYTE:
                                save_arg(char);
                                break;
                        case FORMAT_TYPE_USHORT:
                        case FORMAT_TYPE_SHORT:
                                save_arg(short);
                                break;
                        default:
                                save_arg(int);
                        }
                }
        }

out:
        return (u32 *)(PTR_ALIGN(str, sizeof(u32))) - bin_buf;
#undef save_arg
}
EXPORT_SYMBOL_GPL(vbin_printf);

/**
 * bstr_printf - Format a string from binary arguments and place it in a buffer
 * @buf: The buffer to place the result into
 * @size: The size of the buffer, including the trailing null space
 * @fmt: The format string to use
 * @bin_buf: Binary arguments for the format string
 *
 * This function like C99 vsnprintf, but the difference is that vsnprintf gets
 * arguments from stack, and bstr_printf gets arguments from @bin_buf which is
 * a binary buffer that generated by vbin_printf.
 *
 * The format follows C99 vsnprintf, but has some extensions:
 *  see vsnprintf comment for details.
 *
 * The return value is the number of characters which would
 * be generated for the given input, excluding the trailing
 * '\0', as per ISO C99. If you want to have the exact
 * number of characters written into @buf as return value
 * (not including the trailing '\0'), use vscnprintf(). If the
 * return is greater than or equal to @size, the resulting
 * string is truncated.
 */
int bstr_printf(char *buf, size_t size, const char *fmt, const u32 *bin_buf)
{
        struct printf_spec spec = {0};
        char *str, *end;
        const char *args = (const char *)bin_buf;

        if (WARN_ON_ONCE(size > INT_MAX))
                return 0;

        str = buf;
        end = buf + size;

#define get_arg(type)                                                        \
({                                                                        \
        typeof(type) value;                                                \
        if (sizeof(type) == 8) {                                        \
                args = PTR_ALIGN(args, sizeof(u32));                        \
                *(u32 *)&value = *(u32 *)args;                                \
                *((u32 *)&value + 1) = *(u32 *)(args + 4);                \
        } else {                                                        \
                args = PTR_ALIGN(args, sizeof(type));                        \
                value = *(typeof(type) *)args;                                \
        }                                                                \
        args += sizeof(type);                                                \
        value;                                                                \
})

        /* Make sure end is always >= buf */
        if (end < buf) {
                end = ((void *)-1);
                size = end - buf;
        }

        while (*fmt) {
                const char *old_fmt = fmt;
                int read = format_decode(fmt, &spec);

                fmt += read;

                switch (spec.type) {
                case FORMAT_TYPE_NONE: {
                        int copy = read;
                        if (str < end) {
                                if (copy > end - str)
                                        copy = end - str;
                                memcpy(str, old_fmt, copy);
                        }
                        str += read;
                        break;
                }

                case FORMAT_TYPE_WIDTH:
                        set_field_width(&spec, get_arg(int));
                        break;

                case FORMAT_TYPE_PRECISION:
                        set_precision(&spec, get_arg(int));
                        break;

                case FORMAT_TYPE_CHAR: {
                        char c;

                        if (!(spec.flags & LEFT)) {
                                while (--spec.field_width > 0) {
                                        if (str < end)
                                                *str = ' ';
                                        ++str;
                                }
                        }
                        c = (unsigned char) get_arg(char);
                        if (str < end)
                                *str = c;
                        ++str;
                        while (--spec.field_width > 0) {
                                if (str < end)
                                        *str = ' ';
                                ++str;
                        }
                        break;
                }

                case FORMAT_TYPE_STR: {
                        const char *str_arg = args;
                        args += strlen(str_arg) + 1;
                        str = string(str, end, (char *)str_arg, spec);
                        break;
                }

                case FORMAT_TYPE_PTR: {
                        bool process = false;
                        int copy, len;
                        /* Non function dereferences were already done */
                        switch (*fmt) {
                        case 'S':
                        case 's':
                        case 'x':
                        case 'K':
                        case 'e':
                                process = true;
                                break;
                        default:
                                if (!isalnum(*fmt)) {
                                        process = true;
                                        break;
                                }
                                /* Pointer dereference was already processed */
                                if (str < end) {
                                        len = copy = strlen(args);
                                        if (copy > end - str)
                                                copy = end - str;
                                        memcpy(str, args, copy);
                                        str += len;
                                        args += len + 1;
                                }
                        }
                        if (process)
                                str = pointer(fmt, str, end, get_arg(void *), spec);

                        while (isalnum(*fmt))
                                fmt++;
                        break;
                }

                case FORMAT_TYPE_PERCENT_CHAR:
                        if (str < end)
                                *str = '%';
                        ++str;
                        break;

                case FORMAT_TYPE_INVALID:
                        goto out;

                default: {
                        unsigned long long num;

                        switch (spec.type) {

                        case FORMAT_TYPE_LONG_LONG:
                                num = get_arg(long long);
                                break;
                        case FORMAT_TYPE_ULONG:
                        case FORMAT_TYPE_LONG:
                                num = get_arg(unsigned long);
                                break;
                        case FORMAT_TYPE_SIZE_T:
                                num = get_arg(size_t);
                                break;
                        case FORMAT_TYPE_PTRDIFF:
                                num = get_arg(ptrdiff_t);
                                break;
                        case FORMAT_TYPE_UBYTE:
                                num = get_arg(unsigned char);
                                break;
                        case FORMAT_TYPE_BYTE:
                                num = get_arg(signed char);
                                break;
                        case FORMAT_TYPE_USHORT:
                                num = get_arg(unsigned short);
                                break;
                        case FORMAT_TYPE_SHORT:
                                num = get_arg(short);
                                break;
                        case FORMAT_TYPE_UINT:
                                num = get_arg(unsigned int);
                                break;
                        default:
                                num = get_arg(int);
                        }

                        str = number(str, end, num, spec);
                } /* default: */
                } /* switch(spec.type) */
        } /* while(*fmt) */

out:
        if (size > 0) {
                if (str < end)
                        *str = '\0';
                else
                        end[-1] = '\0';
        }

#undef get_arg

        /* the trailing null byte doesn't count towards the total */
        return str - buf;
}
EXPORT_SYMBOL_GPL(bstr_printf);

/**
 * bprintf - Parse a format string and place args' binary value in a buffer
 * @bin_buf: The buffer to place args' binary value
 * @size: The size of the buffer(by words(32bits), not characters)
 * @fmt: The format string to use
 * @...: Arguments for the format string
 *
 * The function returns the number of words(u32) written
 * into @bin_buf.
 */
int bprintf(u32 *bin_buf, size_t size, const char *fmt, ...)
{
        va_list args;
        int ret;

        va_start(args, fmt);
        ret = vbin_printf(bin_buf, size, fmt, args);
        va_end(args);

        return ret;
}
EXPORT_SYMBOL_GPL(bprintf);

#endif /* CONFIG_BINARY_PRINTF */

/**
 * vsscanf - Unformat a buffer into a list of arguments
 * @buf:        input buffer
 * @fmt:        format of buffer
 * @args:        arguments
 */
int vsscanf(const char *buf, const char *fmt, va_list args)
{
        const char *str = buf;
        char *next;
        char digit;
        int num = 0;
        u8 qualifier;
        unsigned int base;
        union {
                long long s;
                unsigned long long u;
        } val;
        s16 field_width;
        bool is_sign;

        while (*fmt) {
                /* skip any white space in format */
                /* white space in format matches any amount of
                 * white space, including none, in the input.
                 */
                if (isspace(*fmt)) {
                        fmt = skip_spaces(++fmt);
                        str = skip_spaces(str);
                }

                /* anything that is not a conversion must match exactly */
                if (*fmt != '%' && *fmt) {
                        if (*fmt++ != *str++)
                                break;
                        continue;
                }

                if (!*fmt)
                        break;
                ++fmt;

                /* skip this conversion.
                 * advance both strings to next white space
                 */
                if (*fmt == '*') {
                        if (!*str)
                                break;
                        while (!isspace(*fmt) && *fmt != '%' && *fmt) {
                                /* '%*[' not yet supported, invalid format */
                                if (*fmt == '[')
                                        return num;
                                fmt++;
                        }
                        while (!isspace(*str) && *str)
                                str++;
                        continue;
                }

                /* get field width */
                field_width = -1;
                if (isdigit(*fmt)) {
                        field_width = skip_atoi(&fmt);
                        if (field_width <= 0)
                                break;
                }

                /* get conversion qualifier */
                qualifier = -1;
                if (*fmt == 'h' || _tolower(*fmt) == 'l' ||
                    *fmt == 'z') {
                        qualifier = *fmt++;
                        if (unlikely(qualifier == *fmt)) {
                                if (qualifier == 'h') {
                                        qualifier = 'H';
                                        fmt++;
                                } else if (qualifier == 'l') {
                                        qualifier = 'L';
                                        fmt++;
                                }
                        }
                }

                if (!*fmt)
                        break;

                if (*fmt == 'n') {
                        /* return number of characters read so far */
                        *va_arg(args, int *) = str - buf;
                        ++fmt;
                        continue;
                }

                if (!*str)
                        break;

                base = 10;
                is_sign = false;

                switch (*fmt++) {
                case 'c':
                {
                        char *s = (char *)va_arg(args, char*);
                        if (field_width == -1)
                                field_width = 1;
                        do {
                                *s++ = *str++;
                        } while (--field_width > 0 && *str);
                        num++;
                }
                continue;
                case 's':
                {
                        char *s = (char *)va_arg(args, char *);
                        if (field_width == -1)
                                field_width = SHRT_MAX;
                        /* first, skip leading white space in buffer */
                        str = skip_spaces(str);

                        /* now copy until next white space */
                        while (*str && !isspace(*str) && field_width--)
                                *s++ = *str++;
                        *s = '\0';
                        num++;
                }
                continue;
                /*
                 * Warning: This implementation of the '[' conversion specifier
                 * deviates from its glibc counterpart in the following ways:
                 * (1) It does NOT support ranges i.e. '-' is NOT a special
                 *     character
                 * (2) It cannot match the closing bracket ']' itself
                 * (3) A field width is required
                 * (4) '%*[' (discard matching input) is currently not supported
                 *
                 * Example usage:
                 * ret = sscanf("00:0a:95","%2[^:]:%2[^:]:%2[^:]",
                 *                buf1, buf2, buf3);
                 * if (ret < 3)
                 *    // etc..
                 */
                case '[':
                {
                        char *s = (char *)va_arg(args, char *);
                        DECLARE_BITMAP(set, 256) = {0};
                        unsigned int len = 0;
                        bool negate = (*fmt == '^');

                        /* field width is required */
                        if (field_width == -1)
                                return num;

                        if (negate)
                                ++fmt;

                        for ( ; *fmt && *fmt != ']'; ++fmt, ++len)
                                set_bit((u8)*fmt, set);

                        /* no ']' or no character set found */
                        if (!*fmt || !len)
                                return num;
                        ++fmt;

                        if (negate) {
                                bitmap_complement(set, set, 256);
                                /* exclude null '\0' byte */
                                clear_bit(0, set);
                        }

                        /* match must be non-empty */
                        if (!test_bit((u8)*str, set))
                                return num;

                        while (test_bit((u8)*str, set) && field_width--)
                                *s++ = *str++;
                        *s = '\0';
                        ++num;
                }
                continue;
                case 'o':
                        base = 8;
                        break;
                case 'x':
                case 'X':
                        base = 16;
                        break;
                case 'i':
                        base = 0;
                        fallthrough;
                case 'd':
                        is_sign = true;
                        fallthrough;
                case 'u':
                        break;
                case '%':
                        /* looking for '%' in str */
                        if (*str++ != '%')
                                return num;
                        continue;
                default:
                        /* invalid format; stop here */
                        return num;
                }

                /* have some sort of integer conversion.
                 * first, skip white space in buffer.
                 */
                str = skip_spaces(str);

                digit = *str;
                if (is_sign && digit == '-') {
                        if (field_width == 1)
                                break;

                        digit = *(str + 1);
                }

                if (!digit
                    || (base == 16 && !isxdigit(digit))
                    || (base == 10 && !isdigit(digit))
                    || (base == 8 && (!isdigit(digit) || digit > '7'))
                    || (base == 0 && !isdigit(digit)))
                        break;

                if (is_sign)
                        val.s = simple_strntoll(str,
                                                field_width >= 0 ? field_width : INT_MAX,
                                                &next, base);
                else
                        val.u = simple_strntoull(str,
                                                 field_width >= 0 ? field_width : INT_MAX,
                                                 &next, base);

                switch (qualifier) {
                case 'H':        /* that's 'hh' in format */
                        if (is_sign)
                                *va_arg(args, signed char *) = val.s;
                        else
                                *va_arg(args, unsigned char *) = val.u;
                        break;
                case 'h':
                        if (is_sign)
                                *va_arg(args, short *) = val.s;
                        else
                                *va_arg(args, unsigned short *) = val.u;
                        break;
                case 'l':
                        if (is_sign)
                                *va_arg(args, long *) = val.s;
                        else
                                *va_arg(args, unsigned long *) = val.u;
                        break;
                case 'L':
                        if (is_sign)
                                *va_arg(args, long long *) = val.s;
                        else
                                *va_arg(args, unsigned long long *) = val.u;
                        break;
                case 'z':
                        *va_arg(args, size_t *) = val.u;
                        break;
                default:
                        if (is_sign)
                                *va_arg(args, int *) = val.s;
                        else
                                *va_arg(args, unsigned int *) = val.u;
                        break;
                }
                num++;

                if (!next)
                        break;
                str = next;
        }

        return num;
}
EXPORT_SYMBOL(vsscanf);

/**
 * sscanf - Unformat a buffer into a list of arguments
 * @buf:        input buffer
 * @fmt:        formatting of buffer
 * @...:        resulting arguments
 */
int sscanf(const char *buf, const char *fmt, ...)
{
        va_list args;
        int i;

        va_start(args, fmt);
        i = vsscanf(buf, fmt, args);
        va_end(args);

        return i;
}
EXPORT_SYMBOL(sscanf);

























































































































































































































































































 1843 













































































































































































































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
// SPDX-License-Identifier: GPL-2.0
/*
 * Detect hard and soft lockups on a system
 *
 * started by Don Zickus, Copyright (C) 2010 Red Hat, Inc.
 *
 * Note: Most of this code is borrowed heavily from the original softlockup
 * detector, so thanks to Ingo for the initial implementation.
 * Some chunks also taken from the old x86-specific nmi watchdog code, thanks
 * to those contributors as well.
 */

#define pr_fmt(fmt) "watchdog: " fmt

#include <linux/mm.h>
#include <linux/cpu.h>
#include <linux/nmi.h>
#include <linux/init.h>
#include <linux/module.h>
#include <linux/sysctl.h>
#include <linux/tick.h>
#include <linux/sched/clock.h>
#include <linux/sched/debug.h>
#include <linux/sched/isolation.h>
#include <linux/stop_machine.h>

#include <asm/irq_regs.h>
#include <linux/kvm_para.h>

static DEFINE_MUTEX(watchdog_mutex);

#if defined(CONFIG_HARDLOCKUP_DETECTOR) || defined(CONFIG_HAVE_NMI_WATCHDOG)
# define WATCHDOG_DEFAULT        (SOFT_WATCHDOG_ENABLED | NMI_WATCHDOG_ENABLED)
# define NMI_WATCHDOG_DEFAULT        1
#else
# define WATCHDOG_DEFAULT        (SOFT_WATCHDOG_ENABLED)
# define NMI_WATCHDOG_DEFAULT        0
#endif

unsigned long __read_mostly watchdog_enabled;
int __read_mostly watchdog_user_enabled = 1;
int __read_mostly nmi_watchdog_user_enabled = NMI_WATCHDOG_DEFAULT;
int __read_mostly soft_watchdog_user_enabled = 1;
int __read_mostly watchdog_thresh = 10;
static int __read_mostly nmi_watchdog_available;

struct cpumask watchdog_cpumask __read_mostly;
unsigned long *watchdog_cpumask_bits = cpumask_bits(&watchdog_cpumask);

#ifdef CONFIG_HARDLOCKUP_DETECTOR

# ifdef CONFIG_SMP
int __read_mostly sysctl_hardlockup_all_cpu_backtrace;
# endif /* CONFIG_SMP */

/*
 * Should we panic when a soft-lockup or hard-lockup occurs:
 */
unsigned int __read_mostly hardlockup_panic =
                        CONFIG_BOOTPARAM_HARDLOCKUP_PANIC_VALUE;
/*
 * We may not want to enable hard lockup detection by default in all cases,
 * for example when running the kernel as a guest on a hypervisor. In these
 * cases this function can be called to disable hard lockup detection. This
 * function should only be executed once by the boot processor before the
 * kernel command line parameters are parsed, because otherwise it is not
 * possible to override this in hardlockup_panic_setup().
 */
void __init hardlockup_detector_disable(void)
{
        nmi_watchdog_user_enabled = 0;
}

static int __init hardlockup_panic_setup(char *str)
{
        if (!strncmp(str, "panic", 5))
                hardlockup_panic = 1;
        else if (!strncmp(str, "nopanic", 7))
                hardlockup_panic = 0;
        else if (!strncmp(str, "0", 1))
                nmi_watchdog_user_enabled = 0;
        else if (!strncmp(str, "1", 1))
                nmi_watchdog_user_enabled = 1;
        return 1;
}
__setup("nmi_watchdog=", hardlockup_panic_setup);

#endif /* CONFIG_HARDLOCKUP_DETECTOR */

/*
 * These functions can be overridden if an architecture implements its
 * own hardlockup detector.
 *
 * watchdog_nmi_enable/disable can be implemented to start and stop when
 * softlockup watchdog start and stop. The arch must select the
 * SOFTLOCKUP_DETECTOR Kconfig.
 */
int __weak watchdog_nmi_enable(unsigned int cpu)
{
        hardlockup_detector_perf_enable();
        return 0;
}

void __weak watchdog_nmi_disable(unsigned int cpu)
{
        hardlockup_detector_perf_disable();
}

/* Return 0, if a NMI watchdog is available. Error code otherwise */
int __weak __init watchdog_nmi_probe(void)
{
        return hardlockup_detector_perf_init();
}

/**
 * watchdog_nmi_stop - Stop the watchdog for reconfiguration
 *
 * The reconfiguration steps are:
 * watchdog_nmi_stop();
 * update_variables();
 * watchdog_nmi_start();
 */
void __weak watchdog_nmi_stop(void) { }

/**
 * watchdog_nmi_start - Start the watchdog after reconfiguration
 *
 * Counterpart to watchdog_nmi_stop().
 *
 * The following variables have been updated in update_variables() and
 * contain the currently valid configuration:
 * - watchdog_enabled
 * - watchdog_thresh
 * - watchdog_cpumask
 */
void __weak watchdog_nmi_start(void) { }

/**
 * lockup_detector_update_enable - Update the sysctl enable bit
 *
 * Caller needs to make sure that the NMI/perf watchdogs are off, so this
 * can't race with watchdog_nmi_disable().
 */
static void lockup_detector_update_enable(void)
{
        watchdog_enabled = 0;
        if (!watchdog_user_enabled)
                return;
        if (nmi_watchdog_available && nmi_watchdog_user_enabled)
                watchdog_enabled |= NMI_WATCHDOG_ENABLED;
        if (soft_watchdog_user_enabled)
                watchdog_enabled |= SOFT_WATCHDOG_ENABLED;
}

#ifdef CONFIG_SOFTLOCKUP_DETECTOR

/*
 * Delay the soflockup report when running a known slow code.
 * It does _not_ affect the timestamp of the last successdul reschedule.
 */
#define SOFTLOCKUP_DELAY_REPORT        ULONG_MAX

#ifdef CONFIG_SMP
int __read_mostly sysctl_softlockup_all_cpu_backtrace;
#endif

static struct cpumask watchdog_allowed_mask __read_mostly;

/* Global variables, exported for sysctl */
unsigned int __read_mostly softlockup_panic =
                        CONFIG_BOOTPARAM_SOFTLOCKUP_PANIC_VALUE;

static bool softlockup_initialized __read_mostly;
static u64 __read_mostly sample_period;

/* Timestamp taken after the last successful reschedule. */
static DEFINE_PER_CPU(unsigned long, watchdog_touch_ts);
/* Timestamp of the last softlockup report. */
static DEFINE_PER_CPU(unsigned long, watchdog_report_ts);
static DEFINE_PER_CPU(struct hrtimer, watchdog_hrtimer);
static DEFINE_PER_CPU(bool, softlockup_touch_sync);
static DEFINE_PER_CPU(unsigned long, hrtimer_interrupts);
static DEFINE_PER_CPU(unsigned long, hrtimer_interrupts_saved);
static unsigned long soft_lockup_nmi_warn;

static int __init softlockup_panic_setup(char *str)
{
        softlockup_panic = simple_strtoul(str, NULL, 0);
        return 1;
}
__setup("softlockup_panic=", softlockup_panic_setup);

static int __init nowatchdog_setup(char *str)
{
        watchdog_user_enabled = 0;
        return 1;
}
__setup("nowatchdog", nowatchdog_setup);

static int __init nosoftlockup_setup(char *str)
{
        soft_watchdog_user_enabled = 0;
        return 1;
}
__setup("nosoftlockup", nosoftlockup_setup);

static int __init watchdog_thresh_setup(char *str)
{
        get_option(&str, &watchdog_thresh);
        return 1;
}
__setup("watchdog_thresh=", watchdog_thresh_setup);

static void __lockup_detector_cleanup(void);

/*
 * Hard-lockup warnings should be triggered after just a few seconds. Soft-
 * lockups can have false positives under extreme conditions. So we generally
 * want a higher threshold for soft lockups than for hard lockups. So we couple
 * the thresholds with a factor: we make the soft threshold twice the amount of
 * time the hard threshold is.
 */
static int get_softlockup_thresh(void)
{
        return watchdog_thresh * 2;
}

/*
 * Returns seconds, approximately.  We don't need nanosecond
 * resolution, and we don't need to waste time with a big divide when
 * 2^30ns == 1.074s.
 */
static unsigned long get_timestamp(void)
{
        return running_clock() >> 30LL;  /* 2^30 ~= 10^9 */
}

static void set_sample_period(void)
{
        /*
         * convert watchdog_thresh from seconds to ns
         * the divide by 5 is to give hrtimer several chances (two
         * or three with the current relation between the soft
         * and hard thresholds) to increment before the
         * hardlockup detector generates a warning
         */
        sample_period = get_softlockup_thresh() * ((u64)NSEC_PER_SEC / 5);
        watchdog_update_hrtimer_threshold(sample_period);
}

static void update_report_ts(void)
{
        __this_cpu_write(watchdog_report_ts, get_timestamp());
}

/* Commands for resetting the watchdog */
static void update_touch_ts(void)
{
        __this_cpu_write(watchdog_touch_ts, get_timestamp());
        update_report_ts();
}

/**
 * touch_softlockup_watchdog_sched - touch watchdog on scheduler stalls
 *
 * Call when the scheduler may have stalled for legitimate reasons
 * preventing the watchdog task from executing - e.g. the scheduler
 * entering idle state.  This should only be used for scheduler events.
 * Use touch_softlockup_watchdog() for everything else.
 */
notrace void touch_softlockup_watchdog_sched(void)
{
        /*
         * Preemption can be enabled.  It doesn't matter which CPU's watchdog
         * report period gets restarted here, so use the raw_ operation.
         */
        raw_cpu_write(watchdog_report_ts, SOFTLOCKUP_DELAY_REPORT);
}

notrace void touch_softlockup_watchdog(void)
{
        touch_softlockup_watchdog_sched();
        wq_watchdog_touch(raw_smp_processor_id());
}
EXPORT_SYMBOL(touch_softlockup_watchdog);

void touch_all_softlockup_watchdogs(void)
{
        int cpu;

        /*
         * watchdog_mutex cannpt be taken here, as this might be called
         * from (soft)interrupt context, so the access to
         * watchdog_allowed_cpumask might race with a concurrent update.
         *
         * The watchdog time stamp can race against a concurrent real
         * update as well, the only side effect might be a cycle delay for
         * the softlockup check.
         */
        for_each_cpu(cpu, &watchdog_allowed_mask) {
                per_cpu(watchdog_report_ts, cpu) = SOFTLOCKUP_DELAY_REPORT;
                wq_watchdog_touch(cpu);
        }
}

void touch_softlockup_watchdog_sync(void)
{
        __this_cpu_write(softlockup_touch_sync, true);
        __this_cpu_write(watchdog_report_ts, SOFTLOCKUP_DELAY_REPORT);
}

static int is_softlockup(unsigned long touch_ts,
                         unsigned long period_ts,
                         unsigned long now)
{
        if ((watchdog_enabled & SOFT_WATCHDOG_ENABLED) && watchdog_thresh){
                /* Warn about unreasonable delays. */
                if (time_after(now, period_ts + get_softlockup_thresh()))
                        return now - touch_ts;
        }
        return 0;
}

/* watchdog detector functions */
bool is_hardlockup(void)
{
        unsigned long hrint = __this_cpu_read(hrtimer_interrupts);

        if (__this_cpu_read(hrtimer_interrupts_saved) == hrint)
                return true;

        __this_cpu_write(hrtimer_interrupts_saved, hrint);
        return false;
}

static void watchdog_interrupt_count(void)
{
        __this_cpu_inc(hrtimer_interrupts);
}

static DEFINE_PER_CPU(struct completion, softlockup_completion);
static DEFINE_PER_CPU(struct cpu_stop_work, softlockup_stop_work);

/*
 * The watchdog feed function - touches the timestamp.
 *
 * It only runs once every sample_period seconds (4 seconds by
 * default) to reset the softlockup timestamp. If this gets delayed
 * for more than 2*watchdog_thresh seconds then the debug-printout
 * triggers in watchdog_timer_fn().
 */
static int softlockup_fn(void *data)
{
        update_touch_ts();
        complete(this_cpu_ptr(&softlockup_completion));

        return 0;
}

/* watchdog kicker functions */
static enum hrtimer_restart watchdog_timer_fn(struct hrtimer *hrtimer)
{
        unsigned long touch_ts, period_ts, now;
        struct pt_regs *regs = get_irq_regs();
        int duration;
        int softlockup_all_cpu_backtrace = sysctl_softlockup_all_cpu_backtrace;

        if (!watchdog_enabled)
                return HRTIMER_NORESTART;

        /* kick the hardlockup detector */
        watchdog_interrupt_count();

        /* kick the softlockup detector */
        if (completion_done(this_cpu_ptr(&softlockup_completion))) {
                reinit_completion(this_cpu_ptr(&softlockup_completion));
                stop_one_cpu_nowait(smp_processor_id(),
                                softlockup_fn, NULL,
                                this_cpu_ptr(&softlockup_stop_work));
        }

        /* .. and repeat */
        hrtimer_forward_now(hrtimer, ns_to_ktime(sample_period));

        /*
         * Read the current timestamp first. It might become invalid anytime
         * when a virtual machine is stopped by the host or when the watchog
         * is touched from NMI.
         */
        now = get_timestamp();
        /*
         * If a virtual machine is stopped by the host it can look to
         * the watchdog like a soft lockup. This function touches the watchdog.
         */
        kvm_check_and_clear_guest_paused();
        /*
         * The stored timestamp is comparable with @now only when not touched.
         * It might get touched anytime from NMI. Make sure that is_softlockup()
         * uses the same (valid) value.
         */
        period_ts = READ_ONCE(*this_cpu_ptr(&watchdog_report_ts));

        /* Reset the interval when touched by known problematic code. */
        if (period_ts == SOFTLOCKUP_DELAY_REPORT) {
                if (unlikely(__this_cpu_read(softlockup_touch_sync))) {
                        /*
                         * If the time stamp was touched atomically
                         * make sure the scheduler tick is up to date.
                         */
                        __this_cpu_write(softlockup_touch_sync, false);
                        sched_clock_tick();
                }

                update_report_ts();
                return HRTIMER_RESTART;
        }

        /* Check for a softlockup. */
        touch_ts = __this_cpu_read(watchdog_touch_ts);
        duration = is_softlockup(touch_ts, period_ts, now);
        if (unlikely(duration)) {
                /*
                 * Prevent multiple soft-lockup reports if one cpu is already
                 * engaged in dumping all cpu back traces.
                 */
                if (softlockup_all_cpu_backtrace) {
                        if (test_and_set_bit_lock(0, &soft_lockup_nmi_warn))
                                return HRTIMER_RESTART;
                }

                /* Start period for the next softlockup warning. */
                update_report_ts();

                pr_emerg("BUG: soft lockup - CPU#%d stuck for %us! [%s:%d]\n",
                        smp_processor_id(), duration,
                        current->comm, task_pid_nr(current));
                print_modules();
                print_irqtrace_events(current);
                if (regs)
                        show_regs(regs);
                else
                        dump_stack();

                if (softlockup_all_cpu_backtrace) {
                        trigger_allbutself_cpu_backtrace();
                        clear_bit_unlock(0, &soft_lockup_nmi_warn);
                }

                add_taint(TAINT_SOFTLOCKUP, LOCKDEP_STILL_OK);
                if (softlockup_panic)
                        panic("softlockup: hung tasks");
        }

        return HRTIMER_RESTART;
}

static void watchdog_enable(unsigned int cpu)
{
        struct hrtimer *hrtimer = this_cpu_ptr(&watchdog_hrtimer);
        struct completion *done = this_cpu_ptr(&softlockup_completion);

        WARN_ON_ONCE(cpu != smp_processor_id());

        init_completion(done);
        complete(done);

        /*
         * Start the timer first to prevent the NMI watchdog triggering
         * before the timer has a chance to fire.
         */
        hrtimer_init(hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_HARD);
        hrtimer->function = watchdog_timer_fn;
        hrtimer_start(hrtimer, ns_to_ktime(sample_period),
                      HRTIMER_MODE_REL_PINNED_HARD);

        /* Initialize timestamp */
        update_touch_ts();
        /* Enable the perf event */
        if (watchdog_enabled & NMI_WATCHDOG_ENABLED)
                watchdog_nmi_enable(cpu);
}

static void watchdog_disable(unsigned int cpu)
{
        struct hrtimer *hrtimer = this_cpu_ptr(&watchdog_hrtimer);

        WARN_ON_ONCE(cpu != smp_processor_id());

        /*
         * Disable the perf event first. That prevents that a large delay
         * between disabling the timer and disabling the perf event causes
         * the perf NMI to detect a false positive.
         */
        watchdog_nmi_disable(cpu);
        hrtimer_cancel(hrtimer);
        wait_for_completion(this_cpu_ptr(&softlockup_completion));
}

static int softlockup_stop_fn(void *data)
{
        watchdog_disable(smp_processor_id());
        return 0;
}

static void softlockup_stop_all(void)
{
        int cpu;

        if (!softlockup_initialized)
                return;

        for_each_cpu(cpu, &watchdog_allowed_mask)
                smp_call_on_cpu(cpu, softlockup_stop_fn, NULL, false);

        cpumask_clear(&watchdog_allowed_mask);
}

static int softlockup_start_fn(void *data)
{
        watchdog_enable(smp_processor_id());
        return 0;
}

static void softlockup_start_all(void)
{
        int cpu;

        cpumask_copy(&watchdog_allowed_mask, &watchdog_cpumask);
        for_each_cpu(cpu, &watchdog_allowed_mask)
                smp_call_on_cpu(cpu, softlockup_start_fn, NULL, false);
}

int lockup_detector_online_cpu(unsigned int cpu)
{
        if (cpumask_test_cpu(cpu, &watchdog_allowed_mask))
                watchdog_enable(cpu);
        return 0;
}

int lockup_detector_offline_cpu(unsigned int cpu)
{
        if (cpumask_test_cpu(cpu, &watchdog_allowed_mask))
                watchdog_disable(cpu);
        return 0;
}

static void __lockup_detector_reconfigure(void)
{
        cpus_read_lock();
        watchdog_nmi_stop();

        softlockup_stop_all();
        set_sample_period();
        lockup_detector_update_enable();
        if (watchdog_enabled && watchdog_thresh)
                softlockup_start_all();

        watchdog_nmi_start();
        cpus_read_unlock();
        /*
         * Must be called outside the cpus locked section to prevent
         * recursive locking in the perf code.
         */
        __lockup_detector_cleanup();
}

void lockup_detector_reconfigure(void)
{
        mutex_lock(&watchdog_mutex);
        __lockup_detector_reconfigure();
        mutex_unlock(&watchdog_mutex);
}

/*
 * Create the watchdog infrastructure and configure the detector(s).
 */
static __init void lockup_detector_setup(void)
{
        /*
         * If sysctl is off and watchdog got disabled on the command line,
         * nothing to do here.
         */
        lockup_detector_update_enable();

        if (!IS_ENABLED(CONFIG_SYSCTL) &&
            !(watchdog_enabled && watchdog_thresh))
                return;

        mutex_lock(&watchdog_mutex);
        __lockup_detector_reconfigure();
        softlockup_initialized = true;
        mutex_unlock(&watchdog_mutex);
}

#else /* CONFIG_SOFTLOCKUP_DETECTOR */
static void __lockup_detector_reconfigure(void)
{
        cpus_read_lock();
        watchdog_nmi_stop();
        lockup_detector_update_enable();
        watchdog_nmi_start();
        cpus_read_unlock();
}
void lockup_detector_reconfigure(void)
{
        __lockup_detector_reconfigure();
}
static inline void lockup_detector_setup(void)
{
        __lockup_detector_reconfigure();
}
#endif /* !CONFIG_SOFTLOCKUP_DETECTOR */

static void __lockup_detector_cleanup(void)
{
        lockdep_assert_held(&watchdog_mutex);
        hardlockup_detector_perf_cleanup();
}

/**
 * lockup_detector_cleanup - Cleanup after cpu hotplug or sysctl changes
 *
 * Caller must not hold the cpu hotplug rwsem.
 */
void lockup_detector_cleanup(void)
{
        mutex_lock(&watchdog_mutex);
        __lockup_detector_cleanup();
        mutex_unlock(&watchdog_mutex);
}

/**
 * lockup_detector_soft_poweroff - Interface to stop lockup detector(s)
 *
 * Special interface for parisc. It prevents lockup detector warnings from
 * the default pm_poweroff() function which busy loops forever.
 */
void lockup_detector_soft_poweroff(void)
{
        watchdog_enabled = 0;
}

#ifdef CONFIG_SYSCTL

/* Propagate any changes to the watchdog infrastructure */
static void proc_watchdog_update(void)
{
        /* Remove impossible cpus to keep sysctl output clean. */
        cpumask_and(&watchdog_cpumask, &watchdog_cpumask, cpu_possible_mask);
        __lockup_detector_reconfigure();
}

/*
 * common function for watchdog, nmi_watchdog and soft_watchdog parameter
 *
 * caller             | table->data points to      | 'which'
 * -------------------|----------------------------|--------------------------
 * proc_watchdog      | watchdog_user_enabled      | NMI_WATCHDOG_ENABLED |
 *                    |                            | SOFT_WATCHDOG_ENABLED
 * -------------------|----------------------------|--------------------------
 * proc_nmi_watchdog  | nmi_watchdog_user_enabled  | NMI_WATCHDOG_ENABLED
 * -------------------|----------------------------|--------------------------
 * proc_soft_watchdog | soft_watchdog_user_enabled | SOFT_WATCHDOG_ENABLED
 */
static int proc_watchdog_common(int which, struct ctl_table *table, int write,
                                void *buffer, size_t *lenp, loff_t *ppos)
{
        int err, old, *param = table->data;

        mutex_lock(&watchdog_mutex);

        if (!write) {
                /*
                 * On read synchronize the userspace interface. This is a
                 * racy snapshot.
                 */
                *param = (watchdog_enabled & which) != 0;
                err = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
        } else {
                old = READ_ONCE(*param);
                err = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
                if (!err && old != READ_ONCE(*param))
                        proc_watchdog_update();
        }
        mutex_unlock(&watchdog_mutex);
        return err;
}

/*
 * /proc/sys/kernel/watchdog
 */
int proc_watchdog(struct ctl_table *table, int write,
                  void *buffer, size_t *lenp, loff_t *ppos)
{
        return proc_watchdog_common(NMI_WATCHDOG_ENABLED|SOFT_WATCHDOG_ENABLED,
                                    table, write, buffer, lenp, ppos);
}

/*
 * /proc/sys/kernel/nmi_watchdog
 */
int proc_nmi_watchdog(struct ctl_table *table, int write,
                      void *buffer, size_t *lenp, loff_t *ppos)
{
        if (!nmi_watchdog_available && write)
                return -ENOTSUPP;
        return proc_watchdog_common(NMI_WATCHDOG_ENABLED,
                                    table, write, buffer, lenp, ppos);
}

/*
 * /proc/sys/kernel/soft_watchdog
 */
int proc_soft_watchdog(struct ctl_table *table, int write,
                        void *buffer, size_t *lenp, loff_t *ppos)
{
        return proc_watchdog_common(SOFT_WATCHDOG_ENABLED,
                                    table, write, buffer, lenp, ppos);
}

/*
 * /proc/sys/kernel/watchdog_thresh
 */
int proc_watchdog_thresh(struct ctl_table *table, int write,
                         void *buffer, size_t *lenp, loff_t *ppos)
{
        int err, old;

        mutex_lock(&watchdog_mutex);

        old = READ_ONCE(watchdog_thresh);
        err = proc_dointvec_minmax(table, write, buffer, lenp, ppos);

        if (!err && write && old != READ_ONCE(watchdog_thresh))
                proc_watchdog_update();

        mutex_unlock(&watchdog_mutex);
        return err;
}

/*
 * The cpumask is the mask of possible cpus that the watchdog can run
 * on, not the mask of cpus it is actually running on.  This allows the
 * user to specify a mask that will include cpus that have not yet
 * been brought online, if desired.
 */
int proc_watchdog_cpumask(struct ctl_table *table, int write,
                          void *buffer, size_t *lenp, loff_t *ppos)
{
        int err;

        mutex_lock(&watchdog_mutex);

        err = proc_do_large_bitmap(table, write, buffer, lenp, ppos);
        if (!err && write)
                proc_watchdog_update();

        mutex_unlock(&watchdog_mutex);
        return err;
}
#endif /* CONFIG_SYSCTL */

void __init lockup_detector_init(void)
{
        if (tick_nohz_full_enabled())
                pr_info("Disabling watchdog on nohz_full cores by default\n");

        cpumask_copy(&watchdog_cpumask,
                     housekeeping_cpumask(HK_FLAG_TIMER));

        if (!watchdog_nmi_probe())
                nmi_watchdog_available = true;
        lockup_detector_setup();
}






















































   82 

















   82 

















   22 



    3 
















   58 
   16 

    3 
   27 







   20 
   63 











   78 


   18 
   75 




   22 

   57 


















   78 








   59 


   60 


   17 

   20 









   82 
   82 







   22 




   60 
   60 


   57 

   21 



   77 




   38 






   35 
















































   81 










    4 






   78 




   78 


    1 






   78 





































































































   13 


   68 
    1 





    4 







   79 




    5 


   76 








































   12 



   12 












    2 
    4 



















































































































   24 




  112 


   33 





    1 



   16 


   65 





   34 
   33 


   26 
    8 







   15 




    1 




    1 



















































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
// SPDX-License-Identifier: GPL-2.0+
/*
 * Copyright (C) 2007 Alan Stern
 * Copyright (C) IBM Corporation, 2009
 * Copyright (C) 2009, Frederic Weisbecker <fweisbec@gmail.com>
 *
 * Thanks to Ingo Molnar for his many suggestions.
 *
 * Authors: Alan Stern <stern@rowland.harvard.edu>
 *          K.Prasad <prasad@linux.vnet.ibm.com>
 *          Frederic Weisbecker <fweisbec@gmail.com>
 */

/*
 * HW_breakpoint: a unified kernel/user-space hardware breakpoint facility,
 * using the CPU's debug registers.
 * This file contains the arch-independent routines.
 */

#include <linux/irqflags.h>
#include <linux/kallsyms.h>
#include <linux/notifier.h>
#include <linux/kprobes.h>
#include <linux/kdebug.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/percpu.h>
#include <linux/sched.h>
#include <linux/init.h>
#include <linux/slab.h>
#include <linux/list.h>
#include <linux/cpu.h>
#include <linux/smp.h>
#include <linux/bug.h>

#include <linux/hw_breakpoint.h>
/*
 * Constraints data
 */
struct bp_cpuinfo {
        /* Number of pinned cpu breakpoints in a cpu */
        unsigned int        cpu_pinned;
        /* tsk_pinned[n] is the number of tasks having n+1 breakpoints */
        unsigned int        *tsk_pinned;
        /* Number of non-pinned cpu/task breakpoints in a cpu */
        unsigned int        flexible; /* XXX: placeholder, see fetch_this_slot() */
};

static DEFINE_PER_CPU(struct bp_cpuinfo, bp_cpuinfo[TYPE_MAX]);
static int nr_slots[TYPE_MAX];

static struct bp_cpuinfo *get_bp_info(int cpu, enum bp_type_idx type)
{
        return per_cpu_ptr(bp_cpuinfo + type, cpu);
}

/* Keep track of the breakpoints attached to tasks */
static LIST_HEAD(bp_task_head);

static int constraints_initialized;

/* Gather the number of total pinned and un-pinned bp in a cpuset */
struct bp_busy_slots {
        unsigned int pinned;
        unsigned int flexible;
};

/* Serialize accesses to the above constraints */
static DEFINE_MUTEX(nr_bp_mutex);

__weak int hw_breakpoint_weight(struct perf_event *bp)
{
        return 1;
}

static inline enum bp_type_idx find_slot_idx(u64 bp_type)
{
        if (bp_type & HW_BREAKPOINT_RW)
                return TYPE_DATA;

        return TYPE_INST;
}

/*
 * Report the maximum number of pinned breakpoints a task
 * have in this cpu
 */
static unsigned int max_task_bp_pinned(int cpu, enum bp_type_idx type)
{
        unsigned int *tsk_pinned = get_bp_info(cpu, type)->tsk_pinned;
        int i;

        for (i = nr_slots[type] - 1; i >= 0; i--) {
                if (tsk_pinned[i] > 0)
                        return i + 1;
        }

        return 0;
}

/*
 * Count the number of breakpoints of the same type and same task.
 * The given event must be not on the list.
 */
static int task_bp_pinned(int cpu, struct perf_event *bp, enum bp_type_idx type)
{
        struct task_struct *tsk = bp->hw.target;
        struct perf_event *iter;
        int count = 0;

        list_for_each_entry(iter, &bp_task_head, hw.bp_list) {
                if (iter->hw.target == tsk &&
                    find_slot_idx(iter->attr.bp_type) == type &&
                    (iter->cpu < 0 || cpu == iter->cpu))
                        count += hw_breakpoint_weight(iter);
        }

        return count;
}

static const struct cpumask *cpumask_of_bp(struct perf_event *bp)
{
        if (bp->cpu >= 0)
                return cpumask_of(bp->cpu);
        return cpu_possible_mask;
}

/*
 * Report the number of pinned/un-pinned breakpoints we have in
 * a given cpu (cpu > -1) or in all of them (cpu = -1).
 */
static void
fetch_bp_busy_slots(struct bp_busy_slots *slots, struct perf_event *bp,
                    enum bp_type_idx type)
{
        const struct cpumask *cpumask = cpumask_of_bp(bp);
        int cpu;

        for_each_cpu(cpu, cpumask) {
                struct bp_cpuinfo *info = get_bp_info(cpu, type);
                int nr;

                nr = info->cpu_pinned;
                if (!bp->hw.target)
                        nr += max_task_bp_pinned(cpu, type);
                else
                        nr += task_bp_pinned(cpu, bp, type);

                if (nr > slots->pinned)
                        slots->pinned = nr;

                nr = info->flexible;
                if (nr > slots->flexible)
                        slots->flexible = nr;
        }
}

/*
 * For now, continue to consider flexible as pinned, until we can
 * ensure no flexible event can ever be scheduled before a pinned event
 * in a same cpu.
 */
static void
fetch_this_slot(struct bp_busy_slots *slots, int weight)
{
        slots->pinned += weight;
}

/*
 * Add a pinned breakpoint for the given task in our constraint table
 */
static void toggle_bp_task_slot(struct perf_event *bp, int cpu,
                                enum bp_type_idx type, int weight)
{
        unsigned int *tsk_pinned = get_bp_info(cpu, type)->tsk_pinned;
        int old_idx, new_idx;

        old_idx = task_bp_pinned(cpu, bp, type) - 1;
        new_idx = old_idx + weight;

        if (old_idx >= 0)
                tsk_pinned[old_idx]--;
        if (new_idx >= 0)
                tsk_pinned[new_idx]++;
}

/*
 * Add/remove the given breakpoint in our constraint table
 */
static void
toggle_bp_slot(struct perf_event *bp, bool enable, enum bp_type_idx type,
               int weight)
{
        const struct cpumask *cpumask = cpumask_of_bp(bp);
        int cpu;

        if (!enable)
                weight = -weight;

        /* Pinned counter cpu profiling */
        if (!bp->hw.target) {
                get_bp_info(bp->cpu, type)->cpu_pinned += weight;
                return;
        }

        /* Pinned counter task profiling */
        for_each_cpu(cpu, cpumask)
                toggle_bp_task_slot(bp, cpu, type, weight);

        if (enable)
                list_add_tail(&bp->hw.bp_list, &bp_task_head);
        else
                list_del(&bp->hw.bp_list);
}

__weak int arch_reserve_bp_slot(struct perf_event *bp)
{
        return 0;
}

__weak void arch_release_bp_slot(struct perf_event *bp)
{
}

/*
 * Function to perform processor-specific cleanup during unregistration
 */
__weak void arch_unregister_hw_breakpoint(struct perf_event *bp)
{
        /*
         * A weak stub function here for those archs that don't define
         * it inside arch/.../kernel/hw_breakpoint.c
         */
}

/*
 * Constraints to check before allowing this new breakpoint counter:
 *
 *  == Non-pinned counter == (Considered as pinned for now)
 *
 *   - If attached to a single cpu, check:
 *
 *       (per_cpu(info->flexible, cpu) || (per_cpu(info->cpu_pinned, cpu)
 *           + max(per_cpu(info->tsk_pinned, cpu)))) < HBP_NUM
 *
 *       -> If there are already non-pinned counters in this cpu, it means
 *          there is already a free slot for them.
 *          Otherwise, we check that the maximum number of per task
 *          breakpoints (for this cpu) plus the number of per cpu breakpoint
 *          (for this cpu) doesn't cover every registers.
 *
 *   - If attached to every cpus, check:
 *
 *       (per_cpu(info->flexible, *) || (max(per_cpu(info->cpu_pinned, *))
 *           + max(per_cpu(info->tsk_pinned, *)))) < HBP_NUM
 *
 *       -> This is roughly the same, except we check the number of per cpu
 *          bp for every cpu and we keep the max one. Same for the per tasks
 *          breakpoints.
 *
 *
 * == Pinned counter ==
 *
 *   - If attached to a single cpu, check:
 *
 *       ((per_cpu(info->flexible, cpu) > 1) + per_cpu(info->cpu_pinned, cpu)
 *            + max(per_cpu(info->tsk_pinned, cpu))) < HBP_NUM
 *
 *       -> Same checks as before. But now the info->flexible, if any, must keep
 *          one register at least (or they will never be fed).
 *
 *   - If attached to every cpus, check:
 *
 *       ((per_cpu(info->flexible, *) > 1) + max(per_cpu(info->cpu_pinned, *))
 *            + max(per_cpu(info->tsk_pinned, *))) < HBP_NUM
 */
static int __reserve_bp_slot(struct perf_event *bp, u64 bp_type)
{
        struct bp_busy_slots slots = {0};
        enum bp_type_idx type;
        int weight;
        int ret;

        /* We couldn't initialize breakpoint constraints on boot */
        if (!constraints_initialized)
                return -ENOMEM;

        /* Basic checks */
        if (bp_type == HW_BREAKPOINT_EMPTY ||
            bp_type == HW_BREAKPOINT_INVALID)
                return -EINVAL;

        type = find_slot_idx(bp_type);
        weight = hw_breakpoint_weight(bp);

        fetch_bp_busy_slots(&slots, bp, type);
        /*
         * Simulate the addition of this breakpoint to the constraints
         * and see the result.
         */
        fetch_this_slot(&slots, weight);

        /* Flexible counters need to keep at least one slot */
        if (slots.pinned + (!!slots.flexible) > nr_slots[type])
                return -ENOSPC;

        ret = arch_reserve_bp_slot(bp);
        if (ret)
                return ret;

        toggle_bp_slot(bp, true, type, weight);

        return 0;
}

int reserve_bp_slot(struct perf_event *bp)
{
        int ret;

        mutex_lock(&nr_bp_mutex);

        ret = __reserve_bp_slot(bp, bp->attr.bp_type);

        mutex_unlock(&nr_bp_mutex);

        return ret;
}

static void __release_bp_slot(struct perf_event *bp, u64 bp_type)
{
        enum bp_type_idx type;
        int weight;

        arch_release_bp_slot(bp);

        type = find_slot_idx(bp_type);
        weight = hw_breakpoint_weight(bp);
        toggle_bp_slot(bp, false, type, weight);
}

void release_bp_slot(struct perf_event *bp)
{
        mutex_lock(&nr_bp_mutex);

        arch_unregister_hw_breakpoint(bp);
        __release_bp_slot(bp, bp->attr.bp_type);

        mutex_unlock(&nr_bp_mutex);
}

static int __modify_bp_slot(struct perf_event *bp, u64 old_type, u64 new_type)
{
        int err;

        __release_bp_slot(bp, old_type);

        err = __reserve_bp_slot(bp, new_type);
        if (err) {
                /*
                 * Reserve the old_type slot back in case
                 * there's no space for the new type.
                 *
                 * This must succeed, because we just released
                 * the old_type slot in the __release_bp_slot
                 * call above. If not, something is broken.
                 */
                WARN_ON(__reserve_bp_slot(bp, old_type));
        }

        return err;
}

static int modify_bp_slot(struct perf_event *bp, u64 old_type, u64 new_type)
{
        int ret;

        mutex_lock(&nr_bp_mutex);
        ret = __modify_bp_slot(bp, old_type, new_type);
        mutex_unlock(&nr_bp_mutex);
        return ret;
}

/*
 * Allow the kernel debugger to reserve breakpoint slots without
 * taking a lock using the dbg_* variant of for the reserve and
 * release breakpoint slots.
 */
int dbg_reserve_bp_slot(struct perf_event *bp)
{
        if (mutex_is_locked(&nr_bp_mutex))
                return -1;

        return __reserve_bp_slot(bp, bp->attr.bp_type);
}

int dbg_release_bp_slot(struct perf_event *bp)
{
        if (mutex_is_locked(&nr_bp_mutex))
                return -1;

        __release_bp_slot(bp, bp->attr.bp_type);

        return 0;
}

static int hw_breakpoint_parse(struct perf_event *bp,
                               const struct perf_event_attr *attr,
                               struct arch_hw_breakpoint *hw)
{
        int err;

        err = hw_breakpoint_arch_parse(bp, attr, hw);
        if (err)
                return err;

        if (arch_check_bp_in_kernelspace(hw)) {
                if (attr->exclude_kernel)
                        return -EINVAL;
                /*
                 * Don't let unprivileged users set a breakpoint in the trap
                 * path to avoid trap recursion attacks.
                 */
                if (!capable(CAP_SYS_ADMIN))
                        return -EPERM;
        }

        return 0;
}

int register_perf_hw_breakpoint(struct perf_event *bp)
{
        struct arch_hw_breakpoint hw = { };
        int err;

        err = reserve_bp_slot(bp);
        if (err)
                return err;

        err = hw_breakpoint_parse(bp, &bp->attr, &hw);
        if (err) {
                release_bp_slot(bp);
                return err;
        }

        bp->hw.info = hw;

        return 0;
}

/**
 * register_user_hw_breakpoint - register a hardware breakpoint for user space
 * @attr: breakpoint attributes
 * @triggered: callback to trigger when we hit the breakpoint
 * @context: context data could be used in the triggered callback
 * @tsk: pointer to 'task_struct' of the process to which the address belongs
 */
struct perf_event *
register_user_hw_breakpoint(struct perf_event_attr *attr,
                            perf_overflow_handler_t triggered,
                            void *context,
                            struct task_struct *tsk)
{
        return perf_event_create_kernel_counter(attr, -1, tsk, triggered,
                                                context);
}
EXPORT_SYMBOL_GPL(register_user_hw_breakpoint);

static void hw_breakpoint_copy_attr(struct perf_event_attr *to,
                                    struct perf_event_attr *from)
{
        to->bp_addr = from->bp_addr;
        to->bp_type = from->bp_type;
        to->bp_len  = from->bp_len;
        to->disabled = from->disabled;
}

int
modify_user_hw_breakpoint_check(struct perf_event *bp, struct perf_event_attr *attr,
                                bool check)
{
        struct arch_hw_breakpoint hw = { };
        int err;

        err = hw_breakpoint_parse(bp, attr, &hw);
        if (err)
                return err;

        if (check) {
                struct perf_event_attr old_attr;

                old_attr = bp->attr;
                hw_breakpoint_copy_attr(&old_attr, attr);
                if (memcmp(&old_attr, attr, sizeof(*attr)))
                        return -EINVAL;
        }

        if (bp->attr.bp_type != attr->bp_type) {
                err = modify_bp_slot(bp, bp->attr.bp_type, attr->bp_type);
                if (err)
                        return err;
        }

        hw_breakpoint_copy_attr(&bp->attr, attr);
        bp->hw.info = hw;

        return 0;
}

/**
 * modify_user_hw_breakpoint - modify a user-space hardware breakpoint
 * @bp: the breakpoint structure to modify
 * @attr: new breakpoint attributes
 */
int modify_user_hw_breakpoint(struct perf_event *bp, struct perf_event_attr *attr)
{
        int err;

        /*
         * modify_user_hw_breakpoint can be invoked with IRQs disabled and hence it
         * will not be possible to raise IPIs that invoke __perf_event_disable.
         * So call the function directly after making sure we are targeting the
         * current task.
         */
        if (irqs_disabled() && bp->ctx && bp->ctx->task == current)
                perf_event_disable_local(bp);
        else
                perf_event_disable(bp);

        err = modify_user_hw_breakpoint_check(bp, attr, false);

        if (!bp->attr.disabled)
                perf_event_enable(bp);

        return err;
}
EXPORT_SYMBOL_GPL(modify_user_hw_breakpoint);

/**
 * unregister_hw_breakpoint - unregister a user-space hardware breakpoint
 * @bp: the breakpoint structure to unregister
 */
void unregister_hw_breakpoint(struct perf_event *bp)
{
        if (!bp)
                return;
        perf_event_release_kernel(bp);
}
EXPORT_SYMBOL_GPL(unregister_hw_breakpoint);

/**
 * register_wide_hw_breakpoint - register a wide breakpoint in the kernel
 * @attr: breakpoint attributes
 * @triggered: callback to trigger when we hit the breakpoint
 * @context: context data could be used in the triggered callback
 *
 * @return a set of per_cpu pointers to perf events
 */
struct perf_event * __percpu *
register_wide_hw_breakpoint(struct perf_event_attr *attr,
                            perf_overflow_handler_t triggered,
                            void *context)
{
        struct perf_event * __percpu *cpu_events, *bp;
        long err = 0;
        int cpu;

        cpu_events = alloc_percpu(typeof(*cpu_events));
        if (!cpu_events)
                return (void __percpu __force *)ERR_PTR(-ENOMEM);

        cpus_read_lock();
        for_each_online_cpu(cpu) {
                bp = perf_event_create_kernel_counter(attr, cpu, NULL,
                                                      triggered, context);
                if (IS_ERR(bp)) {
                        err = PTR_ERR(bp);
                        break;
                }

                per_cpu(*cpu_events, cpu) = bp;
        }
        cpus_read_unlock();

        if (likely(!err))
                return cpu_events;

        unregister_wide_hw_breakpoint(cpu_events);
        return (void __percpu __force *)ERR_PTR(err);
}
EXPORT_SYMBOL_GPL(register_wide_hw_breakpoint);

/**
 * unregister_wide_hw_breakpoint - unregister a wide breakpoint in the kernel
 * @cpu_events: the per cpu set of events to unregister
 */
void unregister_wide_hw_breakpoint(struct perf_event * __percpu *cpu_events)
{
        int cpu;

        for_each_possible_cpu(cpu)
                unregister_hw_breakpoint(per_cpu(*cpu_events, cpu));

        free_percpu(cpu_events);
}
EXPORT_SYMBOL_GPL(unregister_wide_hw_breakpoint);

static struct notifier_block hw_breakpoint_exceptions_nb = {
        .notifier_call = hw_breakpoint_exceptions_notify,
        /* we need to be notified first */
        .priority = 0x7fffffff
};

static void bp_perf_event_destroy(struct perf_event *event)
{
        release_bp_slot(event);
}

static int hw_breakpoint_event_init(struct perf_event *bp)
{
        int err;

        if (bp->attr.type != PERF_TYPE_BREAKPOINT)
                return -ENOENT;

        /*
         * no branch sampling for breakpoint events
         */
        if (has_branch_stack(bp))
                return -EOPNOTSUPP;

        err = register_perf_hw_breakpoint(bp);
        if (err)
                return err;

        bp->destroy = bp_perf_event_destroy;

        return 0;
}

static int hw_breakpoint_add(struct perf_event *bp, int flags)
{
        if (!(flags & PERF_EF_START))
                bp->hw.state = PERF_HES_STOPPED;

        if (is_sampling_event(bp)) {
                bp->hw.last_period = bp->hw.sample_period;
                perf_swevent_set_period(bp);
        }

        return arch_install_hw_breakpoint(bp);
}

static void hw_breakpoint_del(struct perf_event *bp, int flags)
{
        arch_uninstall_hw_breakpoint(bp);
}

static void hw_breakpoint_start(struct perf_event *bp, int flags)
{
        bp->hw.state = 0;
}

static void hw_breakpoint_stop(struct perf_event *bp, int flags)
{
        bp->hw.state = PERF_HES_STOPPED;
}

static struct pmu perf_breakpoint = {
        .task_ctx_nr        = perf_sw_context, /* could eventually get its own */

        .event_init        = hw_breakpoint_event_init,
        .add                = hw_breakpoint_add,
        .del                = hw_breakpoint_del,
        .start                = hw_breakpoint_start,
        .stop                = hw_breakpoint_stop,
        .read                = hw_breakpoint_pmu_read,
};

int __init init_hw_breakpoint(void)
{
        int cpu, err_cpu;
        int i;

        for (i = 0; i < TYPE_MAX; i++)
                nr_slots[i] = hw_breakpoint_slots(i);

        for_each_possible_cpu(cpu) {
                for (i = 0; i < TYPE_MAX; i++) {
                        struct bp_cpuinfo *info = get_bp_info(cpu, i);

                        info->tsk_pinned = kcalloc(nr_slots[i], sizeof(int),
                                                        GFP_KERNEL);
                        if (!info->tsk_pinned)
                                goto err_alloc;
                }
        }

        constraints_initialized = 1;

        perf_pmu_register(&perf_breakpoint, "breakpoint", PERF_TYPE_BREAKPOINT);

        return register_die_notifier(&hw_breakpoint_exceptions_nb);

 err_alloc:
        for_each_possible_cpu(err_cpu) {
                for (i = 0; i < TYPE_MAX; i++)
                        kfree(get_bp_info(err_cpu, i)->tsk_pinned);
                if (err_cpu == cpu)
                        break;
        }

        return -ENOMEM;
}


















































































































































































































































































































































   11 




   10 

































































   11 


















   11 








































    7 

    7 



    7 
    7 
    7 
























    7 



















    7 



    7 


    7 





















































































    7 




    7 
    6 



    7 
    7 


















    7 



    7 
























    7 














    7 
    7 







































    7 



































































































































    6 











    6 

















































    6 








    6 
    6 
    6 


    6 


    6 

    6 











    6 

    6 









































    6 
















































































































    7 












    6 





    7 




    7 

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
// SPDX-License-Identifier: GPL-2.0
/* Copyright (C) B.A.T.M.A.N. contributors:
 *
 * Marek Lindner, Simon Wunderlich
 */

#include "send.h"
#include "main.h"

#include <linux/atomic.h>
#include <linux/bug.h>
#include <linux/byteorder/generic.h>
#include <linux/errno.h>
#include <linux/etherdevice.h>
#include <linux/gfp.h>
#include <linux/if.h>
#include <linux/if_ether.h>
#include <linux/jiffies.h>
#include <linux/kernel.h>
#include <linux/kref.h>
#include <linux/list.h>
#include <linux/netdevice.h>
#include <linux/printk.h>
#include <linux/rculist.h>
#include <linux/rcupdate.h>
#include <linux/skbuff.h>
#include <linux/slab.h>
#include <linux/spinlock.h>
#include <linux/stddef.h>
#include <linux/workqueue.h>

#include "distributed-arp-table.h"
#include "fragmentation.h"
#include "gateway_client.h"
#include "hard-interface.h"
#include "log.h"
#include "network-coding.h"
#include "originator.h"
#include "routing.h"
#include "soft-interface.h"
#include "translation-table.h"

static void batadv_send_outstanding_bcast_packet(struct work_struct *work);

/**
 * batadv_send_skb_packet() - send an already prepared packet
 * @skb: the packet to send
 * @hard_iface: the interface to use to send the broadcast packet
 * @dst_addr: the payload destination
 *
 * Send out an already prepared packet to the given neighbor or broadcast it
 * using the specified interface. Either hard_iface or neigh_node must be not
 * NULL.
 * If neigh_node is NULL, then the packet is broadcasted using hard_iface,
 * otherwise it is sent as unicast to the given neighbor.
 *
 * Regardless of the return value, the skb is consumed.
 *
 * Return: A negative errno code is returned on a failure. A success does not
 * guarantee the frame will be transmitted as it may be dropped due
 * to congestion or traffic shaping.
 */
int batadv_send_skb_packet(struct sk_buff *skb,
                           struct batadv_hard_iface *hard_iface,
                           const u8 *dst_addr)
{
        struct batadv_priv *bat_priv;
        struct ethhdr *ethhdr;
        int ret;

        bat_priv = netdev_priv(hard_iface->soft_iface);

        if (hard_iface->if_status != BATADV_IF_ACTIVE)
                goto send_skb_err;

        if (unlikely(!hard_iface->net_dev))
                goto send_skb_err;

        if (!(hard_iface->net_dev->flags & IFF_UP)) {
                pr_warn("Interface %s is not up - can't send packet via that interface!\n",
                        hard_iface->net_dev->name);
                goto send_skb_err;
        }

        /* push to the ethernet header. */
        if (batadv_skb_head_push(skb, ETH_HLEN) < 0)
                goto send_skb_err;

        skb_reset_mac_header(skb);

        ethhdr = eth_hdr(skb);
        ether_addr_copy(ethhdr->h_source, hard_iface->net_dev->dev_addr);
        ether_addr_copy(ethhdr->h_dest, dst_addr);
        ethhdr->h_proto = htons(ETH_P_BATMAN);

        skb_set_network_header(skb, ETH_HLEN);
        skb->protocol = htons(ETH_P_BATMAN);

        skb->dev = hard_iface->net_dev;

        /* Save a clone of the skb to use when decoding coded packets */
        batadv_nc_skb_store_for_decoding(bat_priv, skb);

        /* dev_queue_xmit() returns a negative result on error.         However on
         * congestion and traffic shaping, it drops and returns NET_XMIT_DROP
         * (which is > 0). This will not be treated as an error.
         */
        ret = dev_queue_xmit(skb);
        return net_xmit_eval(ret);
send_skb_err:
        kfree_skb(skb);
        return NET_XMIT_DROP;
}

/**
 * batadv_send_broadcast_skb() - Send broadcast packet via hard interface
 * @skb: packet to be transmitted (with batadv header and no outer eth header)
 * @hard_iface: outgoing interface
 *
 * Return: A negative errno code is returned on a failure. A success does not
 * guarantee the frame will be transmitted as it may be dropped due
 * to congestion or traffic shaping.
 */
int batadv_send_broadcast_skb(struct sk_buff *skb,
                              struct batadv_hard_iface *hard_iface)
{
        return batadv_send_skb_packet(skb, hard_iface, batadv_broadcast_addr);
}

/**
 * batadv_send_unicast_skb() - Send unicast packet to neighbor
 * @skb: packet to be transmitted (with batadv header and no outer eth header)
 * @neigh: neighbor which is used as next hop to destination
 *
 * Return: A negative errno code is returned on a failure. A success does not
 * guarantee the frame will be transmitted as it may be dropped due
 * to congestion or traffic shaping.
 */
int batadv_send_unicast_skb(struct sk_buff *skb,
                            struct batadv_neigh_node *neigh)
{
#ifdef CONFIG_BATMAN_ADV_BATMAN_V
        struct batadv_hardif_neigh_node *hardif_neigh;
#endif
        int ret;

        ret = batadv_send_skb_packet(skb, neigh->if_incoming, neigh->addr);

#ifdef CONFIG_BATMAN_ADV_BATMAN_V
        hardif_neigh = batadv_hardif_neigh_get(neigh->if_incoming, neigh->addr);

        if (hardif_neigh && ret != NET_XMIT_DROP)
                hardif_neigh->bat_v.last_unicast_tx = jiffies;

        batadv_hardif_neigh_put(hardif_neigh);
#endif

        return ret;
}

/**
 * batadv_send_skb_to_orig() - Lookup next-hop and transmit skb.
 * @skb: Packet to be transmitted.
 * @orig_node: Final destination of the packet.
 * @recv_if: Interface used when receiving the packet (can be NULL).
 *
 * Looks up the best next-hop towards the passed originator and passes the
 * skb on for preparation of MAC header. If the packet originated from this
 * host, NULL can be passed as recv_if and no interface alternating is
 * attempted.
 *
 * Return: negative errno code on a failure, -EINPROGRESS if the skb is
 * buffered for later transmit or the NET_XMIT status returned by the
 * lower routine if the packet has been passed down.
 */
int batadv_send_skb_to_orig(struct sk_buff *skb,
                            struct batadv_orig_node *orig_node,
                            struct batadv_hard_iface *recv_if)
{
        struct batadv_priv *bat_priv = orig_node->bat_priv;
        struct batadv_neigh_node *neigh_node;
        int ret;

        /* batadv_find_router() increases neigh_nodes refcount if found. */
        neigh_node = batadv_find_router(bat_priv, orig_node, recv_if);
        if (!neigh_node) {
                ret = -EINVAL;
                goto free_skb;
        }

        /* Check if the skb is too large to send in one piece and fragment
         * it if needed.
         */
        if (atomic_read(&bat_priv->fragmentation) &&
            skb->len > neigh_node->if_incoming->net_dev->mtu) {
                /* Fragment and send packet. */
                ret = batadv_frag_send_packet(skb, orig_node, neigh_node);
                /* skb was consumed */
                skb = NULL;

                goto put_neigh_node;
        }

        /* try to network code the packet, if it is received on an interface
         * (i.e. being forwarded). If the packet originates from this node or if
         * network coding fails, then send the packet as usual.
         */
        if (recv_if && batadv_nc_skb_forward(skb, neigh_node))
                ret = -EINPROGRESS;
        else
                ret = batadv_send_unicast_skb(skb, neigh_node);

        /* skb was consumed */
        skb = NULL;

put_neigh_node:
        batadv_neigh_node_put(neigh_node);
free_skb:
        kfree_skb(skb);

        return ret;
}

/**
 * batadv_send_skb_push_fill_unicast() - extend the buffer and initialize the
 *  common fields for unicast packets
 * @skb: the skb carrying the unicast header to initialize
 * @hdr_size: amount of bytes to push at the beginning of the skb
 * @orig_node: the destination node
 *
 * Return: false if the buffer extension was not possible or true otherwise.
 */
static bool
batadv_send_skb_push_fill_unicast(struct sk_buff *skb, int hdr_size,
                                  struct batadv_orig_node *orig_node)
{
        struct batadv_unicast_packet *unicast_packet;
        u8 ttvn = (u8)atomic_read(&orig_node->last_ttvn);

        if (batadv_skb_head_push(skb, hdr_size) < 0)
                return false;

        unicast_packet = (struct batadv_unicast_packet *)skb->data;
        unicast_packet->version = BATADV_COMPAT_VERSION;
        /* batman packet type: unicast */
        unicast_packet->packet_type = BATADV_UNICAST;
        /* set unicast ttl */
        unicast_packet->ttl = BATADV_TTL;
        /* copy the destination for faster routing */
        ether_addr_copy(unicast_packet->dest, orig_node->orig);
        /* set the destination tt version number */
        unicast_packet->ttvn = ttvn;

        return true;
}

/**
 * batadv_send_skb_prepare_unicast() - encapsulate an skb with a unicast header
 * @skb: the skb containing the payload to encapsulate
 * @orig_node: the destination node
 *
 * Return: false if the payload could not be encapsulated or true otherwise.
 */
static bool batadv_send_skb_prepare_unicast(struct sk_buff *skb,
                                            struct batadv_orig_node *orig_node)
{
        size_t uni_size = sizeof(struct batadv_unicast_packet);

        return batadv_send_skb_push_fill_unicast(skb, uni_size, orig_node);
}

/**
 * batadv_send_skb_prepare_unicast_4addr() - encapsulate an skb with a
 *  unicast 4addr header
 * @bat_priv: the bat priv with all the soft interface information
 * @skb: the skb containing the payload to encapsulate
 * @orig: the destination node
 * @packet_subtype: the unicast 4addr packet subtype to use
 *
 * Return: false if the payload could not be encapsulated or true otherwise.
 */
bool batadv_send_skb_prepare_unicast_4addr(struct batadv_priv *bat_priv,
                                           struct sk_buff *skb,
                                           struct batadv_orig_node *orig,
                                           int packet_subtype)
{
        struct batadv_hard_iface *primary_if;
        struct batadv_unicast_4addr_packet *uc_4addr_packet;
        bool ret = false;

        primary_if = batadv_primary_if_get_selected(bat_priv);
        if (!primary_if)
                goto out;

        /* Pull the header space and fill the unicast_packet substructure.
         * We can do that because the first member of the uc_4addr_packet
         * is of type struct unicast_packet
         */
        if (!batadv_send_skb_push_fill_unicast(skb, sizeof(*uc_4addr_packet),
                                               orig))
                goto out;

        uc_4addr_packet = (struct batadv_unicast_4addr_packet *)skb->data;
        uc_4addr_packet->u.packet_type = BATADV_UNICAST_4ADDR;
        ether_addr_copy(uc_4addr_packet->src, primary_if->net_dev->dev_addr);
        uc_4addr_packet->subtype = packet_subtype;
        uc_4addr_packet->reserved = 0;

        ret = true;
out:
        batadv_hardif_put(primary_if);
        return ret;
}

/**
 * batadv_send_skb_unicast() - encapsulate and send an skb via unicast
 * @bat_priv: the bat priv with all the soft interface information
 * @skb: payload to send
 * @packet_type: the batman unicast packet type to use
 * @packet_subtype: the unicast 4addr packet subtype (only relevant for unicast
 *  4addr packets)
 * @orig_node: the originator to send the packet to
 * @vid: the vid to be used to search the translation table
 *
 * Wrap the given skb into a batman-adv unicast or unicast-4addr header
 * depending on whether BATADV_UNICAST or BATADV_UNICAST_4ADDR was supplied
 * as packet_type. Then send this frame to the given orig_node.
 *
 * Return: NET_XMIT_DROP in case of error or NET_XMIT_SUCCESS otherwise.
 */
int batadv_send_skb_unicast(struct batadv_priv *bat_priv,
                            struct sk_buff *skb, int packet_type,
                            int packet_subtype,
                            struct batadv_orig_node *orig_node,
                            unsigned short vid)
{
        struct batadv_unicast_packet *unicast_packet;
        struct ethhdr *ethhdr;
        int ret = NET_XMIT_DROP;

        if (!orig_node)
                goto out;

        switch (packet_type) {
        case BATADV_UNICAST:
                if (!batadv_send_skb_prepare_unicast(skb, orig_node))
                        goto out;
                break;
        case BATADV_UNICAST_4ADDR:
                if (!batadv_send_skb_prepare_unicast_4addr(bat_priv, skb,
                                                           orig_node,
                                                           packet_subtype))
                        goto out;
                break;
        default:
                /* this function supports UNICAST and UNICAST_4ADDR only. It
                 * should never be invoked with any other packet type
                 */
                goto out;
        }

        /* skb->data might have been reallocated by
         * batadv_send_skb_prepare_unicast{,_4addr}()
         */
        ethhdr = eth_hdr(skb);
        unicast_packet = (struct batadv_unicast_packet *)skb->data;

        /* inform the destination node that we are still missing a correct route
         * for this client. The destination will receive this packet and will
         * try to reroute it because the ttvn contained in the header is less
         * than the current one
         */
        if (batadv_tt_global_client_is_roaming(bat_priv, ethhdr->h_dest, vid))
                unicast_packet->ttvn = unicast_packet->ttvn - 1;

        ret = batadv_send_skb_to_orig(skb, orig_node, NULL);
         /* skb was consumed */
        skb = NULL;

out:
        kfree_skb(skb);
        return ret;
}

/**
 * batadv_send_skb_via_tt_generic() - send an skb via TT lookup
 * @bat_priv: the bat priv with all the soft interface information
 * @skb: payload to send
 * @packet_type: the batman unicast packet type to use
 * @packet_subtype: the unicast 4addr packet subtype (only relevant for unicast
 *  4addr packets)
 * @dst_hint: can be used to override the destination contained in the skb
 * @vid: the vid to be used to search the translation table
 *
 * Look up the recipient node for the destination address in the ethernet
 * header via the translation table. Wrap the given skb into a batman-adv
 * unicast or unicast-4addr header depending on whether BATADV_UNICAST or
 * BATADV_UNICAST_4ADDR was supplied as packet_type. Then send this frame
 * to the according destination node.
 *
 * Return: NET_XMIT_DROP in case of error or NET_XMIT_SUCCESS otherwise.
 */
int batadv_send_skb_via_tt_generic(struct batadv_priv *bat_priv,
                                   struct sk_buff *skb, int packet_type,
                                   int packet_subtype, u8 *dst_hint,
                                   unsigned short vid)
{
        struct ethhdr *ethhdr = (struct ethhdr *)skb->data;
        struct batadv_orig_node *orig_node;
        u8 *src, *dst;
        int ret;

        src = ethhdr->h_source;
        dst = ethhdr->h_dest;

        /* if we got an hint! let's send the packet to this client (if any) */
        if (dst_hint) {
                src = NULL;
                dst = dst_hint;
        }
        orig_node = batadv_transtable_search(bat_priv, src, dst, vid);

        ret = batadv_send_skb_unicast(bat_priv, skb, packet_type,
                                      packet_subtype, orig_node, vid);

        batadv_orig_node_put(orig_node);

        return ret;
}

/**
 * batadv_send_skb_via_gw() - send an skb via gateway lookup
 * @bat_priv: the bat priv with all the soft interface information
 * @skb: payload to send
 * @vid: the vid to be used to search the translation table
 *
 * Look up the currently selected gateway. Wrap the given skb into a batman-adv
 * unicast header and send this frame to this gateway node.
 *
 * Return: NET_XMIT_DROP in case of error or NET_XMIT_SUCCESS otherwise.
 */
int batadv_send_skb_via_gw(struct batadv_priv *bat_priv, struct sk_buff *skb,
                           unsigned short vid)
{
        struct batadv_orig_node *orig_node;
        int ret;

        orig_node = batadv_gw_get_selected_orig(bat_priv);
        ret = batadv_send_skb_unicast(bat_priv, skb, BATADV_UNICAST_4ADDR,
                                      BATADV_P_DATA, orig_node, vid);

        batadv_orig_node_put(orig_node);

        return ret;
}

/**
 * batadv_forw_packet_free() - free a forwarding packet
 * @forw_packet: The packet to free
 * @dropped: whether the packet is freed because is dropped
 *
 * This frees a forwarding packet and releases any resources it might
 * have claimed.
 */
void batadv_forw_packet_free(struct batadv_forw_packet *forw_packet,
                             bool dropped)
{
        if (dropped)
                kfree_skb(forw_packet->skb);
        else
                consume_skb(forw_packet->skb);

        batadv_hardif_put(forw_packet->if_incoming);
        batadv_hardif_put(forw_packet->if_outgoing);
        if (forw_packet->queue_left)
                atomic_inc(forw_packet->queue_left);
        kfree(forw_packet);
}

/**
 * batadv_forw_packet_alloc() - allocate a forwarding packet
 * @if_incoming: The (optional) if_incoming to be grabbed
 * @if_outgoing: The (optional) if_outgoing to be grabbed
 * @queue_left: The (optional) queue counter to decrease
 * @bat_priv: The bat_priv for the mesh of this forw_packet
 * @skb: The raw packet this forwarding packet shall contain
 *
 * Allocates a forwarding packet and tries to get a reference to the
 * (optional) if_incoming, if_outgoing and queue_left. If queue_left
 * is NULL then bat_priv is optional, too.
 *
 * Return: An allocated forwarding packet on success, NULL otherwise.
 */
struct batadv_forw_packet *
batadv_forw_packet_alloc(struct batadv_hard_iface *if_incoming,
                         struct batadv_hard_iface *if_outgoing,
                         atomic_t *queue_left,
                         struct batadv_priv *bat_priv,
                         struct sk_buff *skb)
{
        struct batadv_forw_packet *forw_packet;
        const char *qname;

        if (queue_left && !batadv_atomic_dec_not_zero(queue_left)) {
                qname = "unknown";

                if (queue_left == &bat_priv->bcast_queue_left)
                        qname = "bcast";

                if (queue_left == &bat_priv->batman_queue_left)
                        qname = "batman";

                batadv_dbg(BATADV_DBG_BATMAN, bat_priv,
                           "%s queue is full\n", qname);

                return NULL;
        }

        forw_packet = kmalloc(sizeof(*forw_packet), GFP_ATOMIC);
        if (!forw_packet)
                goto err;

        if (if_incoming)
                kref_get(&if_incoming->refcount);

        if (if_outgoing)
                kref_get(&if_outgoing->refcount);

        INIT_HLIST_NODE(&forw_packet->list);
        INIT_HLIST_NODE(&forw_packet->cleanup_list);
        forw_packet->skb = skb;
        forw_packet->queue_left = queue_left;
        forw_packet->if_incoming = if_incoming;
        forw_packet->if_outgoing = if_outgoing;
        forw_packet->num_packets = 0;

        return forw_packet;

err:
        if (queue_left)
                atomic_inc(queue_left);

        return NULL;
}

/**
 * batadv_forw_packet_was_stolen() - check whether someone stole this packet
 * @forw_packet: the forwarding packet to check
 *
 * This function checks whether the given forwarding packet was claimed by
 * someone else for free().
 *
 * Return: True if someone stole it, false otherwise.
 */
static bool
batadv_forw_packet_was_stolen(struct batadv_forw_packet *forw_packet)
{
        return !hlist_unhashed(&forw_packet->cleanup_list);
}

/**
 * batadv_forw_packet_steal() - claim a forw_packet for free()
 * @forw_packet: the forwarding packet to steal
 * @lock: a key to the store to steal from (e.g. forw_{bat,bcast}_list_lock)
 *
 * This function tries to steal a specific forw_packet from global
 * visibility for the purpose of getting it for free(). That means
 * the caller is *not* allowed to requeue it afterwards.
 *
 * Return: True if stealing was successful. False if someone else stole it
 * before us.
 */
bool batadv_forw_packet_steal(struct batadv_forw_packet *forw_packet,
                              spinlock_t *lock)
{
        /* did purging routine steal it earlier? */
        spin_lock_bh(lock);
        if (batadv_forw_packet_was_stolen(forw_packet)) {
                spin_unlock_bh(lock);
                return false;
        }

        hlist_del_init(&forw_packet->list);

        /* Just to spot misuse of this function */
        hlist_add_fake(&forw_packet->cleanup_list);

        spin_unlock_bh(lock);
        return true;
}

/**
 * batadv_forw_packet_list_steal() - claim a list of forward packets for free()
 * @forw_list: the to be stolen forward packets
 * @cleanup_list: a backup pointer, to be able to dispose the packet later
 * @hard_iface: the interface to steal forward packets from
 *
 * This function claims responsibility to free any forw_packet queued on the
 * given hard_iface. If hard_iface is NULL forwarding packets on all hard
 * interfaces will be claimed.
 *
 * The packets are being moved from the forw_list to the cleanup_list. This
 * makes it possible for already running threads to notice the claim.
 */
static void
batadv_forw_packet_list_steal(struct hlist_head *forw_list,
                              struct hlist_head *cleanup_list,
                              const struct batadv_hard_iface *hard_iface)
{
        struct batadv_forw_packet *forw_packet;
        struct hlist_node *safe_tmp_node;

        hlist_for_each_entry_safe(forw_packet, safe_tmp_node,
                                  forw_list, list) {
                /* if purge_outstanding_packets() was called with an argument
                 * we delete only packets belonging to the given interface
                 */
                if (hard_iface &&
                    forw_packet->if_incoming != hard_iface &&
                    forw_packet->if_outgoing != hard_iface)
                        continue;

                hlist_del(&forw_packet->list);
                hlist_add_head(&forw_packet->cleanup_list, cleanup_list);
        }
}

/**
 * batadv_forw_packet_list_free() - free a list of forward packets
 * @head: a list of to be freed forw_packets
 *
 * This function cancels the scheduling of any packet in the provided list,
 * waits for any possibly running packet forwarding thread to finish and
 * finally, safely frees this forward packet.
 *
 * This function might sleep.
 */
static void batadv_forw_packet_list_free(struct hlist_head *head)
{
        struct batadv_forw_packet *forw_packet;
        struct hlist_node *safe_tmp_node;

        hlist_for_each_entry_safe(forw_packet, safe_tmp_node, head,
                                  cleanup_list) {
                cancel_delayed_work_sync(&forw_packet->delayed_work);

                hlist_del(&forw_packet->cleanup_list);
                batadv_forw_packet_free(forw_packet, true);
        }
}

/**
 * batadv_forw_packet_queue() - try to queue a forwarding packet
 * @forw_packet: the forwarding packet to queue
 * @lock: a key to the store (e.g. forw_{bat,bcast}_list_lock)
 * @head: the shelve to queue it on (e.g. forw_{bat,bcast}_list)
 * @send_time: timestamp (jiffies) when the packet is to be sent
 *
 * This function tries to (re)queue a forwarding packet. Requeuing
 * is prevented if the according interface is shutting down
 * (e.g. if batadv_forw_packet_list_steal() was called for this
 * packet earlier).
 *
 * Calling batadv_forw_packet_queue() after a call to
 * batadv_forw_packet_steal() is forbidden!
 *
 * Caller needs to ensure that forw_packet->delayed_work was initialized.
 */
static void batadv_forw_packet_queue(struct batadv_forw_packet *forw_packet,
                                     spinlock_t *lock, struct hlist_head *head,
                                     unsigned long send_time)
{
        spin_lock_bh(lock);

        /* did purging routine steal it from us? */
        if (batadv_forw_packet_was_stolen(forw_packet)) {
                /* If you got it for free() without trouble, then
                 * don't get back into the queue after stealing...
                 */
                WARN_ONCE(hlist_fake(&forw_packet->cleanup_list),
                          "Requeuing after batadv_forw_packet_steal() not allowed!\n");

                spin_unlock_bh(lock);
                return;
        }

        hlist_del_init(&forw_packet->list);
        hlist_add_head(&forw_packet->list, head);

        queue_delayed_work(batadv_event_workqueue,
                           &forw_packet->delayed_work,
                           send_time - jiffies);
        spin_unlock_bh(lock);
}

/**
 * batadv_forw_packet_bcast_queue() - try to queue a broadcast packet
 * @bat_priv: the bat priv with all the soft interface information
 * @forw_packet: the forwarding packet to queue
 * @send_time: timestamp (jiffies) when the packet is to be sent
 *
 * This function tries to (re)queue a broadcast packet.
 *
 * Caller needs to ensure that forw_packet->delayed_work was initialized.
 */
static void
batadv_forw_packet_bcast_queue(struct batadv_priv *bat_priv,
                               struct batadv_forw_packet *forw_packet,
                               unsigned long send_time)
{
        batadv_forw_packet_queue(forw_packet, &bat_priv->forw_bcast_list_lock,
                                 &bat_priv->forw_bcast_list, send_time);
}

/**
 * batadv_forw_packet_ogmv1_queue() - try to queue an OGMv1 packet
 * @bat_priv: the bat priv with all the soft interface information
 * @forw_packet: the forwarding packet to queue
 * @send_time: timestamp (jiffies) when the packet is to be sent
 *
 * This function tries to (re)queue an OGMv1 packet.
 *
 * Caller needs to ensure that forw_packet->delayed_work was initialized.
 */
void batadv_forw_packet_ogmv1_queue(struct batadv_priv *bat_priv,
                                    struct batadv_forw_packet *forw_packet,
                                    unsigned long send_time)
{
        batadv_forw_packet_queue(forw_packet, &bat_priv->forw_bat_list_lock,
                                 &bat_priv->forw_bat_list, send_time);
}

/**
 * batadv_forw_bcast_packet_to_list() - queue broadcast packet for transmissions
 * @bat_priv: the bat priv with all the soft interface information
 * @skb: broadcast packet to add
 * @delay: number of jiffies to wait before sending
 * @own_packet: true if it is a self-generated broadcast packet
 * @if_in: the interface where the packet was received on
 * @if_out: the outgoing interface to queue on
 *
 * Adds a broadcast packet to the queue and sets up timers. Broadcast packets
 * are sent multiple times to increase probability for being received.
 *
 * This call clones the given skb, hence the caller needs to take into
 * account that the data segment of the original skb might not be
 * modifiable anymore.
 *
 * Return: NETDEV_TX_OK on success and NETDEV_TX_BUSY on errors.
 */
static int batadv_forw_bcast_packet_to_list(struct batadv_priv *bat_priv,
                                            struct sk_buff *skb,
                                            unsigned long delay,
                                            bool own_packet,
                                            struct batadv_hard_iface *if_in,
                                            struct batadv_hard_iface *if_out)
{
        struct batadv_forw_packet *forw_packet;
        unsigned long send_time = jiffies;
        struct sk_buff *newskb;

        newskb = skb_clone(skb, GFP_ATOMIC);
        if (!newskb)
                goto err;

        forw_packet = batadv_forw_packet_alloc(if_in, if_out,
                                               &bat_priv->bcast_queue_left,
                                               bat_priv, newskb);
        if (!forw_packet)
                goto err_packet_free;

        forw_packet->own = own_packet;

        INIT_DELAYED_WORK(&forw_packet->delayed_work,
                          batadv_send_outstanding_bcast_packet);

        send_time += delay ? delay : msecs_to_jiffies(5);

        batadv_forw_packet_bcast_queue(bat_priv, forw_packet, send_time);
        return NETDEV_TX_OK;

err_packet_free:
        kfree_skb(newskb);
err:
        return NETDEV_TX_BUSY;
}

/**
 * batadv_forw_bcast_packet_if() - forward and queue a broadcast packet
 * @bat_priv: the bat priv with all the soft interface information
 * @skb: broadcast packet to add
 * @delay: number of jiffies to wait before sending
 * @own_packet: true if it is a self-generated broadcast packet
 * @if_in: the interface where the packet was received on
 * @if_out: the outgoing interface to forward to
 *
 * Transmits a broadcast packet on the specified interface either immediately
 * or if a delay is given after that. Furthermore, queues additional
 * retransmissions if this interface is a wireless one.
 *
 * This call clones the given skb, hence the caller needs to take into
 * account that the data segment of the original skb might not be
 * modifiable anymore.
 *
 * Return: NETDEV_TX_OK on success and NETDEV_TX_BUSY on errors.
 */
static int batadv_forw_bcast_packet_if(struct batadv_priv *bat_priv,
                                       struct sk_buff *skb,
                                       unsigned long delay,
                                       bool own_packet,
                                       struct batadv_hard_iface *if_in,
                                       struct batadv_hard_iface *if_out)
{
        unsigned int num_bcasts = if_out->num_bcasts;
        struct sk_buff *newskb;
        int ret = NETDEV_TX_OK;

        if (!delay) {
                newskb = skb_clone(skb, GFP_ATOMIC);
                if (!newskb)
                        return NETDEV_TX_BUSY;

                batadv_send_broadcast_skb(newskb, if_out);
                num_bcasts--;
        }

        /* delayed broadcast or rebroadcasts? */
        if (num_bcasts >= 1) {
                BATADV_SKB_CB(skb)->num_bcasts = num_bcasts;

                ret = batadv_forw_bcast_packet_to_list(bat_priv, skb, delay,
                                                       own_packet, if_in,
                                                       if_out);
        }

        return ret;
}

/**
 * batadv_send_no_broadcast() - check whether (re)broadcast is necessary
 * @bat_priv: the bat priv with all the soft interface information
 * @skb: broadcast packet to check
 * @own_packet: true if it is a self-generated broadcast packet
 * @if_out: the outgoing interface checked and considered for (re)broadcast
 *
 * Return: False if a packet needs to be (re)broadcasted on the given interface,
 * true otherwise.
 */
static bool batadv_send_no_broadcast(struct batadv_priv *bat_priv,
                                     struct sk_buff *skb, bool own_packet,
                                     struct batadv_hard_iface *if_out)
{
        struct batadv_hardif_neigh_node *neigh_node = NULL;
        struct batadv_bcast_packet *bcast_packet;
        u8 *orig_neigh;
        u8 *neigh_addr;
        char *type;
        int ret;

        if (!own_packet) {
                neigh_addr = eth_hdr(skb)->h_source;
                neigh_node = batadv_hardif_neigh_get(if_out,
                                                     neigh_addr);
        }

        bcast_packet = (struct batadv_bcast_packet *)skb->data;
        orig_neigh = neigh_node ? neigh_node->orig : NULL;

        ret = batadv_hardif_no_broadcast(if_out, bcast_packet->orig,
                                         orig_neigh);

        batadv_hardif_neigh_put(neigh_node);

        /* ok, may broadcast */
        if (!ret)
                return false;

        /* no broadcast */
        switch (ret) {
        case BATADV_HARDIF_BCAST_NORECIPIENT:
                type = "no neighbor";
                break;
        case BATADV_HARDIF_BCAST_DUPFWD:
                type = "single neighbor is source";
                break;
        case BATADV_HARDIF_BCAST_DUPORIG:
                type = "single neighbor is originator";
                break;
        default:
                type = "unknown";
        }

        batadv_dbg(BATADV_DBG_BATMAN, bat_priv,
                   "BCAST packet from orig %pM on %s suppressed: %s\n",
                   bcast_packet->orig,
                   if_out->net_dev->name, type);

        return true;
}

/**
 * __batadv_forw_bcast_packet() - forward and queue a broadcast packet
 * @bat_priv: the bat priv with all the soft interface information
 * @skb: broadcast packet to add
 * @delay: number of jiffies to wait before sending
 * @own_packet: true if it is a self-generated broadcast packet
 *
 * Transmits a broadcast packet either immediately or if a delay is given
 * after that. Furthermore, queues additional retransmissions on wireless
 * interfaces.
 *
 * This call clones the given skb, hence the caller needs to take into
 * account that the data segment of the given skb might not be
 * modifiable anymore.
 *
 * Return: NETDEV_TX_OK on success and NETDEV_TX_BUSY on errors.
 */
static int __batadv_forw_bcast_packet(struct batadv_priv *bat_priv,
                                      struct sk_buff *skb,
                                      unsigned long delay,
                                      bool own_packet)
{
        struct batadv_hard_iface *hard_iface;
        struct batadv_hard_iface *primary_if;
        int ret = NETDEV_TX_OK;

        primary_if = batadv_primary_if_get_selected(bat_priv);
        if (!primary_if)
                return NETDEV_TX_BUSY;

        rcu_read_lock();
        list_for_each_entry_rcu(hard_iface, &batadv_hardif_list, list) {
                if (hard_iface->soft_iface != bat_priv->soft_iface)
                        continue;

                if (!kref_get_unless_zero(&hard_iface->refcount))
                        continue;

                if (batadv_send_no_broadcast(bat_priv, skb, own_packet,
                                             hard_iface)) {
                        batadv_hardif_put(hard_iface);
                        continue;
                }

                ret = batadv_forw_bcast_packet_if(bat_priv, skb, delay,
                                                  own_packet, primary_if,
                                                  hard_iface);
                batadv_hardif_put(hard_iface);

                if (ret == NETDEV_TX_BUSY)
                        break;
        }
        rcu_read_unlock();

        batadv_hardif_put(primary_if);
        return ret;
}

/**
 * batadv_forw_bcast_packet() - forward and queue a broadcast packet
 * @bat_priv: the bat priv with all the soft interface information
 * @skb: broadcast packet to add
 * @delay: number of jiffies to wait before sending
 * @own_packet: true if it is a self-generated broadcast packet
 *
 * Transmits a broadcast packet either immediately or if a delay is given
 * after that. Furthermore, queues additional retransmissions on wireless
 * interfaces.
 *
 * Return: NETDEV_TX_OK on success and NETDEV_TX_BUSY on errors.
 */
int batadv_forw_bcast_packet(struct batadv_priv *bat_priv,
                             struct sk_buff *skb,
                             unsigned long delay,
                             bool own_packet)
{
        return __batadv_forw_bcast_packet(bat_priv, skb, delay, own_packet);
}

/**
 * batadv_send_bcast_packet() - send and queue a broadcast packet
 * @bat_priv: the bat priv with all the soft interface information
 * @skb: broadcast packet to add
 * @delay: number of jiffies to wait before sending
 * @own_packet: true if it is a self-generated broadcast packet
 *
 * Transmits a broadcast packet either immediately or if a delay is given
 * after that. Furthermore, queues additional retransmissions on wireless
 * interfaces.
 *
 * Consumes the provided skb.
 */
void batadv_send_bcast_packet(struct batadv_priv *bat_priv,
                              struct sk_buff *skb,
                              unsigned long delay,
                              bool own_packet)
{
        __batadv_forw_bcast_packet(bat_priv, skb, delay, own_packet);
        consume_skb(skb);
}

/**
 * batadv_forw_packet_bcasts_left() - check if a retransmission is necessary
 * @forw_packet: the forwarding packet to check
 *
 * Checks whether a given packet has any (re)transmissions left on the provided
 * interface.
 *
 * hard_iface may be NULL: In that case the number of transmissions this skb had
 * so far is compared with the maximum amount of retransmissions independent of
 * any interface instead.
 *
 * Return: True if (re)transmissions are left, false otherwise.
 */
static bool
batadv_forw_packet_bcasts_left(struct batadv_forw_packet *forw_packet)
{
        return BATADV_SKB_CB(forw_packet->skb)->num_bcasts;
}

/**
 * batadv_forw_packet_bcasts_dec() - decrement retransmission counter of a
 *  packet
 * @forw_packet: the packet to decrease the counter for
 */
static void
batadv_forw_packet_bcasts_dec(struct batadv_forw_packet *forw_packet)
{
        BATADV_SKB_CB(forw_packet->skb)->num_bcasts--;
}

/**
 * batadv_forw_packet_is_rebroadcast() - check packet for previous transmissions
 * @forw_packet: the packet to check
 *
 * Return: True if this packet was transmitted before, false otherwise.
 */
bool batadv_forw_packet_is_rebroadcast(struct batadv_forw_packet *forw_packet)
{
        unsigned char num_bcasts = BATADV_SKB_CB(forw_packet->skb)->num_bcasts;

        return num_bcasts != forw_packet->if_outgoing->num_bcasts;
}

/**
 * batadv_send_outstanding_bcast_packet() - transmit a queued broadcast packet
 * @work: work queue item
 *
 * Transmits a queued broadcast packet and if necessary reschedules it.
 */
static void batadv_send_outstanding_bcast_packet(struct work_struct *work)
{
        unsigned long send_time = jiffies + msecs_to_jiffies(5);
        struct batadv_forw_packet *forw_packet;
        struct delayed_work *delayed_work;
        struct batadv_priv *bat_priv;
        struct sk_buff *skb1;
        bool dropped = false;

        delayed_work = to_delayed_work(work);
        forw_packet = container_of(delayed_work, struct batadv_forw_packet,
                                   delayed_work);
        bat_priv = netdev_priv(forw_packet->if_incoming->soft_iface);

        if (atomic_read(&bat_priv->mesh_state) == BATADV_MESH_DEACTIVATING) {
                dropped = true;
                goto out;
        }

        if (batadv_dat_drop_broadcast_packet(bat_priv, forw_packet)) {
                dropped = true;
                goto out;
        }

        /* send a copy of the saved skb */
        skb1 = skb_clone(forw_packet->skb, GFP_ATOMIC);
        if (!skb1)
                goto out;

        batadv_send_broadcast_skb(skb1, forw_packet->if_outgoing);
        batadv_forw_packet_bcasts_dec(forw_packet);

        if (batadv_forw_packet_bcasts_left(forw_packet)) {
                batadv_forw_packet_bcast_queue(bat_priv, forw_packet,
                                               send_time);
                return;
        }

out:
        /* do we get something for free()? */
        if (batadv_forw_packet_steal(forw_packet,
                                     &bat_priv->forw_bcast_list_lock))
                batadv_forw_packet_free(forw_packet, dropped);
}

/**
 * batadv_purge_outstanding_packets() - stop/purge scheduled bcast/OGMv1 packets
 * @bat_priv: the bat priv with all the soft interface information
 * @hard_iface: the hard interface to cancel and purge bcast/ogm packets on
 *
 * This method cancels and purges any broadcast and OGMv1 packet on the given
 * hard_iface. If hard_iface is NULL, broadcast and OGMv1 packets on all hard
 * interfaces will be canceled and purged.
 *
 * This function might sleep.
 */
void
batadv_purge_outstanding_packets(struct batadv_priv *bat_priv,
                                 const struct batadv_hard_iface *hard_iface)
{
        struct hlist_head head = HLIST_HEAD_INIT;

        if (hard_iface)
                batadv_dbg(BATADV_DBG_BATMAN, bat_priv,
                           "%s(): %s\n",
                           __func__, hard_iface->net_dev->name);
        else
                batadv_dbg(BATADV_DBG_BATMAN, bat_priv,
                           "%s()\n", __func__);

        /* claim bcast list for free() */
        spin_lock_bh(&bat_priv->forw_bcast_list_lock);
        batadv_forw_packet_list_steal(&bat_priv->forw_bcast_list, &head,
                                      hard_iface);
        spin_unlock_bh(&bat_priv->forw_bcast_list_lock);

        /* claim batman packet list for free() */
        spin_lock_bh(&bat_priv->forw_bat_list_lock);
        batadv_forw_packet_list_steal(&bat_priv->forw_bat_list, &head,
                                      hard_iface);
        spin_unlock_bh(&bat_priv->forw_bat_list_lock);

        /* then cancel or wait for packet workers to finish and free */
        batadv_forw_packet_list_free(&head);
}













































































































































































































































































































































































































































































































































































    5 



    5 









































































































    5 


    5 








    5 



















    5 








    5 




















    5 





    5 

    5 
    5 







    5 










    5 






    5 








    5 




    5 



    5 
    5 






    5 






























































    5 




    5 




    5 

























































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
// SPDX-License-Identifier: GPL-2.0
/*
 *  MQ Deadline i/o scheduler - adaptation of the legacy deadline scheduler,
 *  for the blk-mq scheduling framework
 *
 *  Copyright (C) 2016 Jens Axboe <axboe@kernel.dk>
 */
#include <linux/kernel.h>
#include <linux/fs.h>
#include <linux/blkdev.h>
#include <linux/blk-mq.h>
#include <linux/elevator.h>
#include <linux/bio.h>
#include <linux/module.h>
#include <linux/slab.h>
#include <linux/init.h>
#include <linux/compiler.h>
#include <linux/rbtree.h>
#include <linux/sbitmap.h>

#include <trace/events/block.h>

#include "blk.h"
#include "blk-mq.h"
#include "blk-mq-debugfs.h"
#include "blk-mq-tag.h"
#include "blk-mq-sched.h"

/*
 * See Documentation/block/deadline-iosched.rst
 */
static const int read_expire = HZ / 2;  /* max time before a read is submitted. */
static const int write_expire = 5 * HZ; /* ditto for writes, these limits are SOFT! */
static const int writes_starved = 2;    /* max times reads can starve a write */
static const int fifo_batch = 16;       /* # of sequential requests treated as one
                                     by the above parameters. For throughput. */

enum dd_data_dir {
        DD_READ                = READ,
        DD_WRITE        = WRITE,
};

enum { DD_DIR_COUNT = 2 };

enum dd_prio {
        DD_RT_PRIO        = 0,
        DD_BE_PRIO        = 1,
        DD_IDLE_PRIO        = 2,
        DD_PRIO_MAX        = 2,
};

enum { DD_PRIO_COUNT = 3 };

/* I/O statistics per I/O priority. */
struct io_stats_per_prio {
        local_t inserted;
        local_t merged;
        local_t dispatched;
        local_t completed;
};

/* I/O statistics for all I/O priorities (enum dd_prio). */
struct io_stats {
        struct io_stats_per_prio stats[DD_PRIO_COUNT];
};

/*
 * Deadline scheduler data per I/O priority (enum dd_prio). Requests are
 * present on both sort_list[] and fifo_list[].
 */
struct dd_per_prio {
        struct list_head dispatch;
        struct rb_root sort_list[DD_DIR_COUNT];
        struct list_head fifo_list[DD_DIR_COUNT];
        /* Next request in FIFO order. Read, write or both are NULL. */
        struct request *next_rq[DD_DIR_COUNT];
};

struct deadline_data {
        /*
         * run time data
         */

        struct dd_per_prio per_prio[DD_PRIO_COUNT];

        /* Data direction of latest dispatched request. */
        enum dd_data_dir last_dir;
        unsigned int batching;                /* number of sequential requests made */
        unsigned int starved;                /* times reads have starved writes */

        struct io_stats __percpu *stats;

        /*
         * settings that change how the i/o scheduler behaves
         */
        int fifo_expire[DD_DIR_COUNT];
        int fifo_batch;
        int writes_starved;
        int front_merges;
        u32 async_depth;

        spinlock_t lock;
        spinlock_t zone_lock;
};

/* Count one event of type 'event_type' and with I/O priority 'prio' */
#define dd_count(dd, event_type, prio) do {                                \
        struct io_stats *io_stats = get_cpu_ptr((dd)->stats);                \
                                                                        \
        BUILD_BUG_ON(!__same_type((dd), struct deadline_data *));        \
        BUILD_BUG_ON(!__same_type((prio), enum dd_prio));                \
        local_inc(&io_stats->stats[(prio)].event_type);                        \
        put_cpu_ptr(io_stats);                                                \
} while (0)

/*
 * Returns the total number of dd_count(dd, event_type, prio) calls across all
 * CPUs. No locking or barriers since it is fine if the returned sum is slightly
 * outdated.
 */
#define dd_sum(dd, event_type, prio) ({                                        \
        unsigned int cpu;                                                \
        u32 sum = 0;                                                        \
                                                                        \
        BUILD_BUG_ON(!__same_type((dd), struct deadline_data *));        \
        BUILD_BUG_ON(!__same_type((prio), enum dd_prio));                \
        for_each_present_cpu(cpu)                                        \
                sum += local_read(&per_cpu_ptr((dd)->stats, cpu)->        \
                                  stats[(prio)].event_type);                \
        sum;                                                                \
})

/* Maps an I/O priority class to a deadline scheduler priority. */
static const enum dd_prio ioprio_class_to_prio[] = {
        [IOPRIO_CLASS_NONE]        = DD_BE_PRIO,
        [IOPRIO_CLASS_RT]        = DD_RT_PRIO,
        [IOPRIO_CLASS_BE]        = DD_BE_PRIO,
        [IOPRIO_CLASS_IDLE]        = DD_IDLE_PRIO,
};

static inline struct rb_root *
deadline_rb_root(struct dd_per_prio *per_prio, struct request *rq)
{
        return &per_prio->sort_list[rq_data_dir(rq)];
}

/*
 * Returns the I/O priority class (IOPRIO_CLASS_*) that has been assigned to a
 * request.
 */
static u8 dd_rq_ioclass(struct request *rq)
{
        return IOPRIO_PRIO_CLASS(req_get_ioprio(rq));
}

/*
 * get the request before `rq' in sector-sorted order
 */
static inline struct request *
deadline_earlier_request(struct request *rq)
{
        struct rb_node *node = rb_prev(&rq->rb_node);

        if (node)
                return rb_entry_rq(node);

        return NULL;
}

/*
 * get the request after `rq' in sector-sorted order
 */
static inline struct request *
deadline_latter_request(struct request *rq)
{
        struct rb_node *node = rb_next(&rq->rb_node);

        if (node)
                return rb_entry_rq(node);

        return NULL;
}

static void
deadline_add_rq_rb(struct dd_per_prio *per_prio, struct request *rq)
{
        struct rb_root *root = deadline_rb_root(per_prio, rq);

        elv_rb_add(root, rq);
}

static inline void
deadline_del_rq_rb(struct dd_per_prio *per_prio, struct request *rq)
{
        const enum dd_data_dir data_dir = rq_data_dir(rq);

        if (per_prio->next_rq[data_dir] == rq)
                per_prio->next_rq[data_dir] = deadline_latter_request(rq);

        elv_rb_del(deadline_rb_root(per_prio, rq), rq);
}

/*
 * remove rq from rbtree and fifo.
 */
static void deadline_remove_request(struct request_queue *q,
                                    struct dd_per_prio *per_prio,
                                    struct request *rq)
{
        list_del_init(&rq->queuelist);

        /*
         * We might not be on the rbtree, if we are doing an insert merge
         */
        if (!RB_EMPTY_NODE(&rq->rb_node))
                deadline_del_rq_rb(per_prio, rq);

        elv_rqhash_del(q, rq);
        if (q->last_merge == rq)
                q->last_merge = NULL;
}

static void dd_request_merged(struct request_queue *q, struct request *req,
                              enum elv_merge type)
{
        struct deadline_data *dd = q->elevator->elevator_data;
        const u8 ioprio_class = dd_rq_ioclass(req);
        const enum dd_prio prio = ioprio_class_to_prio[ioprio_class];
        struct dd_per_prio *per_prio = &dd->per_prio[prio];

        /*
         * if the merge was a front merge, we need to reposition request
         */
        if (type == ELEVATOR_FRONT_MERGE) {
                elv_rb_del(deadline_rb_root(per_prio, req), req);
                deadline_add_rq_rb(per_prio, req);
        }
}

/*
 * Callback function that is invoked after @next has been merged into @req.
 */
static void dd_merged_requests(struct request_queue *q, struct request *req,
                               struct request *next)
{
        struct deadline_data *dd = q->elevator->elevator_data;
        const u8 ioprio_class = dd_rq_ioclass(next);
        const enum dd_prio prio = ioprio_class_to_prio[ioprio_class];

        dd_count(dd, merged, prio);

        /*
         * if next expires before rq, assign its expire time to rq
         * and move into next position (next will be deleted) in fifo
         */
        if (!list_empty(&req->queuelist) && !list_empty(&next->queuelist)) {
                if (time_before((unsigned long)next->fifo_time,
                                (unsigned long)req->fifo_time)) {
                        list_move(&req->queuelist, &next->queuelist);
                        req->fifo_time = next->fifo_time;
                }
        }

        /*
         * kill knowledge of next, this one is a goner
         */
        deadline_remove_request(q, &dd->per_prio[prio], next);
}

/*
 * move an entry to dispatch queue
 */
static void
deadline_move_request(struct deadline_data *dd, struct dd_per_prio *per_prio,
                      struct request *rq)
{
        const enum dd_data_dir data_dir = rq_data_dir(rq);

        per_prio->next_rq[data_dir] = deadline_latter_request(rq);

        /*
         * take it off the sort and fifo list
         */
        deadline_remove_request(rq->q, per_prio, rq);
}

/*
 * deadline_check_fifo returns 0 if there are no expired requests on the fifo,
 * 1 otherwise. Requires !list_empty(&dd->fifo_list[data_dir])
 */
static inline int deadline_check_fifo(struct dd_per_prio *per_prio,
                                      enum dd_data_dir data_dir)
{
        struct request *rq = rq_entry_fifo(per_prio->fifo_list[data_dir].next);

        /*
         * rq is expired!
         */
        if (time_after_eq(jiffies, (unsigned long)rq->fifo_time))
                return 1;

        return 0;
}

/*
 * Check if rq has a sequential request preceding it.
 */
static bool deadline_is_seq_write(struct deadline_data *dd, struct request *rq)
{
        struct request *prev = deadline_earlier_request(rq);

        if (!prev)
                return false;

        return blk_rq_pos(prev) + blk_rq_sectors(prev) == blk_rq_pos(rq);
}

/*
 * Skip all write requests that are sequential from @rq, even if we cross
 * a zone boundary.
 */
static struct request *deadline_skip_seq_writes(struct deadline_data *dd,
                                                struct request *rq)
{
        sector_t pos = blk_rq_pos(rq);
        sector_t skipped_sectors = 0;

        while (rq) {
                if (blk_rq_pos(rq) != pos + skipped_sectors)
                        break;
                skipped_sectors += blk_rq_sectors(rq);
                rq = deadline_latter_request(rq);
        }

        return rq;
}

/*
 * For the specified data direction, return the next request to
 * dispatch using arrival ordered lists.
 */
static struct request *
deadline_fifo_request(struct deadline_data *dd, struct dd_per_prio *per_prio,
                      enum dd_data_dir data_dir)
{
        struct request *rq;
        unsigned long flags;

        if (list_empty(&per_prio->fifo_list[data_dir]))
                return NULL;

        rq = rq_entry_fifo(per_prio->fifo_list[data_dir].next);
        if (data_dir == DD_READ || !blk_queue_is_zoned(rq->q))
                return rq;

        /*
         * Look for a write request that can be dispatched, that is one with
         * an unlocked target zone. For some HDDs, breaking a sequential
         * write stream can lead to lower throughput, so make sure to preserve
         * sequential write streams, even if that stream crosses into the next
         * zones and these zones are unlocked.
         */
        spin_lock_irqsave(&dd->zone_lock, flags);
        list_for_each_entry(rq, &per_prio->fifo_list[DD_WRITE], queuelist) {
                if (blk_req_can_dispatch_to_zone(rq) &&
                    (blk_queue_nonrot(rq->q) ||
                     !deadline_is_seq_write(dd, rq)))
                        goto out;
        }
        rq = NULL;
out:
        spin_unlock_irqrestore(&dd->zone_lock, flags);

        return rq;
}

/*
 * For the specified data direction, return the next request to
 * dispatch using sector position sorted lists.
 */
static struct request *
deadline_next_request(struct deadline_data *dd, struct dd_per_prio *per_prio,
                      enum dd_data_dir data_dir)
{
        struct request *rq;
        unsigned long flags;

        rq = per_prio->next_rq[data_dir];
        if (!rq)
                return NULL;

        if (data_dir == DD_READ || !blk_queue_is_zoned(rq->q))
                return rq;

        /*
         * Look for a write request that can be dispatched, that is one with
         * an unlocked target zone. For some HDDs, breaking a sequential
         * write stream can lead to lower throughput, so make sure to preserve
         * sequential write streams, even if that stream crosses into the next
         * zones and these zones are unlocked.
         */
        spin_lock_irqsave(&dd->zone_lock, flags);
        while (rq) {
                if (blk_req_can_dispatch_to_zone(rq))
                        break;
                if (blk_queue_nonrot(rq->q))
                        rq = deadline_latter_request(rq);
                else
                        rq = deadline_skip_seq_writes(dd, rq);
        }
        spin_unlock_irqrestore(&dd->zone_lock, flags);

        return rq;
}

/*
 * deadline_dispatch_requests selects the best request according to
 * read/write expire, fifo_batch, etc
 */
static struct request *__dd_dispatch_request(struct deadline_data *dd,
                                             struct dd_per_prio *per_prio)
{
        struct request *rq, *next_rq;
        enum dd_data_dir data_dir;
        enum dd_prio prio;
        u8 ioprio_class;

        lockdep_assert_held(&dd->lock);

        if (!list_empty(&per_prio->dispatch)) {
                rq = list_first_entry(&per_prio->dispatch, struct request,
                                      queuelist);
                list_del_init(&rq->queuelist);
                goto done;
        }

        /*
         * batches are currently reads XOR writes
         */
        rq = deadline_next_request(dd, per_prio, dd->last_dir);
        if (rq && dd->batching < dd->fifo_batch)
                /* we have a next request are still entitled to batch */
                goto dispatch_request;

        /*
         * at this point we are not running a batch. select the appropriate
         * data direction (read / write)
         */

        if (!list_empty(&per_prio->fifo_list[DD_READ])) {
                BUG_ON(RB_EMPTY_ROOT(&per_prio->sort_list[DD_READ]));

                if (deadline_fifo_request(dd, per_prio, DD_WRITE) &&
                    (dd->starved++ >= dd->writes_starved))
                        goto dispatch_writes;

                data_dir = DD_READ;

                goto dispatch_find_request;
        }

        /*
         * there are either no reads or writes have been starved
         */

        if (!list_empty(&per_prio->fifo_list[DD_WRITE])) {
dispatch_writes:
                BUG_ON(RB_EMPTY_ROOT(&per_prio->sort_list[DD_WRITE]));

                dd->starved = 0;

                data_dir = DD_WRITE;

                goto dispatch_find_request;
        }

        return NULL;

dispatch_find_request:
        /*
         * we are not running a batch, find best request for selected data_dir
         */
        next_rq = deadline_next_request(dd, per_prio, data_dir);
        if (deadline_check_fifo(per_prio, data_dir) || !next_rq) {
                /*
                 * A deadline has expired, the last request was in the other
                 * direction, or we have run out of higher-sectored requests.
                 * Start again from the request with the earliest expiry time.
                 */
                rq = deadline_fifo_request(dd, per_prio, data_dir);
        } else {
                /*
                 * The last req was the same dir and we have a next request in
                 * sort order. No expired requests so continue on from here.
                 */
                rq = next_rq;
        }

        /*
         * For a zoned block device, if we only have writes queued and none of
         * them can be dispatched, rq will be NULL.
         */
        if (!rq)
                return NULL;

        dd->last_dir = data_dir;
        dd->batching = 0;

dispatch_request:
        /*
         * rq is the selected appropriate request.
         */
        dd->batching++;
        deadline_move_request(dd, per_prio, rq);
done:
        ioprio_class = dd_rq_ioclass(rq);
        prio = ioprio_class_to_prio[ioprio_class];
        dd_count(dd, dispatched, prio);
        /*
         * If the request needs its target zone locked, do it.
         */
        blk_req_zone_write_lock(rq);
        rq->rq_flags |= RQF_STARTED;
        return rq;
}

/*
 * Called from blk_mq_run_hw_queue() -> __blk_mq_sched_dispatch_requests().
 *
 * One confusing aspect here is that we get called for a specific
 * hardware queue, but we may return a request that is for a
 * different hardware queue. This is because mq-deadline has shared
 * state for all hardware queues, in terms of sorting, FIFOs, etc.
 */
static struct request *dd_dispatch_request(struct blk_mq_hw_ctx *hctx)
{
        struct deadline_data *dd = hctx->queue->elevator->elevator_data;
        struct request *rq;
        enum dd_prio prio;

        spin_lock(&dd->lock);
        for (prio = 0; prio <= DD_PRIO_MAX; prio++) {
                rq = __dd_dispatch_request(dd, &dd->per_prio[prio]);
                if (rq)
                        break;
        }
        spin_unlock(&dd->lock);

        return rq;
}

/*
 * Called by __blk_mq_alloc_request(). The shallow_depth value set by this
 * function is used by __blk_mq_get_tag().
 */
static void dd_limit_depth(unsigned int op, struct blk_mq_alloc_data *data)
{
        struct deadline_data *dd = data->q->elevator->elevator_data;

        /* Do not throttle synchronous reads. */
        if (op_is_sync(op) && !op_is_write(op))
                return;

        /*
         * Throttle asynchronous requests and writes such that these requests
         * do not block the allocation of synchronous requests.
         */
        data->shallow_depth = dd->async_depth;
}

/* Called by blk_mq_update_nr_requests(). */
static void dd_depth_updated(struct blk_mq_hw_ctx *hctx)
{
        struct request_queue *q = hctx->queue;
        struct deadline_data *dd = q->elevator->elevator_data;
        struct blk_mq_tags *tags = hctx->sched_tags;

        dd->async_depth = max(1UL, 3 * q->nr_requests / 4);

        sbitmap_queue_min_shallow_depth(tags->bitmap_tags, dd->async_depth);
}

/* Called by blk_mq_init_hctx() and blk_mq_init_sched(). */
static int dd_init_hctx(struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx)
{
        dd_depth_updated(hctx);
        return 0;
}

static void dd_exit_sched(struct elevator_queue *e)
{
        struct deadline_data *dd = e->elevator_data;
        enum dd_prio prio;

        for (prio = 0; prio <= DD_PRIO_MAX; prio++) {
                struct dd_per_prio *per_prio = &dd->per_prio[prio];

                WARN_ON_ONCE(!list_empty(&per_prio->fifo_list[DD_READ]));
                WARN_ON_ONCE(!list_empty(&per_prio->fifo_list[DD_WRITE]));
        }

        free_percpu(dd->stats);

        kfree(dd);
}

/*
 * initialize elevator private data (deadline_data).
 */
static int dd_init_sched(struct request_queue *q, struct elevator_type *e)
{
        struct deadline_data *dd;
        struct elevator_queue *eq;
        enum dd_prio prio;
        int ret = -ENOMEM;

        eq = elevator_alloc(q, e);
        if (!eq)
                return ret;

        dd = kzalloc_node(sizeof(*dd), GFP_KERNEL, q->node);
        if (!dd)
                goto put_eq;

        eq->elevator_data = dd;

        dd->stats = alloc_percpu_gfp(typeof(*dd->stats),
                                     GFP_KERNEL | __GFP_ZERO);
        if (!dd->stats)
                goto free_dd;

        for (prio = 0; prio <= DD_PRIO_MAX; prio++) {
                struct dd_per_prio *per_prio = &dd->per_prio[prio];

                INIT_LIST_HEAD(&per_prio->dispatch);
                INIT_LIST_HEAD(&per_prio->fifo_list[DD_READ]);
                INIT_LIST_HEAD(&per_prio->fifo_list[DD_WRITE]);
                per_prio->sort_list[DD_READ] = RB_ROOT;
                per_prio->sort_list[DD_WRITE] = RB_ROOT;
        }
        dd->fifo_expire[DD_READ] = read_expire;
        dd->fifo_expire[DD_WRITE] = write_expire;
        dd->writes_starved = writes_starved;
        dd->front_merges = 1;
        dd->last_dir = DD_WRITE;
        dd->fifo_batch = fifo_batch;
        spin_lock_init(&dd->lock);
        spin_lock_init(&dd->zone_lock);

        q->elevator = eq;
        return 0;

free_dd:
        kfree(dd);

put_eq:
        kobject_put(&eq->kobj);
        return ret;
}

/*
 * Try to merge @bio into an existing request. If @bio has been merged into
 * an existing request, store the pointer to that request into *@rq.
 */
static int dd_request_merge(struct request_queue *q, struct request **rq,
                            struct bio *bio)
{
        struct deadline_data *dd = q->elevator->elevator_data;
        const u8 ioprio_class = IOPRIO_PRIO_CLASS(bio->bi_ioprio);
        const enum dd_prio prio = ioprio_class_to_prio[ioprio_class];
        struct dd_per_prio *per_prio = &dd->per_prio[prio];
        sector_t sector = bio_end_sector(bio);
        struct request *__rq;

        if (!dd->front_merges)
                return ELEVATOR_NO_MERGE;

        __rq = elv_rb_find(&per_prio->sort_list[bio_data_dir(bio)], sector);
        if (__rq) {
                BUG_ON(sector != blk_rq_pos(__rq));

                if (elv_bio_merge_ok(__rq, bio)) {
                        *rq = __rq;
                        if (blk_discard_mergable(__rq))
                                return ELEVATOR_DISCARD_MERGE;
                        return ELEVATOR_FRONT_MERGE;
                }
        }

        return ELEVATOR_NO_MERGE;
}

/*
 * Attempt to merge a bio into an existing request. This function is called
 * before @bio is associated with a request.
 */
static bool dd_bio_merge(struct request_queue *q, struct bio *bio,
                unsigned int nr_segs)
{
        struct deadline_data *dd = q->elevator->elevator_data;
        struct request *free = NULL;
        bool ret;

        spin_lock(&dd->lock);
        ret = blk_mq_sched_try_merge(q, bio, nr_segs, &free);
        spin_unlock(&dd->lock);

        if (free)
                blk_mq_free_request(free);

        return ret;
}

/*
 * add rq to rbtree and fifo
 */
static void dd_insert_request(struct blk_mq_hw_ctx *hctx, struct request *rq,
                              bool at_head)
{
        struct request_queue *q = hctx->queue;
        struct deadline_data *dd = q->elevator->elevator_data;
        const enum dd_data_dir data_dir = rq_data_dir(rq);
        u16 ioprio = req_get_ioprio(rq);
        u8 ioprio_class = IOPRIO_PRIO_CLASS(ioprio);
        struct dd_per_prio *per_prio;
        enum dd_prio prio;
        LIST_HEAD(free);

        lockdep_assert_held(&dd->lock);

        /*
         * This may be a requeue of a write request that has locked its
         * target zone. If it is the case, this releases the zone lock.
         */
        blk_req_zone_write_unlock(rq);

        prio = ioprio_class_to_prio[ioprio_class];
        dd_count(dd, inserted, prio);
        rq->elv.priv[0] = (void *)(uintptr_t)1;

        if (blk_mq_sched_try_insert_merge(q, rq, &free)) {
                blk_mq_free_requests(&free);
                return;
        }

        trace_block_rq_insert(rq);

        per_prio = &dd->per_prio[prio];
        if (at_head) {
                list_add(&rq->queuelist, &per_prio->dispatch);
        } else {
                deadline_add_rq_rb(per_prio, rq);

                if (rq_mergeable(rq)) {
                        elv_rqhash_add(q, rq);
                        if (!q->last_merge)
                                q->last_merge = rq;
                }

                /*
                 * set expire time and add to fifo list
                 */
                rq->fifo_time = jiffies + dd->fifo_expire[data_dir];
                list_add_tail(&rq->queuelist, &per_prio->fifo_list[data_dir]);
        }
}

/*
 * Called from blk_mq_sched_insert_request() or blk_mq_sched_insert_requests().
 */
static void dd_insert_requests(struct blk_mq_hw_ctx *hctx,
                               struct list_head *list, bool at_head)
{
        struct request_queue *q = hctx->queue;
        struct deadline_data *dd = q->elevator->elevator_data;

        spin_lock(&dd->lock);
        while (!list_empty(list)) {
                struct request *rq;

                rq = list_first_entry(list, struct request, queuelist);
                list_del_init(&rq->queuelist);
                dd_insert_request(hctx, rq, at_head);
        }
        spin_unlock(&dd->lock);
}

/* Callback from inside blk_mq_rq_ctx_init(). */
static void dd_prepare_request(struct request *rq)
{
        rq->elv.priv[0] = NULL;
}

static bool dd_has_write_work(struct blk_mq_hw_ctx *hctx)
{
        struct deadline_data *dd = hctx->queue->elevator->elevator_data;
        enum dd_prio p;

        for (p = 0; p <= DD_PRIO_MAX; p++)
                if (!list_empty_careful(&dd->per_prio[p].fifo_list[DD_WRITE]))
                        return true;

        return false;
}

/*
 * Callback from inside blk_mq_free_request().
 *
 * For zoned block devices, write unlock the target zone of
 * completed write requests. Do this while holding the zone lock
 * spinlock so that the zone is never unlocked while deadline_fifo_request()
 * or deadline_next_request() are executing. This function is called for
 * all requests, whether or not these requests complete successfully.
 *
 * For a zoned block device, __dd_dispatch_request() may have stopped
 * dispatching requests if all the queued requests are write requests directed
 * at zones that are already locked due to on-going write requests. To ensure
 * write request dispatch progress in this case, mark the queue as needing a
 * restart to ensure that the queue is run again after completion of the
 * request and zones being unlocked.
 */
static void dd_finish_request(struct request *rq)
{
        struct request_queue *q = rq->q;
        struct deadline_data *dd = q->elevator->elevator_data;
        const u8 ioprio_class = dd_rq_ioclass(rq);
        const enum dd_prio prio = ioprio_class_to_prio[ioprio_class];

        /*
         * The block layer core may call dd_finish_request() without having
         * called dd_insert_requests(). Hence only update statistics for
         * requests for which dd_insert_requests() has been called. See also
         * blk_mq_request_bypass_insert().
         */
        if (rq->elv.priv[0])
                dd_count(dd, completed, prio);

        if (blk_queue_is_zoned(q)) {
                unsigned long flags;

                spin_lock_irqsave(&dd->zone_lock, flags);
                blk_req_zone_write_unlock(rq);
                spin_unlock_irqrestore(&dd->zone_lock, flags);

                if (dd_has_write_work(rq->mq_hctx))
                        blk_mq_sched_mark_restart_hctx(rq->mq_hctx);
        }
}

static bool dd_has_work_for_prio(struct dd_per_prio *per_prio)
{
        return !list_empty_careful(&per_prio->dispatch) ||
                !list_empty_careful(&per_prio->fifo_list[DD_READ]) ||
                !list_empty_careful(&per_prio->fifo_list[DD_WRITE]);
}

static bool dd_has_work(struct blk_mq_hw_ctx *hctx)
{
        struct deadline_data *dd = hctx->queue->elevator->elevator_data;
        enum dd_prio prio;

        for (prio = 0; prio <= DD_PRIO_MAX; prio++)
                if (dd_has_work_for_prio(&dd->per_prio[prio]))
                        return true;

        return false;
}

/*
 * sysfs parts below
 */
#define SHOW_INT(__FUNC, __VAR)                                                \
static ssize_t __FUNC(struct elevator_queue *e, char *page)                \
{                                                                        \
        struct deadline_data *dd = e->elevator_data;                        \
                                                                        \
        return sysfs_emit(page, "%d\n", __VAR);                                \
}
#define SHOW_JIFFIES(__FUNC, __VAR) SHOW_INT(__FUNC, jiffies_to_msecs(__VAR))
SHOW_JIFFIES(deadline_read_expire_show, dd->fifo_expire[DD_READ]);
SHOW_JIFFIES(deadline_write_expire_show, dd->fifo_expire[DD_WRITE]);
SHOW_INT(deadline_writes_starved_show, dd->writes_starved);
SHOW_INT(deadline_front_merges_show, dd->front_merges);
SHOW_INT(deadline_async_depth_show, dd->async_depth);
SHOW_INT(deadline_fifo_batch_show, dd->fifo_batch);
#undef SHOW_INT
#undef SHOW_JIFFIES

#define STORE_FUNCTION(__FUNC, __PTR, MIN, MAX, __CONV)                        \
static ssize_t __FUNC(struct elevator_queue *e, const char *page, size_t count)        \
{                                                                        \
        struct deadline_data *dd = e->elevator_data;                        \
        int __data, __ret;                                                \
                                                                        \
        __ret = kstrtoint(page, 0, &__data);                                \
        if (__ret < 0)                                                        \
                return __ret;                                                \
        if (__data < (MIN))                                                \
                __data = (MIN);                                                \
        else if (__data > (MAX))                                        \
                __data = (MAX);                                                \
        *(__PTR) = __CONV(__data);                                        \
        return count;                                                        \
}
#define STORE_INT(__FUNC, __PTR, MIN, MAX)                                \
        STORE_FUNCTION(__FUNC, __PTR, MIN, MAX, )
#define STORE_JIFFIES(__FUNC, __PTR, MIN, MAX)                                \
        STORE_FUNCTION(__FUNC, __PTR, MIN, MAX, msecs_to_jiffies)
STORE_JIFFIES(deadline_read_expire_store, &dd->fifo_expire[DD_READ], 0, INT_MAX);
STORE_JIFFIES(deadline_write_expire_store, &dd->fifo_expire[DD_WRITE], 0, INT_MAX);
STORE_INT(deadline_writes_starved_store, &dd->writes_starved, INT_MIN, INT_MAX);
STORE_INT(deadline_front_merges_store, &dd->front_merges, 0, 1);
STORE_INT(deadline_async_depth_store, &dd->async_depth, 1, INT_MAX);
STORE_INT(deadline_fifo_batch_store, &dd->fifo_batch, 0, INT_MAX);
#undef STORE_FUNCTION
#undef STORE_INT
#undef STORE_JIFFIES

#define DD_ATTR(name) \
        __ATTR(name, 0644, deadline_##name##_show, deadline_##name##_store)

static struct elv_fs_entry deadline_attrs[] = {
        DD_ATTR(read_expire),
        DD_ATTR(write_expire),
        DD_ATTR(writes_starved),
        DD_ATTR(front_merges),
        DD_ATTR(async_depth),
        DD_ATTR(fifo_batch),
        __ATTR_NULL
};

#ifdef CONFIG_BLK_DEBUG_FS
#define DEADLINE_DEBUGFS_DDIR_ATTRS(prio, data_dir, name)                \
static void *deadline_##name##_fifo_start(struct seq_file *m,                \
                                          loff_t *pos)                        \
        __acquires(&dd->lock)                                                \
{                                                                        \
        struct request_queue *q = m->private;                                \
        struct deadline_data *dd = q->elevator->elevator_data;                \
        struct dd_per_prio *per_prio = &dd->per_prio[prio];                \
                                                                        \
        spin_lock(&dd->lock);                                                \
        return seq_list_start(&per_prio->fifo_list[data_dir], *pos);        \
}                                                                        \
                                                                        \
static void *deadline_##name##_fifo_next(struct seq_file *m, void *v,        \
                                         loff_t *pos)                        \
{                                                                        \
        struct request_queue *q = m->private;                                \
        struct deadline_data *dd = q->elevator->elevator_data;                \
        struct dd_per_prio *per_prio = &dd->per_prio[prio];                \
                                                                        \
        return seq_list_next(v, &per_prio->fifo_list[data_dir], pos);        \
}                                                                        \
                                                                        \
static void deadline_##name##_fifo_stop(struct seq_file *m, void *v)        \
        __releases(&dd->lock)                                                \
{                                                                        \
        struct request_queue *q = m->private;                                \
        struct deadline_data *dd = q->elevator->elevator_data;                \
                                                                        \
        spin_unlock(&dd->lock);                                                \
}                                                                        \
                                                                        \
static const struct seq_operations deadline_##name##_fifo_seq_ops = {        \
        .start        = deadline_##name##_fifo_start,                                \
        .next        = deadline_##name##_fifo_next,                                \
        .stop        = deadline_##name##_fifo_stop,                                \
        .show        = blk_mq_debugfs_rq_show,                                \
};                                                                        \
                                                                        \
static int deadline_##name##_next_rq_show(void *data,                        \
                                          struct seq_file *m)                \
{                                                                        \
        struct request_queue *q = data;                                        \
        struct deadline_data *dd = q->elevator->elevator_data;                \
        struct dd_per_prio *per_prio = &dd->per_prio[prio];                \
        struct request *rq = per_prio->next_rq[data_dir];                \
                                                                        \
        if (rq)                                                                \
                __blk_mq_debugfs_rq_show(m, rq);                        \
        return 0;                                                        \
}

DEADLINE_DEBUGFS_DDIR_ATTRS(DD_RT_PRIO, DD_READ, read0);
DEADLINE_DEBUGFS_DDIR_ATTRS(DD_RT_PRIO, DD_WRITE, write0);
DEADLINE_DEBUGFS_DDIR_ATTRS(DD_BE_PRIO, DD_READ, read1);
DEADLINE_DEBUGFS_DDIR_ATTRS(DD_BE_PRIO, DD_WRITE, write1);
DEADLINE_DEBUGFS_DDIR_ATTRS(DD_IDLE_PRIO, DD_READ, read2);
DEADLINE_DEBUGFS_DDIR_ATTRS(DD_IDLE_PRIO, DD_WRITE, write2);
#undef DEADLINE_DEBUGFS_DDIR_ATTRS

static int deadline_batching_show(void *data, struct seq_file *m)
{
        struct request_queue *q = data;
        struct deadline_data *dd = q->elevator->elevator_data;

        seq_printf(m, "%u\n", dd->batching);
        return 0;
}

static int deadline_starved_show(void *data, struct seq_file *m)
{
        struct request_queue *q = data;
        struct deadline_data *dd = q->elevator->elevator_data;

        seq_printf(m, "%u\n", dd->starved);
        return 0;
}

static int dd_async_depth_show(void *data, struct seq_file *m)
{
        struct request_queue *q = data;
        struct deadline_data *dd = q->elevator->elevator_data;

        seq_printf(m, "%u\n", dd->async_depth);
        return 0;
}

/* Number of requests queued for a given priority level. */
static u32 dd_queued(struct deadline_data *dd, enum dd_prio prio)
{
        return dd_sum(dd, inserted, prio) - dd_sum(dd, completed, prio);
}

static int dd_queued_show(void *data, struct seq_file *m)
{
        struct request_queue *q = data;
        struct deadline_data *dd = q->elevator->elevator_data;

        seq_printf(m, "%u %u %u\n", dd_queued(dd, DD_RT_PRIO),
                   dd_queued(dd, DD_BE_PRIO),
                   dd_queued(dd, DD_IDLE_PRIO));
        return 0;
}

/* Number of requests owned by the block driver for a given priority. */
static u32 dd_owned_by_driver(struct deadline_data *dd, enum dd_prio prio)
{
        return dd_sum(dd, dispatched, prio) + dd_sum(dd, merged, prio)
                - dd_sum(dd, completed, prio);
}

static int dd_owned_by_driver_show(void *data, struct seq_file *m)
{
        struct request_queue *q = data;
        struct deadline_data *dd = q->elevator->elevator_data;

        seq_printf(m, "%u %u %u\n", dd_owned_by_driver(dd, DD_RT_PRIO),
                   dd_owned_by_driver(dd, DD_BE_PRIO),
                   dd_owned_by_driver(dd, DD_IDLE_PRIO));
        return 0;
}

#define DEADLINE_DISPATCH_ATTR(prio)                                        \
static void *deadline_dispatch##prio##_start(struct seq_file *m,        \
                                             loff_t *pos)                \
        __acquires(&dd->lock)                                                \
{                                                                        \
        struct request_queue *q = m->private;                                \
        struct deadline_data *dd = q->elevator->elevator_data;                \
        struct dd_per_prio *per_prio = &dd->per_prio[prio];                \
                                                                        \
        spin_lock(&dd->lock);                                                \
        return seq_list_start(&per_prio->dispatch, *pos);                \
}                                                                        \
                                                                        \
static void *deadline_dispatch##prio##_next(struct seq_file *m,                \
                                            void *v, loff_t *pos)        \
{                                                                        \
        struct request_queue *q = m->private;                                \
        struct deadline_data *dd = q->elevator->elevator_data;                \
        struct dd_per_prio *per_prio = &dd->per_prio[prio];                \
                                                                        \
        return seq_list_next(v, &per_prio->dispatch, pos);                \
}                                                                        \
                                                                        \
static void deadline_dispatch##prio##_stop(struct seq_file *m, void *v)        \
        __releases(&dd->lock)                                                \
{                                                                        \
        struct request_queue *q = m->private;                                \
        struct deadline_data *dd = q->elevator->elevator_data;                \
                                                                        \
        spin_unlock(&dd->lock);                                                \
}                                                                        \
                                                                        \
static const struct seq_operations deadline_dispatch##prio##_seq_ops = { \
        .start        = deadline_dispatch##prio##_start,                        \
        .next        = deadline_dispatch##prio##_next,                        \
        .stop        = deadline_dispatch##prio##_stop,                        \
        .show        = blk_mq_debugfs_rq_show,                                \
}

DEADLINE_DISPATCH_ATTR(0);
DEADLINE_DISPATCH_ATTR(1);
DEADLINE_DISPATCH_ATTR(2);
#undef DEADLINE_DISPATCH_ATTR

#define DEADLINE_QUEUE_DDIR_ATTRS(name)                                        \
        {#name "_fifo_list", 0400,                                        \
                        .seq_ops = &deadline_##name##_fifo_seq_ops}
#define DEADLINE_NEXT_RQ_ATTR(name)                                        \
        {#name "_next_rq", 0400, deadline_##name##_next_rq_show}
static const struct blk_mq_debugfs_attr deadline_queue_debugfs_attrs[] = {
        DEADLINE_QUEUE_DDIR_ATTRS(read0),
        DEADLINE_QUEUE_DDIR_ATTRS(write0),
        DEADLINE_QUEUE_DDIR_ATTRS(read1),
        DEADLINE_QUEUE_DDIR_ATTRS(write1),
        DEADLINE_QUEUE_DDIR_ATTRS(read2),
        DEADLINE_QUEUE_DDIR_ATTRS(write2),
        DEADLINE_NEXT_RQ_ATTR(read0),
        DEADLINE_NEXT_RQ_ATTR(write0),
        DEADLINE_NEXT_RQ_ATTR(read1),
        DEADLINE_NEXT_RQ_ATTR(write1),
        DEADLINE_NEXT_RQ_ATTR(read2),
        DEADLINE_NEXT_RQ_ATTR(write2),
        {"batching", 0400, deadline_batching_show},
        {"starved", 0400, deadline_starved_show},
        {"async_depth", 0400, dd_async_depth_show},
        {"dispatch0", 0400, .seq_ops = &deadline_dispatch0_seq_ops},
        {"dispatch1", 0400, .seq_ops = &deadline_dispatch1_seq_ops},
        {"dispatch2", 0400, .seq_ops = &deadline_dispatch2_seq_ops},
        {"owned_by_driver", 0400, dd_owned_by_driver_show},
        {"queued", 0400, dd_queued_show},
        {},
};
#undef DEADLINE_QUEUE_DDIR_ATTRS
#endif

static struct elevator_type mq_deadline = {
        .ops = {
                .depth_updated                = dd_depth_updated,
                .limit_depth                = dd_limit_depth,
                .insert_requests        = dd_insert_requests,
                .dispatch_request        = dd_dispatch_request,
                .prepare_request        = dd_prepare_request,
                .finish_request                = dd_finish_request,
                .next_request                = elv_rb_latter_request,
                .former_request                = elv_rb_former_request,
                .bio_merge                = dd_bio_merge,
                .request_merge                = dd_request_merge,
                .requests_merged        = dd_merged_requests,
                .request_merged                = dd_request_merged,
                .has_work                = dd_has_work,
                .init_sched                = dd_init_sched,
                .exit_sched                = dd_exit_sched,
                .init_hctx                = dd_init_hctx,
        },

#ifdef CONFIG_BLK_DEBUG_FS
        .queue_debugfs_attrs = deadline_queue_debugfs_attrs,
#endif
        .elevator_attrs = deadline_attrs,
        .elevator_name = "mq-deadline",
        .elevator_alias = "deadline",
        .elevator_features = ELEVATOR_F_ZBD_SEQ_WRITE,
        .elevator_owner = THIS_MODULE,
};
MODULE_ALIAS("mq-deadline-iosched");

static int __init deadline_init(void)
{
        return elv_register(&mq_deadline);
}

static void __exit deadline_exit(void)
{
        elv_unregister(&mq_deadline);
}

module_init(deadline_init);
module_exit(deadline_exit);

MODULE_AUTHOR("Jens Axboe, Damien Le Moal and Bart Van Assche");
MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("MQ deadline IO scheduler");



























































































































































    8 

 2257 




 2260 





































 12267 




















  127 
 1694 




 2261 

















 2245 




 12196 















   96 
 12186 










 3114 



 11650 



























 1564 




 12071 
















   28 
 12065 









 4798 















 4798 










1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
/* SPDX-License-Identifier: GPL-2.0 */
/*
 * Variant of atomic_t specialized for reference counts.
 *
 * The interface matches the atomic_t interface (to aid in porting) but only
 * provides the few functions one should use for reference counting.
 *
 * Saturation semantics
 * ====================
 *
 * refcount_t differs from atomic_t in that the counter saturates at
 * REFCOUNT_SATURATED and will not move once there. This avoids wrapping the
 * counter and causing 'spurious' use-after-free issues. In order to avoid the
 * cost associated with introducing cmpxchg() loops into all of the saturating
 * operations, we temporarily allow the counter to take on an unchecked value
 * and then explicitly set it to REFCOUNT_SATURATED on detecting that underflow
 * or overflow has occurred. Although this is racy when multiple threads
 * access the refcount concurrently, by placing REFCOUNT_SATURATED roughly
 * equidistant from 0 and INT_MAX we minimise the scope for error:
 *
 *                                    INT_MAX     REFCOUNT_SATURATED   UINT_MAX
 *   0                          (0x7fff_ffff)    (0xc000_0000)    (0xffff_ffff)
 *   +--------------------------------+----------------+----------------+
 *                                     <---------- bad value! ---------->
 *
 * (in a signed view of the world, the "bad value" range corresponds to
 * a negative counter value).
 *
 * As an example, consider a refcount_inc() operation that causes the counter
 * to overflow:
 *
 *         int old = atomic_fetch_add_relaxed(r);
 *        // old is INT_MAX, refcount now INT_MIN (0x8000_0000)
 *        if (old < 0)
 *                atomic_set(r, REFCOUNT_SATURATED);
 *
 * If another thread also performs a refcount_inc() operation between the two
 * atomic operations, then the count will continue to edge closer to 0. If it
 * reaches a value of 1 before /any/ of the threads reset it to the saturated
 * value, then a concurrent refcount_dec_and_test() may erroneously free the
 * underlying object.
 * Linux limits the maximum number of tasks to PID_MAX_LIMIT, which is currently
 * 0x400000 (and can't easily be raised in the future beyond FUTEX_TID_MASK).
 * With the current PID limit, if no batched refcounting operations are used and
 * the attacker can't repeatedly trigger kernel oopses in the middle of refcount
 * operations, this makes it impossible for a saturated refcount to leave the
 * saturation range, even if it is possible for multiple uses of the same
 * refcount to nest in the context of a single task:
 *
 *     (UINT_MAX+1-REFCOUNT_SATURATED) / PID_MAX_LIMIT =
 *     0x40000000 / 0x400000 = 0x100 = 256
 *
 * If hundreds of references are added/removed with a single refcounting
 * operation, it may potentially be possible to leave the saturation range; but
 * given the precise timing details involved with the round-robin scheduling of
 * each thread manipulating the refcount and the need to hit the race multiple
 * times in succession, there doesn't appear to be a practical avenue of attack
 * even if using refcount_add() operations with larger increments.
 *
 * Memory ordering
 * ===============
 *
 * Memory ordering rules are slightly relaxed wrt regular atomic_t functions
 * and provide only what is strictly required for refcounts.
 *
 * The increments are fully relaxed; these will not provide ordering. The
 * rationale is that whatever is used to obtain the object we're increasing the
 * reference count on will provide the ordering. For locked data structures,
 * its the lock acquire, for RCU/lockless data structures its the dependent
 * load.
 *
 * Do note that inc_not_zero() provides a control dependency which will order
 * future stores against the inc, this ensures we'll never modify the object
 * if we did not in fact acquire a reference.
 *
 * The decrements will provide release order, such that all the prior loads and
 * stores will be issued before, it also provides a control dependency, which
 * will order us against the subsequent free().
 *
 * The control dependency is against the load of the cmpxchg (ll/sc) that
 * succeeded. This means the stores aren't fully ordered, but this is fine
 * because the 1->0 transition indicates no concurrency.
 *
 * Note that the allocator is responsible for ordering things between free()
 * and alloc().
 *
 * The decrements dec_and_test() and sub_and_test() also provide acquire
 * ordering on success.
 *
 */

#ifndef _LINUX_REFCOUNT_H
#define _LINUX_REFCOUNT_H

#include <linux/atomic.h>
#include <linux/bug.h>
#include <linux/compiler.h>
#include <linux/limits.h>
#include <linux/spinlock_types.h>

struct mutex;

/**
 * typedef refcount_t - variant of atomic_t specialized for reference counts
 * @refs: atomic_t counter field
 *
 * The counter saturates at REFCOUNT_SATURATED and will not move once
 * there. This avoids wrapping the counter and causing 'spurious'
 * use-after-free bugs.
 */
typedef struct refcount_struct {
        atomic_t refs;
} refcount_t;

#define REFCOUNT_INIT(n)        { .refs = ATOMIC_INIT(n), }
#define REFCOUNT_MAX                INT_MAX
#define REFCOUNT_SATURATED        (INT_MIN / 2)

enum refcount_saturation_type {
        REFCOUNT_ADD_NOT_ZERO_OVF,
        REFCOUNT_ADD_OVF,
        REFCOUNT_ADD_UAF,
        REFCOUNT_SUB_UAF,
        REFCOUNT_DEC_LEAK,
};

void refcount_warn_saturate(refcount_t *r, enum refcount_saturation_type t);

/**
 * refcount_set - set a refcount's value
 * @r: the refcount
 * @n: value to which the refcount will be set
 */
static inline void refcount_set(refcount_t *r, int n)
{
        atomic_set(&r->refs, n);
}

/**
 * refcount_read - get a refcount's value
 * @r: the refcount
 *
 * Return: the refcount's value
 */
static inline unsigned int refcount_read(const refcount_t *r)
{
        return atomic_read(&r->refs);
}

static inline __must_check bool __refcount_add_not_zero(int i, refcount_t *r, int *oldp)
{
        int old = refcount_read(r);

        do {
                if (!old)
                        break;
        } while (!atomic_try_cmpxchg_relaxed(&r->refs, &old, old + i));

        if (oldp)
                *oldp = old;

        if (unlikely(old < 0 || old + i < 0))
                refcount_warn_saturate(r, REFCOUNT_ADD_NOT_ZERO_OVF);

        return old;
}

/**
 * refcount_add_not_zero - add a value to a refcount unless it is 0
 * @i: the value to add to the refcount
 * @r: the refcount
 *
 * Will saturate at REFCOUNT_SATURATED and WARN.
 *
 * Provides no memory ordering, it is assumed the caller has guaranteed the
 * object memory to be stable (RCU, etc.). It does provide a control dependency
 * and thereby orders future stores. See the comment on top.
 *
 * Use of this function is not recommended for the normal reference counting
 * use case in which references are taken and released one at a time.  In these
 * cases, refcount_inc(), or one of its variants, should instead be used to
 * increment a reference count.
 *
 * Return: false if the passed refcount is 0, true otherwise
 */
static inline __must_check bool refcount_add_not_zero(int i, refcount_t *r)
{
        return __refcount_add_not_zero(i, r, NULL);
}

static inline void __refcount_add(int i, refcount_t *r, int *oldp)
{
        int old = atomic_fetch_add_relaxed(i, &r->refs);

        if (oldp)
                *oldp = old;

        if (unlikely(!old))
                refcount_warn_saturate(r, REFCOUNT_ADD_UAF);
        else if (unlikely(old < 0 || old + i < 0))
                refcount_warn_saturate(r, REFCOUNT_ADD_OVF);
}

/**
 * refcount_add - add a value to a refcount
 * @i: the value to add to the refcount
 * @r: the refcount
 *
 * Similar to atomic_add(), but will saturate at REFCOUNT_SATURATED and WARN.
 *
 * Provides no memory ordering, it is assumed the caller has guaranteed the
 * object memory to be stable (RCU, etc.). It does provide a control dependency
 * and thereby orders future stores. See the comment on top.
 *
 * Use of this function is not recommended for the normal reference counting
 * use case in which references are taken and released one at a time.  In these
 * cases, refcount_inc(), or one of its variants, should instead be used to
 * increment a reference count.
 */
static inline void refcount_add(int i, refcount_t *r)
{
        __refcount_add(i, r, NULL);
}

static inline __must_check bool __refcount_inc_not_zero(refcount_t *r, int *oldp)
{
        return __refcount_add_not_zero(1, r, oldp);
}

/**
 * refcount_inc_not_zero - increment a refcount unless it is 0
 * @r: the refcount to increment
 *
 * Similar to atomic_inc_not_zero(), but will saturate at REFCOUNT_SATURATED
 * and WARN.
 *
 * Provides no memory ordering, it is assumed the caller has guaranteed the
 * object memory to be stable (RCU, etc.). It does provide a control dependency
 * and thereby orders future stores. See the comment on top.
 *
 * Return: true if the increment was successful, false otherwise
 */
static inline __must_check bool refcount_inc_not_zero(refcount_t *r)
{
        return __refcount_inc_not_zero(r, NULL);
}

static inline void __refcount_inc(refcount_t *r, int *oldp)
{
        __refcount_add(1, r, oldp);
}

/**
 * refcount_inc - increment a refcount
 * @r: the refcount to increment
 *
 * Similar to atomic_inc(), but will saturate at REFCOUNT_SATURATED and WARN.
 *
 * Provides no memory ordering, it is assumed the caller already has a
 * reference on the object.
 *
 * Will WARN if the refcount is 0, as this represents a possible use-after-free
 * condition.
 */
static inline void refcount_inc(refcount_t *r)
{
        __refcount_inc(r, NULL);
}

static inline __must_check bool __refcount_sub_and_test(int i, refcount_t *r, int *oldp)
{
        int old = atomic_fetch_sub_release(i, &r->refs);

        if (oldp)
                *oldp = old;

        if (old == i) {
                smp_acquire__after_ctrl_dep();
                return true;
        }

        if (unlikely(old < 0 || old - i < 0))
                refcount_warn_saturate(r, REFCOUNT_SUB_UAF);

        return false;
}

/**
 * refcount_sub_and_test - subtract from a refcount and test if it is 0
 * @i: amount to subtract from the refcount
 * @r: the refcount
 *
 * Similar to atomic_dec_and_test(), but it will WARN, return false and
 * ultimately leak on underflow and will fail to decrement when saturated
 * at REFCOUNT_SATURATED.
 *
 * Provides release memory ordering, such that prior loads and stores are done
 * before, and provides an acquire ordering on success such that free()
 * must come after.
 *
 * Use of this function is not recommended for the normal reference counting
 * use case in which references are taken and released one at a time.  In these
 * cases, refcount_dec(), or one of its variants, should instead be used to
 * decrement a reference count.
 *
 * Return: true if the resulting refcount is 0, false otherwise
 */
static inline __must_check bool refcount_sub_and_test(int i, refcount_t *r)
{
        return __refcount_sub_and_test(i, r, NULL);
}

static inline __must_check bool __refcount_dec_and_test(refcount_t *r, int *oldp)
{
        return __refcount_sub_and_test(1, r, oldp);
}

/**
 * refcount_dec_and_test - decrement a refcount and test if it is 0
 * @r: the refcount
 *
 * Similar to atomic_dec_and_test(), it will WARN on underflow and fail to
 * decrement when saturated at REFCOUNT_SATURATED.
 *
 * Provides release memory ordering, such that prior loads and stores are done
 * before, and provides an acquire ordering on success such that free()
 * must come after.
 *
 * Return: true if the resulting refcount is 0, false otherwise
 */
static inline __must_check bool refcount_dec_and_test(refcount_t *r)
{
        return __refcount_dec_and_test(r, NULL);
}

static inline void __refcount_dec(refcount_t *r, int *oldp)
{
        int old = atomic_fetch_sub_release(1, &r->refs);

        if (oldp)
                *oldp = old;

        if (unlikely(old <= 1))
                refcount_warn_saturate(r, REFCOUNT_DEC_LEAK);
}

/**
 * refcount_dec - decrement a refcount
 * @r: the refcount
 *
 * Similar to atomic_dec(), it will WARN on underflow and fail to decrement
 * when saturated at REFCOUNT_SATURATED.
 *
 * Provides release memory ordering, such that prior loads and stores are done
 * before.
 */
static inline void refcount_dec(refcount_t *r)
{
        __refcount_dec(r, NULL);
}

extern __must_check bool refcount_dec_if_one(refcount_t *r);
extern __must_check bool refcount_dec_not_one(refcount_t *r);
extern __must_check bool refcount_dec_and_mutex_lock(refcount_t *r, struct mutex *lock);
extern __must_check bool refcount_dec_and_lock(refcount_t *r, spinlock_t *lock);
extern __must_check bool refcount_dec_and_lock_irqsave(refcount_t *r,
                                                       spinlock_t *lock,
                                                       unsigned long *flags);
#endif /* _LINUX_REFCOUNT_H */



































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































   28 








   14 




   14 

























   27 










   28 

   11 













   28 

















   28 

















































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































   38 






































    4 



    4 
    4 
    4 


    4 














































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































    8 















    8 




    8 


    8 


































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965
2966
2967
2968
2969
2970
2971
2972
2973
2974
2975
2976
2977
2978
2979
2980
2981
2982
2983
2984
2985
2986
2987
2988
2989
2990
2991
2992
2993
2994
2995
2996
2997
2998
2999
3000
3001
3002
3003
3004
3005
3006
3007
3008
3009
3010
3011
3012
3013
3014
3015
3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029
3030
3031
3032
3033
3034
3035
3036
3037
3038
3039
3040
3041
3042
3043
3044
3045
3046
3047
3048
3049
3050
3051
3052
3053
3054
3055
3056
3057
3058
3059
3060
3061
3062
3063
3064
3065
3066
3067
3068
3069
3070
3071
3072
3073
3074
3075
3076
3077
3078
3079
3080
3081
3082
3083
3084
3085
3086
3087
3088
3089
3090
3091
3092
3093
3094
3095
3096
3097
3098
3099
3100
3101
3102
3103
3104
3105
3106
3107
3108
3109
3110
3111
3112
3113
3114
3115
3116
3117
3118
3119
3120
3121
3122
3123
3124
3125
3126
3127
3128
3129
3130
3131
3132
3133
3134
3135
3136
3137
3138
3139
3140
3141
3142
3143
3144
3145
3146
3147
3148
3149
3150
3151
3152
3153
3154
3155
3156
3157
3158
3159
3160
3161
3162
3163
3164
3165
3166
3167
3168
3169
3170
3171
3172
3173
3174
3175
3176
3177
3178
3179
3180
3181
3182
3183
3184
3185
3186
3187
3188
3189
3190
3191
3192
3193
3194
3195
3196
3197
3198
3199
3200
3201
3202
3203
3204
3205
3206
3207
3208
3209
3210
3211
3212
3213
3214
3215
3216
3217
3218
3219
3220
3221
3222
3223
3224
3225
3226
3227
3228
3229
3230
3231
3232
3233
3234
3235
3236
3237
3238
3239
3240
3241
3242
3243
3244
3245
3246
3247
3248
3249
3250
3251
3252
3253
3254
3255
3256
3257
3258
3259
3260
3261
3262
3263
3264
3265
3266
3267
3268
3269
3270
3271
3272
3273
3274
3275
3276
3277
3278
3279
3280
3281
3282
3283
3284
3285
3286
3287
3288
3289
3290
3291
3292
3293
3294
3295
3296
3297
3298
3299
3300
3301
3302
3303
3304
3305
3306
3307
3308
3309
3310
3311
3312
3313
3314
3315
3316
3317
3318
3319
3320
3321
3322
3323
3324
3325
3326
3327
3328
3329
3330
3331
3332
3333
3334
3335
3336
3337
3338
3339
3340
3341
3342
3343
3344
3345
3346
3347
3348
3349
3350
3351
3352
3353
3354
3355
3356
3357
3358
3359
3360
3361
3362
3363
3364
3365
3366
3367
3368
3369
3370
3371
3372
3373
3374
3375
3376
3377
3378
3379
3380
3381
3382
3383
3384
3385
3386
3387
3388
3389
3390
3391
3392
3393
3394
3395
3396
3397
3398
3399
3400
3401
3402
3403
3404
3405
3406
3407
3408
3409
3410
3411
3412
3413
3414
3415
3416
3417
3418
3419
3420
3421
3422
3423
3424
3425
3426
3427
3428
3429
3430
3431
3432
3433
3434
3435
3436
3437
3438
3439
3440
3441
3442
3443
3444
3445
3446
3447
3448
3449
3450
3451
3452
3453
3454
3455
3456
3457
3458
3459
3460
3461
3462
3463
3464
3465
3466
3467
3468
3469
3470
3471
3472
3473
3474
3475
3476
3477
3478
3479
3480
3481
3482
3483
3484
3485
3486
3487
3488
3489
3490
3491
3492
3493
3494
3495
3496
3497
3498
3499
3500
3501
3502
3503
3504
3505
3506
3507
3508
3509
3510
3511
3512
3513
3514
3515
3516
3517
3518
3519
3520
3521
3522
3523
3524
3525
3526
3527
3528
3529
3530
3531
3532
3533
3534
3535
3536
3537
3538
3539
3540
3541
3542
3543
3544
3545
3546
3547
3548
3549
3550
3551
3552
3553
3554
3555
3556
3557
3558
3559
3560
3561
3562
3563
3564
3565
3566
3567
3568
3569
3570
3571
3572
3573
3574
3575
3576
3577
3578
3579
3580
3581
3582
3583
3584
3585
3586
3587
3588
3589
3590
3591
3592
3593
3594
3595
3596
3597
3598
3599
3600
3601
3602
3603
3604
3605
3606
3607
3608
3609
3610
3611
3612
3613
3614
3615
3616
3617
3618
3619
3620
3621
3622
3623
3624
3625
3626
3627
3628
3629
3630
3631
3632
3633
3634
3635
3636
3637
3638
3639
3640
3641
3642
3643
3644
3645
3646
3647
3648
3649
3650
3651
3652
3653
3654
3655
3656
3657
3658
3659
3660
3661
3662
3663
3664
3665
3666
3667
3668
3669
3670
3671
3672
3673
3674
3675
3676
3677
3678
3679
3680
3681
3682
3683
3684
3685
3686
3687
3688
3689
3690
3691
3692
3693
3694
3695
3696
3697
3698
3699
3700
3701
3702
3703
3704
3705
3706
3707
3708
3709
3710
3711
3712
3713
3714
3715
3716
3717
3718
3719
3720
3721
3722
3723
3724
3725
3726
3727
3728
3729
3730
3731
3732
3733
3734
3735
3736
3737
3738
3739
3740
3741
3742
3743
3744
3745
3746
3747
3748
3749
3750
3751
3752
3753
3754
3755
3756
3757
3758
3759
3760
3761
3762
3763
3764
3765
3766
3767
3768
3769
3770
3771
3772
3773
3774
3775
3776
3777
3778
3779
3780
3781
3782
3783
3784
3785
3786
3787
3788
3789
3790
3791
3792
3793
3794
3795
3796
3797
3798
3799
3800
3801
3802
3803
3804
3805
3806
3807
3808
3809
3810
3811
3812
3813
3814
3815
3816
3817
3818
3819
3820
3821
3822
3823
3824
3825
3826
3827
3828
3829
3830
3831
3832
3833
3834
3835
3836
3837
3838
3839
3840
3841
3842
3843
3844
3845
3846
3847
3848
3849
3850
3851
3852
3853
3854
3855
3856
3857
3858
3859
3860
3861
3862
3863
3864
3865
3866
3867
3868
3869
3870
3871
3872
3873
3874
3875
3876
3877
3878
3879
3880
3881
3882
3883
3884
3885
3886
3887
3888
3889
3890
3891
3892
3893
3894
3895
3896
3897
3898
3899
3900
3901
3902
3903
3904
3905
3906
3907
3908
3909
3910
3911
3912
3913
3914
3915
3916
3917
3918
3919
3920
3921
3922
3923
3924
3925
3926
3927
3928
3929
3930
3931
3932
3933
3934
3935
3936
3937
3938
3939
3940
3941
3942
3943
3944
3945
3946
3947
3948
3949
3950
3951
3952
3953
3954
3955
3956
3957
3958
3959
3960
3961
3962
3963
3964
3965
3966
3967
3968
3969
3970
3971
3972
3973
3974
3975
3976
3977
3978
3979
3980
3981
3982
3983
3984
3985
3986
3987
3988
3989
3990
3991
3992
3993
3994
3995
3996
3997
3998
3999
4000
4001
4002
4003
4004
4005
4006
4007
4008
4009
4010
4011
4012
4013
4014
4015
4016
4017
4018
4019
4020
4021
4022
4023
4024
4025
4026
4027
4028
4029
4030
4031
4032
4033
4034
4035
4036
4037
4038
4039
4040
4041
4042
4043
4044
4045
4046
4047
4048
4049
4050
4051
4052
4053
4054
4055
4056
4057
4058
4059
4060
4061
4062
4063
4064
4065
4066
4067
4068
4069
4070
4071
4072
4073
4074
4075
4076
4077
4078
4079
4080
4081
4082
4083
4084
4085
4086
4087
4088
4089
4090
4091
4092
4093
4094
4095
4096
4097
4098
4099
4100
4101
4102
4103
4104
4105
4106
4107
4108
4109
4110
4111
4112
4113
4114
4115
4116
4117
4118
4119
4120
4121
4122
4123
4124
4125
4126
4127
4128
4129
4130
4131
4132
4133
4134
4135
4136
4137
4138
4139
4140
4141
4142
4143
4144
4145
4146
4147
4148
4149
4150
4151
4152
4153
4154
4155
4156
4157
4158
4159
4160
4161
4162
4163
4164
4165
4166
4167
4168
4169
4170
4171
4172
4173
4174
4175
4176
4177
4178
4179
4180
4181
4182
4183
4184
4185
4186
4187
4188
4189
4190
4191
4192
4193
4194
4195
4196
4197
4198
4199
4200
4201
4202
4203
4204
4205
4206
4207
4208
4209
4210
4211
4212
4213
4214
4215
4216
4217
4218
4219
4220
4221
4222
4223
4224
4225
4226
4227
4228
4229
4230
4231
4232
4233
4234
4235
4236
4237
4238
4239
4240
4241
4242
4243
4244
4245
4246
4247
4248
4249
4250
4251
4252
4253
4254
4255
4256
4257
4258
4259
4260
4261
4262
4263
4264
4265
4266
4267
4268
4269
4270
4271
4272
4273
4274
4275
4276
4277
4278
4279
4280
4281
4282
4283
4284
4285
4286
4287
4288
4289
4290
4291
4292
4293
4294
4295
4296
4297
4298
4299
4300
4301
4302
4303
4304
4305
4306
4307
4308
4309
4310
4311
4312
4313
4314
4315
4316
4317
4318
4319
4320
4321
4322
4323
4324
4325
4326
4327
4328
4329
4330
4331
4332
4333
4334
4335
4336
4337
4338
4339
4340
4341
4342
4343
4344
4345
4346
4347
4348
4349
4350
4351
4352
4353
4354
4355
4356
4357
4358
4359
4360
4361
4362
4363
4364
4365
4366
4367
4368
4369
4370
4371
4372
4373
4374
4375
4376
4377
4378
4379
4380
4381
4382
4383
4384
4385
4386
4387
4388
4389
4390
4391
4392
4393
4394
4395
4396
4397
4398
4399
4400
4401
4402
4403
4404
4405
4406
4407
4408
4409
4410
4411
4412
4413
4414
4415
4416
4417
4418
4419
4420
4421
4422
4423
4424
4425
4426
4427
4428
4429
4430
4431
4432
4433
4434
4435
4436
4437
4438
4439
4440
4441
4442
4443
4444
4445
4446
4447
4448
4449
4450
4451
4452
4453
4454
4455
4456
4457
4458
4459
4460
4461
4462
4463
4464
4465
4466
4467
4468
4469
4470
4471
4472
4473
4474
4475
4476
4477
4478
4479
4480
4481
4482
4483
4484
4485
4486
4487
4488
4489
4490
4491
4492
4493
4494
4495
4496
4497
4498
4499
4500
4501
4502
4503
4504
4505
4506
4507
4508
4509
4510
4511
4512
4513
4514
4515
4516
4517
4518
4519
4520
4521
4522
4523
4524
4525
4526
4527
4528
4529
4530
4531
4532
4533
4534
4535
4536
4537
4538
4539
4540
4541
4542
4543
4544
4545
4546
4547
4548
4549
4550
4551
4552
4553
4554
4555
4556
4557
4558
4559
4560
4561
4562
4563
4564
4565
4566
4567
4568
4569
4570
4571
4572
4573
4574
4575
4576
4577
4578
4579
4580
4581
4582
4583
4584
4585
4586
4587
4588
4589
4590
4591
4592
4593
4594
4595
4596
4597
4598
4599
4600
4601
4602
4603
4604
4605
4606
4607
4608
4609
4610
4611
4612
4613
4614
4615
4616
4617
4618
4619
4620
4621
4622
4623
4624
4625
4626
4627
4628
4629
4630
4631
4632
4633
4634
4635
4636
4637
4638
4639
4640
4641
4642
4643
4644
4645
4646
4647
4648
4649
4650
4651
4652
4653
4654
4655
4656
4657
4658
4659
4660
4661
4662
4663
4664
4665
4666
4667
4668
4669
4670
4671
4672
4673
4674
4675
4676
4677
4678
4679
4680
4681
4682
4683
4684
4685
4686
4687
4688
4689
4690
4691
4692
4693
4694
4695
4696
4697
4698
4699
4700
4701
4702
4703
4704
4705
4706
4707
4708
4709
4710
4711
4712
4713
4714
4715
4716
4717
4718
4719
4720
4721
4722
4723
4724
4725
4726
4727
4728
4729
4730
4731
4732
4733
4734
4735
4736
4737
4738
4739
4740
4741
4742
4743
4744
4745
4746
4747
4748
4749
4750
4751
4752
4753
4754
4755
4756
4757
4758
4759
4760
4761
4762
4763
4764
4765
4766
4767
4768
4769
4770
4771
4772
4773
4774
4775
4776
4777
4778
4779
4780
4781
4782
4783
4784
4785
4786
4787
4788
4789
4790
4791
4792
4793
4794
4795
4796
4797
4798
4799
4800
4801
4802
4803
4804
4805
4806
4807
4808
4809
4810
4811
4812
4813
4814
4815
4816
4817
4818
4819
4820
4821
4822
4823
4824
4825
4826
4827
4828
4829
4830
4831
4832
4833
4834
4835
4836
4837
4838
4839
4840
4841
4842
4843
4844
4845
4846
4847
4848
4849
4850
4851
4852
4853
4854
4855
4856
4857
4858
4859
4860
4861
4862
4863
4864
4865
4866
4867
4868
4869
4870
4871
4872
4873
4874
4875
4876
4877
4878
4879
4880
4881
4882
4883
4884
4885
4886
4887
4888
4889
4890
4891
4892
4893
4894
4895
4896
4897
4898
4899
4900
4901
4902
4903
4904
4905
4906
4907
4908
4909
4910
4911
4912
4913
4914
4915
4916
4917
4918
4919
4920
4921
4922
4923
4924
4925
4926
4927
4928
4929
4930
4931
4932
4933
4934
4935
4936
4937
4938
4939
4940
4941
4942
4943
4944
4945
4946
4947
4948
4949
4950
4951
4952
4953
4954
4955
4956
4957
4958
4959
4960
4961
4962
4963
4964
4965
4966
4967
4968
4969
4970
4971
4972
4973
4974
4975
4976
4977
4978
4979
4980
4981
4982
4983
4984
4985
4986
4987
4988
4989
4990
4991
4992
4993
4994
4995
4996
4997
4998
4999
5000
5001
5002
5003
5004
5005
5006
5007
5008
5009
5010
5011
5012
5013
5014
5015
5016
5017
5018
5019
5020
5021
5022
5023
5024
5025
5026
5027
5028
5029
5030
5031
5032
5033
5034
5035
5036
5037
5038
5039
5040
5041
5042
5043
5044
5045
5046
5047
5048
5049
5050
5051
5052
5053
5054
5055
5056
5057
5058
5059
5060
5061
5062
5063
5064
5065
5066
5067
5068
5069
5070
5071
5072
5073
5074
5075
5076
5077
5078
5079
5080
5081
5082
5083
5084
5085
5086
5087
5088
5089
5090
5091
5092
5093
5094
5095
5096
5097
5098
5099
5100
5101
5102
5103
5104
5105
5106
5107
5108
5109
5110
5111
5112
5113
5114
5115
5116
5117
5118
5119
5120
5121
5122
5123
5124
5125
5126
5127
5128
5129
5130
5131
5132
5133
5134
5135
5136
5137
5138
5139
5140
5141
5142
5143
5144
5145
5146
5147
5148
5149
5150
5151
5152
5153
5154
5155
5156
5157
5158
5159
5160
5161
5162
5163
5164
5165
5166
5167
5168
5169
5170
5171
5172
5173
5174
5175
5176
5177
5178
5179
5180
5181
5182
5183
5184
5185
5186
5187
5188
5189
5190
5191
5192
5193
5194
5195
5196
5197
5198
5199
5200
5201
5202
5203
5204
5205
5206
5207
5208
5209
5210
5211
5212
5213
5214
5215
5216
5217
5218
5219
5220
5221
5222
5223
5224
5225
5226
5227
5228
5229
5230
5231
5232
5233
5234
5235
5236
5237
5238
5239
5240
5241
5242
5243
5244
5245
5246
5247
5248
5249
5250
5251
5252
5253
5254
5255
5256
5257
5258
5259
5260
5261
5262
5263
5264
5265
5266
5267
5268
5269
5270
5271
5272
5273
5274
5275
5276
5277
5278
5279
5280
5281
5282
5283
5284
5285
5286
5287
5288
5289
5290
5291
5292
5293
5294
5295
5296
5297
5298
5299
5300
5301
5302
5303
5304
5305
5306
5307
5308
5309
5310
5311
5312
5313
5314
5315
5316
5317
5318
5319
5320
5321
5322
5323
5324
5325
5326
5327
5328
5329
5330
5331
5332
5333
5334
5335
5336
5337
5338
5339
5340
5341
5342
5343
5344
5345
5346
5347
5348
5349
5350
5351
5352
5353
5354
5355
5356
5357
5358
5359
5360
5361
5362
5363
5364
5365
5366
5367
5368
5369
5370
5371
5372
5373
5374
5375
5376
5377
5378
5379
5380
5381
5382
5383
5384
5385
5386
5387
5388
5389
5390
5391
5392
5393
5394
5395
5396
5397
5398
5399
5400
5401
5402
5403
5404
5405
5406
5407
5408
5409
5410
5411
5412
5413
5414
5415
5416
5417
5418
5419
5420
5421
5422
5423
5424
5425
5426
5427
5428
5429
5430
5431
5432
5433
5434
5435
5436
5437
5438
5439
5440
5441
5442
5443
5444
5445
5446
5447
5448
5449
5450
5451
5452
5453
5454
5455
5456
5457
5458
5459
5460
5461
5462
5463
5464
5465
5466
5467
5468
5469
5470
5471
5472
5473
5474
5475
5476
5477
5478
5479
5480
5481
5482
5483
5484
5485
5486
5487
5488
5489
5490
5491
5492
5493
5494
5495
5496
5497
5498
5499
5500
5501
5502
5503
5504
5505
5506
5507
5508
5509
5510
5511
5512
5513
5514
5515
5516
5517
5518
5519
5520
5521
5522
5523
5524
5525
5526
5527
5528
5529
5530
5531
5532
5533
5534
5535
5536
5537
5538
5539
5540
5541
5542
5543
5544
5545
5546
5547
5548
5549
5550
5551
5552
5553
5554
5555
5556
5557
5558
5559
5560
5561
5562
5563
5564
5565
5566
5567
5568
5569
5570
5571
5572
5573
5574
5575
5576
5577
5578
5579
5580
5581
5582
5583
5584
5585
5586
5587
5588
5589
5590
5591
5592
5593
5594
5595
5596
5597
5598
5599
5600
5601
5602
5603
5604
5605
5606
5607
5608
5609
5610
5611
5612
5613
5614
5615
5616
5617
5618
5619
5620
5621
5622
5623
5624
5625
5626
5627
5628
5629
5630
5631
5632
5633
5634
5635
5636
5637
5638
5639
5640
5641
5642
5643
5644
5645
5646
5647
5648
5649
5650
5651
5652
5653
5654
5655
5656
5657
5658
5659
5660
5661
5662
5663
5664
5665
5666
5667
5668
5669
5670
5671
5672
5673
5674
5675
5676
5677
5678
5679
5680
5681
5682
5683
5684
5685
5686
5687
5688
5689
5690
5691
5692
5693
5694
5695
5696
5697
5698
5699
5700
5701
5702
5703
5704
5705
5706
5707
5708
5709
5710
5711
5712
5713
5714
5715
5716
5717
5718
5719
5720
5721
5722
5723
5724
5725
5726
5727
5728
5729
5730
5731
5732
5733
5734
5735
5736
5737
5738
5739
5740
5741
5742
5743
5744
5745
5746
5747
5748
5749
5750
5751
5752
5753
5754
5755
5756
5757
5758
5759
5760
5761
5762
5763
5764
5765
5766
5767
5768
5769
5770
5771
5772
5773
5774
5775
5776
5777
5778
5779
5780
5781
5782
5783
5784
5785
5786
5787
5788
5789
5790
5791
5792
5793
5794
5795
5796
5797
5798
5799
5800
5801
5802
5803
5804
5805
5806
5807
5808
5809
5810
5811
5812
5813
5814
5815
5816
5817
5818
5819
5820
5821
5822
5823
5824
5825
5826
5827
5828
5829
5830
5831
5832
5833
5834
5835
5836
5837
5838
5839
5840
5841
5842
5843
5844
5845
5846
5847
5848
5849
5850
5851
5852
5853
5854
5855
5856
5857
5858
5859
5860
5861
5862
5863
5864
5865
5866
5867
5868
5869
5870
5871
5872
5873
5874
5875
5876
5877
5878
5879
5880
5881
5882
5883
5884
5885
5886
5887
5888
5889
5890
5891
5892
5893
5894
5895
5896
5897
5898
5899
5900
5901
5902
5903
5904
5905
5906
5907
5908
5909
5910
5911
5912
5913
5914
5915
5916
5917
5918
5919
5920
5921
5922
5923
5924
5925
5926
5927
5928
5929
5930
5931
5932
5933
5934
5935
5936
5937
5938
5939
5940
5941
5942
5943
5944
5945
5946
5947
5948
5949
5950
5951
5952
5953
5954
5955
5956
5957
5958
5959
5960
5961
5962
5963
5964
5965
5966
5967
5968
5969
5970
5971
5972
5973
5974
5975
5976
5977
5978
5979
5980
5981
5982
5983
5984
5985
5986
5987
5988
5989
5990
5991
5992
5993
5994
5995
5996
5997
5998
5999
6000
6001
6002
6003
6004
6005
6006
6007
6008
6009
6010
6011
6012
6013
6014
6015
6016
6017
6018
6019
6020
6021
6022
6023
6024
6025
6026
6027
6028
6029
6030
6031
6032
6033
6034
6035
6036
6037
6038
6039
6040
6041
6042
6043
6044
6045
6046
6047
6048
6049
6050
6051
// SPDX-License-Identifier: GPL-2.0-only
/*
 * BSS client mode implementation
 * Copyright 2003-2008, Jouni Malinen <j@w1.fi>
 * Copyright 2004, Instant802 Networks, Inc.
 * Copyright 2005, Devicescape Software, Inc.
 * Copyright 2006-2007        Jiri Benc <jbenc@suse.cz>
 * Copyright 2007, Michael Wu <flamingice@sourmilk.net>
 * Copyright 2013-2014  Intel Mobile Communications GmbH
 * Copyright (C) 2015 - 2017 Intel Deutschland GmbH
 * Copyright (C) 2018 - 2021 Intel Corporation
 */

#include <linux/delay.h>
#include <linux/fips.h>
#include <linux/if_ether.h>
#include <linux/skbuff.h>
#include <linux/if_arp.h>
#include <linux/etherdevice.h>
#include <linux/moduleparam.h>
#include <linux/rtnetlink.h>
#include <linux/crc32.h>
#include <linux/slab.h>
#include <linux/export.h>
#include <net/mac80211.h>
#include <asm/unaligned.h>

#include "ieee80211_i.h"
#include "driver-ops.h"
#include "rate.h"
#include "led.h"
#include "fils_aead.h"

#define IEEE80211_AUTH_TIMEOUT                (HZ / 5)
#define IEEE80211_AUTH_TIMEOUT_LONG        (HZ / 2)
#define IEEE80211_AUTH_TIMEOUT_SHORT        (HZ / 10)
#define IEEE80211_AUTH_TIMEOUT_SAE        (HZ * 2)
#define IEEE80211_AUTH_MAX_TRIES        3
#define IEEE80211_AUTH_WAIT_ASSOC        (HZ * 5)
#define IEEE80211_AUTH_WAIT_SAE_RETRY        (HZ * 2)
#define IEEE80211_ASSOC_TIMEOUT                (HZ / 5)
#define IEEE80211_ASSOC_TIMEOUT_LONG        (HZ / 2)
#define IEEE80211_ASSOC_TIMEOUT_SHORT        (HZ / 10)
#define IEEE80211_ASSOC_MAX_TRIES        3

static int max_nullfunc_tries = 2;
module_param(max_nullfunc_tries, int, 0644);
MODULE_PARM_DESC(max_nullfunc_tries,
                 "Maximum nullfunc tx tries before disconnecting (reason 4).");

static int max_probe_tries = 5;
module_param(max_probe_tries, int, 0644);
MODULE_PARM_DESC(max_probe_tries,
                 "Maximum probe tries before disconnecting (reason 4).");

/*
 * Beacon loss timeout is calculated as N frames times the
 * advertised beacon interval.  This may need to be somewhat
 * higher than what hardware might detect to account for
 * delays in the host processing frames. But since we also
 * probe on beacon miss before declaring the connection lost
 * default to what we want.
 */
static int beacon_loss_count = 7;
module_param(beacon_loss_count, int, 0644);
MODULE_PARM_DESC(beacon_loss_count,
                 "Number of beacon intervals before we decide beacon was lost.");

/*
 * Time the connection can be idle before we probe
 * it to see if we can still talk to the AP.
 */
#define IEEE80211_CONNECTION_IDLE_TIME        (30 * HZ)
/*
 * Time we wait for a probe response after sending
 * a probe request because of beacon loss or for
 * checking the connection still works.
 */
static int probe_wait_ms = 500;
module_param(probe_wait_ms, int, 0644);
MODULE_PARM_DESC(probe_wait_ms,
                 "Maximum time(ms) to wait for probe response"
                 " before disconnecting (reason 4).");

/*
 * How many Beacon frames need to have been used in average signal strength
 * before starting to indicate signal change events.
 */
#define IEEE80211_SIGNAL_AVE_MIN_COUNT        4

/*
 * We can have multiple work items (and connection probing)
 * scheduling this timer, but we need to take care to only
 * reschedule it when it should fire _earlier_ than it was
 * asked for before, or if it's not pending right now. This
 * function ensures that. Note that it then is required to
 * run this function for all timeouts after the first one
 * has happened -- the work that runs from this timer will
 * do that.
 */
static void run_again(struct ieee80211_sub_if_data *sdata,
                      unsigned long timeout)
{
        sdata_assert_lock(sdata);

        if (!timer_pending(&sdata->u.mgd.timer) ||
            time_before(timeout, sdata->u.mgd.timer.expires))
                mod_timer(&sdata->u.mgd.timer, timeout);
}

void ieee80211_sta_reset_beacon_monitor(struct ieee80211_sub_if_data *sdata)
{
        if (sdata->vif.driver_flags & IEEE80211_VIF_BEACON_FILTER)
                return;

        if (ieee80211_hw_check(&sdata->local->hw, CONNECTION_MONITOR))
                return;

        mod_timer(&sdata->u.mgd.bcn_mon_timer,
                  round_jiffies_up(jiffies + sdata->u.mgd.beacon_timeout));
}

void ieee80211_sta_reset_conn_monitor(struct ieee80211_sub_if_data *sdata)
{
        struct ieee80211_if_managed *ifmgd = &sdata->u.mgd;

        if (unlikely(!ifmgd->associated))
                return;

        if (ifmgd->probe_send_count)
                ifmgd->probe_send_count = 0;

        if (ieee80211_hw_check(&sdata->local->hw, CONNECTION_MONITOR))
                return;

        mod_timer(&ifmgd->conn_mon_timer,
                  round_jiffies_up(jiffies + IEEE80211_CONNECTION_IDLE_TIME));
}

static int ecw2cw(int ecw)
{
        return (1 << ecw) - 1;
}

static u32
ieee80211_determine_chantype(struct ieee80211_sub_if_data *sdata,
                             struct ieee80211_supported_band *sband,
                             struct ieee80211_channel *channel,
                             u32 vht_cap_info,
                             const struct ieee80211_ht_operation *ht_oper,
                             const struct ieee80211_vht_operation *vht_oper,
                             const struct ieee80211_he_operation *he_oper,
                             const struct ieee80211_s1g_oper_ie *s1g_oper,
                             struct cfg80211_chan_def *chandef, bool tracking)
{
        struct ieee80211_if_managed *ifmgd = &sdata->u.mgd;
        struct cfg80211_chan_def vht_chandef;
        struct ieee80211_sta_ht_cap sta_ht_cap;
        u32 ht_cfreq, ret;

        memset(chandef, 0, sizeof(struct cfg80211_chan_def));
        chandef->chan = channel;
        chandef->width = NL80211_CHAN_WIDTH_20_NOHT;
        chandef->center_freq1 = channel->center_freq;
        chandef->freq1_offset = channel->freq_offset;

        if (channel->band == NL80211_BAND_6GHZ) {
                if (!ieee80211_chandef_he_6ghz_oper(sdata, he_oper, chandef))
                        ret = IEEE80211_STA_DISABLE_HT |
                              IEEE80211_STA_DISABLE_VHT |
                              IEEE80211_STA_DISABLE_HE;
                else
                        ret = 0;
                vht_chandef = *chandef;
                goto out;
        } else if (sband->band == NL80211_BAND_S1GHZ) {
                if (!ieee80211_chandef_s1g_oper(s1g_oper, chandef)) {
                        sdata_info(sdata,
                                   "Missing S1G Operation Element? Trying operating == primary\n");
                        chandef->width = ieee80211_s1g_channel_width(channel);
                }

                ret = IEEE80211_STA_DISABLE_HT | IEEE80211_STA_DISABLE_40MHZ |
                      IEEE80211_STA_DISABLE_VHT |
                      IEEE80211_STA_DISABLE_80P80MHZ |
                      IEEE80211_STA_DISABLE_160MHZ;
                goto out;
        }

        memcpy(&sta_ht_cap, &sband->ht_cap, sizeof(sta_ht_cap));
        ieee80211_apply_htcap_overrides(sdata, &sta_ht_cap);

        if (!ht_oper || !sta_ht_cap.ht_supported) {
                ret = IEEE80211_STA_DISABLE_HT |
                      IEEE80211_STA_DISABLE_VHT |
                      IEEE80211_STA_DISABLE_HE;
                goto out;
        }

        chandef->width = NL80211_CHAN_WIDTH_20;

        ht_cfreq = ieee80211_channel_to_frequency(ht_oper->primary_chan,
                                                  channel->band);
        /* check that channel matches the right operating channel */
        if (!tracking && channel->center_freq != ht_cfreq) {
                /*
                 * It's possible that some APs are confused here;
                 * Netgear WNDR3700 sometimes reports 4 higher than
                 * the actual channel in association responses, but
                 * since we look at probe response/beacon data here
                 * it should be OK.
                 */
                sdata_info(sdata,
                           "Wrong control channel: center-freq: %d ht-cfreq: %d ht->primary_chan: %d band: %d - Disabling HT\n",
                           channel->center_freq, ht_cfreq,
                           ht_oper->primary_chan, channel->band);
                ret = IEEE80211_STA_DISABLE_HT |
                      IEEE80211_STA_DISABLE_VHT |
                      IEEE80211_STA_DISABLE_HE;
                goto out;
        }

        /* check 40 MHz support, if we have it */
        if (sta_ht_cap.cap & IEEE80211_HT_CAP_SUP_WIDTH_20_40) {
                ieee80211_chandef_ht_oper(ht_oper, chandef);
        } else {
                /* 40 MHz (and 80 MHz) must be supported for VHT */
                ret = IEEE80211_STA_DISABLE_VHT;
                /* also mark 40 MHz disabled */
                ret |= IEEE80211_STA_DISABLE_40MHZ;
                goto out;
        }

        if (!vht_oper || !sband->vht_cap.vht_supported) {
                ret = IEEE80211_STA_DISABLE_VHT;
                goto out;
        }

        vht_chandef = *chandef;
        if (!(ifmgd->flags & IEEE80211_STA_DISABLE_HE) && he_oper &&
            (le32_to_cpu(he_oper->he_oper_params) &
             IEEE80211_HE_OPERATION_VHT_OPER_INFO)) {
                struct ieee80211_vht_operation he_oper_vht_cap;

                /*
                 * Set only first 3 bytes (other 2 aren't used in
                 * ieee80211_chandef_vht_oper() anyway)
                 */
                memcpy(&he_oper_vht_cap, he_oper->optional, 3);
                he_oper_vht_cap.basic_mcs_set = cpu_to_le16(0);

                if (!ieee80211_chandef_vht_oper(&sdata->local->hw, vht_cap_info,
                                                &he_oper_vht_cap, ht_oper,
                                                &vht_chandef)) {
                        if (!(ifmgd->flags & IEEE80211_STA_DISABLE_HE))
                                sdata_info(sdata,
                                           "HE AP VHT information is invalid, disable HE\n");
                        ret = IEEE80211_STA_DISABLE_HE;
                        goto out;
                }
        } else if (!ieee80211_chandef_vht_oper(&sdata->local->hw,
                                               vht_cap_info,
                                               vht_oper, ht_oper,
                                               &vht_chandef)) {
                if (!(ifmgd->flags & IEEE80211_STA_DISABLE_VHT))
                        sdata_info(sdata,
                                   "AP VHT information is invalid, disable VHT\n");
                ret = IEEE80211_STA_DISABLE_VHT;
                goto out;
        }

        if (!cfg80211_chandef_valid(&vht_chandef)) {
                if (!(ifmgd->flags & IEEE80211_STA_DISABLE_VHT))
                        sdata_info(sdata,
                                   "AP VHT information is invalid, disable VHT\n");
                ret = IEEE80211_STA_DISABLE_VHT;
                goto out;
        }

        if (cfg80211_chandef_identical(chandef, &vht_chandef)) {
                ret = 0;
                goto out;
        }

        if (!cfg80211_chandef_compatible(chandef, &vht_chandef)) {
                if (!(ifmgd->flags & IEEE80211_STA_DISABLE_VHT))
                        sdata_info(sdata,
                                   "AP VHT information doesn't match HT, disable VHT\n");
                ret = IEEE80211_STA_DISABLE_VHT;
                goto out;
        }

        *chandef = vht_chandef;

        ret = 0;

out:
        /*
         * When tracking the current AP, don't do any further checks if the
         * new chandef is identical to the one we're currently using for the
         * connection. This keeps us from playing ping-pong with regulatory,
         * without it the following can happen (for example):
         *  - connect to an AP with 80 MHz, world regdom allows 80 MHz
         *  - AP advertises regdom US
         *  - CRDA loads regdom US with 80 MHz prohibited (old database)
         *  - the code below detects an unsupported channel, downgrades, and
         *    we disconnect from the AP in the caller
         *  - disconnect causes CRDA to reload world regdomain and the game
         *    starts anew.
         * (see https://bugzilla.kernel.org/show_bug.cgi?id=70881)
         *
         * It seems possible that there are still scenarios with CSA or real
         * bandwidth changes where a this could happen, but those cases are
         * less common and wouldn't completely prevent using the AP.
         */
        if (tracking &&
            cfg80211_chandef_identical(chandef, &sdata->vif.bss_conf.chandef))
                return ret;

        /* don't print the message below for VHT mismatch if VHT is disabled */
        if (ret & IEEE80211_STA_DISABLE_VHT)
                vht_chandef = *chandef;

        /*
         * Ignore the DISABLED flag when we're already connected and only
         * tracking the APs beacon for bandwidth changes - otherwise we
         * might get disconnected here if we connect to an AP, update our
         * regulatory information based on the AP's country IE and the
         * information we have is wrong/outdated and disables the channel
         * that we're actually using for the connection to the AP.
         */
        while (!cfg80211_chandef_usable(sdata->local->hw.wiphy, chandef,
                                        tracking ? 0 :
                                                   IEEE80211_CHAN_DISABLED)) {
                if (WARN_ON(chandef->width == NL80211_CHAN_WIDTH_20_NOHT)) {
                        ret = IEEE80211_STA_DISABLE_HT |
                              IEEE80211_STA_DISABLE_VHT |
                              IEEE80211_STA_DISABLE_HE;
                        break;
                }

                ret |= ieee80211_chandef_downgrade(chandef);
        }

        if (!he_oper || !cfg80211_chandef_usable(sdata->wdev.wiphy, chandef,
                                                 IEEE80211_CHAN_NO_HE))
                ret |= IEEE80211_STA_DISABLE_HE;

        if (chandef->width != vht_chandef.width && !tracking)
                sdata_info(sdata,
                           "capabilities/regulatory prevented using AP HT/VHT configuration, downgraded\n");

        WARN_ON_ONCE(!cfg80211_chandef_valid(chandef));
        return ret;
}

static int ieee80211_config_bw(struct ieee80211_sub_if_data *sdata,
                               struct sta_info *sta,
                               const struct ieee80211_ht_cap *ht_cap,
                               const struct ieee80211_vht_cap *vht_cap,
                               const struct ieee80211_ht_operation *ht_oper,
                               const struct ieee80211_vht_operation *vht_oper,
                               const struct ieee80211_he_operation *he_oper,
                               const struct ieee80211_s1g_oper_ie *s1g_oper,
                               const u8 *bssid, u32 *changed)
{
        struct ieee80211_local *local = sdata->local;
        struct ieee80211_if_managed *ifmgd = &sdata->u.mgd;
        struct ieee80211_channel *chan = sdata->vif.bss_conf.chandef.chan;
        struct ieee80211_supported_band *sband =
                local->hw.wiphy->bands[chan->band];
        struct cfg80211_chan_def chandef;
        u16 ht_opmode;
        u32 flags;
        u32 vht_cap_info = 0;
        int ret;

        /* if HT was/is disabled, don't track any bandwidth changes */
        if (ifmgd->flags & IEEE80211_STA_DISABLE_HT || !ht_oper)
                return 0;

        /* don't check VHT if we associated as non-VHT station */
        if (ifmgd->flags & IEEE80211_STA_DISABLE_VHT)
                vht_oper = NULL;

        /* don't check HE if we associated as non-HE station */
        if (ifmgd->flags & IEEE80211_STA_DISABLE_HE ||
            !ieee80211_get_he_iftype_cap(sband,
                                         ieee80211_vif_type_p2p(&sdata->vif)))

                he_oper = NULL;

        if (WARN_ON_ONCE(!sta))
                return -EINVAL;

        /*
         * if bss configuration changed store the new one -
         * this may be applicable even if channel is identical
         */
        ht_opmode = le16_to_cpu(ht_oper->operation_mode);
        if (sdata->vif.bss_conf.ht_operation_mode != ht_opmode) {
                *changed |= BSS_CHANGED_HT;
                sdata->vif.bss_conf.ht_operation_mode = ht_opmode;
        }

        if (vht_cap)
                vht_cap_info = le32_to_cpu(vht_cap->vht_cap_info);

        /* calculate new channel (type) based on HT/VHT/HE operation IEs */
        flags = ieee80211_determine_chantype(sdata, sband, chan, vht_cap_info,
                                             ht_oper, vht_oper, he_oper,
                                             s1g_oper, &chandef, true);

        /*
         * Downgrade the new channel if we associated with restricted
         * capabilities. For example, if we associated as a 20 MHz STA
         * to a 40 MHz AP (due to regulatory, capabilities or config
         * reasons) then switching to a 40 MHz channel now won't do us
         * any good -- we couldn't use it with the AP.
         */
        if (ifmgd->flags & IEEE80211_STA_DISABLE_80P80MHZ &&
            chandef.width == NL80211_CHAN_WIDTH_80P80)
                flags |= ieee80211_chandef_downgrade(&chandef);
        if (ifmgd->flags & IEEE80211_STA_DISABLE_160MHZ &&
            chandef.width == NL80211_CHAN_WIDTH_160)
                flags |= ieee80211_chandef_downgrade(&chandef);
        if (ifmgd->flags & IEEE80211_STA_DISABLE_40MHZ &&
            chandef.width > NL80211_CHAN_WIDTH_20)
                flags |= ieee80211_chandef_downgrade(&chandef);

        if (cfg80211_chandef_identical(&chandef, &sdata->vif.bss_conf.chandef))
                return 0;

        sdata_info(sdata,
                   "AP %pM changed bandwidth, new config is %d.%03d MHz, "
                   "width %d (%d.%03d/%d MHz)\n",
                   ifmgd->bssid, chandef.chan->center_freq,
                   chandef.chan->freq_offset, chandef.width,
                   chandef.center_freq1, chandef.freq1_offset,
                   chandef.center_freq2);

        if (flags != (ifmgd->flags & (IEEE80211_STA_DISABLE_HT |
                                      IEEE80211_STA_DISABLE_VHT |
                                      IEEE80211_STA_DISABLE_HE |
                                      IEEE80211_STA_DISABLE_40MHZ |
                                      IEEE80211_STA_DISABLE_80P80MHZ |
                                      IEEE80211_STA_DISABLE_160MHZ)) ||
            !cfg80211_chandef_valid(&chandef)) {
                sdata_info(sdata,
                           "AP %pM changed caps/bw in a way we can't support (0x%x/0x%x) - disconnect\n",
                           ifmgd->bssid, flags, ifmgd->flags);
                return -EINVAL;
        }

        ret = ieee80211_vif_change_bandwidth(sdata, &chandef, changed);

        if (ret) {
                sdata_info(sdata,
                           "AP %pM changed bandwidth to incompatible one - disconnect\n",
                           ifmgd->bssid);
                return ret;
        }

        return 0;
}

/* frame sending functions */

static void ieee80211_add_ht_ie(struct ieee80211_sub_if_data *sdata,
                                struct sk_buff *skb, u8 ap_ht_param,
                                struct ieee80211_supported_band *sband,
                                struct ieee80211_channel *channel,
                                enum ieee80211_smps_mode smps)
{
        u8 *pos;
        u32 flags = channel->flags;
        u16 cap;
        struct ieee80211_sta_ht_cap ht_cap;

        BUILD_BUG_ON(sizeof(ht_cap) != sizeof(sband->ht_cap));

        memcpy(&ht_cap, &sband->ht_cap, sizeof(ht_cap));
        ieee80211_apply_htcap_overrides(sdata, &ht_cap);

        /* determine capability flags */
        cap = ht_cap.cap;

        switch (ap_ht_param & IEEE80211_HT_PARAM_CHA_SEC_OFFSET) {
        case IEEE80211_HT_PARAM_CHA_SEC_ABOVE:
                if (flags & IEEE80211_CHAN_NO_HT40PLUS) {
                        cap &= ~IEEE80211_HT_CAP_SUP_WIDTH_20_40;
                        cap &= ~IEEE80211_HT_CAP_SGI_40;
                }
                break;
        case IEEE80211_HT_PARAM_CHA_SEC_BELOW:
                if (flags & IEEE80211_CHAN_NO_HT40MINUS) {
                        cap &= ~IEEE80211_HT_CAP_SUP_WIDTH_20_40;
                        cap &= ~IEEE80211_HT_CAP_SGI_40;
                }
                break;
        }

        /*
         * If 40 MHz was disabled associate as though we weren't
         * capable of 40 MHz -- some broken APs will never fall
         * back to trying to transmit in 20 MHz.
         */
        if (sdata->u.mgd.flags & IEEE80211_STA_DISABLE_40MHZ) {
                cap &= ~IEEE80211_HT_CAP_SUP_WIDTH_20_40;
                cap &= ~IEEE80211_HT_CAP_SGI_40;
        }

        /* set SM PS mode properly */
        cap &= ~IEEE80211_HT_CAP_SM_PS;
        switch (smps) {
        case IEEE80211_SMPS_AUTOMATIC:
        case IEEE80211_SMPS_NUM_MODES:
                WARN_ON(1);
                fallthrough;
        case IEEE80211_SMPS_OFF:
                cap |= WLAN_HT_CAP_SM_PS_DISABLED <<
                        IEEE80211_HT_CAP_SM_PS_SHIFT;
                break;
        case IEEE80211_SMPS_STATIC:
                cap |= WLAN_HT_CAP_SM_PS_STATIC <<
                        IEEE80211_HT_CAP_SM_PS_SHIFT;
                break;
        case IEEE80211_SMPS_DYNAMIC:
                cap |= WLAN_HT_CAP_SM_PS_DYNAMIC <<
                        IEEE80211_HT_CAP_SM_PS_SHIFT;
                break;
        }

        /* reserve and fill IE */
        pos = skb_put(skb, sizeof(struct ieee80211_ht_cap) + 2);
        ieee80211_ie_build_ht_cap(pos, &ht_cap, cap);
}

/* This function determines vht capability flags for the association
 * and builds the IE.
 * Note - the function may set the owner of the MU-MIMO capability
 */
static void ieee80211_add_vht_ie(struct ieee80211_sub_if_data *sdata,
                                 struct sk_buff *skb,
                                 struct ieee80211_supported_band *sband,
                                 struct ieee80211_vht_cap *ap_vht_cap)
{
        struct ieee80211_local *local = sdata->local;
        u8 *pos;
        u32 cap;
        struct ieee80211_sta_vht_cap vht_cap;
        u32 mask, ap_bf_sts, our_bf_sts;

        BUILD_BUG_ON(sizeof(vht_cap) != sizeof(sband->vht_cap));

        memcpy(&vht_cap, &sband->vht_cap, sizeof(vht_cap));
        ieee80211_apply_vhtcap_overrides(sdata, &vht_cap);

        /* determine capability flags */
        cap = vht_cap.cap;

        if (sdata->u.mgd.flags & IEEE80211_STA_DISABLE_80P80MHZ) {
                u32 bw = cap & IEEE80211_VHT_CAP_SUPP_CHAN_WIDTH_MASK;

                cap &= ~IEEE80211_VHT_CAP_SUPP_CHAN_WIDTH_MASK;
                if (bw == IEEE80211_VHT_CAP_SUPP_CHAN_WIDTH_160MHZ ||
                    bw == IEEE80211_VHT_CAP_SUPP_CHAN_WIDTH_160_80PLUS80MHZ)
                        cap |= IEEE80211_VHT_CAP_SUPP_CHAN_WIDTH_160MHZ;
        }

        if (sdata->u.mgd.flags & IEEE80211_STA_DISABLE_160MHZ) {
                cap &= ~IEEE80211_VHT_CAP_SHORT_GI_160;
                cap &= ~IEEE80211_VHT_CAP_SUPP_CHAN_WIDTH_MASK;
        }

        /*
         * Some APs apparently get confused if our capabilities are better
         * than theirs, so restrict what we advertise in the assoc request.
         */
        if (!(ap_vht_cap->vht_cap_info &
                        cpu_to_le32(IEEE80211_VHT_CAP_SU_BEAMFORMER_CAPABLE)))
                cap &= ~(IEEE80211_VHT_CAP_SU_BEAMFORMEE_CAPABLE |
                         IEEE80211_VHT_CAP_MU_BEAMFORMEE_CAPABLE);
        else if (!(ap_vht_cap->vht_cap_info &
                        cpu_to_le32(IEEE80211_VHT_CAP_MU_BEAMFORMER_CAPABLE)))
                cap &= ~IEEE80211_VHT_CAP_MU_BEAMFORMEE_CAPABLE;

        /*
         * If some other vif is using the MU-MIMO capability we cannot associate
         * using MU-MIMO - this will lead to contradictions in the group-id
         * mechanism.
         * Ownership is defined since association request, in order to avoid
         * simultaneous associations with MU-MIMO.
         */
        if (cap & IEEE80211_VHT_CAP_MU_BEAMFORMEE_CAPABLE) {
                bool disable_mu_mimo = false;
                struct ieee80211_sub_if_data *other;

                list_for_each_entry_rcu(other, &local->interfaces, list) {
                        if (other->vif.mu_mimo_owner) {
                                disable_mu_mimo = true;
                                break;
                        }
                }
                if (disable_mu_mimo)
                        cap &= ~IEEE80211_VHT_CAP_MU_BEAMFORMEE_CAPABLE;
                else
                        sdata->vif.mu_mimo_owner = true;
        }

        mask = IEEE80211_VHT_CAP_BEAMFORMEE_STS_MASK;

        ap_bf_sts = le32_to_cpu(ap_vht_cap->vht_cap_info) & mask;
        our_bf_sts = cap & mask;

        if (ap_bf_sts < our_bf_sts) {
                cap &= ~mask;
                cap |= ap_bf_sts;
        }

        /* reserve and fill IE */
        pos = skb_put(skb, sizeof(struct ieee80211_vht_cap) + 2);
        ieee80211_ie_build_vht_cap(pos, &vht_cap, cap);
}

/* This function determines HE capability flags for the association
 * and builds the IE.
 */
static void ieee80211_add_he_ie(struct ieee80211_sub_if_data *sdata,
                                struct sk_buff *skb,
                                struct ieee80211_supported_band *sband)
{
        u8 *pos, *pre_he_pos;
        const struct ieee80211_sta_he_cap *he_cap = NULL;
        struct ieee80211_chanctx_conf *chanctx_conf;
        u8 he_cap_size;
        bool reg_cap = false;

        rcu_read_lock();
        chanctx_conf = rcu_dereference(sdata->vif.chanctx_conf);
        if (!WARN_ON_ONCE(!chanctx_conf))
                reg_cap = cfg80211_chandef_usable(sdata->wdev.wiphy,
                                                  &chanctx_conf->def,
                                                  IEEE80211_CHAN_NO_HE);

        rcu_read_unlock();

        he_cap = ieee80211_get_he_iftype_cap(sband,
                                             ieee80211_vif_type_p2p(&sdata->vif));
        if (!he_cap || !chanctx_conf || !reg_cap)
                return;

        /* get a max size estimate */
        he_cap_size =
                2 + 1 + sizeof(he_cap->he_cap_elem) +
                ieee80211_he_mcs_nss_size(&he_cap->he_cap_elem) +
                ieee80211_he_ppe_size(he_cap->ppe_thres[0],
                                      he_cap->he_cap_elem.phy_cap_info);
        pos = skb_put(skb, he_cap_size);
        pre_he_pos = pos;
        pos = ieee80211_ie_build_he_cap(sdata->u.mgd.flags,
                                        pos, he_cap, pos + he_cap_size);
        /* trim excess if any */
        skb_trim(skb, skb->len - (pre_he_pos + he_cap_size - pos));

        ieee80211_ie_build_he_6ghz_cap(sdata, skb);
}

static int ieee80211_send_assoc(struct ieee80211_sub_if_data *sdata)
{
        struct ieee80211_local *local = sdata->local;
        struct ieee80211_if_managed *ifmgd = &sdata->u.mgd;
        struct ieee80211_mgd_assoc_data *assoc_data = ifmgd->assoc_data;
        struct sk_buff *skb;
        struct ieee80211_mgmt *mgmt;
        u8 *pos, qos_info, *ie_start;
        size_t offset = 0, noffset;
        int i, count, rates_len, supp_rates_len, shift;
        u16 capab;
        struct ieee80211_supported_band *sband;
        struct ieee80211_chanctx_conf *chanctx_conf;
        struct ieee80211_channel *chan;
        u32 rates = 0;
        __le16 listen_int;
        struct element *ext_capa = NULL;
        enum nl80211_iftype iftype = ieee80211_vif_type_p2p(&sdata->vif);
        const struct ieee80211_sband_iftype_data *iftd;
        struct ieee80211_prep_tx_info info = {};
        int ret;

        /* we know it's writable, cast away the const */
        if (assoc_data->ie_len)
                ext_capa = (void *)cfg80211_find_elem(WLAN_EID_EXT_CAPABILITY,
                                                      assoc_data->ie,
                                                      assoc_data->ie_len);

        sdata_assert_lock(sdata);

        rcu_read_lock();
        chanctx_conf = rcu_dereference(sdata->vif.chanctx_conf);
        if (WARN_ON(!chanctx_conf)) {
                rcu_read_unlock();
                return -EINVAL;
        }
        chan = chanctx_conf->def.chan;
        rcu_read_unlock();
        sband = local->hw.wiphy->bands[chan->band];
        shift = ieee80211_vif_get_shift(&sdata->vif);

        if (assoc_data->supp_rates_len) {
                /*
                 * Get all rates supported by the device and the AP as
                 * some APs don't like getting a superset of their rates
                 * in the association request (e.g. D-Link DAP 1353 in
                 * b-only mode)...
                 */
                rates_len = ieee80211_parse_bitrates(&chanctx_conf->def, sband,
                                                     assoc_data->supp_rates,
                                                     assoc_data->supp_rates_len,
                                                     &rates);
        } else {
                /*
                 * In case AP not provide any supported rates information
                 * before association, we send information element(s) with
                 * all rates that we support.
                 */
                rates_len = 0;
                for (i = 0; i < sband->n_bitrates; i++) {
                        rates |= BIT(i);
                        rates_len++;
                }
        }

        iftd = ieee80211_get_sband_iftype_data(sband, iftype);

        skb = alloc_skb(local->hw.extra_tx_headroom +
                        sizeof(*mgmt) + /* bit too much but doesn't matter */
                        2 + assoc_data->ssid_len + /* SSID */
                        4 + rates_len + /* (extended) rates */
                        4 + /* power capability */
                        2 + 2 * sband->n_channels + /* supported channels */
                        2 + sizeof(struct ieee80211_ht_cap) + /* HT */
                        2 + sizeof(struct ieee80211_vht_cap) + /* VHT */
                        2 + 1 + sizeof(struct ieee80211_he_cap_elem) + /* HE */
                                sizeof(struct ieee80211_he_mcs_nss_supp) +
                                IEEE80211_HE_PPE_THRES_MAX_LEN +
                        2 + 1 + sizeof(struct ieee80211_he_6ghz_capa) +
                        assoc_data->ie_len + /* extra IEs */
                        (assoc_data->fils_kek_len ? 16 /* AES-SIV */ : 0) +
                        9 + /* WMM */
                        (iftd ? iftd->vendor_elems.len : 0),
                        GFP_KERNEL);
        if (!skb)
                return -ENOMEM;

        skb_reserve(skb, local->hw.extra_tx_headroom);

        capab = WLAN_CAPABILITY_ESS;

        if (sband->band == NL80211_BAND_2GHZ) {
                capab |= WLAN_CAPABILITY_SHORT_SLOT_TIME;
                capab |= WLAN_CAPABILITY_SHORT_PREAMBLE;
        }

        if (assoc_data->capability & WLAN_CAPABILITY_PRIVACY)
                capab |= WLAN_CAPABILITY_PRIVACY;

        if ((assoc_data->capability & WLAN_CAPABILITY_SPECTRUM_MGMT) &&
            ieee80211_hw_check(&local->hw, SPECTRUM_MGMT))
                capab |= WLAN_CAPABILITY_SPECTRUM_MGMT;

        if (ifmgd->flags & IEEE80211_STA_ENABLE_RRM)
                capab |= WLAN_CAPABILITY_RADIO_MEASURE;

        mgmt = skb_put_zero(skb, 24);
        memcpy(mgmt->da, assoc_data->bss->bssid, ETH_ALEN);
        memcpy(mgmt->sa, sdata->vif.addr, ETH_ALEN);
        memcpy(mgmt->bssid, assoc_data->bss->bssid, ETH_ALEN);

        listen_int = cpu_to_le16(sband->band == NL80211_BAND_S1GHZ ?
                        ieee80211_encode_usf(local->hw.conf.listen_interval) :
                        local->hw.conf.listen_interval);
        if (!is_zero_ether_addr(assoc_data->prev_bssid)) {
                skb_put(skb, 10);
                mgmt->frame_control = cpu_to_le16(IEEE80211_FTYPE_MGMT |
                                                  IEEE80211_STYPE_REASSOC_REQ);
                mgmt->u.reassoc_req.capab_info = cpu_to_le16(capab);
                mgmt->u.reassoc_req.listen_interval = listen_int;
                memcpy(mgmt->u.reassoc_req.current_ap, assoc_data->prev_bssid,
                       ETH_ALEN);
                info.subtype = IEEE80211_STYPE_REASSOC_REQ;
        } else {
                skb_put(skb, 4);
                mgmt->frame_control = cpu_to_le16(IEEE80211_FTYPE_MGMT |
                                                  IEEE80211_STYPE_ASSOC_REQ);
                mgmt->u.assoc_req.capab_info = cpu_to_le16(capab);
                mgmt->u.assoc_req.listen_interval = listen_int;
                info.subtype = IEEE80211_STYPE_ASSOC_REQ;
        }

        /* SSID */
        pos = skb_put(skb, 2 + assoc_data->ssid_len);
        ie_start = pos;
        *pos++ = WLAN_EID_SSID;
        *pos++ = assoc_data->ssid_len;
        memcpy(pos, assoc_data->ssid, assoc_data->ssid_len);

        if (sband->band == NL80211_BAND_S1GHZ)
                goto skip_rates;

        /* add all rates which were marked to be used above */
        supp_rates_len = rates_len;
        if (supp_rates_len > 8)
                supp_rates_len = 8;

        pos = skb_put(skb, supp_rates_len + 2);
        *pos++ = WLAN_EID_SUPP_RATES;
        *pos++ = supp_rates_len;

        count = 0;
        for (i = 0; i < sband->n_bitrates; i++) {
                if (BIT(i) & rates) {
                        int rate = DIV_ROUND_UP(sband->bitrates[i].bitrate,
                                                5 * (1 << shift));
                        *pos++ = (u8) rate;
                        if (++count == 8)
                                break;
                }
        }

        if (rates_len > count) {
                pos = skb_put(skb, rates_len - count + 2);
                *pos++ = WLAN_EID_EXT_SUPP_RATES;
                *pos++ = rates_len - count;

                for (i++; i < sband->n_bitrates; i++) {
                        if (BIT(i) & rates) {
                                int rate;
                                rate = DIV_ROUND_UP(sband->bitrates[i].bitrate,
                                                    5 * (1 << shift));
                                *pos++ = (u8) rate;
                        }
                }
        }

skip_rates:
        if (capab & WLAN_CAPABILITY_SPECTRUM_MGMT ||
            capab & WLAN_CAPABILITY_RADIO_MEASURE) {
                pos = skb_put(skb, 4);
                *pos++ = WLAN_EID_PWR_CAPABILITY;
                *pos++ = 2;
                *pos++ = 0; /* min tx power */
                 /* max tx power */
                *pos++ = ieee80211_chandef_max_power(&chanctx_conf->def);
        }

        /*
         * Per spec, we shouldn't include the list of channels if we advertise
         * support for extended channel switching, but we've always done that;
         * (for now?) apply this restriction only on the (new) 6 GHz band.
         */
        if (capab & WLAN_CAPABILITY_SPECTRUM_MGMT &&
            (sband->band != NL80211_BAND_6GHZ ||
             !ext_capa || ext_capa->datalen < 1 ||
             !(ext_capa->data[0] & WLAN_EXT_CAPA1_EXT_CHANNEL_SWITCHING))) {
                /* TODO: get this in reg domain format */
                pos = skb_put(skb, 2 * sband->n_channels + 2);
                *pos++ = WLAN_EID_SUPPORTED_CHANNELS;
                *pos++ = 2 * sband->n_channels;
                for (i = 0; i < sband->n_channels; i++) {
                        *pos++ = ieee80211_frequency_to_channel(
                                        sband->channels[i].center_freq);
                        *pos++ = 1; /* one channel in the subband*/
                }
        }

        /* Set MBSSID support for HE AP if needed */
        if (ieee80211_hw_check(&local->hw, SUPPORTS_ONLY_HE_MULTI_BSSID) &&
            !(ifmgd->flags & IEEE80211_STA_DISABLE_HE) && assoc_data->ie_len &&
            ext_capa && ext_capa->datalen >= 3)
                ext_capa->data[2] |= WLAN_EXT_CAPA3_MULTI_BSSID_SUPPORT;

        /* if present, add any custom IEs that go before HT */
        if (assoc_data->ie_len) {
                static const u8 before_ht[] = {
                        WLAN_EID_SSID,
                        WLAN_EID_SUPP_RATES,
                        WLAN_EID_EXT_SUPP_RATES,
                        WLAN_EID_PWR_CAPABILITY,
                        WLAN_EID_SUPPORTED_CHANNELS,
                        WLAN_EID_RSN,
                        WLAN_EID_QOS_CAPA,
                        WLAN_EID_RRM_ENABLED_CAPABILITIES,
                        WLAN_EID_MOBILITY_DOMAIN,
                        WLAN_EID_FAST_BSS_TRANSITION,        /* reassoc only */
                        WLAN_EID_RIC_DATA,                /* reassoc only */
                        WLAN_EID_SUPPORTED_REGULATORY_CLASSES,
                };
                static const u8 after_ric[] = {
                        WLAN_EID_SUPPORTED_REGULATORY_CLASSES,
                        WLAN_EID_HT_CAPABILITY,
                        WLAN_EID_BSS_COEX_2040,
                        /* luckily this is almost always there */
                        WLAN_EID_EXT_CAPABILITY,
                        WLAN_EID_QOS_TRAFFIC_CAPA,
                        WLAN_EID_TIM_BCAST_REQ,
                        WLAN_EID_INTERWORKING,
                        /* 60 GHz (Multi-band, DMG, MMS) can't happen */
                        WLAN_EID_VHT_CAPABILITY,
                        WLAN_EID_OPMODE_NOTIF,
                };

                noffset = ieee80211_ie_split_ric(assoc_data->ie,
                                                 assoc_data->ie_len,
                                                 before_ht,
                                                 ARRAY_SIZE(before_ht),
                                                 after_ric,
                                                 ARRAY_SIZE(after_ric),
                                                 offset);
                skb_put_data(skb, assoc_data->ie + offset, noffset - offset);
                offset = noffset;
        }

        if (WARN_ON_ONCE((ifmgd->flags & IEEE80211_STA_DISABLE_HT) &&
                         !(ifmgd->flags & IEEE80211_STA_DISABLE_VHT)))
                ifmgd->flags |= IEEE80211_STA_DISABLE_VHT;

        if (sband->band != NL80211_BAND_6GHZ &&
            !(ifmgd->flags & IEEE80211_STA_DISABLE_HT))
                ieee80211_add_ht_ie(sdata, skb, assoc_data->ap_ht_param,
                                    sband, chan, sdata->smps_mode);

        /* if present, add any custom IEs that go before VHT */
        if (assoc_data->ie_len) {
                static const u8 before_vht[] = {
                        /*
                         * no need to list the ones split off before HT
                         * or generated here
                         */
                        WLAN_EID_BSS_COEX_2040,
                        WLAN_EID_EXT_CAPABILITY,
                        WLAN_EID_QOS_TRAFFIC_CAPA,
                        WLAN_EID_TIM_BCAST_REQ,
                        WLAN_EID_INTERWORKING,
                        /* 60 GHz (Multi-band, DMG, MMS) can't happen */
                };

                /* RIC already taken above, so no need to handle here anymore */
                noffset = ieee80211_ie_split(assoc_data->ie, assoc_data->ie_len,
                                             before_vht, ARRAY_SIZE(before_vht),
                                             offset);
                skb_put_data(skb, assoc_data->ie + offset, noffset - offset);
                offset = noffset;
        }

        /* if present, add any custom IEs that go before HE */
        if (assoc_data->ie_len) {
                static const u8 before_he[] = {
                        /*
                         * no need to list the ones split off before VHT
                         * or generated here
                         */
                        WLAN_EID_OPMODE_NOTIF,
                        WLAN_EID_EXTENSION, WLAN_EID_EXT_FUTURE_CHAN_GUIDANCE,
                        /* 11ai elements */
                        WLAN_EID_EXTENSION, WLAN_EID_EXT_FILS_SESSION,
                        WLAN_EID_EXTENSION, WLAN_EID_EXT_FILS_PUBLIC_KEY,
                        WLAN_EID_EXTENSION, WLAN_EID_EXT_FILS_KEY_CONFIRM,
                        WLAN_EID_EXTENSION, WLAN_EID_EXT_FILS_HLP_CONTAINER,
                        WLAN_EID_EXTENSION, WLAN_EID_EXT_FILS_IP_ADDR_ASSIGN,
                        /* TODO: add 11ah/11aj/11ak elements */
                };

                /* RIC already taken above, so no need to handle here anymore */
                noffset = ieee80211_ie_split(assoc_data->ie, assoc_data->ie_len,
                                             before_he, ARRAY_SIZE(before_he),
                                             offset);
                pos = skb_put(skb, noffset - offset);
                memcpy(pos, assoc_data->ie + offset, noffset - offset);
                offset = noffset;
        }

        if (sband->band != NL80211_BAND_6GHZ &&
            !(ifmgd->flags & IEEE80211_STA_DISABLE_VHT))
                ieee80211_add_vht_ie(sdata, skb, sband,
                                     &assoc_data->ap_vht_cap);

        /*
         * If AP doesn't support HT, mark HE as disabled.
         * If on the 5GHz band, make sure it supports VHT.
         */
        if (ifmgd->flags & IEEE80211_STA_DISABLE_HT ||
            (sband->band == NL80211_BAND_5GHZ &&
             ifmgd->flags & IEEE80211_STA_DISABLE_VHT))
                ifmgd->flags |= IEEE80211_STA_DISABLE_HE;

        if (!(ifmgd->flags & IEEE80211_STA_DISABLE_HE))
                ieee80211_add_he_ie(sdata, skb, sband);

        /* if present, add any custom non-vendor IEs that go after HE */
        if (assoc_data->ie_len) {
                noffset = ieee80211_ie_split_vendor(assoc_data->ie,
                                                    assoc_data->ie_len,
                                                    offset);
                skb_put_data(skb, assoc_data->ie + offset, noffset - offset);
                offset = noffset;
        }

        if (assoc_data->wmm) {
                if (assoc_data->uapsd) {
                        qos_info = ifmgd->uapsd_queues;
                        qos_info |= (ifmgd->uapsd_max_sp_len <<
                                     IEEE80211_WMM_IE_STA_QOSINFO_SP_SHIFT);
                } else {
                        qos_info = 0;
                }

                pos = ieee80211_add_wmm_info_ie(skb_put(skb, 9), qos_info);
        }

        if (sband->band == NL80211_BAND_S1GHZ) {
                ieee80211_add_aid_request_ie(sdata, skb);
                ieee80211_add_s1g_capab_ie(sdata, &sband->s1g_cap, skb);
        }

        if (iftd && iftd->vendor_elems.data && iftd->vendor_elems.len)
                skb_put_data(skb, iftd->vendor_elems.data, iftd->vendor_elems.len);

        /* add any remaining custom (i.e. vendor specific here) IEs */
        if (assoc_data->ie_len) {
                noffset = assoc_data->ie_len;
                skb_put_data(skb, assoc_data->ie + offset, noffset - offset);
        }

        if (assoc_data->fils_kek_len) {
                ret = fils_encrypt_assoc_req(skb, assoc_data);
                if (ret < 0) {
                        dev_kfree_skb(skb);
                        return ret;
                }
        }

        pos = skb_tail_pointer(skb);
        kfree(ifmgd->assoc_req_ies);
        ifmgd->assoc_req_ies = kmemdup(ie_start, pos - ie_start, GFP_ATOMIC);
        if (!ifmgd->assoc_req_ies) {
                dev_kfree_skb(skb);
                return -ENOMEM;
        }

        ifmgd->assoc_req_ies_len = pos - ie_start;

        drv_mgd_prepare_tx(local, sdata, &info);

        IEEE80211_SKB_CB(skb)->flags |= IEEE80211_TX_INTFL_DONT_ENCRYPT;
        if (ieee80211_hw_check(&local->hw, REPORTS_TX_ACK_STATUS))
                IEEE80211_SKB_CB(skb)->flags |= IEEE80211_TX_CTL_REQ_TX_STATUS |
                                                IEEE80211_TX_INTFL_MLME_CONN_TX;
        ieee80211_tx_skb(sdata, skb);

        return 0;
}

void ieee80211_send_pspoll(struct ieee80211_local *local,
                           struct ieee80211_sub_if_data *sdata)
{
        struct ieee80211_pspoll *pspoll;
        struct sk_buff *skb;

        skb = ieee80211_pspoll_get(&local->hw, &sdata->vif);
        if (!skb)
                return;

        pspoll = (struct ieee80211_pspoll *) skb->data;
        pspoll->frame_control |= cpu_to_le16(IEEE80211_FCTL_PM);

        IEEE80211_SKB_CB(skb)->flags |= IEEE80211_TX_INTFL_DONT_ENCRYPT;
        ieee80211_tx_skb(sdata, skb);
}

void ieee80211_send_nullfunc(struct ieee80211_local *local,
                             struct ieee80211_sub_if_data *sdata,
                             bool powersave)
{
        struct sk_buff *skb;
        struct ieee80211_hdr_3addr *nullfunc;
        struct ieee80211_if_managed *ifmgd = &sdata->u.mgd;

        skb = ieee80211_nullfunc_get(&local->hw, &sdata->vif,
                !ieee80211_hw_check(&local->hw, DOESNT_SUPPORT_QOS_NDP));
        if (!skb)
                return;

        nullfunc = (struct ieee80211_hdr_3addr *) skb->data;
        if (powersave)
                nullfunc->frame_control |= cpu_to_le16(IEEE80211_FCTL_PM);

        IEEE80211_SKB_CB(skb)->flags |= IEEE80211_TX_INTFL_DONT_ENCRYPT |
                                        IEEE80211_TX_INTFL_OFFCHAN_TX_OK;

        if (ieee80211_hw_check(&local->hw, REPORTS_TX_ACK_STATUS))
                IEEE80211_SKB_CB(skb)->flags |= IEEE80211_TX_CTL_REQ_TX_STATUS;

        if (ifmgd->flags & IEEE80211_STA_CONNECTION_POLL)
                IEEE80211_SKB_CB(skb)->flags |= IEEE80211_TX_CTL_USE_MINRATE;

        ieee80211_tx_skb(sdata, skb);
}

void ieee80211_send_4addr_nullfunc(struct ieee80211_local *local,
                                   struct ieee80211_sub_if_data *sdata)
{
        struct sk_buff *skb;
        struct ieee80211_hdr *nullfunc;
        __le16 fc;

        if (WARN_ON(sdata->vif.type != NL80211_IFTYPE_STATION))
                return;

        skb = dev_alloc_skb(local->hw.extra_tx_headroom + 30);
        if (!skb)
                return;

        skb_reserve(skb, local->hw.extra_tx_headroom);

        nullfunc = skb_put_zero(skb, 30);
        fc = cpu_to_le16(IEEE80211_FTYPE_DATA | IEEE80211_STYPE_NULLFUNC |
                         IEEE80211_FCTL_FROMDS | IEEE80211_FCTL_TODS);
        nullfunc->frame_control = fc;
        memcpy(nullfunc->addr1, sdata->u.mgd.bssid, ETH_ALEN);
        memcpy(nullfunc->addr2, sdata->vif.addr, ETH_ALEN);
        memcpy(nullfunc->addr3, sdata->u.mgd.bssid, ETH_ALEN);
        memcpy(nullfunc->addr4, sdata->vif.addr, ETH_ALEN);

        IEEE80211_SKB_CB(skb)->flags |= IEEE80211_TX_INTFL_DONT_ENCRYPT;
        ieee80211_tx_skb(sdata, skb);
}

/* spectrum management related things */
static void ieee80211_chswitch_work(struct work_struct *work)
{
        struct ieee80211_sub_if_data *sdata =
                container_of(work, struct ieee80211_sub_if_data, u.mgd.chswitch_work);
        struct ieee80211_local *local = sdata->local;
        struct ieee80211_if_managed *ifmgd = &sdata->u.mgd;
        int ret;

        if (!ieee80211_sdata_running(sdata))
                return;

        sdata_lock(sdata);
        mutex_lock(&local->mtx);
        mutex_lock(&local->chanctx_mtx);

        if (!ifmgd->associated)
                goto out;

        if (!sdata->vif.csa_active)
                goto out;

        /*
         * using reservation isn't immediate as it may be deferred until later
         * with multi-vif. once reservation is complete it will re-schedule the
         * work with no reserved_chanctx so verify chandef to check if it
         * completed successfully
         */

        if (sdata->reserved_chanctx) {
                /*
                 * with multi-vif csa driver may call ieee80211_csa_finish()
                 * many times while waiting for other interfaces to use their
                 * reservations
                 */
                if (sdata->reserved_ready)
                        goto out;

                ret = ieee80211_vif_use_reserved_context(sdata);
                if (ret) {
                        sdata_info(sdata,
                                   "failed to use reserved channel context, disconnecting (err=%d)\n",
                                   ret);
                        ieee80211_queue_work(&sdata->local->hw,
                                             &ifmgd->csa_connection_drop_work);
                        goto out;
                }

                goto out;
        }

        if (!cfg80211_chandef_identical(&sdata->vif.bss_conf.chandef,
                                        &sdata->csa_chandef)) {
                sdata_info(sdata,
                           "failed to finalize channel switch, disconnecting\n");
                ieee80211_queue_work(&sdata->local->hw,
                                     &ifmgd->csa_connection_drop_work);
                goto out;
        }

        ifmgd->csa_waiting_bcn = true;

        ieee80211_sta_reset_beacon_monitor(sdata);
        ieee80211_sta_reset_conn_monitor(sdata);

out:
        mutex_unlock(&local->chanctx_mtx);
        mutex_unlock(&local->mtx);
        sdata_unlock(sdata);
}

static void ieee80211_chswitch_post_beacon(struct ieee80211_sub_if_data *sdata)
{
        struct ieee80211_local *local = sdata->local;
        struct ieee80211_if_managed *ifmgd = &sdata->u.mgd;
        int ret;

        sdata_assert_lock(sdata);

        WARN_ON(!sdata->vif.csa_active);

        if (sdata->csa_block_tx) {
                ieee80211_wake_vif_queues(local, sdata,
                                          IEEE80211_QUEUE_STOP_REASON_CSA);
                sdata->csa_block_tx = false;
        }

        sdata->vif.csa_active = false;
        ifmgd->csa_waiting_bcn = false;
        /*
         * If the CSA IE is still present on the beacon after the switch,
         * we need to consider it as a new CSA (possibly to self).
         */
        ifmgd->beacon_crc_valid = false;

        ret = drv_post_channel_switch(sdata);
        if (ret) {
                sdata_info(sdata,
                           "driver post channel switch failed, disconnecting\n");
                ieee80211_queue_work(&local->hw,
                                     &ifmgd->csa_connection_drop_work);
                return;
        }

        cfg80211_ch_switch_notify(sdata->dev, &sdata->reserved_chandef);
}

void ieee80211_chswitch_done(struct ieee80211_vif *vif, bool success)
{
        struct ieee80211_sub_if_data *sdata = vif_to_sdata(vif);
        struct ieee80211_if_managed *ifmgd = &sdata->u.mgd;

        trace_api_chswitch_done(sdata, success);
        if (!success) {
                sdata_info(sdata,
                           "driver channel switch failed, disconnecting\n");
                ieee80211_queue_work(&sdata->local->hw,
                                     &ifmgd->csa_connection_drop_work);
        } else {
                ieee80211_queue_work(&sdata->local->hw, &ifmgd->chswitch_work);
        }
}
EXPORT_SYMBOL(ieee80211_chswitch_done);

static void ieee80211_chswitch_timer(struct timer_list *t)
{
        struct ieee80211_sub_if_data *sdata =
                from_timer(sdata, t, u.mgd.chswitch_timer);

        ieee80211_queue_work(&sdata->local->hw, &sdata->u.mgd.chswitch_work);
}

static void
ieee80211_sta_abort_chanswitch(struct ieee80211_sub_if_data *sdata)
{
        struct ieee80211_local *local = sdata->local;

        if (!local->ops->abort_channel_switch)
                return;

        mutex_lock(&local->mtx);

        mutex_lock(&local->chanctx_mtx);
        ieee80211_vif_unreserve_chanctx(sdata);
        mutex_unlock(&local->chanctx_mtx);

        if (sdata->csa_block_tx)
                ieee80211_wake_vif_queues(local, sdata,
                                          IEEE80211_QUEUE_STOP_REASON_CSA);

        sdata->csa_block_tx = false;
        sdata->vif.csa_active = false;

        mutex_unlock(&local->mtx);

        drv_abort_channel_switch(sdata);
}

static void
ieee80211_sta_process_chanswitch(struct ieee80211_sub_if_data *sdata,
                                 u64 timestamp, u32 device_timestamp,
                                 struct ieee802_11_elems *elems,
                                 bool beacon)
{
        struct ieee80211_local *local = sdata->local;
        struct ieee80211_if_managed *ifmgd = &sdata->u.mgd;
        struct cfg80211_bss *cbss = ifmgd->associated;
        struct ieee80211_chanctx_conf *conf;
        struct ieee80211_chanctx *chanctx;
        enum nl80211_band current_band;
        struct ieee80211_csa_ie csa_ie;
        struct ieee80211_channel_switch ch_switch;
        struct ieee80211_bss *bss;
        int res;

        sdata_assert_lock(sdata);

        if (!cbss)
                return;

        if (local->scanning)
                return;

        current_band = cbss->channel->band;
        bss = (void *)cbss->priv;
        res = ieee80211_parse_ch_switch_ie(sdata, elems, current_band,
                                           bss->vht_cap_info,
                                           ifmgd->flags,
                                           ifmgd->associated->bssid, &csa_ie);

        if (!res) {
                ch_switch.timestamp = timestamp;
                ch_switch.device_timestamp = device_timestamp;
                ch_switch.block_tx = csa_ie.mode;
                ch_switch.chandef = csa_ie.chandef;
                ch_switch.count = csa_ie.count;
                ch_switch.delay = csa_ie.max_switch_time;
        }

        if (res < 0)
                goto lock_and_drop_connection;

        if (beacon && sdata->vif.csa_active && !ifmgd->csa_waiting_bcn) {
                if (res)
                        ieee80211_sta_abort_chanswitch(sdata);
                else
                        drv_channel_switch_rx_beacon(sdata, &ch_switch);
                return;
        } else if (sdata->vif.csa_active || res) {
                /* disregard subsequent announcements if already processing */
                return;
        }

        if (sdata->vif.bss_conf.chandef.chan->band !=
            csa_ie.chandef.chan->band) {
                sdata_info(sdata,
                           "AP %pM switches to different band (%d MHz, width:%d, CF1/2: %d/%d MHz), disconnecting\n",
                           ifmgd->associated->bssid,
                           csa_ie.chandef.chan->center_freq,
                           csa_ie.chandef.width, csa_ie.chandef.center_freq1,
                           csa_ie.chandef.center_freq2);
                goto lock_and_drop_connection;
        }

        if (!cfg80211_chandef_usable(local->hw.wiphy, &csa_ie.chandef,
                                     IEEE80211_CHAN_DISABLED)) {
                sdata_info(sdata,
                           "AP %pM switches to unsupported channel "
                           "(%d.%03d MHz, width:%d, CF1/2: %d.%03d/%d MHz), "
                           "disconnecting\n",
                           ifmgd->associated->bssid,
                           csa_ie.chandef.chan->center_freq,
                           csa_ie.chandef.chan->freq_offset,
                           csa_ie.chandef.width, csa_ie.chandef.center_freq1,
                           csa_ie.chandef.freq1_offset,
                           csa_ie.chandef.center_freq2);
                goto lock_and_drop_connection;
        }

        if (cfg80211_chandef_identical(&csa_ie.chandef,
                                       &sdata->vif.bss_conf.chandef) &&
            (!csa_ie.mode || !beacon)) {
                if (ifmgd->csa_ignored_same_chan)
                        return;
                sdata_info(sdata,
                           "AP %pM tries to chanswitch to same channel, ignore\n",
                           ifmgd->associated->bssid);
                ifmgd->csa_ignored_same_chan = true;
                return;
        }

        /*
         * Drop all TDLS peers - either we disconnect or move to a different
         * channel from this point on. There's no telling what our peer will do.
         * The TDLS WIDER_BW scenario is also problematic, as peers might now
         * have an incompatible wider chandef.
         */
        ieee80211_teardown_tdls_peers(sdata);

        mutex_lock(&local->mtx);
        mutex_lock(&local->chanctx_mtx);
        conf = rcu_dereference_protected(sdata->vif.chanctx_conf,
                                         lockdep_is_held(&local->chanctx_mtx));
        if (!conf) {
                sdata_info(sdata,
                           "no channel context assigned to vif?, disconnecting\n");
                goto drop_connection;
        }

        chanctx = container_of(conf, struct ieee80211_chanctx, conf);

        if (local->use_chanctx &&
            !ieee80211_hw_check(&local->hw, CHANCTX_STA_CSA)) {
                sdata_info(sdata,
                           "driver doesn't support chan-switch with channel contexts\n");
                goto drop_connection;
        }

        if (drv_pre_channel_switch(sdata, &ch_switch)) {
                sdata_info(sdata,
                           "preparing for channel switch failed, disconnecting\n");
                goto drop_connection;
        }

        res = ieee80211_vif_reserve_chanctx(sdata, &csa_ie.chandef,
                                            chanctx->mode, false);
        if (res) {
                sdata_info(sdata,
                           "failed to reserve channel context for channel switch, disconnecting (err=%d)\n",
                           res);
                goto drop_connection;
        }
        mutex_unlock(&local->chanctx_mtx);

        sdata->vif.csa_active = true;
        sdata->csa_chandef = csa_ie.chandef;
        sdata->csa_block_tx = csa_ie.mode;
        ifmgd->csa_ignored_same_chan = false;
        ifmgd->beacon_crc_valid = false;

        if (sdata->csa_block_tx)
                ieee80211_stop_vif_queues(local, sdata,
                                          IEEE80211_QUEUE_STOP_REASON_CSA);
        mutex_unlock(&local->mtx);

        cfg80211_ch_switch_started_notify(sdata->dev, &csa_ie.chandef,
                                          csa_ie.count, csa_ie.mode);

        if (local->ops->channel_switch) {
                /* use driver's channel switch callback */
                drv_channel_switch(local, sdata, &ch_switch);
                return;
        }

        /* channel switch handled in software */
        if (csa_ie.count <= 1)
                ieee80211_queue_work(&local->hw, &ifmgd->chswitch_work);
        else
                mod_timer(&ifmgd->chswitch_timer,
                          TU_TO_EXP_TIME((csa_ie.count - 1) *
                                         cbss->beacon_interval));
        return;
 lock_and_drop_connection:
        mutex_lock(&local->mtx);
        mutex_lock(&local->chanctx_mtx);
 drop_connection:
        /*
         * This is just so that the disconnect flow will know that
         * we were trying to switch channel and failed. In case the
         * mode is 1 (we are not allowed to Tx), we will know not to
         * send a deauthentication frame. Those two fields will be
         * reset when the disconnection worker runs.
         */
        sdata->vif.csa_active = true;
        sdata->csa_block_tx = csa_ie.mode;

        ieee80211_queue_work(&local->hw, &ifmgd->csa_connection_drop_work);
        mutex_unlock(&local->chanctx_mtx);
        mutex_unlock(&local->mtx);
}

static bool
ieee80211_find_80211h_pwr_constr(struct ieee80211_sub_if_data *sdata,
                                 struct ieee80211_channel *channel,
                                 const u8 *country_ie, u8 country_ie_len,
                                 const u8 *pwr_constr_elem,
                                 int *chan_pwr, int *pwr_reduction)
{
        struct ieee80211_country_ie_triplet *triplet;
        int chan = ieee80211_frequency_to_channel(channel->center_freq);
        int i, chan_increment;
        bool have_chan_pwr = false;

        /* Invalid IE */
        if (country_ie_len % 2 || country_ie_len < IEEE80211_COUNTRY_IE_MIN_LEN)
                return false;

        triplet = (void *)(country_ie + 3);
        country_ie_len -= 3;

        switch (channel->band) {
        default:
                WARN_ON_ONCE(1);
                fallthrough;
        case NL80211_BAND_2GHZ:
        case NL80211_BAND_60GHZ:
                chan_increment = 1;
                break;
        case NL80211_BAND_5GHZ:
                chan_increment = 4;
                break;
        case NL80211_BAND_6GHZ:
                /*
                 * In the 6 GHz band, the "maximum transmit power level"
                 * field in the triplets is reserved, and thus will be
                 * zero and we shouldn't use it to control TX power.
                 * The actual TX power will be given in the transmit
                 * power envelope element instead.
                 */
                return false;
        }

        /* find channel */
        while (country_ie_len >= 3) {
                u8 first_channel = triplet->chans.first_channel;

                if (first_channel >= IEEE80211_COUNTRY_EXTENSION_ID)
                        goto next;

                for (i = 0; i < triplet->chans.num_channels; i++) {
                        if (first_channel + i * chan_increment == chan) {
                                have_chan_pwr = true;
                                *chan_pwr = triplet->chans.max_power;
                                break;
                        }
                }
                if (have_chan_pwr)
                        break;

 next:
                triplet++;
                country_ie_len -= 3;
        }

        if (have_chan_pwr && pwr_constr_elem)
                *pwr_reduction = *pwr_constr_elem;
        else
                *pwr_reduction = 0;

        return have_chan_pwr;
}

static void ieee80211_find_cisco_dtpc(struct ieee80211_sub_if_data *sdata,
                                      struct ieee80211_channel *channel,
                                      const u8 *cisco_dtpc_ie,
                                      int *pwr_level)
{
        /* From practical testing, the first data byte of the DTPC element
         * seems to contain the requested dBm level, and the CLI on Cisco
         * APs clearly state the range is -127 to 127 dBm, which indicates
         * a signed byte, although it seemingly never actually goes negative.
         * The other byte seems to always be zero.
         */
        *pwr_level = (__s8)cisco_dtpc_ie[4];
}

static u32 ieee80211_handle_pwr_constr(struct ieee80211_sub_if_data *sdata,
                                       struct ieee80211_channel *channel,
                                       struct ieee80211_mgmt *mgmt,
                                       const u8 *country_ie, u8 country_ie_len,
                                       const u8 *pwr_constr_ie,
                                       const u8 *cisco_dtpc_ie)
{
        bool has_80211h_pwr = false, has_cisco_pwr = false;
        int chan_pwr = 0, pwr_reduction_80211h = 0;
        int pwr_level_cisco, pwr_level_80211h;
        int new_ap_level;
        __le16 capab = mgmt->u.probe_resp.capab_info;

        if (ieee80211_is_s1g_beacon(mgmt->frame_control))
                return 0;        /* TODO */

        if (country_ie &&
            (capab & cpu_to_le16(WLAN_CAPABILITY_SPECTRUM_MGMT) ||
             capab & cpu_to_le16(WLAN_CAPABILITY_RADIO_MEASURE))) {
                has_80211h_pwr = ieee80211_find_80211h_pwr_constr(
                        sdata, channel, country_ie, country_ie_len,
                        pwr_constr_ie, &chan_pwr, &pwr_reduction_80211h);
                pwr_level_80211h =
                        max_t(int, 0, chan_pwr - pwr_reduction_80211h);
        }

        if (cisco_dtpc_ie) {
                ieee80211_find_cisco_dtpc(
                        sdata, channel, cisco_dtpc_ie, &pwr_level_cisco);
                has_cisco_pwr = true;
        }

        if (!has_80211h_pwr && !has_cisco_pwr)
                return 0;

        /* If we have both 802.11h and Cisco DTPC, apply both limits
         * by picking the smallest of the two power levels advertised.
         */
        if (has_80211h_pwr &&
            (!has_cisco_pwr || pwr_level_80211h <= pwr_level_cisco)) {
                new_ap_level = pwr_level_80211h;

                if (sdata->ap_power_level == new_ap_level)
                        return 0;

                sdata_dbg(sdata,
                          "Limiting TX power to %d (%d - %d) dBm as advertised by %pM\n",
                          pwr_level_80211h, chan_pwr, pwr_reduction_80211h,
                          sdata->u.mgd.bssid);
        } else {  /* has_cisco_pwr is always true here. */
                new_ap_level = pwr_level_cisco;

                if (sdata->ap_power_level == new_ap_level)
                        return 0;

                sdata_dbg(sdata,
                          "Limiting TX power to %d dBm as advertised by %pM\n",
                          pwr_level_cisco, sdata->u.mgd.bssid);
        }

        sdata->ap_power_level = new_ap_level;
        if (__ieee80211_recalc_txpower(sdata))
                return BSS_CHANGED_TXPOWER;
        return 0;
}

/* powersave */
static void ieee80211_enable_ps(struct ieee80211_local *local,
                                struct ieee80211_sub_if_data *sdata)
{
        struct ieee80211_conf *conf = &local->hw.conf;

        /*
         * If we are scanning right now then the parameters will
         * take effect when scan finishes.
         */
        if (local->scanning)
                return;

        if (conf->dynamic_ps_timeout > 0 &&
            !ieee80211_hw_check(&local->hw, SUPPORTS_DYNAMIC_PS)) {
                mod_timer(&local->dynamic_ps_timer, jiffies +
                          msecs_to_jiffies(conf->dynamic_ps_timeout));
        } else {
                if (ieee80211_hw_check(&local->hw, PS_NULLFUNC_STACK))
                        ieee80211_send_nullfunc(local, sdata, true);

                if (ieee80211_hw_check(&local->hw, PS_NULLFUNC_STACK) &&
                    ieee80211_hw_check(&local->hw, REPORTS_TX_ACK_STATUS))
                        return;

                conf->flags |= IEEE80211_CONF_PS;
                ieee80211_hw_config(local, IEEE80211_CONF_CHANGE_PS);
        }
}

static void ieee80211_change_ps(struct ieee80211_local *local)
{
        struct ieee80211_conf *conf = &local->hw.conf;

        if (local->ps_sdata) {
                ieee80211_enable_ps(local, local->ps_sdata);
        } else if (conf->flags & IEEE80211_CONF_PS) {
                conf->flags &= ~IEEE80211_CONF_PS;
                ieee80211_hw_config(local, IEEE80211_CONF_CHANGE_PS);
                del_timer_sync(&local->dynamic_ps_timer);
                cancel_work_sync(&local->dynamic_ps_enable_work);
        }
}

static bool ieee80211_powersave_allowed(struct ieee80211_sub_if_data *sdata)
{
        struct ieee80211_if_managed *mgd = &sdata->u.mgd;
        struct sta_info *sta = NULL;
        bool authorized = false;

        if (!mgd->powersave)
                return false;

        if (mgd->broken_ap)
                return false;

        if (!mgd->associated)
                return false;

        if (mgd->flags & IEEE80211_STA_CONNECTION_POLL)
                return false;

        if (!mgd->have_beacon)
                return false;

        rcu_read_lock();
        sta = sta_info_get(sdata, mgd->bssid);
        if (sta)
                authorized = test_sta_flag(sta, WLAN_STA_AUTHORIZED);
        rcu_read_unlock();

        return authorized;
}

/* need to hold RTNL or interface lock */
void ieee80211_recalc_ps(struct ieee80211_local *local)
{
        struct ieee80211_sub_if_data *sdata, *found = NULL;
        int count = 0;
        int timeout;

        if (!ieee80211_hw_check(&local->hw, SUPPORTS_PS)) {
                local->ps_sdata = NULL;
                return;
        }

        list_for_each_entry(sdata, &local->interfaces, list) {
                if (!ieee80211_sdata_running(sdata))
                        continue;
                if (sdata->vif.type == NL80211_IFTYPE_AP) {
                        /* If an AP vif is found, then disable PS
                         * by setting the count to zero thereby setting
                         * ps_sdata to NULL.
                         */
                        count = 0;
                        break;
                }
                if (sdata->vif.type != NL80211_IFTYPE_STATION)
                        continue;
                found = sdata;
                count++;
        }

        if (count == 1 && ieee80211_powersave_allowed(found)) {
                u8 dtimper = found->u.mgd.dtim_period;

                timeout = local->dynamic_ps_forced_timeout;
                if (timeout < 0)
                        timeout = 100;
                local->hw.conf.dynamic_ps_timeout = timeout;

                /* If the TIM IE is invalid, pretend the value is 1 */
                if (!dtimper)
                        dtimper = 1;

                local->hw.conf.ps_dtim_period = dtimper;
                local->ps_sdata = found;
        } else {
                local->ps_sdata = NULL;
        }

        ieee80211_change_ps(local);
}

void ieee80211_recalc_ps_vif(struct ieee80211_sub_if_data *sdata)
{
        bool ps_allowed = ieee80211_powersave_allowed(sdata);

        if (sdata->vif.bss_conf.ps != ps_allowed) {
                sdata->vif.bss_conf.ps = ps_allowed;
                ieee80211_bss_info_change_notify(sdata, BSS_CHANGED_PS);
        }
}

void ieee80211_dynamic_ps_disable_work(struct work_struct *work)
{
        struct ieee80211_local *local =
                container_of(work, struct ieee80211_local,
                             dynamic_ps_disable_work);

        if (local->hw.conf.flags & IEEE80211_CONF_PS) {
                local->hw.conf.flags &= ~IEEE80211_CONF_PS;
                ieee80211_hw_config(local, IEEE80211_CONF_CHANGE_PS);
        }

        ieee80211_wake_queues_by_reason(&local->hw,
                                        IEEE80211_MAX_QUEUE_MAP,
                                        IEEE80211_QUEUE_STOP_REASON_PS,
                                        false);
}

void ieee80211_dynamic_ps_enable_work(struct work_struct *work)
{
        struct ieee80211_local *local =
                container_of(work, struct ieee80211_local,
                             dynamic_ps_enable_work);
        struct ieee80211_sub_if_data *sdata = local->ps_sdata;
        struct ieee80211_if_managed *ifmgd;
        unsigned long flags;
        int q;

        /* can only happen when PS was just disabled anyway */
        if (!sdata)
                return;

        ifmgd = &sdata->u.mgd;

        if (local->hw.conf.flags & IEEE80211_CONF_PS)
                return;

        if (local->hw.conf.dynamic_ps_timeout > 0) {
                /* don't enter PS if TX frames are pending */
                if (drv_tx_frames_pending(local)) {
                        mod_timer(&local->dynamic_ps_timer, jiffies +
                                  msecs_to_jiffies(
                                  local->hw.conf.dynamic_ps_timeout));
                        return;
                }

                /*
                 * transmission can be stopped by others which leads to
                 * dynamic_ps_timer expiry. Postpone the ps timer if it
                 * is not the actual idle state.
                 */
                spin_lock_irqsave(&local->queue_stop_reason_lock, flags);
                for (q = 0; q < local->hw.queues; q++) {
                        if (local->queue_stop_reasons[q]) {
                                spin_unlock_irqrestore(&local->queue_stop_reason_lock,
                                                       flags);
                                mod_timer(&local->dynamic_ps_timer, jiffies +
                                          msecs_to_jiffies(
                                          local->hw.conf.dynamic_ps_timeout));
                                return;
                        }
                }
                spin_unlock_irqrestore(&local->queue_stop_reason_lock, flags);
        }

        if (ieee80211_hw_check(&local->hw, PS_NULLFUNC_STACK) &&
            !(ifmgd->flags & IEEE80211_STA_NULLFUNC_ACKED)) {
                if (drv_tx_frames_pending(local)) {
                        mod_timer(&local->dynamic_ps_timer, jiffies +
                                  msecs_to_jiffies(
                                  local->hw.conf.dynamic_ps_timeout));
                } else {
                        ieee80211_send_nullfunc(local, sdata, true);
                        /* Flush to get the tx status of nullfunc frame */
                        ieee80211_flush_queues(local, sdata, false);
                }
        }

        if (!(ieee80211_hw_check(&local->hw, REPORTS_TX_ACK_STATUS) &&
              ieee80211_hw_check(&local->hw, PS_NULLFUNC_STACK)) ||
            (ifmgd->flags & IEEE80211_STA_NULLFUNC_ACKED)) {
                ifmgd->flags &= ~IEEE80211_STA_NULLFUNC_ACKED;
                local->hw.conf.flags |= IEEE80211_CONF_PS;
                ieee80211_hw_config(local, IEEE80211_CONF_CHANGE_PS);
        }
}

void ieee80211_dynamic_ps_timer(struct timer_list *t)
{
        struct ieee80211_local *local = from_timer(local, t, dynamic_ps_timer);

        ieee80211_queue_work(&local->hw, &local->dynamic_ps_enable_work);
}

void ieee80211_dfs_cac_timer_work(struct work_struct *work)
{
        struct delayed_work *delayed_work = to_delayed_work(work);
        struct ieee80211_sub_if_data *sdata =
                container_of(delayed_work, struct ieee80211_sub_if_data,
                             dfs_cac_timer_work);
        struct cfg80211_chan_def chandef = sdata->vif.bss_conf.chandef;

        mutex_lock(&sdata->local->mtx);
        if (sdata->wdev.cac_started) {
                ieee80211_vif_release_channel(sdata);
                cfg80211_cac_event(sdata->dev, &chandef,
                                   NL80211_RADAR_CAC_FINISHED,
                                   GFP_KERNEL);
        }
        mutex_unlock(&sdata->local->mtx);
}

static bool
__ieee80211_sta_handle_tspec_ac_params(struct ieee80211_sub_if_data *sdata)
{
        struct ieee80211_local *local = sdata->local;
        struct ieee80211_if_managed *ifmgd = &sdata->u.mgd;
        bool ret = false;
        int ac;

        if (local->hw.queues < IEEE80211_NUM_ACS)
                return false;

        for (ac = 0; ac < IEEE80211_NUM_ACS; ac++) {
                struct ieee80211_sta_tx_tspec *tx_tspec = &ifmgd->tx_tspec[ac];
                int non_acm_ac;
                unsigned long now = jiffies;

                if (tx_tspec->action == TX_TSPEC_ACTION_NONE &&
                    tx_tspec->admitted_time &&
                    time_after(now, tx_tspec->time_slice_start + HZ)) {
                        tx_tspec->consumed_tx_time = 0;
                        tx_tspec->time_slice_start = now;

                        if (tx_tspec->downgraded)
                                tx_tspec->action =
                                        TX_TSPEC_ACTION_STOP_DOWNGRADE;
                }

                switch (tx_tspec->action) {
                case TX_TSPEC_ACTION_STOP_DOWNGRADE:
                        /* take the original parameters */
                        if (drv_conf_tx(local, sdata, ac, &sdata->tx_conf[ac]))
                                sdata_err(sdata,
                                          "failed to set TX queue parameters for queue %d\n",
                                          ac);
                        tx_tspec->action = TX_TSPEC_ACTION_NONE;
                        tx_tspec->downgraded = false;
                        ret = true;
                        break;
                case TX_TSPEC_ACTION_DOWNGRADE:
                        if (time_after(now, tx_tspec->time_slice_start + HZ)) {
                                tx_tspec->action = TX_TSPEC_ACTION_NONE;
                                ret = true;
                                break;
                        }
                        /* downgrade next lower non-ACM AC */
                        for (non_acm_ac = ac + 1;
                             non_acm_ac < IEEE80211_NUM_ACS;
                             non_acm_ac++)
                                if (!(sdata->wmm_acm & BIT(7 - 2 * non_acm_ac)))
                                        break;
                        /* Usually the loop will result in using BK even if it
                         * requires admission control, but such a configuration
                         * makes no sense and we have to transmit somehow - the
                         * AC selection does the same thing.
                         * If we started out trying to downgrade from BK, then
                         * the extra condition here might be needed.
                         */
                        if (non_acm_ac >= IEEE80211_NUM_ACS)
                                non_acm_ac = IEEE80211_AC_BK;
                        if (drv_conf_tx(local, sdata, ac,
                                        &sdata->tx_conf[non_acm_ac]))
                                sdata_err(sdata,
                                          "failed to set TX queue parameters for queue %d\n",
                                          ac);
                        tx_tspec->action = TX_TSPEC_ACTION_NONE;
                        ret = true;
                        schedule_delayed_work(&ifmgd->tx_tspec_wk,
                                tx_tspec->time_slice_start + HZ - now + 1);
                        break;
                case TX_TSPEC_ACTION_NONE:
                        /* nothing now */
                        break;
                }
        }

        return ret;
}

void ieee80211_sta_handle_tspec_ac_params(struct ieee80211_sub_if_data *sdata)
{
        if (__ieee80211_sta_handle_tspec_ac_params(sdata))
                ieee80211_bss_info_change_notify(sdata, BSS_CHANGED_QOS);
}

static void ieee80211_sta_handle_tspec_ac_params_wk(struct work_struct *work)
{
        struct ieee80211_sub_if_data *sdata;

        sdata = container_of(work, struct ieee80211_sub_if_data,
                             u.mgd.tx_tspec_wk.work);
        ieee80211_sta_handle_tspec_ac_params(sdata);
}

/* MLME */
static bool
ieee80211_sta_wmm_params(struct ieee80211_local *local,
                         struct ieee80211_sub_if_data *sdata,
                         const u8 *wmm_param, size_t wmm_param_len,
                         const struct ieee80211_mu_edca_param_set *mu_edca)
{
        struct ieee80211_tx_queue_params params[IEEE80211_NUM_ACS];
        struct ieee80211_if_managed *ifmgd = &sdata->u.mgd;
        size_t left;
        int count, mu_edca_count, ac;
        const u8 *pos;
        u8 uapsd_queues = 0;

        if (!local->ops->conf_tx)
                return false;

        if (local->hw.queues < IEEE80211_NUM_ACS)
                return false;

        if (!wmm_param)
                return false;

        if (wmm_param_len < 8 || wmm_param[5] /* version */ != 1)
                return false;

        if (ifmgd->flags & IEEE80211_STA_UAPSD_ENABLED)
                uapsd_queues = ifmgd->uapsd_queues;

        count = wmm_param[6] & 0x0f;
        /* -1 is the initial value of ifmgd->mu_edca_last_param_set.
         * if mu_edca was preset before and now it disappeared tell
         * the driver about it.
         */
        mu_edca_count = mu_edca ? mu_edca->mu_qos_info & 0x0f : -1;
        if (count == ifmgd->wmm_last_param_set &&
            mu_edca_count == ifmgd->mu_edca_last_param_set)
                return false;
        ifmgd->wmm_last_param_set = count;
        ifmgd->mu_edca_last_param_set = mu_edca_count;

        pos = wmm_param + 8;
        left = wmm_param_len - 8;

        memset(&params, 0, sizeof(params));

        sdata->wmm_acm = 0;
        for (; left >= 4; left -= 4, pos += 4) {
                int aci = (pos[0] >> 5) & 0x03;
                int acm = (pos[0] >> 4) & 0x01;
                bool uapsd = false;

                switch (aci) {
                case 1: /* AC_BK */
                        ac = IEEE80211_AC_BK;
                        if (acm)
                                sdata->wmm_acm |= BIT(1) | BIT(2); /* BK/- */
                        if (uapsd_queues & IEEE80211_WMM_IE_STA_QOSINFO_AC_BK)
                                uapsd = true;
                        params[ac].mu_edca = !!mu_edca;
                        if (mu_edca)
                                params[ac].mu_edca_param_rec = mu_edca->ac_bk;
                        break;
                case 2: /* AC_VI */
                        ac = IEEE80211_AC_VI;
                        if (acm)
                                sdata->wmm_acm |= BIT(4) | BIT(5); /* CL/VI */
                        if (uapsd_queues & IEEE80211_WMM_IE_STA_QOSINFO_AC_VI)
                                uapsd = true;
                        params[ac].mu_edca = !!mu_edca;
                        if (mu_edca)
                                params[ac].mu_edca_param_rec = mu_edca->ac_vi;
                        break;
                case 3: /* AC_VO */
                        ac = IEEE80211_AC_VO;
                        if (acm)
                                sdata->wmm_acm |= BIT(6) | BIT(7); /* VO/NC */
                        if (uapsd_queues & IEEE80211_WMM_IE_STA_QOSINFO_AC_VO)
                                uapsd = true;
                        params[ac].mu_edca = !!mu_edca;
                        if (mu_edca)
                                params[ac].mu_edca_param_rec = mu_edca->ac_vo;
                        break;
                case 0: /* AC_BE */
                default:
                        ac = IEEE80211_AC_BE;
                        if (acm)
                                sdata->wmm_acm |= BIT(0) | BIT(3); /* BE/EE */
                        if (uapsd_queues & IEEE80211_WMM_IE_STA_QOSINFO_AC_BE)
                                uapsd = true;
                        params[ac].mu_edca = !!mu_edca;
                        if (mu_edca)
                                params[ac].mu_edca_param_rec = mu_edca->ac_be;
                        break;
                }

                params[ac].aifs = pos[0] & 0x0f;

                if (params[ac].aifs < 2) {
                        sdata_info(sdata,
                                   "AP has invalid WMM params (AIFSN=%d for ACI %d), will use 2\n",
                                   params[ac].aifs, aci);
                        params[ac].aifs = 2;
                }
                params[ac].cw_max = ecw2cw((pos[1] & 0xf0) >> 4);
                params[ac].cw_min = ecw2cw(pos[1] & 0x0f);
                params[ac].txop = get_unaligned_le16(pos + 2);
                params[ac].acm = acm;
                params[ac].uapsd = uapsd;

                if (params[ac].cw_min == 0 ||
                    params[ac].cw_min > params[ac].cw_max) {
                        sdata_info(sdata,
                                   "AP has invalid WMM params (CWmin/max=%d/%d for ACI %d), using defaults\n",
                                   params[ac].cw_min, params[ac].cw_max, aci);
                        return false;
                }
                ieee80211_regulatory_limit_wmm_params(sdata, &params[ac], ac);
        }

        /* WMM specification requires all 4 ACIs. */
        for (ac = 0; ac < IEEE80211_NUM_ACS; ac++) {
                if (params[ac].cw_min == 0) {
                        sdata_info(sdata,
                                   "AP has invalid WMM params (missing AC %d), using defaults\n",
                                   ac);
                        return false;
                }
        }

        for (ac = 0; ac < IEEE80211_NUM_ACS; ac++) {
                mlme_dbg(sdata,
                         "WMM AC=%d acm=%d aifs=%d cWmin=%d cWmax=%d txop=%d uapsd=%d, downgraded=%d\n",
                         ac, params[ac].acm,
                         params[ac].aifs, params[ac].cw_min, params[ac].cw_max,
                         params[ac].txop, params[ac].uapsd,
                         ifmgd->tx_tspec[ac].downgraded);
                sdata->tx_conf[ac] = params[ac];
                if (!ifmgd->tx_tspec[ac].downgraded &&
                    drv_conf_tx(local, sdata, ac, &params[ac]))
                        sdata_err(sdata,
                                  "failed to set TX queue parameters for AC %d\n",
                                  ac);
        }

        /* enable WMM or activate new settings */
        sdata->vif.bss_conf.qos = true;
        return true;
}

static void __ieee80211_stop_poll(struct ieee80211_sub_if_data *sdata)
{
        lockdep_assert_held(&sdata->local->mtx);

        sdata->u.mgd.flags &= ~IEEE80211_STA_CONNECTION_POLL;
        ieee80211_run_deferred_scan(sdata->local);
}

static void ieee80211_stop_poll(struct ieee80211_sub_if_data *sdata)
{
        mutex_lock(&sdata->local->mtx);
        __ieee80211_stop_poll(sdata);
        mutex_unlock(&sdata->local->mtx);
}

static u32 ieee80211_handle_bss_capability(struct ieee80211_sub_if_data *sdata,
                                           u16 capab, bool erp_valid, u8 erp)
{
        struct ieee80211_bss_conf *bss_conf = &sdata->vif.bss_conf;
        struct ieee80211_supported_band *sband;
        u32 changed = 0;
        bool use_protection;
        bool use_short_preamble;
        bool use_short_slot;

        sband = ieee80211_get_sband(sdata);
        if (!sband)
                return changed;

        if (erp_valid) {
                use_protection = (erp & WLAN_ERP_USE_PROTECTION) != 0;
                use_short_preamble = (erp & WLAN_ERP_BARKER_PREAMBLE) == 0;
        } else {
                use_protection = false;
                use_short_preamble = !!(capab & WLAN_CAPABILITY_SHORT_PREAMBLE);
        }

        use_short_slot = !!(capab & WLAN_CAPABILITY_SHORT_SLOT_TIME);
        if (sband->band == NL80211_BAND_5GHZ ||
            sband->band == NL80211_BAND_6GHZ)
                use_short_slot = true;

        if (use_protection != bss_conf->use_cts_prot) {
                bss_conf->use_cts_prot = use_protection;
                changed |= BSS_CHANGED_ERP_CTS_PROT;
        }

        if (use_short_preamble != bss_conf->use_short_preamble) {
                bss_conf->use_short_preamble = use_short_preamble;
                changed |= BSS_CHANGED_ERP_PREAMBLE;
        }

        if (use_short_slot != bss_conf->use_short_slot) {
                bss_conf->use_short_slot = use_short_slot;
                changed |= BSS_CHANGED_ERP_SLOT;
        }

        return changed;
}

static void ieee80211_set_associated(struct ieee80211_sub_if_data *sdata,
                                     struct cfg80211_bss *cbss,
                                     u32 bss_info_changed)
{
        struct ieee80211_bss *bss = (void *)cbss->priv;
        struct ieee80211_local *local = sdata->local;
        struct ieee80211_bss_conf *bss_conf = &sdata->vif.bss_conf;

        bss_info_changed |= BSS_CHANGED_ASSOC;
        bss_info_changed |= ieee80211_handle_bss_capability(sdata,
                bss_conf->assoc_capability, bss->has_erp_value, bss->erp_value);

        sdata->u.mgd.beacon_timeout = usecs_to_jiffies(ieee80211_tu_to_usec(
                beacon_loss_count * bss_conf->beacon_int));

        sdata->u.mgd.associated = cbss;
        memcpy(sdata->u.mgd.bssid, cbss->bssid, ETH_ALEN);

        ieee80211_check_rate_mask(sdata);

        sdata->u.mgd.flags |= IEEE80211_STA_RESET_SIGNAL_AVE;

        if (sdata->vif.p2p ||
            sdata->vif.driver_flags & IEEE80211_VIF_GET_NOA_UPDATE) {
                const struct cfg80211_bss_ies *ies;

                rcu_read_lock();
                ies = rcu_dereference(cbss->ies);
                if (ies) {
                        int ret;

                        ret = cfg80211_get_p2p_attr(
                                        ies->data, ies->len,
                                        IEEE80211_P2P_ATTR_ABSENCE_NOTICE,
                                        (u8 *) &bss_conf->p2p_noa_attr,
                                        sizeof(bss_conf->p2p_noa_attr));
                        if (ret >= 2) {
                                sdata->u.mgd.p2p_noa_index =
                                        bss_conf->p2p_noa_attr.index;
                                bss_info_changed |= BSS_CHANGED_P2P_PS;
                        }
                }
                rcu_read_unlock();
        }

        /* just to be sure */
        ieee80211_stop_poll(sdata);

        ieee80211_led_assoc(local, 1);

        if (sdata->u.mgd.have_beacon) {
                /*
                 * If the AP is buggy we may get here with no DTIM period
                 * known, so assume it's 1 which is the only safe assumption
                 * in that case, although if the TIM IE is broken powersave
                 * probably just won't work at all.
                 */
                bss_conf->dtim_period = sdata->u.mgd.dtim_period ?: 1;
                bss_conf->beacon_rate = bss->beacon_rate;
                bss_info_changed |= BSS_CHANGED_BEACON_INFO;
        } else {
                bss_conf->beacon_rate = NULL;
                bss_conf->dtim_period = 0;
        }

        bss_conf->assoc = 1;

        /* Tell the driver to monitor connection quality (if supported) */
        if (sdata->vif.driver_flags & IEEE80211_VIF_SUPPORTS_CQM_RSSI &&
            bss_conf->cqm_rssi_thold)
                bss_info_changed |= BSS_CHANGED_CQM;

        /* Enable ARP filtering */
        if (bss_conf->arp_addr_cnt)
                bss_info_changed |= BSS_CHANGED_ARP_FILTER;

        ieee80211_bss_info_change_notify(sdata, bss_info_changed);

        mutex_lock(&local->iflist_mtx);
        ieee80211_recalc_ps(local);
        mutex_unlock(&local->iflist_mtx);

        ieee80211_recalc_smps(sdata);
        ieee80211_recalc_ps_vif(sdata);

        netif_carrier_on(sdata->dev);
}

static void ieee80211_set_disassoc(struct ieee80211_sub_if_data *sdata,
                                   u16 stype, u16 reason, bool tx,
                                   u8 *frame_buf)
{
        struct ieee80211_if_managed *ifmgd = &sdata->u.mgd;
        struct ieee80211_local *local = sdata->local;
        u32 changed = 0;
        struct ieee80211_prep_tx_info info = {
                .subtype = stype,
        };

        sdata_assert_lock(sdata);

        if (WARN_ON_ONCE(tx && !frame_buf))
                return;

        if (WARN_ON(!ifmgd->associated))
                return;

        ieee80211_stop_poll(sdata);

        ifmgd->associated = NULL;
        netif_carrier_off(sdata->dev);

        /*
         * if we want to get out of ps before disassoc (why?) we have
         * to do it before sending disassoc, as otherwise the null-packet
         * won't be valid.
         */
        if (local->hw.conf.flags & IEEE80211_CONF_PS) {
                local->hw.conf.flags &= ~IEEE80211_CONF_PS;
                ieee80211_hw_config(local, IEEE80211_CONF_CHANGE_PS);
        }
        local->ps_sdata = NULL;

        /* disable per-vif ps */
        ieee80211_recalc_ps_vif(sdata);

        /* make sure ongoing transmission finishes */
        synchronize_net();

        /*
         * drop any frame before deauth/disassoc, this can be data or
         * management frame. Since we are disconnecting, we should not
         * insist sending these frames which can take time and delay
         * the disconnection and possible the roaming.
         */
        if (tx)
                ieee80211_flush_queues(local, sdata, true);

        /* deauthenticate/disassociate now */
        if (tx || frame_buf) {
                /*
                 * In multi channel scenarios guarantee that the virtual
                 * interface is granted immediate airtime to transmit the
                 * deauthentication frame by calling mgd_prepare_tx, if the
                 * driver requested so.
                 */
                if (ieee80211_hw_check(&local->hw, DEAUTH_NEED_MGD_TX_PREP) &&
                    !ifmgd->have_beacon) {
                        drv_mgd_prepare_tx(sdata->local, sdata, &info);
                }

                ieee80211_send_deauth_disassoc(sdata, ifmgd->bssid,
                                               ifmgd->bssid, stype, reason,
                                               tx, frame_buf);
        }

        /* flush out frame - make sure the deauth was actually sent */
        if (tx)
                ieee80211_flush_queues(local, sdata, false);

        drv_mgd_complete_tx(sdata->local, sdata, &info);

        /* clear bssid only after building the needed mgmt frames */
        eth_zero_addr(ifmgd->bssid);

        sdata->vif.bss_conf.ssid_len = 0;

        /* remove AP and TDLS peers */
        sta_info_flush(sdata);

        /* finally reset all BSS / config parameters */
        changed |= ieee80211_reset_erp_info(sdata);

        ieee80211_led_assoc(local, 0);
        changed |= BSS_CHANGED_ASSOC;
        sdata->vif.bss_conf.assoc = false;

        ifmgd->p2p_noa_index = -1;
        memset(&sdata->vif.bss_conf.p2p_noa_attr, 0,
               sizeof(sdata->vif.bss_conf.p2p_noa_attr));

        /* on the next assoc, re-program HT/VHT parameters */
        memset(&ifmgd->ht_capa, 0, sizeof(ifmgd->ht_capa));
        memset(&ifmgd->ht_capa_mask, 0, sizeof(ifmgd->ht_capa_mask));
        memset(&ifmgd->vht_capa, 0, sizeof(ifmgd->vht_capa));
        memset(&ifmgd->vht_capa_mask, 0, sizeof(ifmgd->vht_capa_mask));

        /* reset MU-MIMO ownership and group data */
        memset(sdata->vif.bss_conf.mu_group.membership, 0,
               sizeof(sdata->vif.bss_conf.mu_group.membership));
        memset(sdata->vif.bss_conf.mu_group.position, 0,
               sizeof(sdata->vif.bss_conf.mu_group.position));
        changed |= BSS_CHANGED_MU_GROUPS;
        sdata->vif.mu_mimo_owner = false;

        sdata->ap_power_level = IEEE80211_UNSET_POWER_LEVEL;

        del_timer_sync(&local->dynamic_ps_timer);
        cancel_work_sync(&local->dynamic_ps_enable_work);

        /* Disable ARP filtering */
        if (sdata->vif.bss_conf.arp_addr_cnt)
                changed |= BSS_CHANGED_ARP_FILTER;

        sdata->vif.bss_conf.qos = false;
        changed |= BSS_CHANGED_QOS;

        /* The BSSID (not really interesting) and HT changed */
        changed |= BSS_CHANGED_BSSID | BSS_CHANGED_HT;
        ieee80211_bss_info_change_notify(sdata, changed);

        /* disassociated - set to defaults now */
        ieee80211_set_wmm_default(sdata, false, false);

        del_timer_sync(&sdata->u.mgd.conn_mon_timer);
        del_timer_sync(&sdata->u.mgd.bcn_mon_timer);
        del_timer_sync(&sdata->u.mgd.timer);
        del_timer_sync(&sdata->u.mgd.chswitch_timer);

        sdata->vif.bss_conf.dtim_period = 0;
        sdata->vif.bss_conf.beacon_rate = NULL;

        ifmgd->have_beacon = false;

        ifmgd->flags = 0;
        mutex_lock(&local->mtx);
        ieee80211_vif_release_channel(sdata);

        sdata->vif.csa_active = false;
        ifmgd->csa_waiting_bcn = false;
        ifmgd->csa_ignored_same_chan = false;
        if (sdata->csa_block_tx) {
                ieee80211_wake_vif_queues(local, sdata,
                                          IEEE80211_QUEUE_STOP_REASON_CSA);
                sdata->csa_block_tx = false;
        }
        mutex_unlock(&local->mtx);

        /* existing TX TSPEC sessions no longer exist */
        memset(ifmgd->tx_tspec, 0, sizeof(ifmgd->tx_tspec));
        cancel_delayed_work_sync(&ifmgd->tx_tspec_wk);

        sdata->encrypt_headroom = IEEE80211_ENCRYPT_HEADROOM;
}

static void ieee80211_reset_ap_probe(struct ieee80211_sub_if_data *sdata)
{
        struct ieee80211_if_managed *ifmgd = &sdata->u.mgd;
        struct ieee80211_local *local = sdata->local;

        mutex_lock(&local->mtx);
        if (!(ifmgd->flags & IEEE80211_STA_CONNECTION_POLL))
                goto out;

        __ieee80211_stop_poll(sdata);

        mutex_lock(&local->iflist_mtx);
        ieee80211_recalc_ps(local);
        mutex_unlock(&local->iflist_mtx);

        if (ieee80211_hw_check(&sdata->local->hw, CONNECTION_MONITOR))
                goto out;

        /*
         * We've received a probe response, but are not sure whether
         * we have or will be receiving any beacons or data, so let's
         * schedule the timers again, just in case.
         */
        ieee80211_sta_reset_beacon_monitor(sdata);

        mod_timer(&ifmgd->conn_mon_timer,
                  round_jiffies_up(jiffies +
                                   IEEE80211_CONNECTION_IDLE_TIME));
out:
        mutex_unlock(&local->mtx);
}

static void ieee80211_sta_tx_wmm_ac_notify(struct ieee80211_sub_if_data *sdata,
                                           struct ieee80211_hdr *hdr,
                                           u16 tx_time)
{
        struct ieee80211_if_managed *ifmgd = &sdata->u.mgd;
        u16 tid;
        int ac;
        struct ieee80211_sta_tx_tspec *tx_tspec;
        unsigned long now = jiffies;

        if (!ieee80211_is_data_qos(hdr->frame_control))
                return;

        tid = ieee80211_get_tid(hdr);
        ac = ieee80211_ac_from_tid(tid);
        tx_tspec = &ifmgd->tx_tspec[ac];

        if (likely(!tx_tspec->admitted_time))
                return;

        if (time_after(now, tx_tspec->time_slice_start + HZ)) {
                tx_tspec->consumed_tx_time = 0;
                tx_tspec->time_slice_start = now;

                if (tx_tspec->downgraded) {
                        tx_tspec->action = TX_TSPEC_ACTION_STOP_DOWNGRADE;
                        schedule_delayed_work(&ifmgd->tx_tspec_wk, 0);
                }
        }

        if (tx_tspec->downgraded)
                return;

        tx_tspec->consumed_tx_time += tx_time;

        if (tx_tspec->consumed_tx_time >= tx_tspec->admitted_time) {
                tx_tspec->downgraded = true;
                tx_tspec->action = TX_TSPEC_ACTION_DOWNGRADE;
                schedule_delayed_work(&ifmgd->tx_tspec_wk, 0);
        }
}

void ieee80211_sta_tx_notify(struct ieee80211_sub_if_data *sdata,
                             struct ieee80211_hdr *hdr, bool ack, u16 tx_time)
{
        ieee80211_sta_tx_wmm_ac_notify(sdata, hdr, tx_time);

        if (!ieee80211_is_any_nullfunc(hdr->frame_control) ||
            !sdata->u.mgd.probe_send_count)
                return;

        if (ack)
                sdata->u.mgd.probe_send_count = 0;
        else
                sdata->u.mgd.nullfunc_failed = true;
        ieee80211_queue_work(&sdata->local->hw, &sdata->work);
}

static void ieee80211_mlme_send_probe_req(struct ieee80211_sub_if_data *sdata,
                                          const u8 *src, const u8 *dst,
                                          const u8 *ssid, size_t ssid_len,
                                          struct ieee80211_channel *channel)
{
        struct sk_buff *skb;

        skb = ieee80211_build_probe_req(sdata, src, dst, (u32)-1, channel,
                                        ssid, ssid_len, NULL, 0,
                                        IEEE80211_PROBE_FLAG_DIRECTED);
        if (skb)
                ieee80211_tx_skb(sdata, skb);
}

static void ieee80211_mgd_probe_ap_send(struct ieee80211_sub_if_data *sdata)
{
        struct ieee80211_if_managed *ifmgd = &sdata->u.mgd;
        const u8 *ssid;
        u8 *dst = ifmgd->associated->bssid;
        u8 unicast_limit = max(1, max_probe_tries - 3);
        struct sta_info *sta;

        /*
         * Try sending broadcast probe requests for the last three
         * probe requests after the first ones failed since some
         * buggy APs only support broadcast probe requests.
         */
        if (ifmgd->probe_send_count >= unicast_limit)
                dst = NULL;

        /*
         * When the hardware reports an accurate Tx ACK status, it's
         * better to send a nullfunc frame instead of a probe request,
         * as it will kick us off the AP quickly if we aren't associated
         * anymore. The timeout will be reset if the frame is ACKed by
         * the AP.
         */
        ifmgd->probe_send_count++;

        if (dst) {
                mutex_lock(&sdata->local->sta_mtx);
                sta = sta_info_get(sdata, dst);
                if (!WARN_ON(!sta))
                        ieee80211_check_fast_rx(sta);
                mutex_unlock(&sdata->local->sta_mtx);
        }

        if (ieee80211_hw_check(&sdata->local->hw, REPORTS_TX_ACK_STATUS)) {
                ifmgd->nullfunc_failed = false;
                ieee80211_send_nullfunc(sdata->local, sdata, false);
        } else {
                int ssid_len;

                rcu_read_lock();
                ssid = ieee80211_bss_get_ie(ifmgd->associated, WLAN_EID_SSID);
                if (WARN_ON_ONCE(ssid == NULL))
                        ssid_len = 0;
                else
                        ssid_len = ssid[1];

                ieee80211_mlme_send_probe_req(sdata, sdata->vif.addr, dst,
                                              ssid + 2, ssid_len,
                                              ifmgd->associated->channel);
                rcu_read_unlock();
        }

        ifmgd->probe_timeout = jiffies + msecs_to_jiffies(probe_wait_ms);
        run_again(sdata, ifmgd->probe_timeout);
}

static void ieee80211_mgd_probe_ap(struct ieee80211_sub_if_data *sdata,
                                   bool beacon)
{
        struct ieee80211_if_managed *ifmgd = &sdata->u.mgd;
        bool already = false;

        if (!ieee80211_sdata_running(sdata))
                return;

        sdata_lock(sdata);

        if (!ifmgd->associated)
                goto out;

        mutex_lock(&sdata->local->mtx);

        if (sdata->local->tmp_channel || sdata->local->scanning) {
                mutex_unlock(&sdata->local->mtx);
                goto out;
        }

        if (beacon) {
                mlme_dbg_ratelimited(sdata,
                                     "detected beacon loss from AP (missed %d beacons) - probing\n",
                                     beacon_loss_count);

                ieee80211_cqm_beacon_loss_notify(&sdata->vif, GFP_KERNEL);
        }

        /*
         * The driver/our work has already reported this event or the
         * connection monitoring has kicked in and we have already sent
         * a probe request. Or maybe the AP died and the driver keeps
         * reporting until we disassociate...
         *
         * In either case we have to ignore the current call to this
         * function (except for setting the correct probe reason bit)
         * because otherwise we would reset the timer every time and
         * never check whether we received a probe response!
         */
        if (ifmgd->flags & IEEE80211_STA_CONNECTION_POLL)
                already = true;

        ifmgd->flags |= IEEE80211_STA_CONNECTION_POLL;

        mutex_unlock(&sdata->local->mtx);

        if (already)
                goto out;

        mutex_lock(&sdata->local->iflist_mtx);
        ieee80211_recalc_ps(sdata->local);
        mutex_unlock(&sdata->local->iflist_mtx);

        ifmgd->probe_send_count = 0;
        ieee80211_mgd_probe_ap_send(sdata);
 out:
        sdata_unlock(sdata);
}

struct sk_buff *ieee80211_ap_probereq_get(struct ieee80211_hw *hw,
                                          struct ieee80211_vif *vif)
{
        struct ieee80211_sub_if_data *sdata = vif_to_sdata(vif);
        struct ieee80211_if_managed *ifmgd = &sdata->u.mgd;
        struct cfg80211_bss *cbss;
        struct sk_buff *skb;
        const u8 *ssid;
        int ssid_len;

        if (WARN_ON(sdata->vif.type != NL80211_IFTYPE_STATION))
                return NULL;

        sdata_assert_lock(sdata);

        if (ifmgd->associated)
                cbss = ifmgd->associated;
        else if (ifmgd->auth_data)
                cbss = ifmgd->auth_data->bss;
        else if (ifmgd->assoc_data)
                cbss = ifmgd->assoc_data->bss;
        else
                return NULL;

        rcu_read_lock();
        ssid = ieee80211_bss_get_ie(cbss, WLAN_EID_SSID);
        if (WARN_ONCE(!ssid || ssid[1] > IEEE80211_MAX_SSID_LEN,
                      "invalid SSID element (len=%d)", ssid ? ssid[1] : -1))
                ssid_len = 0;
        else
                ssid_len = ssid[1];

        skb = ieee80211_build_probe_req(sdata, sdata->vif.addr, cbss->bssid,
                                        (u32) -1, cbss->channel,
                                        ssid + 2, ssid_len,
                                        NULL, 0, IEEE80211_PROBE_FLAG_DIRECTED);
        rcu_read_unlock();

        return skb;
}
EXPORT_SYMBOL(ieee80211_ap_probereq_get);

static void ieee80211_report_disconnect(struct ieee80211_sub_if_data *sdata,
                                        const u8 *buf, size_t len, bool tx,
                                        u16 reason, bool reconnect)
{
        struct ieee80211_event event = {
                .type = MLME_EVENT,
                .u.mlme.data = tx ? DEAUTH_TX_EVENT : DEAUTH_RX_EVENT,
                .u.mlme.reason = reason,
        };

        if (tx)
                cfg80211_tx_mlme_mgmt(sdata->dev, buf, len, reconnect);
        else
                cfg80211_rx_mlme_mgmt(sdata->dev, buf, len);

        drv_event_callback(sdata->local, sdata, &event);
}

static void __ieee80211_disconnect(struct ieee80211_sub_if_data *sdata)
{
        struct ieee80211_local *local = sdata->local;
        struct ieee80211_if_managed *ifmgd = &sdata->u.mgd;
        u8 frame_buf[IEEE80211_DEAUTH_FRAME_LEN];
        bool tx;

        sdata_lock(sdata);
        if (!ifmgd->associated) {
                sdata_unlock(sdata);
                return;
        }

        tx = !sdata->csa_block_tx;

        if (!ifmgd->driver_disconnect) {
                /*
                 * AP is probably out of range (or not reachable for another
                 * reason) so remove the bss struct for that AP.
                 */
                cfg80211_unlink_bss(local->hw.wiphy, ifmgd->associated);
        }

        ieee80211_set_disassoc(sdata, IEEE80211_STYPE_DEAUTH,
                               ifmgd->driver_disconnect ?
                                        WLAN_REASON_DEAUTH_LEAVING :
                                        WLAN_REASON_DISASSOC_DUE_TO_INACTIVITY,
                               tx, frame_buf);
        mutex_lock(&local->mtx);
        sdata->vif.csa_active = false;
        ifmgd->csa_waiting_bcn = false;
        if (sdata->csa_block_tx) {
                ieee80211_wake_vif_queues(local, sdata,
                                          IEEE80211_QUEUE_STOP_REASON_CSA);
                sdata->csa_block_tx = false;
        }
        mutex_unlock(&local->mtx);

        ieee80211_report_disconnect(sdata, frame_buf, sizeof(frame_buf), tx,
                                    WLAN_REASON_DISASSOC_DUE_TO_INACTIVITY,
                                    ifmgd->reconnect);
        ifmgd->reconnect = false;

        sdata_unlock(sdata);
}

static void ieee80211_beacon_connection_loss_work(struct work_struct *work)
{
        struct ieee80211_sub_if_data *sdata =
                container_of(work, struct ieee80211_sub_if_data,
                             u.mgd.beacon_connection_loss_work);
        struct ieee80211_if_managed *ifmgd = &sdata->u.mgd;

        if (ifmgd->associated)
                ifmgd->beacon_loss_count++;

        if (ifmgd->connection_loss) {
                sdata_info(sdata, "Connection to AP %pM lost\n",
                           ifmgd->bssid);
                __ieee80211_disconnect(sdata);
                ifmgd->connection_loss = false;
        } else if (ifmgd->driver_disconnect) {
                sdata_info(sdata,
                           "Driver requested disconnection from AP %pM\n",
                           ifmgd->bssid);
                __ieee80211_disconnect(sdata);
                ifmgd->driver_disconnect = false;
        } else {
                ieee80211_mgd_probe_ap(sdata, true);
        }
}

static void ieee80211_csa_connection_drop_work(struct work_struct *work)
{
        struct ieee80211_sub_if_data *sdata =
                container_of(work, struct ieee80211_sub_if_data,
                             u.mgd.csa_connection_drop_work);

        __ieee80211_disconnect(sdata);
}

void ieee80211_beacon_loss(struct ieee80211_vif *vif)
{
        struct ieee80211_sub_if_data *sdata = vif_to_sdata(vif);
        struct ieee80211_hw *hw = &sdata->local->hw;

        trace_api_beacon_loss(sdata);

        sdata->u.mgd.connection_loss = false;
        ieee80211_queue_work(hw, &sdata->u.mgd.beacon_connection_loss_work);
}
EXPORT_SYMBOL(ieee80211_beacon_loss);

void ieee80211_connection_loss(struct ieee80211_vif *vif)
{
        struct ieee80211_sub_if_data *sdata = vif_to_sdata(vif);
        struct ieee80211_hw *hw = &sdata->local->hw;

        trace_api_connection_loss(sdata);

        sdata->u.mgd.connection_loss = true;
        ieee80211_queue_work(hw, &sdata->u.mgd.beacon_connection_loss_work);
}
EXPORT_SYMBOL(ieee80211_connection_loss);

void ieee80211_disconnect(struct ieee80211_vif *vif, bool reconnect)
{
        struct ieee80211_sub_if_data *sdata = vif_to_sdata(vif);
        struct ieee80211_hw *hw = &sdata->local->hw;

        trace_api_disconnect(sdata, reconnect);

        if (WARN_ON(sdata->vif.type != NL80211_IFTYPE_STATION))
                return;

        sdata->u.mgd.driver_disconnect = true;
        sdata->u.mgd.reconnect = reconnect;
        ieee80211_queue_work(hw, &sdata->u.mgd.beacon_connection_loss_work);
}
EXPORT_SYMBOL(ieee80211_disconnect);

static void ieee80211_destroy_auth_data(struct ieee80211_sub_if_data *sdata,
                                        bool assoc)
{
        struct ieee80211_mgd_auth_data *auth_data = sdata->u.mgd.auth_data;

        sdata_assert_lock(sdata);

        if (!assoc) {
                /*
                 * we are not authenticated yet, the only timer that could be
                 * running is the timeout for the authentication response which
                 * which is not relevant anymore.
                 */
                del_timer_sync(&sdata->u.mgd.timer);
                sta_info_destroy_addr(sdata, auth_data->bss->bssid);

                eth_zero_addr(sdata->u.mgd.bssid);
                ieee80211_bss_info_change_notify(sdata, BSS_CHANGED_BSSID);
                sdata->u.mgd.flags = 0;
                mutex_lock(&sdata->local->mtx);
                ieee80211_vif_release_channel(sdata);
                mutex_unlock(&sdata->local->mtx);
        }

        cfg80211_put_bss(sdata->local->hw.wiphy, auth_data->bss);
        kfree(auth_data);
        sdata->u.mgd.auth_data = NULL;
}

static void ieee80211_destroy_assoc_data(struct ieee80211_sub_if_data *sdata,
                                         bool assoc, bool abandon)
{
        struct ieee80211_mgd_assoc_data *assoc_data = sdata->u.mgd.assoc_data;

        sdata_assert_lock(sdata);

        if (!assoc) {
                /*
                 * we are not associated yet, the only timer that could be
                 * running is the timeout for the association response which
                 * which is not relevant anymore.
                 */
                del_timer_sync(&sdata->u.mgd.timer);
                sta_info_destroy_addr(sdata, assoc_data->bss->bssid);

                eth_zero_addr(sdata->u.mgd.bssid);
                ieee80211_bss_info_change_notify(sdata, BSS_CHANGED_BSSID);
                sdata->u.mgd.flags = 0;
                sdata->vif.mu_mimo_owner = false;

                mutex_lock(&sdata->local->mtx);
                ieee80211_vif_release_channel(sdata);
                mutex_unlock(&sdata->local->mtx);

                if (abandon)
                        cfg80211_abandon_assoc(sdata->dev, assoc_data->bss);
        }

        kfree(assoc_data);
        sdata->u.mgd.assoc_data = NULL;
}

static void ieee80211_auth_challenge(struct ieee80211_sub_if_data *sdata,
                                     struct ieee80211_mgmt *mgmt, size_t len)
{
        struct ieee80211_local *local = sdata->local;
        struct ieee80211_mgd_auth_data *auth_data = sdata->u.mgd.auth_data;
        const struct element *challenge;
        u8 *pos;
        u32 tx_flags = 0;
        struct ieee80211_prep_tx_info info = {
                .subtype = IEEE80211_STYPE_AUTH,
        };

        pos = mgmt->u.auth.variable;
        challenge = cfg80211_find_elem(WLAN_EID_CHALLENGE, pos,
                                       len - (pos - (u8 *)mgmt));
        if (!challenge)
                return;
        auth_data->expected_transaction = 4;
        drv_mgd_prepare_tx(sdata->local, sdata, &info);
        if (ieee80211_hw_check(&local->hw, REPORTS_TX_ACK_STATUS))
                tx_flags = IEEE80211_TX_CTL_REQ_TX_STATUS |
                           IEEE80211_TX_INTFL_MLME_CONN_TX;
        ieee80211_send_auth(sdata, 3, auth_data->algorithm, 0,
                            (void *)challenge,
                            challenge->datalen + sizeof(*challenge),
                            auth_data->bss->bssid, auth_data->bss->bssid,
                            auth_data->key, auth_data->key_len,
                            auth_data->key_idx, tx_flags);
}

static bool ieee80211_mark_sta_auth(struct ieee80211_sub_if_data *sdata,
                                    const u8 *bssid)
{
        struct ieee80211_if_managed *ifmgd = &sdata->u.mgd;
        struct sta_info *sta;
        bool result = true;

        sdata_info(sdata, "authenticated\n");
        ifmgd->auth_data->done = true;
        ifmgd->auth_data->timeout = jiffies + IEEE80211_AUTH_WAIT_ASSOC;
        ifmgd->auth_data->timeout_started = true;
        run_again(sdata, ifmgd->auth_data->timeout);

        /* move station state to auth */
        mutex_lock(&sdata->local->sta_mtx);
        sta = sta_info_get(sdata, bssid);
        if (!sta) {
                WARN_ONCE(1, "%s: STA %pM not found", sdata->name, bssid);
                result = false;
                goto out;
        }
        if (sta_info_move_state(sta, IEEE80211_STA_AUTH)) {
                sdata_info(sdata, "failed moving %pM to auth\n", bssid);
                result = false;
                goto out;
        }

out:
        mutex_unlock(&sdata->local->sta_mtx);
        return result;
}

static void ieee80211_rx_mgmt_auth(struct ieee80211_sub_if_data *sdata,
                                   struct ieee80211_mgmt *mgmt, size_t len)
{
        struct ieee80211_if_managed *ifmgd = &sdata->u.mgd;
        u8 bssid[ETH_ALEN];
        u16 auth_alg, auth_transaction, status_code;
        struct ieee80211_event event = {
                .type = MLME_EVENT,
                .u.mlme.data = AUTH_EVENT,
        };
        struct ieee80211_prep_tx_info info = {
                .subtype = IEEE80211_STYPE_AUTH,
        };

        sdata_assert_lock(sdata);

        if (len < 24 + 6)
                return;

        if (!ifmgd->auth_data || ifmgd->auth_data->done)
                return;

        memcpy(bssid, ifmgd->auth_data->bss->bssid, ETH_ALEN);

        if (!ether_addr_equal(bssid, mgmt->bssid))
                return;

        auth_alg = le16_to_cpu(mgmt->u.auth.auth_alg);
        auth_transaction = le16_to_cpu(mgmt->u.auth.auth_transaction);
        status_code = le16_to_cpu(mgmt->u.auth.status_code);

        if (auth_alg != ifmgd->auth_data->algorithm ||
            (auth_alg != WLAN_AUTH_SAE &&
             auth_transaction != ifmgd->auth_data->expected_transaction) ||
            (auth_alg == WLAN_AUTH_SAE &&
             (auth_transaction < ifmgd->auth_data->expected_transaction ||
              auth_transaction > 2))) {
                sdata_info(sdata, "%pM unexpected authentication state: alg %d (expected %d) transact %d (expected %d)\n",
                           mgmt->sa, auth_alg, ifmgd->auth_data->algorithm,
                           auth_transaction,
                           ifmgd->auth_data->expected_transaction);
                goto notify_driver;
        }

        if (status_code != WLAN_STATUS_SUCCESS) {
                cfg80211_rx_mlme_mgmt(sdata->dev, (u8 *)mgmt, len);

                if (auth_alg == WLAN_AUTH_SAE &&
                    (status_code == WLAN_STATUS_ANTI_CLOG_REQUIRED ||
                     (auth_transaction == 1 &&
                      (status_code == WLAN_STATUS_SAE_HASH_TO_ELEMENT ||
                       status_code == WLAN_STATUS_SAE_PK)))) {
                        /* waiting for userspace now */
                        ifmgd->auth_data->waiting = true;
                        ifmgd->auth_data->timeout =
                                jiffies + IEEE80211_AUTH_WAIT_SAE_RETRY;
                        ifmgd->auth_data->timeout_started = true;
                        run_again(sdata, ifmgd->auth_data->timeout);
                        goto notify_driver;
                }

                sdata_info(sdata, "%pM denied authentication (status %d)\n",
                           mgmt->sa, status_code);
                ieee80211_destroy_auth_data(sdata, false);
                event.u.mlme.status = MLME_DENIED;
                event.u.mlme.reason = status_code;
                drv_event_callback(sdata->local, sdata, &event);
                goto notify_driver;
        }

        switch (ifmgd->auth_data->algorithm) {
        case WLAN_AUTH_OPEN:
        case WLAN_AUTH_LEAP:
        case WLAN_AUTH_FT:
        case WLAN_AUTH_SAE:
        case WLAN_AUTH_FILS_SK:
        case WLAN_AUTH_FILS_SK_PFS:
        case WLAN_AUTH_FILS_PK:
                break;
        case WLAN_AUTH_SHARED_KEY:
                if (ifmgd->auth_data->expected_transaction != 4) {
                        ieee80211_auth_challenge(sdata, mgmt, len);
                        /* need another frame */
                        return;
                }
                break;
        default:
                WARN_ONCE(1, "invalid auth alg %d",
                          ifmgd->auth_data->algorithm);
                goto notify_driver;
        }

        event.u.mlme.status = MLME_SUCCESS;
        info.success = 1;
        drv_event_callback(sdata->local, sdata, &event);
        if (ifmgd->auth_data->algorithm != WLAN_AUTH_SAE ||
            (auth_transaction == 2 &&
             ifmgd->auth_data->expected_transaction == 2)) {
                if (!ieee80211_mark_sta_auth(sdata, bssid))
                        return; /* ignore frame -- wait for timeout */
        } else if (ifmgd->auth_data->algorithm == WLAN_AUTH_SAE &&
                   auth_transaction == 2) {
                sdata_info(sdata, "SAE peer confirmed\n");
                ifmgd->auth_data->peer_confirmed = true;
        }

        cfg80211_rx_mlme_mgmt(sdata->dev, (u8 *)mgmt, len);
notify_driver:
        drv_mgd_complete_tx(sdata->local, sdata, &info);
}

#define case_WLAN(type) \
        case WLAN_REASON_##type: return #type

const char *ieee80211_get_reason_code_string(u16 reason_code)
{
        switch (reason_code) {
        case_WLAN(UNSPECIFIED);
        case_WLAN(PREV_AUTH_NOT_VALID);
        case_WLAN(DEAUTH_LEAVING);
        case_WLAN(DISASSOC_DUE_TO_INACTIVITY);
        case_WLAN(DISASSOC_AP_BUSY);
        case_WLAN(CLASS2_FRAME_FROM_NONAUTH_STA);
        case_WLAN(CLASS3_FRAME_FROM_NONASSOC_STA);
        case_WLAN(DISASSOC_STA_HAS_LEFT);
        case_WLAN(STA_REQ_ASSOC_WITHOUT_AUTH);
        case_WLAN(DISASSOC_BAD_POWER);
        case_WLAN(DISASSOC_BAD_SUPP_CHAN);
        case_WLAN(INVALID_IE);
        case_WLAN(MIC_FAILURE);
        case_WLAN(4WAY_HANDSHAKE_TIMEOUT);
        case_WLAN(GROUP_KEY_HANDSHAKE_TIMEOUT);
        case_WLAN(IE_DIFFERENT);
        case_WLAN(INVALID_GROUP_CIPHER);
        case_WLAN(INVALID_PAIRWISE_CIPHER);
        case_WLAN(INVALID_AKMP);
        case_WLAN(UNSUPP_RSN_VERSION);
        case_WLAN(INVALID_RSN_IE_CAP);
        case_WLAN(IEEE8021X_FAILED);
        case_WLAN(CIPHER_SUITE_REJECTED);
        case_WLAN(DISASSOC_UNSPECIFIED_QOS);
        case_WLAN(DISASSOC_QAP_NO_BANDWIDTH);
        case_WLAN(DISASSOC_LOW_ACK);
        case_WLAN(DISASSOC_QAP_EXCEED_TXOP);
        case_WLAN(QSTA_LEAVE_QBSS);
        case_WLAN(QSTA_NOT_USE);
        case_WLAN(QSTA_REQUIRE_SETUP);
        case_WLAN(QSTA_TIMEOUT);
        case_WLAN(QSTA_CIPHER_NOT_SUPP);
        case_WLAN(MESH_PEER_CANCELED);
        case_WLAN(MESH_MAX_PEERS);
        case_WLAN(MESH_CONFIG);
        case_WLAN(MESH_CLOSE);
        case_WLAN(MESH_MAX_RETRIES);
        case_WLAN(MESH_CONFIRM_TIMEOUT);
        case_WLAN(MESH_INVALID_GTK);
        case_WLAN(MESH_INCONSISTENT_PARAM);
        case_WLAN(MESH_INVALID_SECURITY);
        case_WLAN(MESH_PATH_ERROR);
        case_WLAN(MESH_PATH_NOFORWARD);
        case_WLAN(MESH_PATH_DEST_UNREACHABLE);
        case_WLAN(MAC_EXISTS_IN_MBSS);
        case_WLAN(MESH_CHAN_REGULATORY);
        case_WLAN(MESH_CHAN);
        default: return "<unknown>";
        }
}

static void ieee80211_rx_mgmt_deauth(struct ieee80211_sub_if_data *sdata,
                                     struct ieee80211_mgmt *mgmt, size_t len)
{
        struct ieee80211_if_managed *ifmgd = &sdata->u.mgd;
        u16 reason_code = le16_to_cpu(mgmt->u.deauth.reason_code);

        sdata_assert_lock(sdata);

        if (len < 24 + 2)
                return;

        if (!ether_addr_equal(mgmt->bssid, mgmt->sa)) {
                ieee80211_tdls_handle_disconnect(sdata, mgmt->sa, reason_code);
                return;
        }

        if (ifmgd->associated &&
            ether_addr_equal(mgmt->bssid, ifmgd->associated->bssid)) {
                const u8 *bssid = ifmgd->associated->bssid;

                sdata_info(sdata, "deauthenticated from %pM (Reason: %u=%s)\n",
                           bssid, reason_code,
                           ieee80211_get_reason_code_string(reason_code));

                ieee80211_set_disassoc(sdata, 0, 0, false, NULL);

                ieee80211_report_disconnect(sdata, (u8 *)mgmt, len, false,
                                            reason_code, false);
                return;
        }

        if (ifmgd->assoc_data &&
            ether_addr_equal(mgmt->bssid, ifmgd->assoc_data->bss->bssid)) {
                const u8 *bssid = ifmgd->assoc_data->bss->bssid;

                sdata_info(sdata,
                           "deauthenticated from %pM while associating (Reason: %u=%s)\n",
                           bssid, reason_code,
                           ieee80211_get_reason_code_string(reason_code));

                ieee80211_destroy_assoc_data(sdata, false, true);

                cfg80211_rx_mlme_mgmt(sdata->dev, (u8 *)mgmt, len);
                return;
        }
}


static void ieee80211_rx_mgmt_disassoc(struct ieee80211_sub_if_data *sdata,
                                       struct ieee80211_mgmt *mgmt, size_t len)
{
        struct ieee80211_if_managed *ifmgd = &sdata->u.mgd;
        u16 reason_code;

        sdata_assert_lock(sdata);

        if (len < 24 + 2)
                return;

        if (!ifmgd->associated ||
            !ether_addr_equal(mgmt->bssid, ifmgd->associated->bssid))
                return;

        reason_code = le16_to_cpu(mgmt->u.disassoc.reason_code);

        if (!ether_addr_equal(mgmt->bssid, mgmt->sa)) {
                ieee80211_tdls_handle_disconnect(sdata, mgmt->sa, reason_code);
                return;
        }

        sdata_info(sdata, "disassociated from %pM (Reason: %u=%s)\n",
                   mgmt->sa, reason_code,
                   ieee80211_get_reason_code_string(reason_code));

        ieee80211_set_disassoc(sdata, 0, 0, false, NULL);

        ieee80211_report_disconnect(sdata, (u8 *)mgmt, len, false, reason_code,
                                    false);
}

static void ieee80211_get_rates(struct ieee80211_supported_band *sband,
                                u8 *supp_rates, unsigned int supp_rates_len,
                                u32 *rates, u32 *basic_rates,
                                bool *have_higher_than_11mbit,
                                int *min_rate, int *min_rate_index,
                                int shift)
{
        int i, j;

        for (i = 0; i < supp_rates_len; i++) {
                int rate = supp_rates[i] & 0x7f;
                bool is_basic = !!(supp_rates[i] & 0x80);

                if ((rate * 5 * (1 << shift)) > 110)
                        *have_higher_than_11mbit = true;

                /*
                 * Skip HT, VHT, HE and SAE H2E only BSS membership selectors
                 * since they're not rates.
                 *
                 * Note: Even though the membership selector and the basic
                 *         rate flag share the same bit, they are not exactly
                 *         the same.
                 */
                if (supp_rates[i] == (0x80 | BSS_MEMBERSHIP_SELECTOR_HT_PHY) ||
                    supp_rates[i] == (0x80 | BSS_MEMBERSHIP_SELECTOR_VHT_PHY) ||
                    supp_rates[i] == (0x80 | BSS_MEMBERSHIP_SELECTOR_HE_PHY) ||
                    supp_rates[i] == (0x80 | BSS_MEMBERSHIP_SELECTOR_SAE_H2E))
                        continue;

                for (j = 0; j < sband->n_bitrates; j++) {
                        struct ieee80211_rate *br;
                        int brate;

                        br = &sband->bitrates[j];

                        brate = DIV_ROUND_UP(br->bitrate, (1 << shift) * 5);
                        if (brate == rate) {
                                *rates |= BIT(j);
                                if (is_basic)
                                        *basic_rates |= BIT(j);
                                if ((rate * 5) < *min_rate) {
                                        *min_rate = rate * 5;
                                        *min_rate_index = j;
                                }
                                break;
                        }
                }
        }
}

static bool ieee80211_twt_req_supported(const struct sta_info *sta,
                                        const struct ieee802_11_elems *elems)
{
        if (elems->ext_capab_len < 10)
                return false;

        if (!(elems->ext_capab[9] & WLAN_EXT_CAPA10_TWT_RESPONDER_SUPPORT))
                return false;

        return sta->sta.he_cap.he_cap_elem.mac_cap_info[0] &
                IEEE80211_HE_MAC_CAP0_TWT_RES;
}

static int ieee80211_recalc_twt_req(struct ieee80211_sub_if_data *sdata,
                                    struct sta_info *sta,
                                    struct ieee802_11_elems *elems)
{
        bool twt = ieee80211_twt_req_supported(sta, elems);

        if (sdata->vif.bss_conf.twt_requester != twt) {
                sdata->vif.bss_conf.twt_requester = twt;
                return BSS_CHANGED_TWT;
        }
        return 0;
}

static bool ieee80211_twt_bcast_support(struct ieee80211_sub_if_data *sdata,
                                        struct ieee80211_bss_conf *bss_conf,
                                        struct ieee80211_supported_band *sband,
                                        struct sta_info *sta)
{
        const struct ieee80211_sta_he_cap *own_he_cap =
                ieee80211_get_he_iftype_cap(sband,
                                            ieee80211_vif_type_p2p(&sdata->vif));

        return bss_conf->he_support &&
                (sta->sta.he_cap.he_cap_elem.mac_cap_info[2] &
                        IEEE80211_HE_MAC_CAP2_BCAST_TWT) &&
                own_he_cap &&
                (own_he_cap->he_cap_elem.mac_cap_info[2] &
                        IEEE80211_HE_MAC_CAP2_BCAST_TWT);
}

static bool ieee80211_assoc_success(struct ieee80211_sub_if_data *sdata,
                                    struct cfg80211_bss *cbss,
                                    struct ieee80211_mgmt *mgmt, size_t len,
                                    struct ieee802_11_elems *elems)
{
        struct ieee80211_if_managed *ifmgd = &sdata->u.mgd;
        struct ieee80211_local *local = sdata->local;
        struct ieee80211_supported_band *sband;
        struct sta_info *sta;
        u16 capab_info, aid;
        struct ieee80211_bss_conf *bss_conf = &sdata->vif.bss_conf;
        const struct cfg80211_bss_ies *bss_ies = NULL;
        struct ieee80211_mgd_assoc_data *assoc_data = ifmgd->assoc_data;
        bool is_6ghz = cbss->channel->band == NL80211_BAND_6GHZ;
        bool is_s1g = cbss->channel->band == NL80211_BAND_S1GHZ;
        u32 changed = 0;
        u8 *pos;
        int err;
        bool ret;

        /* AssocResp and ReassocResp have identical structure */

        pos = mgmt->u.assoc_resp.variable;
        aid = le16_to_cpu(mgmt->u.assoc_resp.aid);
        if (is_s1g) {
                pos = (u8 *) mgmt->u.s1g_assoc_resp.variable;
                aid = 0; /* TODO */
        }
        capab_info = le16_to_cpu(mgmt->u.assoc_resp.capab_info);
        elems = ieee802_11_parse_elems(pos, len - (pos - (u8 *)mgmt), false,
                                       mgmt->bssid, assoc_data->bss->bssid);

        if (!elems)
                return false;

        if (elems->aid_resp)
                aid = le16_to_cpu(elems->aid_resp->aid);

        /*
         * The 5 MSB of the AID field are reserved
         * (802.11-2016 9.4.1.8 AID field)
         */
        aid &= 0x7ff;

        ifmgd->broken_ap = false;

        if (aid == 0 || aid > IEEE80211_MAX_AID) {
                sdata_info(sdata, "invalid AID value %d (out of range), turn off PS\n",
                           aid);
                aid = 0;
                ifmgd->broken_ap = true;
        }

        if (!is_s1g && !elems->supp_rates) {
                sdata_info(sdata, "no SuppRates element in AssocResp\n");
                ret = false;
                goto out;
        }

        sdata->vif.bss_conf.aid = aid;
        ifmgd->tdls_chan_switch_prohibited =
                elems->ext_capab && elems->ext_capab_len >= 5 &&
                (elems->ext_capab[4] & WLAN_EXT_CAPA5_TDLS_CH_SW_PROHIBITED);

        /*
         * Some APs are erroneously not including some information in their
         * (re)association response frames. Try to recover by using the data
         * from the beacon or probe response. This seems to afflict mobile
         * 2G/3G/4G wifi routers, reported models include the "Onda PN51T",
         * "Vodafone PocketWiFi 2", "ZTE MF60" and a similar T-Mobile device.
         */
        if (!is_6ghz &&
            ((assoc_data->wmm && !elems->wmm_param) ||
             (!(ifmgd->flags & IEEE80211_STA_DISABLE_HT) &&
              (!elems->ht_cap_elem || !elems->ht_operation)) ||
             (!(ifmgd->flags & IEEE80211_STA_DISABLE_VHT) &&
              (!elems->vht_cap_elem || !elems->vht_operation)))) {
                const struct cfg80211_bss_ies *ies;
                struct ieee802_11_elems *bss_elems;

                rcu_read_lock();
                ies = rcu_dereference(cbss->ies);
                if (ies)
                        bss_ies = kmemdup(ies, sizeof(*ies) + ies->len,
                                          GFP_ATOMIC);
                rcu_read_unlock();
                if (!bss_ies) {
                        ret = false;
                        goto out;
                }

                bss_elems = ieee802_11_parse_elems(bss_ies->data, bss_ies->len,
                                                   false, mgmt->bssid,
                                                   assoc_data->bss->bssid);
                if (!bss_elems) {
                        ret = false;
                        goto out;
                }

                if (assoc_data->wmm &&
                    !elems->wmm_param && bss_elems->wmm_param) {
                        elems->wmm_param = bss_elems->wmm_param;
                        sdata_info(sdata,
                                   "AP bug: WMM param missing from AssocResp\n");
                }

                /*
                 * Also check if we requested HT/VHT, otherwise the AP doesn't
                 * have to include the IEs in the (re)association response.
                 */
                if (!elems->ht_cap_elem && bss_elems->ht_cap_elem &&
                    !(ifmgd->flags & IEEE80211_STA_DISABLE_HT)) {
                        elems->ht_cap_elem = bss_elems->ht_cap_elem;
                        sdata_info(sdata,
                                   "AP bug: HT capability missing from AssocResp\n");
                }
                if (!elems->ht_operation && bss_elems->ht_operation &&
                    !(ifmgd->flags & IEEE80211_STA_DISABLE_HT)) {
                        elems->ht_operation = bss_elems->ht_operation;
                        sdata_info(sdata,
                                   "AP bug: HT operation missing from AssocResp\n");
                }
                if (!elems->vht_cap_elem && bss_elems->vht_cap_elem &&
                    !(ifmgd->flags & IEEE80211_STA_DISABLE_VHT)) {
                        elems->vht_cap_elem = bss_elems->vht_cap_elem;
                        sdata_info(sdata,
                                   "AP bug: VHT capa missing from AssocResp\n");
                }
                if (!elems->vht_operation && bss_elems->vht_operation &&
                    !(ifmgd->flags & IEEE80211_STA_DISABLE_VHT)) {
                        elems->vht_operation = bss_elems->vht_operation;
                        sdata_info(sdata,
                                   "AP bug: VHT operation missing from AssocResp\n");
                }

                kfree(bss_elems);
        }

        /*
         * We previously checked these in the beacon/probe response, so
         * they should be present here. This is just a safety net.
         */
        if (!is_6ghz && !(ifmgd->flags & IEEE80211_STA_DISABLE_HT) &&
            (!elems->wmm_param || !elems->ht_cap_elem || !elems->ht_operation)) {
                sdata_info(sdata,
                           "HT AP is missing WMM params or HT capability/operation\n");
                ret = false;
                goto out;
        }

        if (!is_6ghz && !(ifmgd->flags & IEEE80211_STA_DISABLE_VHT) &&
            (!elems->vht_cap_elem || !elems->vht_operation)) {
                sdata_info(sdata,
                           "VHT AP is missing VHT capability/operation\n");
                ret = false;
                goto out;
        }

        if (is_6ghz && !(ifmgd->flags & IEEE80211_STA_DISABLE_HE) &&
            !elems->he_6ghz_capa) {
                sdata_info(sdata,
                           "HE 6 GHz AP is missing HE 6 GHz band capability\n");
                ret = false;
                goto out;
        }

        mutex_lock(&sdata->local->sta_mtx);
        /*
         * station info was already allocated and inserted before
         * the association and should be available to us
         */
        sta = sta_info_get(sdata, cbss->bssid);
        if (WARN_ON(!sta)) {
                mutex_unlock(&sdata->local->sta_mtx);
                ret = false;
                goto out;
        }

        sband = ieee80211_get_sband(sdata);
        if (!sband) {
                mutex_unlock(&sdata->local->sta_mtx);
                ret = false;
                goto out;
        }

        if (!(ifmgd->flags & IEEE80211_STA_DISABLE_HE) &&
            (!elems->he_cap || !elems->he_operation)) {
                mutex_unlock(&sdata->local->sta_mtx);
                sdata_info(sdata,
                           "HE AP is missing HE capability/operation\n");
                ret = false;
                goto out;
        }

        /* Set up internal HT/VHT capabilities */
        if (elems->ht_cap_elem && !(ifmgd->flags & IEEE80211_STA_DISABLE_HT))
                ieee80211_ht_cap_ie_to_sta_ht_cap(sdata, sband,
                                                  elems->ht_cap_elem, sta);

        if (elems->vht_cap_elem && !(ifmgd->flags & IEEE80211_STA_DISABLE_VHT))
                ieee80211_vht_cap_ie_to_sta_vht_cap(sdata, sband,
                                                    elems->vht_cap_elem, sta);

        if (elems->he_operation && !(ifmgd->flags & IEEE80211_STA_DISABLE_HE) &&
            elems->he_cap) {
                ieee80211_he_cap_ie_to_sta_he_cap(sdata, sband,
                                                  elems->he_cap,
                                                  elems->he_cap_len,
                                                  elems->he_6ghz_capa,
                                                  sta);

                bss_conf->he_support = sta->sta.he_cap.has_he;
                if (elems->rsnx && elems->rsnx_len &&
                    (elems->rsnx[0] & WLAN_RSNX_CAPA_PROTECTED_TWT) &&
                    wiphy_ext_feature_isset(local->hw.wiphy,
                                            NL80211_EXT_FEATURE_PROTECTED_TWT))
                        bss_conf->twt_protected = true;
                else
                        bss_conf->twt_protected = false;

                changed |= ieee80211_recalc_twt_req(sdata, sta, elems);
        } else {
                bss_conf->he_support = false;
                bss_conf->twt_requester = false;
                bss_conf->twt_protected = false;
        }

        bss_conf->twt_broadcast =
                ieee80211_twt_bcast_support(sdata, bss_conf, sband, sta);

        if (bss_conf->he_support) {
                bss_conf->he_bss_color.color =
                        le32_get_bits(elems->he_operation->he_oper_params,
                                      IEEE80211_HE_OPERATION_BSS_COLOR_MASK);
                bss_conf->he_bss_color.partial =
                        le32_get_bits(elems->he_operation->he_oper_params,
                                      IEEE80211_HE_OPERATION_PARTIAL_BSS_COLOR);
                bss_conf->he_bss_color.enabled =
                        !le32_get_bits(elems->he_operation->he_oper_params,
                                       IEEE80211_HE_OPERATION_BSS_COLOR_DISABLED);

                if (bss_conf->he_bss_color.enabled)
                        changed |= BSS_CHANGED_HE_BSS_COLOR;

                bss_conf->htc_trig_based_pkt_ext =
                        le32_get_bits(elems->he_operation->he_oper_params,
                              IEEE80211_HE_OPERATION_DFLT_PE_DURATION_MASK);
                bss_conf->frame_time_rts_th =
                        le32_get_bits(elems->he_operation->he_oper_params,
                              IEEE80211_HE_OPERATION_RTS_THRESHOLD_MASK);

                bss_conf->uora_exists = !!elems->uora_element;
                if (elems->uora_element)
                        bss_conf->uora_ocw_range = elems->uora_element[0];

                ieee80211_he_op_ie_to_bss_conf(&sdata->vif, elems->he_operation);
                ieee80211_he_spr_ie_to_bss_conf(&sdata->vif, elems->he_spr);
                /* TODO: OPEN: what happens if BSS color disable is set? */
        }

        if (cbss->transmitted_bss) {
                bss_conf->nontransmitted = true;
                ether_addr_copy(bss_conf->transmitter_bssid,
                                cbss->transmitted_bss->bssid);
                bss_conf->bssid_indicator = cbss->max_bssid_indicator;
                bss_conf->bssid_index = cbss->bssid_index;
        } else {
                bss_conf->nontransmitted = false;
                memset(bss_conf->transmitter_bssid, 0,
                       sizeof(bss_conf->transmitter_bssid));
                bss_conf->bssid_indicator = 0;
                bss_conf->bssid_index = 0;
        }

        /*
         * Some APs, e.g. Netgear WNDR3700, report invalid HT operation data
         * in their association response, so ignore that data for our own
         * configuration. If it changed since the last beacon, we'll get the
         * next beacon and update then.
         */

        /*
         * If an operating mode notification IE is present, override the
         * NSS calculation (that would be done in rate_control_rate_init())
         * and use the # of streams from that element.
         */
        if (elems->opmode_notif &&
            !(*elems->opmode_notif & IEEE80211_OPMODE_NOTIF_RX_NSS_TYPE_BF)) {
                u8 nss;

                nss = *elems->opmode_notif & IEEE80211_OPMODE_NOTIF_RX_NSS_MASK;
                nss >>= IEEE80211_OPMODE_NOTIF_RX_NSS_SHIFT;
                nss += 1;
                sta->sta.rx_nss = nss;
        }

        rate_control_rate_init(sta);

        if (ifmgd->flags & IEEE80211_STA_MFP_ENABLED) {
                set_sta_flag(sta, WLAN_STA_MFP);
                sta->sta.mfp = true;
        } else {
                sta->sta.mfp = false;
        }

        sta->sta.wme = (elems->wmm_param || elems->s1g_capab) &&
                       local->hw.queues >= IEEE80211_NUM_ACS;

        err = sta_info_move_state(sta, IEEE80211_STA_ASSOC);
        if (!err && !(ifmgd->flags & IEEE80211_STA_CONTROL_PORT))
                err = sta_info_move_state(sta, IEEE80211_STA_AUTHORIZED);
        if (err) {
                sdata_info(sdata,
                           "failed to move station %pM to desired state\n",
                           sta->sta.addr);
                WARN_ON(__sta_info_destroy(sta));
                mutex_unlock(&sdata->local->sta_mtx);
                ret = false;
                goto out;
        }

        if (sdata->wdev.use_4addr)
                drv_sta_set_4addr(local, sdata, &sta->sta, true);

        mutex_unlock(&sdata->local->sta_mtx);

        /*
         * Always handle WMM once after association regardless
         * of the first value the AP uses. Setting -1 here has
         * that effect because the AP values is an unsigned
         * 4-bit value.
         */
        ifmgd->wmm_last_param_set = -1;
        ifmgd->mu_edca_last_param_set = -1;

        if (ifmgd->flags & IEEE80211_STA_DISABLE_WMM) {
                ieee80211_set_wmm_default(sdata, false, false);
        } else if (!ieee80211_sta_wmm_params(local, sdata, elems->wmm_param,
                                             elems->wmm_param_len,
                                             elems->mu_edca_param_set)) {
                /* still enable QoS since we might have HT/VHT */
                ieee80211_set_wmm_default(sdata, false, true);
                /* set the disable-WMM flag in this case to disable
                 * tracking WMM parameter changes in the beacon if
                 * the parameters weren't actually valid. Doing so
                 * avoids changing parameters very strangely when
                 * the AP is going back and forth between valid and
                 * invalid parameters.
                 */
                ifmgd->flags |= IEEE80211_STA_DISABLE_WMM;
        }
        changed |= BSS_CHANGED_QOS;

        if (elems->max_idle_period_ie) {
                bss_conf->max_idle_period =
                        le16_to_cpu(elems->max_idle_period_ie->max_idle_period);
                bss_conf->protected_keep_alive =
                        !!(elems->max_idle_period_ie->idle_options &
                           WLAN_IDLE_OPTIONS_PROTECTED_KEEP_ALIVE);
                changed |= BSS_CHANGED_KEEP_ALIVE;
        } else {
                bss_conf->max_idle_period = 0;
                bss_conf->protected_keep_alive = false;
        }

        /* set assoc capability (AID was already set earlier),
         * ieee80211_set_associated() will tell the driver */
        bss_conf->assoc_capability = capab_info;
        ieee80211_set_associated(sdata, cbss, changed);

        /*
         * If we're using 4-addr mode, let the AP know that we're
         * doing so, so that it can create the STA VLAN on its side
         */
        if (ifmgd->use_4addr)
                ieee80211_send_4addr_nullfunc(local, sdata);

        /*
         * Start timer to probe the connection to the AP now.
         * Also start the timer that will detect beacon loss.
         */
        ieee80211_sta_reset_beacon_monitor(sdata);
        ieee80211_sta_reset_conn_monitor(sdata);

        ret = true;
 out:
        kfree(elems);
        kfree(bss_ies);
        return ret;
}

static void ieee80211_rx_mgmt_assoc_resp(struct ieee80211_sub_if_data *sdata,
                                         struct ieee80211_mgmt *mgmt,
                                         size_t len)
{
        struct ieee80211_if_managed *ifmgd = &sdata->u.mgd;
        struct ieee80211_mgd_assoc_data *assoc_data = ifmgd->assoc_data;
        u16 capab_info, status_code, aid;
        struct ieee802_11_elems *elems;
        int ac, uapsd_queues = -1;
        u8 *pos;
        bool reassoc;
        struct cfg80211_bss *cbss;
        struct ieee80211_event event = {
                .type = MLME_EVENT,
                .u.mlme.data = ASSOC_EVENT,
        };
        struct ieee80211_prep_tx_info info = {};

        sdata_assert_lock(sdata);

        if (!assoc_data)
                return;

        if (!ether_addr_equal(assoc_data->bss->bssid, mgmt->bssid))
                return;

        cbss = assoc_data->bss;

        /*
         * AssocResp and ReassocResp have identical structure, so process both
         * of them in this function.
         */

        if (len < 24 + 6)
                return;

        reassoc = ieee80211_is_reassoc_resp(mgmt->frame_control);
        capab_info = le16_to_cpu(mgmt->u.assoc_resp.capab_info);
        status_code = le16_to_cpu(mgmt->u.assoc_resp.status_code);
        pos = mgmt->u.assoc_resp.variable;
        aid = le16_to_cpu(mgmt->u.assoc_resp.aid);
        if (cbss->channel->band == NL80211_BAND_S1GHZ) {
                pos = (u8 *) mgmt->u.s1g_assoc_resp.variable;
                aid = 0; /* TODO */
        }

        /*
         * Note: this may not be perfect, AP might misbehave - if
         * anyone needs to rely on perfect complete notification
         * with the exact right subtype, then we need to track what
         * we actually transmitted.
         */
        info.subtype = reassoc ? IEEE80211_STYPE_REASSOC_REQ :
                                 IEEE80211_STYPE_ASSOC_REQ;

        sdata_info(sdata,
                   "RX %sssocResp from %pM (capab=0x%x status=%d aid=%d)\n",
                   reassoc ? "Rea" : "A", mgmt->sa,
                   capab_info, status_code, (u16)(aid & ~(BIT(15) | BIT(14))));

        if (assoc_data->fils_kek_len &&
            fils_decrypt_assoc_resp(sdata, (u8 *)mgmt, &len, assoc_data) < 0)
                return;

        elems = ieee802_11_parse_elems(pos, len - (pos - (u8 *)mgmt), false,
                                       mgmt->bssid, assoc_data->bss->bssid);
        if (!elems)
                goto notify_driver;

        if (status_code == WLAN_STATUS_ASSOC_REJECTED_TEMPORARILY &&
            elems->timeout_int &&
            elems->timeout_int->type == WLAN_TIMEOUT_ASSOC_COMEBACK) {
                u32 tu, ms;
                tu = le32_to_cpu(elems->timeout_int->value);
                ms = tu * 1024 / 1000;
                sdata_info(sdata,
                           "%pM rejected association temporarily; comeback duration %u TU (%u ms)\n",
                           mgmt->sa, tu, ms);
                assoc_data->timeout = jiffies + msecs_to_jiffies(ms);
                assoc_data->timeout_started = true;
                if (ms > IEEE80211_ASSOC_TIMEOUT)
                        run_again(sdata, assoc_data->timeout);
                goto notify_driver;
        }

        if (status_code != WLAN_STATUS_SUCCESS) {
                sdata_info(sdata, "%pM denied association (code=%d)\n",
                           mgmt->sa, status_code);
                ieee80211_destroy_assoc_data(sdata, false, false);
                event.u.mlme.status = MLME_DENIED;
                event.u.mlme.reason = status_code;
                drv_event_callback(sdata->local, sdata, &event);
        } else {
                if (!ieee80211_assoc_success(sdata, cbss, mgmt, len, elems)) {
                        /* oops -- internal error -- send timeout for now */
                        ieee80211_destroy_assoc_data(sdata, false, false);
                        cfg80211_assoc_timeout(sdata->dev, cbss);
                        goto notify_driver;
                }
                event.u.mlme.status = MLME_SUCCESS;
                drv_event_callback(sdata->local, sdata, &event);
                sdata_info(sdata, "associated\n");

                /*
                 * destroy assoc_data afterwards, as otherwise an idle
                 * recalc after assoc_data is NULL but before associated
                 * is set can cause the interface to go idle
                 */
                ieee80211_destroy_assoc_data(sdata, true, false);

                /* get uapsd queues configuration */
                uapsd_queues = 0;
                for (ac = 0; ac < IEEE80211_NUM_ACS; ac++)
                        if (sdata->tx_conf[ac].uapsd)
                                uapsd_queues |= ieee80211_ac_to_qos_mask[ac];

                info.success = 1;
        }

        cfg80211_rx_assoc_resp(sdata->dev, cbss, (u8 *)mgmt, len, uapsd_queues,
                               ifmgd->assoc_req_ies, ifmgd->assoc_req_ies_len);
notify_driver:
        drv_mgd_complete_tx(sdata->local, sdata, &info);
        kfree(elems);
}

static void ieee80211_rx_bss_info(struct ieee80211_sub_if_data *sdata,
                                  struct ieee80211_mgmt *mgmt, size_t len,
                                  struct ieee80211_rx_status *rx_status)
{
        struct ieee80211_local *local = sdata->local;
        struct ieee80211_bss *bss;
        struct ieee80211_channel *channel;

        sdata_assert_lock(sdata);

        channel = ieee80211_get_channel_khz(local->hw.wiphy,
                                        ieee80211_rx_status_to_khz(rx_status));
        if (!channel)
                return;

        bss = ieee80211_bss_info_update(local, rx_status, mgmt, len, channel);
        if (bss) {
                sdata->vif.bss_conf.beacon_rate = bss->beacon_rate;
                ieee80211_rx_bss_put(local, bss);
        }
}


static void ieee80211_rx_mgmt_probe_resp(struct ieee80211_sub_if_data *sdata,
                                         struct sk_buff *skb)
{
        struct ieee80211_mgmt *mgmt = (void *)skb->data;
        struct ieee80211_if_managed *ifmgd;
        struct ieee80211_rx_status *rx_status = (void *) skb->cb;
        struct ieee80211_channel *channel;
        size_t baselen, len = skb->len;

        ifmgd = &sdata->u.mgd;

        sdata_assert_lock(sdata);

        /*
         * According to Draft P802.11ax D6.0 clause 26.17.2.3.2:
         * "If a 6 GHz AP receives a Probe Request frame  and responds with
         * a Probe Response frame [..], the Address 1 field of the Probe
         * Response frame shall be set to the broadcast address [..]"
         * So, on 6GHz band we should also accept broadcast responses.
         */
        channel = ieee80211_get_channel(sdata->local->hw.wiphy,
                                        rx_status->freq);
        if (!channel)
                return;

        if (!ether_addr_equal(mgmt->da, sdata->vif.addr) &&
            (channel->band != NL80211_BAND_6GHZ ||
             !is_broadcast_ether_addr(mgmt->da)))
                return; /* ignore ProbeResp to foreign address */

        baselen = (u8 *) mgmt->u.probe_resp.variable - (u8 *) mgmt;
        if (baselen > len)
                return;

        ieee80211_rx_bss_info(sdata, mgmt, len, rx_status);

        if (ifmgd->associated &&
            ether_addr_equal(mgmt->bssid, ifmgd->associated->bssid))
                ieee80211_reset_ap_probe(sdata);
}

/*
 * This is the canonical list of information elements we care about,
 * the filter code also gives us all changes to the Microsoft OUI
 * (00:50:F2) vendor IE which is used for WMM which we need to track,
 * as well as the DTPC IE (part of the Cisco OUI) used for signaling
 * changes to requested client power.
 *
 * We implement beacon filtering in software since that means we can
 * avoid processing the frame here and in cfg80211, and userspace
 * will not be able to tell whether the hardware supports it or not.
 *
 * XXX: This list needs to be dynamic -- userspace needs to be able to
 *        add items it requires. It also needs to be able to tell us to
 *        look out for other vendor IEs.
 */
static const u64 care_about_ies =
        (1ULL << WLAN_EID_COUNTRY) |
        (1ULL << WLAN_EID_ERP_INFO) |
        (1ULL << WLAN_EID_CHANNEL_SWITCH) |
        (1ULL << WLAN_EID_PWR_CONSTRAINT) |
        (1ULL << WLAN_EID_HT_CAPABILITY) |
        (1ULL << WLAN_EID_HT_OPERATION) |
        (1ULL << WLAN_EID_EXT_CHANSWITCH_ANN);

static void ieee80211_handle_beacon_sig(struct ieee80211_sub_if_data *sdata,
                                        struct ieee80211_if_managed *ifmgd,
                                        struct ieee80211_bss_conf *bss_conf,
                                        struct ieee80211_local *local,
                                        struct ieee80211_rx_status *rx_status)
{
        /* Track average RSSI from the Beacon frames of the current AP */

        if (ifmgd->flags & IEEE80211_STA_RESET_SIGNAL_AVE) {
                ifmgd->flags &= ~IEEE80211_STA_RESET_SIGNAL_AVE;
                ewma_beacon_signal_init(&ifmgd->ave_beacon_signal);
                ifmgd->last_cqm_event_signal = 0;
                ifmgd->count_beacon_signal = 1;
                ifmgd->last_ave_beacon_signal = 0;
        } else {
                ifmgd->count_beacon_signal++;
        }

        ewma_beacon_signal_add(&ifmgd->ave_beacon_signal, -rx_status->signal);

        if (ifmgd->rssi_min_thold != ifmgd->rssi_max_thold &&
            ifmgd->count_beacon_signal >= IEEE80211_SIGNAL_AVE_MIN_COUNT) {
                int sig = -ewma_beacon_signal_read(&ifmgd->ave_beacon_signal);
                int last_sig = ifmgd->last_ave_beacon_signal;
                struct ieee80211_event event = {
                        .type = RSSI_EVENT,
                };

                /*
                 * if signal crosses either of the boundaries, invoke callback
                 * with appropriate parameters
                 */
                if (sig > ifmgd->rssi_max_thold &&
                    (last_sig <= ifmgd->rssi_min_thold || last_sig == 0)) {
                        ifmgd->last_ave_beacon_signal = sig;
                        event.u.rssi.data = RSSI_EVENT_HIGH;
                        drv_event_callback(local, sdata, &event);
                } else if (sig < ifmgd->rssi_min_thold &&
                           (last_sig >= ifmgd->rssi_max_thold ||
                           last_sig == 0)) {
                        ifmgd->last_ave_beacon_signal = sig;
                        event.u.rssi.data = RSSI_EVENT_LOW;
                        drv_event_callback(local, sdata, &event);
                }
        }

        if (bss_conf->cqm_rssi_thold &&
            ifmgd->count_beacon_signal >= IEEE80211_SIGNAL_AVE_MIN_COUNT &&
            !(sdata->vif.driver_flags & IEEE80211_VIF_SUPPORTS_CQM_RSSI)) {
                int sig = -ewma_beacon_signal_read(&ifmgd->ave_beacon_signal);
                int last_event = ifmgd->last_cqm_event_signal;
                int thold = bss_conf->cqm_rssi_thold;
                int hyst = bss_conf->cqm_rssi_hyst;

                if (sig < thold &&
                    (last_event == 0 || sig < last_event - hyst)) {
                        ifmgd->last_cqm_event_signal = sig;
                        ieee80211_cqm_rssi_notify(
                                &sdata->vif,
                                NL80211_CQM_RSSI_THRESHOLD_EVENT_LOW,
                                sig, GFP_KERNEL);
                } else if (sig > thold &&
                           (last_event == 0 || sig > last_event + hyst)) {
                        ifmgd->last_cqm_event_signal = sig;
                        ieee80211_cqm_rssi_notify(
                                &sdata->vif,
                                NL80211_CQM_RSSI_THRESHOLD_EVENT_HIGH,
                                sig, GFP_KERNEL);
                }
        }

        if (bss_conf->cqm_rssi_low &&
            ifmgd->count_beacon_signal >= IEEE80211_SIGNAL_AVE_MIN_COUNT) {
                int sig = -ewma_beacon_signal_read(&ifmgd->ave_beacon_signal);
                int last_event = ifmgd->last_cqm_event_signal;
                int low = bss_conf->cqm_rssi_low;
                int high = bss_conf->cqm_rssi_high;

                if (sig < low &&
                    (last_event == 0 || last_event >= low)) {
                        ifmgd->last_cqm_event_signal = sig;
                        ieee80211_cqm_rssi_notify(
                                &sdata->vif,
                                NL80211_CQM_RSSI_THRESHOLD_EVENT_LOW,
                                sig, GFP_KERNEL);
                } else if (sig > high &&
                           (last_event == 0 || last_event <= high)) {
                        ifmgd->last_cqm_event_signal = sig;
                        ieee80211_cqm_rssi_notify(
                                &sdata->vif,
                                NL80211_CQM_RSSI_THRESHOLD_EVENT_HIGH,
                                sig, GFP_KERNEL);
                }
        }
}

static bool ieee80211_rx_our_beacon(const u8 *tx_bssid,
                                    struct cfg80211_bss *bss)
{
        if (ether_addr_equal(tx_bssid, bss->bssid))
                return true;
        if (!bss->transmitted_bss)
                return false;
        return ether_addr_equal(tx_bssid, bss->transmitted_bss->bssid);
}

static void ieee80211_rx_mgmt_beacon(struct ieee80211_sub_if_data *sdata,
                                     struct ieee80211_hdr *hdr, size_t len,
                                     struct ieee80211_rx_status *rx_status)
{
        struct ieee80211_if_managed *ifmgd = &sdata->u.mgd;
        struct ieee80211_bss_conf *bss_conf = &sdata->vif.bss_conf;
        struct ieee80211_mgmt *mgmt = (void *) hdr;
        size_t baselen;
        struct ieee802_11_elems *elems;
        struct ieee80211_local *local = sdata->local;
        struct ieee80211_chanctx_conf *chanctx_conf;
        struct ieee80211_channel *chan;
        struct sta_info *sta;
        u32 changed = 0;
        bool erp_valid;
        u8 erp_value = 0;
        u32 ncrc = 0;
        u8 *bssid, *variable = mgmt->u.beacon.variable;
        u8 deauth_buf[IEEE80211_DEAUTH_FRAME_LEN];

        sdata_assert_lock(sdata);

        /* Process beacon from the current BSS */
        bssid = ieee80211_get_bssid(hdr, len, sdata->vif.type);
        if (ieee80211_is_s1g_beacon(mgmt->frame_control)) {
                struct ieee80211_ext *ext = (void *) mgmt;

                if (ieee80211_is_s1g_short_beacon(ext->frame_control))
                        variable = ext->u.s1g_short_beacon.variable;
                else
                        variable = ext->u.s1g_beacon.variable;
        }

        baselen = (u8 *) variable - (u8 *) mgmt;
        if (baselen > len)
                return;

        rcu_read_lock();
        chanctx_conf = rcu_dereference(sdata->vif.chanctx_conf);
        if (!chanctx_conf) {
                rcu_read_unlock();
                return;
        }

        if (ieee80211_rx_status_to_khz(rx_status) !=
            ieee80211_channel_to_khz(chanctx_conf->def.chan)) {
                rcu_read_unlock();
                return;
        }
        chan = chanctx_conf->def.chan;
        rcu_read_unlock();

        if (ifmgd->assoc_data && ifmgd->assoc_data->need_beacon &&
            ieee80211_rx_our_beacon(bssid, ifmgd->assoc_data->bss)) {
                elems = ieee802_11_parse_elems(variable, len - baselen, false,
                                               bssid,
                                               ifmgd->assoc_data->bss->bssid);
                if (!elems)
                        return;

                ieee80211_rx_bss_info(sdata, mgmt, len, rx_status);

                if (elems->dtim_period)
                        ifmgd->dtim_period = elems->dtim_period;
                ifmgd->have_beacon = true;
                ifmgd->assoc_data->need_beacon = false;
                if (ieee80211_hw_check(&local->hw, TIMING_BEACON_ONLY)) {
                        sdata->vif.bss_conf.sync_tsf =
                                le64_to_cpu(mgmt->u.beacon.timestamp);
                        sdata->vif.bss_conf.sync_device_ts =
                                rx_status->device_timestamp;
                        sdata->vif.bss_conf.sync_dtim_count = elems->dtim_count;
                }

                if (elems->mbssid_config_ie)
                        bss_conf->profile_periodicity =
                                elems->mbssid_config_ie->profile_periodicity;
                else
                        bss_conf->profile_periodicity = 0;

                if (elems->ext_capab_len >= 11 &&
                    (elems->ext_capab[10] & WLAN_EXT_CAPA11_EMA_SUPPORT))
                        bss_conf->ema_ap = true;
                else
                        bss_conf->ema_ap = false;

                /* continue assoc process */
                ifmgd->assoc_data->timeout = jiffies;
                ifmgd->assoc_data->timeout_started = true;
                run_again(sdata, ifmgd->assoc_data->timeout);
                kfree(elems);
                return;
        }

        if (!ifmgd->associated ||
            !ieee80211_rx_our_beacon(bssid,  ifmgd->associated))
                return;
        bssid = ifmgd->associated->bssid;

        if (!(rx_status->flag & RX_FLAG_NO_SIGNAL_VAL))
                ieee80211_handle_beacon_sig(sdata, ifmgd, bss_conf,
                                            local, rx_status);

        if (ifmgd->flags & IEEE80211_STA_CONNECTION_POLL) {
                mlme_dbg_ratelimited(sdata,
                                     "cancelling AP probe due to a received beacon\n");
                ieee80211_reset_ap_probe(sdata);
        }

        /*
         * Push the beacon loss detection into the future since
         * we are processing a beacon from the AP just now.
         */
        ieee80211_sta_reset_beacon_monitor(sdata);

        /* TODO: CRC urrently not calculated on S1G Beacon Compatibility
         * element (which carries the beacon interval). Don't forget to add a
         * bit to care_about_ies[] above if mac80211 is interested in a
         * changing S1G element.
         */
        if (!ieee80211_is_s1g_beacon(hdr->frame_control))
                ncrc = crc32_be(0, (void *)&mgmt->u.beacon.beacon_int, 4);
        elems = ieee802_11_parse_elems_crc(variable, len - baselen,
                                           false, care_about_ies, ncrc,
                                           mgmt->bssid, bssid);
        if (!elems)
                return;
        ncrc = elems->crc;

        if (ieee80211_hw_check(&local->hw, PS_NULLFUNC_STACK) &&
            ieee80211_check_tim(elems->tim, elems->tim_len, bss_conf->aid)) {
                if (local->hw.conf.dynamic_ps_timeout > 0) {
                        if (local->hw.conf.flags & IEEE80211_CONF_PS) {
                                local->hw.conf.flags &= ~IEEE80211_CONF_PS;
                                ieee80211_hw_config(local,
                                                    IEEE80211_CONF_CHANGE_PS);
                        }
                        ieee80211_send_nullfunc(local, sdata, false);
                } else if (!local->pspolling && sdata->u.mgd.powersave) {
                        local->pspolling = true;

                        /*
                         * Here is assumed that the driver will be
                         * able to send ps-poll frame and receive a
                         * response even though power save mode is
                         * enabled, but some drivers might require
                         * to disable power save here. This needs
                         * to be investigated.
                         */
                        ieee80211_send_pspoll(local, sdata);
                }
        }

        if (sdata->vif.p2p ||
            sdata->vif.driver_flags & IEEE80211_VIF_GET_NOA_UPDATE) {
                struct ieee80211_p2p_noa_attr noa = {};
                int ret;

                ret = cfg80211_get_p2p_attr(variable,
                                            len - baselen,
                                            IEEE80211_P2P_ATTR_ABSENCE_NOTICE,
                                            (u8 *) &noa, sizeof(noa));
                if (ret >= 2) {
                        if (sdata->u.mgd.p2p_noa_index != noa.index) {
                                /* valid noa_attr and index changed */
                                sdata->u.mgd.p2p_noa_index = noa.index;
                                memcpy(&bss_conf->p2p_noa_attr, &noa, sizeof(noa));
                                changed |= BSS_CHANGED_P2P_PS;
                                /*
                                 * make sure we update all information, the CRC
                                 * mechanism doesn't look at P2P attributes.
                                 */
                                ifmgd->beacon_crc_valid = false;
                        }
                } else if (sdata->u.mgd.p2p_noa_index != -1) {
                        /* noa_attr not found and we had valid noa_attr before */
                        sdata->u.mgd.p2p_noa_index = -1;
                        memset(&bss_conf->p2p_noa_attr, 0, sizeof(bss_conf->p2p_noa_attr));
                        changed |= BSS_CHANGED_P2P_PS;
                        ifmgd->beacon_crc_valid = false;
                }
        }

        if (ifmgd->csa_waiting_bcn)
                ieee80211_chswitch_post_beacon(sdata);

        /*
         * Update beacon timing and dtim count on every beacon appearance. This
         * will allow the driver to use the most updated values. Do it before
         * comparing this one with last received beacon.
         * IMPORTANT: These parameters would possibly be out of sync by the time
         * the driver will use them. The synchronized view is currently
         * guaranteed only in certain callbacks.
         */
        if (ieee80211_hw_check(&local->hw, TIMING_BEACON_ONLY) &&
            !ieee80211_is_s1g_beacon(hdr->frame_control)) {
                sdata->vif.bss_conf.sync_tsf =
                        le64_to_cpu(mgmt->u.beacon.timestamp);
                sdata->vif.bss_conf.sync_device_ts =
                        rx_status->device_timestamp;
                sdata->vif.bss_conf.sync_dtim_count = elems->dtim_count;
        }

        if ((ncrc == ifmgd->beacon_crc && ifmgd->beacon_crc_valid) ||
            ieee80211_is_s1g_short_beacon(mgmt->frame_control))
                goto free;
        ifmgd->beacon_crc = ncrc;
        ifmgd->beacon_crc_valid = true;

        ieee80211_rx_bss_info(sdata, mgmt, len, rx_status);

        ieee80211_sta_process_chanswitch(sdata, rx_status->mactime,
                                         rx_status->device_timestamp,
                                         elems, true);

        if (!(ifmgd->flags & IEEE80211_STA_DISABLE_WMM) &&
            ieee80211_sta_wmm_params(local, sdata, elems->wmm_param,
                                     elems->wmm_param_len,
                                     elems->mu_edca_param_set))
                changed |= BSS_CHANGED_QOS;

        /*
         * If we haven't had a beacon before, tell the driver about the
         * DTIM period (and beacon timing if desired) now.
         */
        if (!ifmgd->have_beacon) {
                /* a few bogus AP send dtim_period = 0 or no TIM IE */
                bss_conf->dtim_period = elems->dtim_period ?: 1;

                changed |= BSS_CHANGED_BEACON_INFO;
                ifmgd->have_beacon = true;

                mutex_lock(&local->iflist_mtx);
                ieee80211_recalc_ps(local);
                mutex_unlock(&local->iflist_mtx);

                ieee80211_recalc_ps_vif(sdata);
        }

        if (elems->erp_info) {
                erp_valid = true;
                erp_value = elems->erp_info[0];
        } else {
                erp_valid = false;
        }

        if (!ieee80211_is_s1g_beacon(hdr->frame_control))
                changed |= ieee80211_handle_bss_capability(sdata,
                                le16_to_cpu(mgmt->u.beacon.capab_info),
                                erp_valid, erp_value);

        mutex_lock(&local->sta_mtx);
        sta = sta_info_get(sdata, bssid);

        changed |= ieee80211_recalc_twt_req(sdata, sta, elems);

        if (ieee80211_config_bw(sdata, sta, elems->ht_cap_elem,
                                elems->vht_cap_elem, elems->ht_operation,
                                elems->vht_operation, elems->he_operation,
                                elems->s1g_oper, bssid, &changed)) {
                mutex_unlock(&local->sta_mtx);
                sdata_info(sdata,
                           "failed to follow AP %pM bandwidth change, disconnect\n",
                           bssid);
                ieee80211_set_disassoc(sdata, IEEE80211_STYPE_DEAUTH,
                                       WLAN_REASON_DEAUTH_LEAVING,
                                       true, deauth_buf);
                ieee80211_report_disconnect(sdata, deauth_buf,
                                            sizeof(deauth_buf), true,
                                            WLAN_REASON_DEAUTH_LEAVING,
                                            false);
                goto free;
        }

        if (sta && elems->opmode_notif)
                ieee80211_vht_handle_opmode(sdata, sta, *elems->opmode_notif,
                                            rx_status->band);
        mutex_unlock(&local->sta_mtx);

        changed |= ieee80211_handle_pwr_constr(sdata, chan, mgmt,
                                               elems->country_elem,
                                               elems->country_elem_len,
                                               elems->pwr_constr_elem,
                                               elems->cisco_dtpc_elem);

        ieee80211_bss_info_change_notify(sdata, changed);
free:
        kfree(elems);
}

void ieee80211_sta_rx_queued_ext(struct ieee80211_sub_if_data *sdata,
                                 struct sk_buff *skb)
{
        struct ieee80211_rx_status *rx_status;
        struct ieee80211_hdr *hdr;
        u16 fc;

        rx_status = (struct ieee80211_rx_status *) skb->cb;
        hdr = (struct ieee80211_hdr *) skb->data;
        fc = le16_to_cpu(hdr->frame_control);

        sdata_lock(sdata);
        switch (fc & IEEE80211_FCTL_STYPE) {
        case IEEE80211_STYPE_S1G_BEACON:
                ieee80211_rx_mgmt_beacon(sdata, hdr, skb->len, rx_status);
                break;
        }
        sdata_unlock(sdata);
}

void ieee80211_sta_rx_queued_mgmt(struct ieee80211_sub_if_data *sdata,
                                  struct sk_buff *skb)
{
        struct ieee80211_rx_status *rx_status;
        struct ieee80211_mgmt *mgmt;
        u16 fc;
        int ies_len;

        rx_status = (struct ieee80211_rx_status *) skb->cb;
        mgmt = (struct ieee80211_mgmt *) skb->data;
        fc = le16_to_cpu(mgmt->frame_control);

        sdata_lock(sdata);

        switch (fc & IEEE80211_FCTL_STYPE) {
        case IEEE80211_STYPE_BEACON:
                ieee80211_rx_mgmt_beacon(sdata, (void *)mgmt,
                                         skb->len, rx_status);
                break;
        case IEEE80211_STYPE_PROBE_RESP:
                ieee80211_rx_mgmt_probe_resp(sdata, skb);
                break;
        case IEEE80211_STYPE_AUTH:
                ieee80211_rx_mgmt_auth(sdata, mgmt, skb->len);
                break;
        case IEEE80211_STYPE_DEAUTH:
                ieee80211_rx_mgmt_deauth(sdata, mgmt, skb->len);
                break;
        case IEEE80211_STYPE_DISASSOC:
                ieee80211_rx_mgmt_disassoc(sdata, mgmt, skb->len);
                break;
        case IEEE80211_STYPE_ASSOC_RESP:
        case IEEE80211_STYPE_REASSOC_RESP:
                ieee80211_rx_mgmt_assoc_resp(sdata, mgmt, skb->len);
                break;
        case IEEE80211_STYPE_ACTION:
                if (mgmt->u.action.category == WLAN_CATEGORY_SPECTRUM_MGMT) {
                        struct ieee802_11_elems *elems;

                        ies_len = skb->len -
                                  offsetof(struct ieee80211_mgmt,
                                           u.action.u.chan_switch.variable);

                        if (ies_len < 0)
                                break;

                        /* CSA IE cannot be overridden, no need for BSSID */
                        elems = ieee802_11_parse_elems(
                                        mgmt->u.action.u.chan_switch.variable,
                                        ies_len, true, mgmt->bssid, NULL);

                        if (elems && !elems->parse_error)
                                ieee80211_sta_process_chanswitch(sdata,
                                                                 rx_status->mactime,
                                                                 rx_status->device_timestamp,
                                                                 elems, false);
                        kfree(elems);
                } else if (mgmt->u.action.category == WLAN_CATEGORY_PUBLIC) {
                        struct ieee802_11_elems *elems;

                        ies_len = skb->len -
                                  offsetof(struct ieee80211_mgmt,
                                           u.action.u.ext_chan_switch.variable);

                        if (ies_len < 0)
                                break;

                        /*
                         * extended CSA IE can't be overridden, no need for
                         * BSSID
                         */
                        elems = ieee802_11_parse_elems(
                                        mgmt->u.action.u.ext_chan_switch.variable,
                                        ies_len, true, mgmt->bssid, NULL);

                        if (elems && !elems->parse_error) {
                                /* for the handling code pretend it was an IE */
                                elems->ext_chansw_ie =
                                        &mgmt->u.action.u.ext_chan_switch.data;

                                ieee80211_sta_process_chanswitch(sdata,
                                                                 rx_status->mactime,
                                                                 rx_status->device_timestamp,
                                                                 elems, false);
                        }

                        kfree(elems);
                }
                break;
        }
        sdata_unlock(sdata);
}

static void ieee80211_sta_timer(struct timer_list *t)
{
        struct ieee80211_sub_if_data *sdata =
                from_timer(sdata, t, u.mgd.timer);

        ieee80211_queue_work(&sdata->local->hw, &sdata->work);
}

void ieee80211_sta_connection_lost(struct ieee80211_sub_if_data *sdata,
                                   u8 *bssid, u8 reason, bool tx)
{
        u8 frame_buf[IEEE80211_DEAUTH_FRAME_LEN];

        ieee80211_set_disassoc(sdata, IEEE80211_STYPE_DEAUTH, reason,
                               tx, frame_buf);

        ieee80211_report_disconnect(sdata, frame_buf, sizeof(frame_buf), true,
                                    reason, false);
}

static int ieee80211_auth(struct ieee80211_sub_if_data *sdata)
{
        struct ieee80211_local *local = sdata->local;
        struct ieee80211_if_managed *ifmgd = &sdata->u.mgd;
        struct ieee80211_mgd_auth_data *auth_data = ifmgd->auth_data;
        u32 tx_flags = 0;
        u16 trans = 1;
        u16 status = 0;
        struct ieee80211_prep_tx_info info = {
                .subtype = IEEE80211_STYPE_AUTH,
        };

        sdata_assert_lock(sdata);

        if (WARN_ON_ONCE(!auth_data))
                return -EINVAL;

        auth_data->tries++;

        if (auth_data->tries > IEEE80211_AUTH_MAX_TRIES) {
                sdata_info(sdata, "authentication with %pM timed out\n",
                           auth_data->bss->bssid);

                /*
                 * Most likely AP is not in the range so remove the
                 * bss struct for that AP.
                 */
                cfg80211_unlink_bss(local->hw.wiphy, auth_data->bss);

                return -ETIMEDOUT;
        }

        if (auth_data->algorithm == WLAN_AUTH_SAE)
                info.duration = jiffies_to_msecs(IEEE80211_AUTH_TIMEOUT_SAE);

        drv_mgd_prepare_tx(local, sdata, &info);

        sdata_info(sdata, "send auth to %pM (try %d/%d)\n",
                   auth_data->bss->bssid, auth_data->tries,
                   IEEE80211_AUTH_MAX_TRIES);

        auth_data->expected_transaction = 2;

        if (auth_data->algorithm == WLAN_AUTH_SAE) {
                trans = auth_data->sae_trans;
                status = auth_data->sae_status;
                auth_data->expected_transaction = trans;
        }

        if (ieee80211_hw_check(&local->hw, REPORTS_TX_ACK_STATUS))
                tx_flags = IEEE80211_TX_CTL_REQ_TX_STATUS |
                           IEEE80211_TX_INTFL_MLME_CONN_TX;

        ieee80211_send_auth(sdata, trans, auth_data->algorithm, status,
                            auth_data->data, auth_data->data_len,
                            auth_data->bss->bssid,
                            auth_data->bss->bssid, NULL, 0, 0,
                            tx_flags);

        if (tx_flags == 0) {
                if (auth_data->algorithm == WLAN_AUTH_SAE)
                        auth_data->timeout = jiffies +
                                IEEE80211_AUTH_TIMEOUT_SAE;
                else
                        auth_data->timeout = jiffies + IEEE80211_AUTH_TIMEOUT;
        } else {
                auth_data->timeout =
                        round_jiffies_up(jiffies + IEEE80211_AUTH_TIMEOUT_LONG);
        }

        auth_data->timeout_started = true;
        run_again(sdata, auth_data->timeout);

        return 0;
}

static int ieee80211_do_assoc(struct ieee80211_sub_if_data *sdata)
{
        struct ieee80211_mgd_assoc_data *assoc_data = sdata->u.mgd.assoc_data;
        struct ieee80211_local *local = sdata->local;
        int ret;

        sdata_assert_lock(sdata);

        assoc_data->tries++;
        if (assoc_data->tries > IEEE80211_ASSOC_MAX_TRIES) {
                sdata_info(sdata, "association with %pM timed out\n",
                           assoc_data->bss->bssid);

                /*
                 * Most likely AP is not in the range so remove the
                 * bss struct for that AP.
                 */
                cfg80211_unlink_bss(local->hw.wiphy, assoc_data->bss);

                return -ETIMEDOUT;
        }

        sdata_info(sdata, "associate with %pM (try %d/%d)\n",
                   assoc_data->bss->bssid, assoc_data->tries,
                   IEEE80211_ASSOC_MAX_TRIES);
        ret = ieee80211_send_assoc(sdata);
        if (ret)
                return ret;

        if (!ieee80211_hw_check(&local->hw, REPORTS_TX_ACK_STATUS)) {
                assoc_data->timeout = jiffies + IEEE80211_ASSOC_TIMEOUT;
                assoc_data->timeout_started = true;
                run_again(sdata, assoc_data->timeout);
        } else {
                assoc_data->timeout =
                        round_jiffies_up(jiffies +
                                         IEEE80211_ASSOC_TIMEOUT_LONG);
                assoc_data->timeout_started = true;
                run_again(sdata, assoc_data->timeout);
        }

        return 0;
}

void ieee80211_mgd_conn_tx_status(struct ieee80211_sub_if_data *sdata,
                                  __le16 fc, bool acked)
{
        struct ieee80211_local *local = sdata->local;

        sdata->u.mgd.status_fc = fc;
        sdata->u.mgd.status_acked = acked;
        sdata->u.mgd.status_received = true;

        ieee80211_queue_work(&local->hw, &sdata->work);
}

void ieee80211_sta_work(struct ieee80211_sub_if_data *sdata)
{
        struct ieee80211_local *local = sdata->local;
        struct ieee80211_if_managed *ifmgd = &sdata->u.mgd;

        sdata_lock(sdata);

        if (ifmgd->status_received) {
                __le16 fc = ifmgd->status_fc;
                bool status_acked = ifmgd->status_acked;

                ifmgd->status_received = false;
                if (ifmgd->auth_data && ieee80211_is_auth(fc)) {
                        if (status_acked) {
                                if (ifmgd->auth_data->algorithm ==
                                    WLAN_AUTH_SAE)
                                        ifmgd->auth_data->timeout =
                                                jiffies +
                                                IEEE80211_AUTH_TIMEOUT_SAE;
                                else
                                        ifmgd->auth_data->timeout =
                                                jiffies +
                                                IEEE80211_AUTH_TIMEOUT_SHORT;
                                run_again(sdata, ifmgd->auth_data->timeout);
                        } else {
                                ifmgd->auth_data->timeout = jiffies - 1;
                        }
                        ifmgd->auth_data->timeout_started = true;
                } else if (ifmgd->assoc_data &&
                           (ieee80211_is_assoc_req(fc) ||
                            ieee80211_is_reassoc_req(fc))) {
                        if (status_acked) {
                                ifmgd->assoc_data->timeout =
                                        jiffies + IEEE80211_ASSOC_TIMEOUT_SHORT;
                                run_again(sdata, ifmgd->assoc_data->timeout);
                        } else {
                                ifmgd->assoc_data->timeout = jiffies - 1;
                        }
                        ifmgd->assoc_data->timeout_started = true;
                }
        }

        if (ifmgd->auth_data && ifmgd->auth_data->timeout_started &&
            time_after(jiffies, ifmgd->auth_data->timeout)) {
                if (ifmgd->auth_data->done || ifmgd->auth_data->waiting) {
                        /*
                         * ok ... we waited for assoc or continuation but
                         * userspace didn't do it, so kill the auth data
                         */
                        ieee80211_destroy_auth_data(sdata, false);
                } else if (ieee80211_auth(sdata)) {
                        u8 bssid[ETH_ALEN];
                        struct ieee80211_event event = {
                                .type = MLME_EVENT,
                                .u.mlme.data = AUTH_EVENT,
                                .u.mlme.status = MLME_TIMEOUT,
                        };

                        memcpy(bssid, ifmgd->auth_data->bss->bssid, ETH_ALEN);

                        ieee80211_destroy_auth_data(sdata, false);

                        cfg80211_auth_timeout(sdata->dev, bssid);
                        drv_event_callback(sdata->local, sdata, &event);
                }
        } else if (ifmgd->auth_data && ifmgd->auth_data->timeout_started)
                run_again(sdata, ifmgd->auth_data->timeout);

        if (ifmgd->assoc_data && ifmgd->assoc_data->timeout_started &&
            time_after(jiffies, ifmgd->assoc_data->timeout)) {
                if ((ifmgd->assoc_data->need_beacon && !ifmgd->have_beacon) ||
                    ieee80211_do_assoc(sdata)) {
                        struct cfg80211_bss *bss = ifmgd->assoc_data->bss;
                        struct ieee80211_event event = {
                                .type = MLME_EVENT,
                                .u.mlme.data = ASSOC_EVENT,
                                .u.mlme.status = MLME_TIMEOUT,
                        };

                        ieee80211_destroy_assoc_data(sdata, false, false);
                        cfg80211_assoc_timeout(sdata->dev, bss);
                        drv_event_callback(sdata->local, sdata, &event);
                }
        } else if (ifmgd->assoc_data && ifmgd->assoc_data->timeout_started)
                run_again(sdata, ifmgd->assoc_data->timeout);

        if (ifmgd->flags & IEEE80211_STA_CONNECTION_POLL &&
            ifmgd->associated) {
                u8 bssid[ETH_ALEN];
                int max_tries;

                memcpy(bssid, ifmgd->associated->bssid, ETH_ALEN);

                if (ieee80211_hw_check(&local->hw, REPORTS_TX_ACK_STATUS))
                        max_tries = max_nullfunc_tries;
                else
                        max_tries = max_probe_tries;

                /* ACK received for nullfunc probing frame */
                if (!ifmgd->probe_send_count)
                        ieee80211_reset_ap_probe(sdata);
                else if (ifmgd->nullfunc_failed) {
                        if (ifmgd->probe_send_count < max_tries) {
                                mlme_dbg(sdata,
                                         "No ack for nullfunc frame to AP %pM, try %d/%i\n",
                                         bssid, ifmgd->probe_send_count,
                                         max_tries);
                                ieee80211_mgd_probe_ap_send(sdata);
                        } else {
                                mlme_dbg(sdata,
                                         "No ack for nullfunc frame to AP %pM, disconnecting.\n",
                                         bssid);
                                ieee80211_sta_connection_lost(sdata, bssid,
                                        WLAN_REASON_DISASSOC_DUE_TO_INACTIVITY,
                                        false);
                        }
                } else if (time_is_after_jiffies(ifmgd->probe_timeout))
                        run_again(sdata, ifmgd->probe_timeout);
                else if (ieee80211_hw_check(&local->hw, REPORTS_TX_ACK_STATUS)) {
                        mlme_dbg(sdata,
                                 "Failed to send nullfunc to AP %pM after %dms, disconnecting\n",
                                 bssid, probe_wait_ms);
                        ieee80211_sta_connection_lost(sdata, bssid,
                                WLAN_REASON_DISASSOC_DUE_TO_INACTIVITY, false);
                } else if (ifmgd->probe_send_count < max_tries) {
                        mlme_dbg(sdata,
                                 "No probe response from AP %pM after %dms, try %d/%i\n",
                                 bssid, probe_wait_ms,
                                 ifmgd->probe_send_count, max_tries);
                        ieee80211_mgd_probe_ap_send(sdata);
                } else {
                        /*
                         * We actually lost the connection ... or did we?
                         * Let's make sure!
                         */
                        mlme_dbg(sdata,
                                 "No probe response from AP %pM after %dms, disconnecting.\n",
                                 bssid, probe_wait_ms);

                        ieee80211_sta_connection_lost(sdata, bssid,
                                WLAN_REASON_DISASSOC_DUE_TO_INACTIVITY, false);
                }
        }

        sdata_unlock(sdata);
}

static void ieee80211_sta_bcn_mon_timer(struct timer_list *t)
{
        struct ieee80211_sub_if_data *sdata =
                from_timer(sdata, t, u.mgd.bcn_mon_timer);
        struct ieee80211_if_managed *ifmgd = &sdata->u.mgd;

        if (sdata->vif.csa_active && !ifmgd->csa_waiting_bcn)
                return;

        if (sdata->vif.driver_flags & IEEE80211_VIF_BEACON_FILTER)
                return;

        sdata->u.mgd.connection_loss = false;
        ieee80211_queue_work(&sdata->local->hw,
                             &sdata->u.mgd.beacon_connection_loss_work);
}

static void ieee80211_sta_conn_mon_timer(struct timer_list *t)
{
        struct ieee80211_sub_if_data *sdata =
                from_timer(sdata, t, u.mgd.conn_mon_timer);
        struct ieee80211_if_managed *ifmgd = &sdata->u.mgd;
        struct ieee80211_local *local = sdata->local;
        struct sta_info *sta;
        unsigned long timeout;

        if (sdata->vif.csa_active && !ifmgd->csa_waiting_bcn)
                return;

        sta = sta_info_get(sdata, ifmgd->bssid);
        if (!sta)
                return;

        timeout = sta->status_stats.last_ack;
        if (time_before(sta->status_stats.last_ack, sta->rx_stats.last_rx))
                timeout = sta->rx_stats.last_rx;
        timeout += IEEE80211_CONNECTION_IDLE_TIME;

        /* If timeout is after now, then update timer to fire at
         * the later date, but do not actually probe at this time.
         */
        if (time_is_after_jiffies(timeout)) {
                mod_timer(&ifmgd->conn_mon_timer, round_jiffies_up(timeout));
                return;
        }

        ieee80211_queue_work(&local->hw, &ifmgd->monitor_work);
}

static void ieee80211_sta_monitor_work(struct work_struct *work)
{
        struct ieee80211_sub_if_data *sdata =
                container_of(work, struct ieee80211_sub_if_data,
                             u.mgd.monitor_work);

        ieee80211_mgd_probe_ap(sdata, false);
}

static void ieee80211_restart_sta_timer(struct ieee80211_sub_if_data *sdata)
{
        if (sdata->vif.type == NL80211_IFTYPE_STATION) {
                __ieee80211_stop_poll(sdata);

                /* let's probe the connection once */
                if (!ieee80211_hw_check(&sdata->local->hw, CONNECTION_MONITOR))
                        ieee80211_queue_work(&sdata->local->hw,
                                             &sdata->u.mgd.monitor_work);
        }
}

#ifdef CONFIG_PM
void ieee80211_mgd_quiesce(struct ieee80211_sub_if_data *sdata)
{
        struct ieee80211_if_managed *ifmgd = &sdata->u.mgd;
        u8 frame_buf[IEEE80211_DEAUTH_FRAME_LEN];

        sdata_lock(sdata);

        if (ifmgd->auth_data || ifmgd->assoc_data) {
                const u8 *bssid = ifmgd->auth_data ?
                                ifmgd->auth_data->bss->bssid :
                                ifmgd->assoc_data->bss->bssid;

                /*
                 * If we are trying to authenticate / associate while suspending,
                 * cfg80211 won't know and won't actually abort those attempts,
                 * thus we need to do that ourselves.
                 */
                ieee80211_send_deauth_disassoc(sdata, bssid, bssid,
                                               IEEE80211_STYPE_DEAUTH,
                                               WLAN_REASON_DEAUTH_LEAVING,
                                               false, frame_buf);
                if (ifmgd->assoc_data)
                        ieee80211_destroy_assoc_data(sdata, false, true);
                if (ifmgd->auth_data)
                        ieee80211_destroy_auth_data(sdata, false);
                cfg80211_tx_mlme_mgmt(sdata->dev, frame_buf,
                                      IEEE80211_DEAUTH_FRAME_LEN,
                                      false);
        }

        /* This is a bit of a hack - we should find a better and more generic
         * solution to this. Normally when suspending, cfg80211 will in fact
         * deauthenticate. However, it doesn't (and cannot) stop an ongoing
         * auth (not so important) or assoc (this is the problem) process.
         *
         * As a consequence, it can happen that we are in the process of both
         * associating and suspending, and receive an association response
         * after cfg80211 has checked if it needs to disconnect, but before
         * we actually set the flag to drop incoming frames. This will then
         * cause the workqueue flush to process the association response in
         * the suspend, resulting in a successful association just before it
         * tries to remove the interface from the driver, which now though
         * has a channel context assigned ... this results in issues.
         *
         * To work around this (for now) simply deauth here again if we're
         * now connected.
         */
        if (ifmgd->associated && !sdata->local->wowlan) {
                u8 bssid[ETH_ALEN];
                struct cfg80211_deauth_request req = {
                        .reason_code = WLAN_REASON_DEAUTH_LEAVING,
                        .bssid = bssid,
                };

                memcpy(bssid, ifmgd->associated->bssid, ETH_ALEN);
                ieee80211_mgd_deauth(sdata, &req);
        }

        sdata_unlock(sdata);
}
#endif

void ieee80211_sta_restart(struct ieee80211_sub_if_data *sdata)
{
        struct ieee80211_if_managed *ifmgd = &sdata->u.mgd;

        sdata_lock(sdata);
        if (!ifmgd->associated) {
                sdata_unlock(sdata);
                return;
        }

        if (sdata->flags & IEEE80211_SDATA_DISCONNECT_RESUME) {
                sdata->flags &= ~IEEE80211_SDATA_DISCONNECT_RESUME;
                mlme_dbg(sdata, "driver requested disconnect after resume\n");
                ieee80211_sta_connection_lost(sdata,
                                              ifmgd->associated->bssid,
                                              WLAN_REASON_UNSPECIFIED,
                                              true);
                sdata_unlock(sdata);
                return;
        }

        if (sdata->flags & IEEE80211_SDATA_DISCONNECT_HW_RESTART) {
                sdata->flags &= ~IEEE80211_SDATA_DISCONNECT_HW_RESTART;
                mlme_dbg(sdata, "driver requested disconnect after hardware restart\n");
                ieee80211_sta_connection_lost(sdata,
                                              ifmgd->associated->bssid,
                                              WLAN_REASON_UNSPECIFIED,
                                              true);
                sdata_unlock(sdata);
                return;
        }

        sdata_unlock(sdata);
}

/* interface setup */
void ieee80211_sta_setup_sdata(struct ieee80211_sub_if_data *sdata)
{
        struct ieee80211_if_managed *ifmgd;

        ifmgd = &sdata->u.mgd;
        INIT_WORK(&ifmgd->monitor_work, ieee80211_sta_monitor_work);
        INIT_WORK(&ifmgd->chswitch_work, ieee80211_chswitch_work);
        INIT_WORK(&ifmgd->beacon_connection_loss_work,
                  ieee80211_beacon_connection_loss_work);
        INIT_WORK(&ifmgd->csa_connection_drop_work,
                  ieee80211_csa_connection_drop_work);
        INIT_WORK(&ifmgd->request_smps_work, ieee80211_request_smps_mgd_work);
        INIT_DELAYED_WORK(&ifmgd->tdls_peer_del_work,
                          ieee80211_tdls_peer_del_work);
        timer_setup(&ifmgd->timer, ieee80211_sta_timer, 0);
        timer_setup(&ifmgd->bcn_mon_timer, ieee80211_sta_bcn_mon_timer, 0);
        timer_setup(&ifmgd->conn_mon_timer, ieee80211_sta_conn_mon_timer, 0);
        timer_setup(&ifmgd->chswitch_timer, ieee80211_chswitch_timer, 0);
        INIT_DELAYED_WORK(&ifmgd->tx_tspec_wk,
                          ieee80211_sta_handle_tspec_ac_params_wk);

        ifmgd->flags = 0;
        ifmgd->powersave = sdata->wdev.ps;
        ifmgd->uapsd_queues = sdata->local->hw.uapsd_queues;
        ifmgd->uapsd_max_sp_len = sdata->local->hw.uapsd_max_sp_len;
        ifmgd->p2p_noa_index = -1;

        if (sdata->local->hw.wiphy->features & NL80211_FEATURE_DYNAMIC_SMPS)
                ifmgd->req_smps = IEEE80211_SMPS_AUTOMATIC;
        else
                ifmgd->req_smps = IEEE80211_SMPS_OFF;

        /* Setup TDLS data */
        spin_lock_init(&ifmgd->teardown_lock);
        ifmgd->teardown_skb = NULL;
        ifmgd->orig_teardown_skb = NULL;
}

/* scan finished notification */
void ieee80211_mlme_notify_scan_completed(struct ieee80211_local *local)
{
        struct ieee80211_sub_if_data *sdata;

        /* Restart STA timers */
        rcu_read_lock();
        list_for_each_entry_rcu(sdata, &local->interfaces, list) {
                if (ieee80211_sdata_running(sdata))
                        ieee80211_restart_sta_timer(sdata);
        }
        rcu_read_unlock();
}

static u8 ieee80211_ht_vht_rx_chains(struct ieee80211_sub_if_data *sdata,
                                     struct cfg80211_bss *cbss)
{
        struct ieee80211_if_managed *ifmgd = &sdata->u.mgd;
        const u8 *ht_cap_ie, *vht_cap_ie;
        const struct ieee80211_ht_cap *ht_cap;
        const struct ieee80211_vht_cap *vht_cap;
        u8 chains = 1;

        if (ifmgd->flags & IEEE80211_STA_DISABLE_HT)
                return chains;

        ht_cap_ie = ieee80211_bss_get_ie(cbss, WLAN_EID_HT_CAPABILITY);
        if (ht_cap_ie && ht_cap_ie[1] >= sizeof(*ht_cap)) {
                ht_cap = (void *)(ht_cap_ie + 2);
                chains = ieee80211_mcs_to_chains(&ht_cap->mcs);
                /*
                 * TODO: use "Tx Maximum Number Spatial Streams Supported" and
                 *         "Tx Unequal Modulation Supported" fields.
                 */
        }

        if (ifmgd->flags & IEEE80211_STA_DISABLE_VHT)
                return chains;

        vht_cap_ie = ieee80211_bss_get_ie(cbss, WLAN_EID_VHT_CAPABILITY);
        if (vht_cap_ie && vht_cap_ie[1] >= sizeof(*vht_cap)) {
                u8 nss;
                u16 tx_mcs_map;

                vht_cap = (void *)(vht_cap_ie + 2);
                tx_mcs_map = le16_to_cpu(vht_cap->supp_mcs.tx_mcs_map);
                for (nss = 8; nss > 0; nss--) {
                        if (((tx_mcs_map >> (2 * (nss - 1))) & 3) !=
                                        IEEE80211_VHT_MCS_NOT_SUPPORTED)
                                break;
                }
                /* TODO: use "Tx Highest Supported Long GI Data Rate" field? */
                chains = max(chains, nss);
        }

        return chains;
}

static bool
ieee80211_verify_sta_he_mcs_support(struct ieee80211_sub_if_data *sdata,
                                    struct ieee80211_supported_band *sband,
                                    const struct ieee80211_he_operation *he_op)
{
        const struct ieee80211_sta_he_cap *sta_he_cap =
                ieee80211_get_he_iftype_cap(sband,
                                            ieee80211_vif_type_p2p(&sdata->vif));
        u16 ap_min_req_set;
        int i;

        if (!sta_he_cap || !he_op)
                return false;

        ap_min_req_set = le16_to_cpu(he_op->he_mcs_nss_set);

        /* Need to go over for 80MHz, 160MHz and for 80+80 */
        for (i = 0; i < 3; i++) {
                const struct ieee80211_he_mcs_nss_supp *sta_mcs_nss_supp =
                        &sta_he_cap->he_mcs_nss_supp;
                u16 sta_mcs_map_rx =
                        le16_to_cpu(((__le16 *)sta_mcs_nss_supp)[2 * i]);
                u16 sta_mcs_map_tx =
                        le16_to_cpu(((__le16 *)sta_mcs_nss_supp)[2 * i + 1]);
                u8 nss;
                bool verified = true;

                /*
                 * For each band there is a maximum of 8 spatial streams
                 * possible. Each of the sta_mcs_map_* is a 16-bit struct built
                 * of 2 bits per NSS (1-8), with the values defined in enum
                 * ieee80211_he_mcs_support. Need to make sure STA TX and RX
                 * capabilities aren't less than the AP's minimum requirements
                 * for this HE BSS per SS.
                 * It is enough to find one such band that meets the reqs.
                 */
                for (nss = 8; nss > 0; nss--) {
                        u8 sta_rx_val = (sta_mcs_map_rx >> (2 * (nss - 1))) & 3;
                        u8 sta_tx_val = (sta_mcs_map_tx >> (2 * (nss - 1))) & 3;
                        u8 ap_val = (ap_min_req_set >> (2 * (nss - 1))) & 3;

                        if (ap_val == IEEE80211_HE_MCS_NOT_SUPPORTED)
                                continue;

                        /*
                         * Make sure the HE AP doesn't require MCSs that aren't
                         * supported by the client
                         */
                        if (sta_rx_val == IEEE80211_HE_MCS_NOT_SUPPORTED ||
                            sta_tx_val == IEEE80211_HE_MCS_NOT_SUPPORTED ||
                            (ap_val > sta_rx_val) || (ap_val > sta_tx_val)) {
                                verified = false;
                                break;
                        }
                }

                if (verified)
                        return true;
        }

        /* If here, STA doesn't meet AP's HE min requirements */
        return false;
}

static int ieee80211_prep_channel(struct ieee80211_sub_if_data *sdata,
                                  struct cfg80211_bss *cbss)
{
        struct ieee80211_local *local = sdata->local;
        struct ieee80211_if_managed *ifmgd = &sdata->u.mgd;
        const struct ieee80211_ht_cap *ht_cap = NULL;
        const struct ieee80211_ht_operation *ht_oper = NULL;
        const struct ieee80211_vht_operation *vht_oper = NULL;
        const struct ieee80211_he_operation *he_oper = NULL;
        const struct ieee80211_s1g_oper_ie *s1g_oper = NULL;
        struct ieee80211_supported_band *sband;
        struct cfg80211_chan_def chandef;
        bool is_6ghz = cbss->channel->band == NL80211_BAND_6GHZ;
        bool is_5ghz = cbss->channel->band == NL80211_BAND_5GHZ;
        struct ieee80211_bss *bss = (void *)cbss->priv;
        int ret;
        u32 i;
        bool have_80mhz;

        sband = local->hw.wiphy->bands[cbss->channel->band];

        ifmgd->flags &= ~(IEEE80211_STA_DISABLE_40MHZ |
                          IEEE80211_STA_DISABLE_80P80MHZ |
                          IEEE80211_STA_DISABLE_160MHZ);

        /* disable HT/VHT/HE if we don't support them */
        if (!sband->ht_cap.ht_supported && !is_6ghz) {
                ifmgd->flags |= IEEE80211_STA_DISABLE_HT;
                ifmgd->flags |= IEEE80211_STA_DISABLE_VHT;
                ifmgd->flags |= IEEE80211_STA_DISABLE_HE;
        }

        if (!sband->vht_cap.vht_supported && is_5ghz) {
                ifmgd->flags |= IEEE80211_STA_DISABLE_VHT;
                ifmgd->flags |= IEEE80211_STA_DISABLE_HE;
        }

        if (!ieee80211_get_he_iftype_cap(sband,
                                         ieee80211_vif_type_p2p(&sdata->vif)))
                ifmgd->flags |= IEEE80211_STA_DISABLE_HE;

        rcu_read_lock();

        if (!(ifmgd->flags & IEEE80211_STA_DISABLE_HT) && !is_6ghz) {
                const u8 *ht_oper_ie, *ht_cap_ie;

                ht_oper_ie = ieee80211_bss_get_ie(cbss, WLAN_EID_HT_OPERATION);
                if (ht_oper_ie && ht_oper_ie[1] >= sizeof(*ht_oper))
                        ht_oper = (void *)(ht_oper_ie + 2);

                ht_cap_ie = ieee80211_bss_get_ie(cbss, WLAN_EID_HT_CAPABILITY);
                if (ht_cap_ie && ht_cap_ie[1] >= sizeof(*ht_cap))
                        ht_cap = (void *)(ht_cap_ie + 2);

                if (!ht_cap) {
                        ifmgd->flags |= IEEE80211_STA_DISABLE_HT;
                        ht_oper = NULL;
                }
        }

        if (!(ifmgd->flags & IEEE80211_STA_DISABLE_VHT) && !is_6ghz) {
                const u8 *vht_oper_ie, *vht_cap;

                vht_oper_ie = ieee80211_bss_get_ie(cbss,
                                                   WLAN_EID_VHT_OPERATION);
                if (vht_oper_ie && vht_oper_ie[1] >= sizeof(*vht_oper))
                        vht_oper = (void *)(vht_oper_ie + 2);
                if (vht_oper && !ht_oper) {
                        vht_oper = NULL;
                        sdata_info(sdata,
                                   "AP advertised VHT without HT, disabling HT/VHT/HE\n");
                        ifmgd->flags |= IEEE80211_STA_DISABLE_HT;
                        ifmgd->flags |= IEEE80211_STA_DISABLE_VHT;
                        ifmgd->flags |= IEEE80211_STA_DISABLE_HE;
                }

                vht_cap = ieee80211_bss_get_ie(cbss, WLAN_EID_VHT_CAPABILITY);
                if (!vht_cap || vht_cap[1] < sizeof(struct ieee80211_vht_cap)) {
                        ifmgd->flags |= IEEE80211_STA_DISABLE_VHT;
                        vht_oper = NULL;
                }
        }

        if (!(ifmgd->flags & IEEE80211_STA_DISABLE_HE)) {
                const struct cfg80211_bss_ies *ies;
                const u8 *he_oper_ie;

                ies = rcu_dereference(cbss->ies);
                he_oper_ie = cfg80211_find_ext_ie(WLAN_EID_EXT_HE_OPERATION,
                                                  ies->data, ies->len);
                if (he_oper_ie &&
                    he_oper_ie[1] >= ieee80211_he_oper_size(&he_oper_ie[3]))
                        he_oper = (void *)(he_oper_ie + 3);
                else
                        he_oper = NULL;

                if (!ieee80211_verify_sta_he_mcs_support(sdata, sband, he_oper))
                        ifmgd->flags |= IEEE80211_STA_DISABLE_HE;
        }

        /* Allow VHT if at least one channel on the sband supports 80 MHz */
        have_80mhz = false;
        for (i = 0; i < sband->n_channels; i++) {
                if (sband->channels[i].flags & (IEEE80211_CHAN_DISABLED |
                                                IEEE80211_CHAN_NO_80MHZ))
                        continue;

                have_80mhz = true;
                break;
        }

        if (!have_80mhz)
                ifmgd->flags |= IEEE80211_STA_DISABLE_VHT;

        if (sband->band == NL80211_BAND_S1GHZ) {
                const u8 *s1g_oper_ie;

                s1g_oper_ie = ieee80211_bss_get_ie(cbss,
                                                   WLAN_EID_S1G_OPERATION);
                if (s1g_oper_ie && s1g_oper_ie[1] >= sizeof(*s1g_oper))
                        s1g_oper = (void *)(s1g_oper_ie + 2);
                else
                        sdata_info(sdata,
                                   "AP missing S1G operation element?\n");
        }

        ifmgd->flags |= ieee80211_determine_chantype(sdata, sband,
                                                     cbss->channel,
                                                     bss->vht_cap_info,
                                                     ht_oper, vht_oper, he_oper,
                                                     s1g_oper,
                                                     &chandef, false);

        sdata->needed_rx_chains = min(ieee80211_ht_vht_rx_chains(sdata, cbss),
                                      local->rx_chains);

        rcu_read_unlock();

        if (ifmgd->flags & IEEE80211_STA_DISABLE_HE && is_6ghz) {
                sdata_info(sdata, "Rejecting non-HE 6/7 GHz connection");
                return -EINVAL;
        }

        /* will change later if needed */
        sdata->smps_mode = IEEE80211_SMPS_OFF;

        mutex_lock(&local->mtx);
        /*
         * If this fails (possibly due to channel context sharing
         * on incompatible channels, e.g. 80+80 and 160 sharing the
         * same control channel) try to use a smaller bandwidth.
         */
        ret = ieee80211_vif_use_channel(sdata, &chandef,
                                        IEEE80211_CHANCTX_SHARED);

        /* don't downgrade for 5 and 10 MHz channels, though. */
        if (chandef.width == NL80211_CHAN_WIDTH_5 ||
            chandef.width == NL80211_CHAN_WIDTH_10)
                goto out;

        while (ret && chandef.width != NL80211_CHAN_WIDTH_20_NOHT) {
                ifmgd->flags |= ieee80211_chandef_downgrade(&chandef);
                ret = ieee80211_vif_use_channel(sdata, &chandef,
                                                IEEE80211_CHANCTX_SHARED);
        }
 out:
        mutex_unlock(&local->mtx);
        return ret;
}

static bool ieee80211_get_dtim(const struct cfg80211_bss_ies *ies,
                               u8 *dtim_count, u8 *dtim_period)
{
        const u8 *tim_ie = cfg80211_find_ie(WLAN_EID_TIM, ies->data, ies->len);
        const u8 *idx_ie = cfg80211_find_ie(WLAN_EID_MULTI_BSSID_IDX, ies->data,
                                         ies->len);
        const struct ieee80211_tim_ie *tim = NULL;
        const struct ieee80211_bssid_index *idx;
        bool valid = tim_ie && tim_ie[1] >= 2;

        if (valid)
                tim = (void *)(tim_ie + 2);

        if (dtim_count)
                *dtim_count = valid ? tim->dtim_count : 0;

        if (dtim_period)
                *dtim_period = valid ? tim->dtim_period : 0;

        /* Check if value is overridden by non-transmitted profile */
        if (!idx_ie || idx_ie[1] < 3)
                return valid;

        idx = (void *)(idx_ie + 2);

        if (dtim_count)
                *dtim_count = idx->dtim_count;

        if (dtim_period)
                *dtim_period = idx->dtim_period;

        return true;
}

static int ieee80211_prep_connection(struct ieee80211_sub_if_data *sdata,
                                     struct cfg80211_bss *cbss, bool assoc,
                                     bool override)
{
        struct ieee80211_local *local = sdata->local;
        struct ieee80211_if_managed *ifmgd = &sdata->u.mgd;
        struct ieee80211_bss *bss = (void *)cbss->priv;
        struct sta_info *new_sta = NULL;
        struct ieee80211_supported_band *sband;
        bool have_sta = false;
        int err;

        sband = local->hw.wiphy->bands[cbss->channel->band];

        if (WARN_ON(!ifmgd->auth_data && !ifmgd->assoc_data))
                return -EINVAL;

        /* If a reconfig is happening, bail out */
        if (local->in_reconfig)
                return -EBUSY;

        if (assoc) {
                rcu_read_lock();
                have_sta = sta_info_get(sdata, cbss->bssid);
                rcu_read_unlock();
        }

        if (!have_sta) {
                new_sta = sta_info_alloc(sdata, cbss->bssid, GFP_KERNEL);
                if (!new_sta)
                        return -ENOMEM;
        }

        /*
         * Set up the information for the new channel before setting the
         * new channel. We can't - completely race-free - change the basic
         * rates bitmap and the channel (sband) that it refers to, but if
         * we set it up before we at least avoid calling into the driver's
         * bss_info_changed() method with invalid information (since we do
         * call that from changing the channel - only for IDLE and perhaps
         * some others, but ...).
         *
         * So to avoid that, just set up all the new information before the
         * channel, but tell the driver to apply it only afterwards, since
         * it might need the new channel for that.
         */
        if (new_sta) {
                u32 rates = 0, basic_rates = 0;
                bool have_higher_than_11mbit = false;
                int min_rate = INT_MAX, min_rate_index = -1;
                const struct cfg80211_bss_ies *ies;
                int shift = ieee80211_vif_get_shift(&sdata->vif);

                /* TODO: S1G Basic Rate Set is expressed elsewhere */
                if (cbss->channel->band == NL80211_BAND_S1GHZ) {
                        ieee80211_s1g_sta_rate_init(new_sta);
                        goto skip_rates;
                }

                ieee80211_get_rates(sband, bss->supp_rates,
                                    bss->supp_rates_len,
                                    &rates, &basic_rates,
                                    &have_higher_than_11mbit,
                                    &min_rate, &min_rate_index,
                                    shift);

                /*
                 * This used to be a workaround for basic rates missing
                 * in the association response frame. Now that we no
                 * longer use the basic rates from there, it probably
                 * doesn't happen any more, but keep the workaround so
                 * in case some *other* APs are buggy in different ways
                 * we can connect -- with a warning.
                 * Allow this workaround only in case the AP provided at least
                 * one rate.
                 */
                if (min_rate_index < 0) {
                        sdata_info(sdata,
                                   "No legacy rates in association response\n");

                        sta_info_free(local, new_sta);
                        return -EINVAL;
                } else if (!basic_rates) {
                        sdata_info(sdata,
                                   "No basic rates, using min rate instead\n");
                        basic_rates = BIT(min_rate_index);
                }

                if (rates)
                        new_sta->sta.supp_rates[cbss->channel->band] = rates;
                else
                        sdata_info(sdata,
                                   "No rates found, keeping mandatory only\n");

                sdata->vif.bss_conf.basic_rates = basic_rates;

                /* cf. IEEE 802.11 9.2.12 */
                if (cbss->channel->band == NL80211_BAND_2GHZ &&
                    have_higher_than_11mbit)
                        sdata->flags |= IEEE80211_SDATA_OPERATING_GMODE;
                else
                        sdata->flags &= ~IEEE80211_SDATA_OPERATING_GMODE;

skip_rates:
                memcpy(ifmgd->bssid, cbss->bssid, ETH_ALEN);

                /* set timing information */
                sdata->vif.bss_conf.beacon_int = cbss->beacon_interval;
                rcu_read_lock();
                ies = rcu_dereference(cbss->beacon_ies);
                if (ies) {
                        sdata->vif.bss_conf.sync_tsf = ies->tsf;
                        sdata->vif.bss_conf.sync_device_ts =
                                bss->device_ts_beacon;

                        ieee80211_get_dtim(ies,
                                           &sdata->vif.bss_conf.sync_dtim_count,
                                           NULL);
                } else if (!ieee80211_hw_check(&sdata->local->hw,
                                               TIMING_BEACON_ONLY)) {
                        ies = rcu_dereference(cbss->proberesp_ies);
                        /* must be non-NULL since beacon IEs were NULL */
                        sdata->vif.bss_conf.sync_tsf = ies->tsf;
                        sdata->vif.bss_conf.sync_device_ts =
                                bss->device_ts_presp;
                        sdata->vif.bss_conf.sync_dtim_count = 0;
                } else {
                        sdata->vif.bss_conf.sync_tsf = 0;
                        sdata->vif.bss_conf.sync_device_ts = 0;
                        sdata->vif.bss_conf.sync_dtim_count = 0;
                }
                rcu_read_unlock();
        }

        if (new_sta || override) {
                err = ieee80211_prep_channel(sdata, cbss);
                if (err) {
                        if (new_sta)
                                sta_info_free(local, new_sta);
                        return -EINVAL;
                }
        }

        if (new_sta) {
                /*
                 * tell driver about BSSID, basic rates and timing
                 * this was set up above, before setting the channel
                 */
                ieee80211_bss_info_change_notify(sdata,
                        BSS_CHANGED_BSSID | BSS_CHANGED_BASIC_RATES |
                        BSS_CHANGED_BEACON_INT);

                if (assoc)
                        sta_info_pre_move_state(new_sta, IEEE80211_STA_AUTH);

                err = sta_info_insert(new_sta);
                new_sta = NULL;
                if (err) {
                        sdata_info(sdata,
                                   "failed to insert STA entry for the AP (error %d)\n",
                                   err);
                        return err;
                }
        } else
                WARN_ON_ONCE(!ether_addr_equal(ifmgd->bssid, cbss->bssid));

        /* Cancel scan to ensure that nothing interferes with connection */
        if (local->scanning)
                ieee80211_scan_cancel(local);

        return 0;
}

/* config hooks */
int ieee80211_mgd_auth(struct ieee80211_sub_if_data *sdata,
                       struct cfg80211_auth_request *req)
{
        struct ieee80211_local *local = sdata->local;
        struct ieee80211_if_managed *ifmgd = &sdata->u.mgd;
        struct ieee80211_mgd_auth_data *auth_data;
        u16 auth_alg;
        int err;
        bool cont_auth;

        /* prepare auth data structure */

        switch (req->auth_type) {
        case NL80211_AUTHTYPE_OPEN_SYSTEM:
                auth_alg = WLAN_AUTH_OPEN;
                break;
        case NL80211_AUTHTYPE_SHARED_KEY:
                if (fips_enabled)
                        return -EOPNOTSUPP;
                auth_alg = WLAN_AUTH_SHARED_KEY;
                break;
        case NL80211_AUTHTYPE_FT:
                auth_alg = WLAN_AUTH_FT;
                break;
        case NL80211_AUTHTYPE_NETWORK_EAP:
                auth_alg = WLAN_AUTH_LEAP;
                break;
        case NL80211_AUTHTYPE_SAE:
                auth_alg = WLAN_AUTH_SAE;
                break;
        case NL80211_AUTHTYPE_FILS_SK:
                auth_alg = WLAN_AUTH_FILS_SK;
                break;
        case NL80211_AUTHTYPE_FILS_SK_PFS:
                auth_alg = WLAN_AUTH_FILS_SK_PFS;
                break;
        case NL80211_AUTHTYPE_FILS_PK:
                auth_alg = WLAN_AUTH_FILS_PK;
                break;
        default:
                return -EOPNOTSUPP;
        }

        if (ifmgd->assoc_data)
                return -EBUSY;

        auth_data = kzalloc(sizeof(*auth_data) + req->auth_data_len +
                            req->ie_len, GFP_KERNEL);
        if (!auth_data)
                return -ENOMEM;

        auth_data->bss = req->bss;

        if (req->auth_data_len >= 4) {
                if (req->auth_type == NL80211_AUTHTYPE_SAE) {
                        __le16 *pos = (__le16 *) req->auth_data;

                        auth_data->sae_trans = le16_to_cpu(pos[0]);
                        auth_data->sae_status = le16_to_cpu(pos[1]);
                }
                memcpy(auth_data->data, req->auth_data + 4,
                       req->auth_data_len - 4);
                auth_data->data_len += req->auth_data_len - 4;
        }

        /* Check if continuing authentication or trying to authenticate with the
         * same BSS that we were in the process of authenticating with and avoid
         * removal and re-addition of the STA entry in
         * ieee80211_prep_connection().
         */
        cont_auth = ifmgd->auth_data && req->bss == ifmgd->auth_data->bss;

        if (req->ie && req->ie_len) {
                memcpy(&auth_data->data[auth_data->data_len],
                       req->ie, req->ie_len);
                auth_data->data_len += req->ie_len;
        }

        if (req->key && req->key_len) {
                auth_data->key_len = req->key_len;
                auth_data->key_idx = req->key_idx;
                memcpy(auth_data->key, req->key, req->key_len);
        }

        auth_data->algorithm = auth_alg;

        /* try to authenticate/probe */

        if (ifmgd->auth_data) {
                if (cont_auth && req->auth_type == NL80211_AUTHTYPE_SAE) {
                        auth_data->peer_confirmed =
                                ifmgd->auth_data->peer_confirmed;
                }
                ieee80211_destroy_auth_data(sdata, cont_auth);
        }

        /* prep auth_data so we don't go into idle on disassoc */
        ifmgd->auth_data = auth_data;

        /* If this is continuation of an ongoing SAE authentication exchange
         * (i.e., request to send SAE Confirm) and the peer has already
         * confirmed, mark authentication completed since we are about to send
         * out SAE Confirm.
         */
        if (cont_auth && req->auth_type == NL80211_AUTHTYPE_SAE &&
            auth_data->peer_confirmed && auth_data->sae_trans == 2)
                ieee80211_mark_sta_auth(sdata, req->bss->bssid);

        if (ifmgd->associated) {
                u8 frame_buf[IEEE80211_DEAUTH_FRAME_LEN];

                sdata_info(sdata,
                           "disconnect from AP %pM for new auth to %pM\n",
                           ifmgd->associated->bssid, req->bss->bssid);
                ieee80211_set_disassoc(sdata, IEEE80211_STYPE_DEAUTH,
                                       WLAN_REASON_UNSPECIFIED,
                                       false, frame_buf);

                ieee80211_report_disconnect(sdata, frame_buf,
                                            sizeof(frame_buf), true,
                                            WLAN_REASON_UNSPECIFIED,
                                            false);
        }

        sdata_info(sdata, "authenticate with %pM\n", req->bss->bssid);

        err = ieee80211_prep_connection(sdata, req->bss, cont_auth, false);
        if (err)
                goto err_clear;

        err = ieee80211_auth(sdata);
        if (err) {
                sta_info_destroy_addr(sdata, req->bss->bssid);
                goto err_clear;
        }

        /* hold our own reference */
        cfg80211_ref_bss(local->hw.wiphy, auth_data->bss);
        return 0;

 err_clear:
        eth_zero_addr(ifmgd->bssid);
        ieee80211_bss_info_change_notify(sdata, BSS_CHANGED_BSSID);
        ifmgd->auth_data = NULL;
        mutex_lock(&sdata->local->mtx);
        ieee80211_vif_release_channel(sdata);
        mutex_unlock(&sdata->local->mtx);
        kfree(auth_data);
        return err;
}

int ieee80211_mgd_assoc(struct ieee80211_sub_if_data *sdata,
                        struct cfg80211_assoc_request *req)
{
        bool is_6ghz = req->bss->channel->band == NL80211_BAND_6GHZ;
        bool is_5ghz = req->bss->channel->band == NL80211_BAND_5GHZ;
        struct ieee80211_local *local = sdata->local;
        struct ieee80211_if_managed *ifmgd = &sdata->u.mgd;
        struct ieee80211_bss *bss = (void *)req->bss->priv;
        struct ieee80211_mgd_assoc_data *assoc_data;
        const struct cfg80211_bss_ies *beacon_ies;
        struct ieee80211_supported_band *sband;
        struct ieee80211_bss_conf *bss_conf = &sdata->vif.bss_conf;
        const u8 *ssidie, *ht_ie, *vht_ie;
        int i, err;
        bool override = false;

        assoc_data = kzalloc(sizeof(*assoc_data) + req->ie_len, GFP_KERNEL);
        if (!assoc_data)
                return -ENOMEM;

        rcu_read_lock();
        ssidie = ieee80211_bss_get_ie(req->bss, WLAN_EID_SSID);
        if (!ssidie || ssidie[1] > sizeof(assoc_data->ssid)) {
                rcu_read_unlock();
                kfree(assoc_data);
                return -EINVAL;
        }
        memcpy(assoc_data->ssid, ssidie + 2, ssidie[1]);
        assoc_data->ssid_len = ssidie[1];
        memcpy(bss_conf->ssid, assoc_data->ssid, assoc_data->ssid_len);
        bss_conf->ssid_len = assoc_data->ssid_len;
        rcu_read_unlock();

        if (ifmgd->associated) {
                u8 frame_buf[IEEE80211_DEAUTH_FRAME_LEN];

                sdata_info(sdata,
                           "disconnect from AP %pM for new assoc to %pM\n",
                           ifmgd->associated->bssid, req->bss->bssid);
                ieee80211_set_disassoc(sdata, IEEE80211_STYPE_DEAUTH,
                                       WLAN_REASON_UNSPECIFIED,
                                       false, frame_buf);

                ieee80211_report_disconnect(sdata, frame_buf,
                                            sizeof(frame_buf), true,
                                            WLAN_REASON_UNSPECIFIED,
                                            false);
        }

        if (ifmgd->auth_data && !ifmgd->auth_data->done) {
                err = -EBUSY;
                goto err_free;
        }

        if (ifmgd->assoc_data) {
                err = -EBUSY;
                goto err_free;
        }

        if (ifmgd->auth_data) {
                bool match;

                /* keep sta info, bssid if matching */
                match = ether_addr_equal(ifmgd->bssid, req->bss->bssid);
                ieee80211_destroy_auth_data(sdata, match);
        }

        /* prepare assoc data */

        ifmgd->beacon_crc_valid = false;

        assoc_data->wmm = bss->wmm_used &&
                          (local->hw.queues >= IEEE80211_NUM_ACS);

        /*
         * IEEE802.11n does not allow TKIP/WEP as pairwise ciphers in HT mode.
         * We still associate in non-HT mode (11a/b/g) if any one of these
         * ciphers is configured as pairwise.
         * We can set this to true for non-11n hardware, that'll be checked
         * separately along with the peer capabilities.
         */
        for (i = 0; i < req->crypto.n_ciphers_pairwise; i++) {
                if (req->crypto.ciphers_pairwise[i] == WLAN_CIPHER_SUITE_WEP40 ||
                    req->crypto.ciphers_pairwise[i] == WLAN_CIPHER_SUITE_TKIP ||
                    req->crypto.ciphers_pairwise[i] == WLAN_CIPHER_SUITE_WEP104) {
                        ifmgd->flags |= IEEE80211_STA_DISABLE_HT;
                        ifmgd->flags |= IEEE80211_STA_DISABLE_VHT;
                        ifmgd->flags |= IEEE80211_STA_DISABLE_HE;
                        netdev_info(sdata->dev,
                                    "disabling HT/VHT/HE due to WEP/TKIP use\n");
                }
        }

        sband = local->hw.wiphy->bands[req->bss->channel->band];

        /* also disable HT/VHT/HE if the AP doesn't use WMM */
        if (!bss->wmm_used) {
                ifmgd->flags |= IEEE80211_STA_DISABLE_HT;
                ifmgd->flags |= IEEE80211_STA_DISABLE_VHT;
                ifmgd->flags |= IEEE80211_STA_DISABLE_HE;
                netdev_info(sdata->dev,
                            "disabling HT/VHT/HE as WMM/QoS is not supported by the AP\n");
        }

        memcpy(&ifmgd->ht_capa, &req->ht_capa, sizeof(ifmgd->ht_capa));
        memcpy(&ifmgd->ht_capa_mask, &req->ht_capa_mask,
               sizeof(ifmgd->ht_capa_mask));

        memcpy(&ifmgd->vht_capa, &req->vht_capa, sizeof(ifmgd->vht_capa));
        memcpy(&ifmgd->vht_capa_mask, &req->vht_capa_mask,
               sizeof(ifmgd->vht_capa_mask));

        memcpy(&ifmgd->s1g_capa, &req->s1g_capa, sizeof(ifmgd->s1g_capa));
        memcpy(&ifmgd->s1g_capa_mask, &req->s1g_capa_mask,
               sizeof(ifmgd->s1g_capa_mask));

        if (req->ie && req->ie_len) {
                memcpy(assoc_data->ie, req->ie, req->ie_len);
                assoc_data->ie_len = req->ie_len;
        }

        if (req->fils_kek) {
                /* should already be checked in cfg80211 - so warn */
                if (WARN_ON(req->fils_kek_len > FILS_MAX_KEK_LEN)) {
                        err = -EINVAL;
                        goto err_free;
                }
                memcpy(assoc_data->fils_kek, req->fils_kek,
                       req->fils_kek_len);
                assoc_data->fils_kek_len = req->fils_kek_len;
        }

        if (req->fils_nonces)
                memcpy(assoc_data->fils_nonces, req->fils_nonces,
                       2 * FILS_NONCE_LEN);

        assoc_data->bss = req->bss;
        assoc_data->capability = req->bss->capability;
        assoc_data->supp_rates = bss->supp_rates;
        assoc_data->supp_rates_len = bss->supp_rates_len;

        rcu_read_lock();
        ht_ie = ieee80211_bss_get_ie(req->bss, WLAN_EID_HT_OPERATION);
        if (ht_ie && ht_ie[1] >= sizeof(struct ieee80211_ht_operation))
                assoc_data->ap_ht_param =
                        ((struct ieee80211_ht_operation *)(ht_ie + 2))->ht_param;
        else if (!is_6ghz)
                ifmgd->flags |= IEEE80211_STA_DISABLE_HT;
        vht_ie = ieee80211_bss_get_ie(req->bss, WLAN_EID_VHT_CAPABILITY);
        if (vht_ie && vht_ie[1] >= sizeof(struct ieee80211_vht_cap))
                memcpy(&assoc_data->ap_vht_cap, vht_ie + 2,
                       sizeof(struct ieee80211_vht_cap));
        else if (is_5ghz)
                ifmgd->flags |= IEEE80211_STA_DISABLE_VHT |
                                IEEE80211_STA_DISABLE_HE;
        rcu_read_unlock();

        if (WARN((sdata->vif.driver_flags & IEEE80211_VIF_SUPPORTS_UAPSD) &&
                 ieee80211_hw_check(&local->hw, PS_NULLFUNC_STACK),
             "U-APSD not supported with HW_PS_NULLFUNC_STACK\n"))
                sdata->vif.driver_flags &= ~IEEE80211_VIF_SUPPORTS_UAPSD;

        if (bss->wmm_used && bss->uapsd_supported &&
            (sdata->vif.driver_flags & IEEE80211_VIF_SUPPORTS_UAPSD)) {
                assoc_data->uapsd = true;
                ifmgd->flags |= IEEE80211_STA_UAPSD_ENABLED;
        } else {
                assoc_data->uapsd = false;
                ifmgd->flags &= ~IEEE80211_STA_UAPSD_ENABLED;
        }

        if (req->prev_bssid)
                memcpy(assoc_data->prev_bssid, req->prev_bssid, ETH_ALEN);

        if (req->use_mfp) {
                ifmgd->mfp = IEEE80211_MFP_REQUIRED;
                ifmgd->flags |= IEEE80211_STA_MFP_ENABLED;
        } else {
                ifmgd->mfp = IEEE80211_MFP_DISABLED;
                ifmgd->flags &= ~IEEE80211_STA_MFP_ENABLED;
        }

        if (req->flags & ASSOC_REQ_USE_RRM)
                ifmgd->flags |= IEEE80211_STA_ENABLE_RRM;
        else
                ifmgd->flags &= ~IEEE80211_STA_ENABLE_RRM;

        if (req->crypto.control_port)
                ifmgd->flags |= IEEE80211_STA_CONTROL_PORT;
        else
                ifmgd->flags &= ~IEEE80211_STA_CONTROL_PORT;

        sdata->control_port_protocol = req->crypto.control_port_ethertype;
        sdata->control_port_no_encrypt = req->crypto.control_port_no_encrypt;
        sdata->control_port_over_nl80211 =
                                        req->crypto.control_port_over_nl80211;
        sdata->control_port_no_preauth = req->crypto.control_port_no_preauth;
        sdata->encrypt_headroom = ieee80211_cs_headroom(local, &req->crypto,
                                                        sdata->vif.type);

        /* kick off associate process */

        ifmgd->assoc_data = assoc_data;
        ifmgd->dtim_period = 0;
        ifmgd->have_beacon = false;

        /* override HT/VHT configuration only if the AP and we support it */
        if (!(ifmgd->flags & IEEE80211_STA_DISABLE_HT)) {
                struct ieee80211_sta_ht_cap sta_ht_cap;

                if (req->flags & ASSOC_REQ_DISABLE_HT)
                        override = true;

                memcpy(&sta_ht_cap, &sband->ht_cap, sizeof(sta_ht_cap));
                ieee80211_apply_htcap_overrides(sdata, &sta_ht_cap);

                /* check for 40 MHz disable override */
                if (!(ifmgd->flags & IEEE80211_STA_DISABLE_40MHZ) &&
                    sband->ht_cap.cap & IEEE80211_HT_CAP_SUP_WIDTH_20_40 &&
                    !(sta_ht_cap.cap & IEEE80211_HT_CAP_SUP_WIDTH_20_40))
                        override = true;

                if (!(ifmgd->flags & IEEE80211_STA_DISABLE_VHT) &&
                    req->flags & ASSOC_REQ_DISABLE_VHT)
                        override = true;
        }

        if (req->flags & ASSOC_REQ_DISABLE_HT) {
                ifmgd->flags |= IEEE80211_STA_DISABLE_HT;
                ifmgd->flags |= IEEE80211_STA_DISABLE_VHT;
                ifmgd->flags |= IEEE80211_STA_DISABLE_HE;
        }

        if (req->flags & ASSOC_REQ_DISABLE_VHT)
                ifmgd->flags |= IEEE80211_STA_DISABLE_VHT;

        if (req->flags & ASSOC_REQ_DISABLE_HE)
                ifmgd->flags |= IEEE80211_STA_DISABLE_HE;

        err = ieee80211_prep_connection(sdata, req->bss, true, override);
        if (err)
                goto err_clear;

        if (ifmgd->req_smps == IEEE80211_SMPS_AUTOMATIC) {
                if (ifmgd->powersave)
                        sdata->smps_mode = IEEE80211_SMPS_DYNAMIC;
                else
                        sdata->smps_mode = IEEE80211_SMPS_OFF;
        } else {
                sdata->smps_mode = ifmgd->req_smps;
        }

        rcu_read_lock();
        beacon_ies = rcu_dereference(req->bss->beacon_ies);

        if (ieee80211_hw_check(&sdata->local->hw, NEED_DTIM_BEFORE_ASSOC) &&
            !beacon_ies) {
                /*
                 * Wait up to one beacon interval ...
                 * should this be more if we miss one?
                 */
                sdata_info(sdata, "waiting for beacon from %pM\n",
                           ifmgd->bssid);
                assoc_data->timeout = TU_TO_EXP_TIME(req->bss->beacon_interval);
                assoc_data->timeout_started = true;
                assoc_data->need_beacon = true;
        } else if (beacon_ies) {
                const struct element *elem;
                u8 dtim_count = 0;

                ieee80211_get_dtim(beacon_ies, &dtim_count,
                                   &ifmgd->dtim_period);

                ifmgd->have_beacon = true;
                assoc_data->timeout = jiffies;
                assoc_data->timeout_started = true;

                if (ieee80211_hw_check(&local->hw, TIMING_BEACON_ONLY)) {
                        sdata->vif.bss_conf.sync_tsf = beacon_ies->tsf;
                        sdata->vif.bss_conf.sync_device_ts =
                                bss->device_ts_beacon;
                        sdata->vif.bss_conf.sync_dtim_count = dtim_count;
                }

                elem = cfg80211_find_ext_elem(WLAN_EID_EXT_MULTIPLE_BSSID_CONFIGURATION,
                                              beacon_ies->data, beacon_ies->len);
                if (elem && elem->datalen >= 3)
                        sdata->vif.bss_conf.profile_periodicity = elem->data[2];
                else
                        sdata->vif.bss_conf.profile_periodicity = 0;

                elem = cfg80211_find_elem(WLAN_EID_EXT_CAPABILITY,
                                          beacon_ies->data, beacon_ies->len);
                if (elem && elem->datalen >= 11 &&
                    (elem->data[10] & WLAN_EXT_CAPA11_EMA_SUPPORT))
                        sdata->vif.bss_conf.ema_ap = true;
                else
                        sdata->vif.bss_conf.ema_ap = false;
        } else {
                assoc_data->timeout = jiffies;
                assoc_data->timeout_started = true;
        }
        rcu_read_unlock();

        run_again(sdata, assoc_data->timeout);

        if (bss->corrupt_data) {
                char *corrupt_type = "data";
                if (bss->corrupt_data & IEEE80211_BSS_CORRUPT_BEACON) {
                        if (bss->corrupt_data &
                                        IEEE80211_BSS_CORRUPT_PROBE_RESP)
                                corrupt_type = "beacon and probe response";
                        else
                                corrupt_type = "beacon";
                } else if (bss->corrupt_data & IEEE80211_BSS_CORRUPT_PROBE_RESP)
                        corrupt_type = "probe response";
                sdata_info(sdata, "associating with AP with corrupt %s\n",
                           corrupt_type);
        }

        return 0;
 err_clear:
        eth_zero_addr(ifmgd->bssid);
        ieee80211_bss_info_change_notify(sdata, BSS_CHANGED_BSSID);
        ifmgd->assoc_data = NULL;
 err_free:
        kfree(assoc_data);
        return err;
}

int ieee80211_mgd_deauth(struct ieee80211_sub_if_data *sdata,
                         struct cfg80211_deauth_request *req)
{
        struct ieee80211_if_managed *ifmgd = &sdata->u.mgd;
        u8 frame_buf[IEEE80211_DEAUTH_FRAME_LEN];
        bool tx = !req->local_state_change;
        struct ieee80211_prep_tx_info info = {
                .subtype = IEEE80211_STYPE_DEAUTH,
        };

        if (ifmgd->auth_data &&
            ether_addr_equal(ifmgd->auth_data->bss->bssid, req->bssid)) {
                sdata_info(sdata,
                           "aborting authentication with %pM by local choice (Reason: %u=%s)\n",
                           req->bssid, req->reason_code,
                           ieee80211_get_reason_code_string(req->reason_code));

                drv_mgd_prepare_tx(sdata->local, sdata, &info);
                ieee80211_send_deauth_disassoc(sdata, req->bssid, req->bssid,
                                               IEEE80211_STYPE_DEAUTH,
                                               req->reason_code, tx,
                                               frame_buf);
                ieee80211_destroy_auth_data(sdata, false);
                ieee80211_report_disconnect(sdata, frame_buf,
                                            sizeof(frame_buf), true,
                                            req->reason_code, false);
                drv_mgd_complete_tx(sdata->local, sdata, &info);
                return 0;
        }

        if (ifmgd->assoc_data &&
            ether_addr_equal(ifmgd->assoc_data->bss->bssid, req->bssid)) {
                sdata_info(sdata,
                           "aborting association with %pM by local choice (Reason: %u=%s)\n",
                           req->bssid, req->reason_code,
                           ieee80211_get_reason_code_string(req->reason_code));

                drv_mgd_prepare_tx(sdata->local, sdata, &info);
                ieee80211_send_deauth_disassoc(sdata, req->bssid, req->bssid,
                                               IEEE80211_STYPE_DEAUTH,
                                               req->reason_code, tx,
                                               frame_buf);
                ieee80211_destroy_assoc_data(sdata, false, true);
                ieee80211_report_disconnect(sdata, frame_buf,
                                            sizeof(frame_buf), true,
                                            req->reason_code, false);
                drv_mgd_complete_tx(sdata->local, sdata, &info);
                return 0;
        }

        if (ifmgd->associated &&
            ether_addr_equal(ifmgd->associated->bssid, req->bssid)) {
                sdata_info(sdata,
                           "deauthenticating from %pM by local choice (Reason: %u=%s)\n",
                           req->bssid, req->reason_code,
                           ieee80211_get_reason_code_string(req->reason_code));

                ieee80211_set_disassoc(sdata, IEEE80211_STYPE_DEAUTH,
                                       req->reason_code, tx, frame_buf);
                ieee80211_report_disconnect(sdata, frame_buf,
                                            sizeof(frame_buf), true,
                                            req->reason_code, false);
                drv_mgd_complete_tx(sdata->local, sdata, &info);
                return 0;
        }

        return -ENOTCONN;
}

int ieee80211_mgd_disassoc(struct ieee80211_sub_if_data *sdata,
                           struct cfg80211_disassoc_request *req)
{
        struct ieee80211_if_managed *ifmgd = &sdata->u.mgd;
        u8 bssid[ETH_ALEN];
        u8 frame_buf[IEEE80211_DEAUTH_FRAME_LEN];

        /*
         * cfg80211 should catch this ... but it's racy since
         * we can receive a disassoc frame, process it, hand it
         * to cfg80211 while that's in a locked section already
         * trying to tell us that the user wants to disconnect.
         */
        if (ifmgd->associated != req->bss)
                return -ENOLINK;

        sdata_info(sdata,
                   "disassociating from %pM by local choice (Reason: %u=%s)\n",
                   req->bss->bssid, req->reason_code, ieee80211_get_reason_code_string(req->reason_code));

        memcpy(bssid, req->bss->bssid, ETH_ALEN);
        ieee80211_set_disassoc(sdata, IEEE80211_STYPE_DISASSOC,
                               req->reason_code, !req->local_state_change,
                               frame_buf);

        ieee80211_report_disconnect(sdata, frame_buf, sizeof(frame_buf), true,
                                    req->reason_code, false);

        return 0;
}

void ieee80211_mgd_stop(struct ieee80211_sub_if_data *sdata)
{
        struct ieee80211_if_managed *ifmgd = &sdata->u.mgd;

        /*
         * Make sure some work items will not run after this,
         * they will not do anything but might not have been
         * cancelled when disconnecting.
         */
        cancel_work_sync(&ifmgd->monitor_work);
        cancel_work_sync(&ifmgd->beacon_connection_loss_work);
        cancel_work_sync(&ifmgd->request_smps_work);
        cancel_work_sync(&ifmgd->csa_connection_drop_work);
        cancel_work_sync(&ifmgd->chswitch_work);
        cancel_delayed_work_sync(&ifmgd->tdls_peer_del_work);

        sdata_lock(sdata);
        if (ifmgd->assoc_data) {
                struct cfg80211_bss *bss = ifmgd->assoc_data->bss;
                ieee80211_destroy_assoc_data(sdata, false, false);
                cfg80211_assoc_timeout(sdata->dev, bss);
        }
        if (ifmgd->auth_data)
                ieee80211_destroy_auth_data(sdata, false);
        spin_lock_bh(&ifmgd->teardown_lock);
        if (ifmgd->teardown_skb) {
                kfree_skb(ifmgd->teardown_skb);
                ifmgd->teardown_skb = NULL;
                ifmgd->orig_teardown_skb = NULL;
        }
        kfree(ifmgd->assoc_req_ies);
        ifmgd->assoc_req_ies = NULL;
        ifmgd->assoc_req_ies_len = 0;
        spin_unlock_bh(&ifmgd->teardown_lock);
        del_timer_sync(&ifmgd->timer);
        sdata_unlock(sdata);
}

void ieee80211_cqm_rssi_notify(struct ieee80211_vif *vif,
                               enum nl80211_cqm_rssi_threshold_event rssi_event,
                               s32 rssi_level,
                               gfp_t gfp)
{
        struct ieee80211_sub_if_data *sdata = vif_to_sdata(vif);

        trace_api_cqm_rssi_notify(sdata, rssi_event, rssi_level);

        cfg80211_cqm_rssi_notify(sdata->dev, rssi_event, rssi_level, gfp);
}
EXPORT_SYMBOL(ieee80211_cqm_rssi_notify);

void ieee80211_cqm_beacon_loss_notify(struct ieee80211_vif *vif, gfp_t gfp)
{
        struct ieee80211_sub_if_data *sdata = vif_to_sdata(vif);

        trace_api_cqm_beacon_loss_notify(sdata->local, sdata);

        cfg80211_cqm_beacon_loss_notify(sdata->dev, gfp);
}
EXPORT_SYMBOL(ieee80211_cqm_beacon_loss_notify);










 4538 
































  330 





















    4 
























   37 




































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
/* SPDX-License-Identifier: GPL-2.0 */
#undef TRACE_SYSTEM
#define TRACE_SYSTEM percpu

#if !defined(_TRACE_PERCPU_H) || defined(TRACE_HEADER_MULTI_READ)
#define _TRACE_PERCPU_H

#include <linux/tracepoint.h>

TRACE_EVENT(percpu_alloc_percpu,

        TP_PROTO(bool reserved, bool is_atomic, size_t size,
                 size_t align, void *base_addr, int off, void __percpu *ptr),

        TP_ARGS(reserved, is_atomic, size, align, base_addr, off, ptr),

        TP_STRUCT__entry(
                __field(        bool,                        reserved        )
                __field(        bool,                        is_atomic        )
                __field(        size_t,                        size                )
                __field(        size_t,                        align                )
                __field(        void *,                        base_addr        )
                __field(        int,                        off                )
                __field(        void __percpu *,        ptr                )
        ),

        TP_fast_assign(
                __entry->reserved        = reserved;
                __entry->is_atomic        = is_atomic;
                __entry->size                = size;
                __entry->align                = align;
                __entry->base_addr        = base_addr;
                __entry->off                = off;
                __entry->ptr                = ptr;
        ),

        TP_printk("reserved=%d is_atomic=%d size=%zu align=%zu base_addr=%p off=%d ptr=%p",
                  __entry->reserved, __entry->is_atomic,
                  __entry->size, __entry->align,
                  __entry->base_addr, __entry->off, __entry->ptr)
);

TRACE_EVENT(percpu_free_percpu,

        TP_PROTO(void *base_addr, int off, void __percpu *ptr),

        TP_ARGS(base_addr, off, ptr),

        TP_STRUCT__entry(
                __field(        void *,                        base_addr        )
                __field(        int,                        off                )
                __field(        void __percpu *,        ptr                )
        ),

        TP_fast_assign(
                __entry->base_addr        = base_addr;
                __entry->off                = off;
                __entry->ptr                = ptr;
        ),

        TP_printk("base_addr=%p off=%d ptr=%p",
                __entry->base_addr, __entry->off, __entry->ptr)
);

TRACE_EVENT(percpu_alloc_percpu_fail,

        TP_PROTO(bool reserved, bool is_atomic, size_t size, size_t align),

        TP_ARGS(reserved, is_atomic, size, align),

        TP_STRUCT__entry(
                __field(        bool,        reserved        )
                __field(        bool,        is_atomic        )
                __field(        size_t,        size                )
                __field(        size_t, align                )
        ),

        TP_fast_assign(
                __entry->reserved        = reserved;
                __entry->is_atomic        = is_atomic;
                __entry->size                = size;
                __entry->align                = align;
        ),

        TP_printk("reserved=%d is_atomic=%d size=%zu align=%zu",
                  __entry->reserved, __entry->is_atomic,
                  __entry->size, __entry->align)
);

TRACE_EVENT(percpu_create_chunk,

        TP_PROTO(void *base_addr),

        TP_ARGS(base_addr),

        TP_STRUCT__entry(
                __field(        void *, base_addr        )
        ),

        TP_fast_assign(
                __entry->base_addr        = base_addr;
        ),

        TP_printk("base_addr=%p", __entry->base_addr)
);

TRACE_EVENT(percpu_destroy_chunk,

        TP_PROTO(void *base_addr),

        TP_ARGS(base_addr),

        TP_STRUCT__entry(
                __field(        void *,        base_addr        )
        ),

        TP_fast_assign(
                __entry->base_addr        = base_addr;
        ),

        TP_printk("base_addr=%p", __entry->base_addr)
);

#endif /* _TRACE_PERCPU_H */

#include <trace/define_trace.h>





































   18 





   18 














  106 











   16 
















   16 









   16 







   16 











   16 


































































































































    8 









    8 




























































   16 




   16 








































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
// SPDX-License-Identifier: GPL-2.0-or-later
/* SCTP kernel implementation
 * (C) Copyright IBM Corp. 2001, 2004
 * Copyright (c) 1999-2000 Cisco, Inc.
 * Copyright (c) 1999-2001 Motorola, Inc.
 * Copyright (c) 2001 Intel Corp.
 *
 * This file is part of the SCTP kernel implementation
 *
 * These functions manipulate sctp tsn mapping array.
 *
 * Please send any bug reports or fixes you make to the
 * email address(es):
 *    lksctp developers <linux-sctp@vger.kernel.org>
 *
 * Written or modified by:
 *    La Monte H.P. Yarroll <piggy@acm.org>
 *    Jon Grimm             <jgrimm@us.ibm.com>
 *    Karl Knutson          <karl@athena.chicago.il.us>
 *    Sridhar Samudrala     <sri@us.ibm.com>
 */

#include <linux/slab.h>
#include <linux/types.h>
#include <linux/bitmap.h>
#include <net/sctp/sctp.h>
#include <net/sctp/sm.h>

static void sctp_tsnmap_update(struct sctp_tsnmap *map);
static void sctp_tsnmap_find_gap_ack(unsigned long *map, __u16 off,
                                     __u16 len, __u16 *start, __u16 *end);
static int sctp_tsnmap_grow(struct sctp_tsnmap *map, u16 size);

/* Initialize a block of memory as a tsnmap.  */
struct sctp_tsnmap *sctp_tsnmap_init(struct sctp_tsnmap *map, __u16 len,
                                     __u32 initial_tsn, gfp_t gfp)
{
        if (!map->tsn_map) {
                map->tsn_map = kzalloc(len>>3, gfp);
                if (map->tsn_map == NULL)
                        return NULL;

                map->len = len;
        } else {
                bitmap_zero(map->tsn_map, map->len);
        }

        /* Keep track of TSNs represented by tsn_map.  */
        map->base_tsn = initial_tsn;
        map->cumulative_tsn_ack_point = initial_tsn - 1;
        map->max_tsn_seen = map->cumulative_tsn_ack_point;
        map->num_dup_tsns = 0;

        return map;
}

void sctp_tsnmap_free(struct sctp_tsnmap *map)
{
        map->len = 0;
        kfree(map->tsn_map);
}

/* Test the tracking state of this TSN.
 * Returns:
 *   0 if the TSN has not yet been seen
 *  >0 if the TSN has been seen (duplicate)
 *  <0 if the TSN is invalid (too large to track)
 */
int sctp_tsnmap_check(const struct sctp_tsnmap *map, __u32 tsn)
{
        u32 gap;

        /* Check to see if this is an old TSN */
        if (TSN_lte(tsn, map->cumulative_tsn_ack_point))
                return 1;

        /* Verify that we can hold this TSN and that it will not
         * overflow our map
         */
        if (!TSN_lt(tsn, map->base_tsn + SCTP_TSN_MAP_SIZE))
                return -1;

        /* Calculate the index into the mapping arrays.  */
        gap = tsn - map->base_tsn;

        /* Check to see if TSN has already been recorded.  */
        if (gap < map->len && test_bit(gap, map->tsn_map))
                return 1;
        else
                return 0;
}


/* Mark this TSN as seen.  */
int sctp_tsnmap_mark(struct sctp_tsnmap *map, __u32 tsn,
                     struct sctp_transport *trans)
{
        u16 gap;

        if (TSN_lt(tsn, map->base_tsn))
                return 0;

        gap = tsn - map->base_tsn;

        if (gap >= map->len && !sctp_tsnmap_grow(map, gap + 1))
                return -ENOMEM;

        if (!sctp_tsnmap_has_gap(map) && gap == 0) {
                /* In this case the map has no gaps and the tsn we are
                 * recording is the next expected tsn.  We don't touch
                 * the map but simply bump the values.
                 */
                map->max_tsn_seen++;
                map->cumulative_tsn_ack_point++;
                if (trans)
                        trans->sack_generation =
                                trans->asoc->peer.sack_generation;
                map->base_tsn++;
        } else {
                /* Either we already have a gap, or about to record a gap, so
                 * have work to do.
                 *
                 * Bump the max.
                 */
                if (TSN_lt(map->max_tsn_seen, tsn))
                        map->max_tsn_seen = tsn;

                /* Mark the TSN as received.  */
                set_bit(gap, map->tsn_map);

                /* Go fixup any internal TSN mapping variables including
                 * cumulative_tsn_ack_point.
                 */
                sctp_tsnmap_update(map);
        }

        return 0;
}


/* Initialize a Gap Ack Block iterator from memory being provided.  */
static void sctp_tsnmap_iter_init(const struct sctp_tsnmap *map,
                                  struct sctp_tsnmap_iter *iter)
{
        /* Only start looking one past the Cumulative TSN Ack Point.  */
        iter->start = map->cumulative_tsn_ack_point + 1;
}

/* Get the next Gap Ack Blocks. Returns 0 if there was not another block
 * to get.
 */
static int sctp_tsnmap_next_gap_ack(const struct sctp_tsnmap *map,
                                    struct sctp_tsnmap_iter *iter,
                                    __u16 *start, __u16 *end)
{
        int ended = 0;
        __u16 start_ = 0, end_ = 0, offset;

        /* If there are no more gap acks possible, get out fast.  */
        if (TSN_lte(map->max_tsn_seen, iter->start))
                return 0;

        offset = iter->start - map->base_tsn;
        sctp_tsnmap_find_gap_ack(map->tsn_map, offset, map->len,
                                 &start_, &end_);

        /* The Gap Ack Block happens to end at the end of the map. */
        if (start_ && !end_)
                end_ = map->len - 1;

        /* If we found a Gap Ack Block, return the start and end and
         * bump the iterator forward.
         */
        if (end_) {
                /* Fix up the start and end based on the
                 * Cumulative TSN Ack which is always 1 behind base.
                 */
                *start = start_ + 1;
                *end = end_ + 1;

                /* Move the iterator forward.  */
                iter->start = map->cumulative_tsn_ack_point + *end + 1;
                ended = 1;
        }

        return ended;
}

/* Mark this and any lower TSN as seen.  */
void sctp_tsnmap_skip(struct sctp_tsnmap *map, __u32 tsn)
{
        u32 gap;

        if (TSN_lt(tsn, map->base_tsn))
                return;
        if (!TSN_lt(tsn, map->base_tsn + SCTP_TSN_MAP_SIZE))
                return;

        /* Bump the max.  */
        if (TSN_lt(map->max_tsn_seen, tsn))
                map->max_tsn_seen = tsn;

        gap = tsn - map->base_tsn + 1;

        map->base_tsn += gap;
        map->cumulative_tsn_ack_point += gap;
        if (gap >= map->len) {
                /* If our gap is larger then the map size, just
                 * zero out the map.
                 */
                bitmap_zero(map->tsn_map, map->len);
        } else {
                /* If the gap is smaller than the map size,
                 * shift the map by 'gap' bits and update further.
                 */
                bitmap_shift_right(map->tsn_map, map->tsn_map, gap, map->len);
                sctp_tsnmap_update(map);
        }
}

/********************************************************************
 * 2nd Level Abstractions
 ********************************************************************/

/* This private helper function updates the tsnmap buffers and
 * the Cumulative TSN Ack Point.
 */
static void sctp_tsnmap_update(struct sctp_tsnmap *map)
{
        u16 len;
        unsigned long zero_bit;


        len = map->max_tsn_seen - map->cumulative_tsn_ack_point;
        zero_bit = find_first_zero_bit(map->tsn_map, len);
        if (!zero_bit)
                return;                /* The first 0-bit is bit 0.  nothing to do */

        map->base_tsn += zero_bit;
        map->cumulative_tsn_ack_point += zero_bit;

        bitmap_shift_right(map->tsn_map, map->tsn_map, zero_bit, map->len);
}

/* How many data chunks  are we missing from our peer?
 */
__u16 sctp_tsnmap_pending(struct sctp_tsnmap *map)
{
        __u32 cum_tsn = map->cumulative_tsn_ack_point;
        __u32 max_tsn = map->max_tsn_seen;
        __u32 base_tsn = map->base_tsn;
        __u16 pending_data;
        u32 gap;

        pending_data = max_tsn - cum_tsn;
        gap = max_tsn - base_tsn;

        if (gap == 0 || gap >= map->len)
                goto out;

        pending_data -= bitmap_weight(map->tsn_map, gap + 1);
out:
        return pending_data;
}

/* This is a private helper for finding Gap Ack Blocks.  It searches a
 * single array for the start and end of a Gap Ack Block.
 *
 * The flags "started" and "ended" tell is if we found the beginning
 * or (respectively) the end of a Gap Ack Block.
 */
static void sctp_tsnmap_find_gap_ack(unsigned long *map, __u16 off,
                                     __u16 len, __u16 *start, __u16 *end)
{
        int i = off;

        /* Look through the entire array, but break out
         * early if we have found the end of the Gap Ack Block.
         */

        /* Also, stop looking past the maximum TSN seen. */

        /* Look for the start. */
        i = find_next_bit(map, len, off);
        if (i < len)
                *start = i;

        /* Look for the end.  */
        if (*start) {
                /* We have found the start, let's find the
                 * end.  If we find the end, break out.
                 */
                i = find_next_zero_bit(map, len, i);
                if (i < len)
                        *end = i - 1;
        }
}

/* Renege that we have seen a TSN.  */
void sctp_tsnmap_renege(struct sctp_tsnmap *map, __u32 tsn)
{
        u32 gap;

        if (TSN_lt(tsn, map->base_tsn))
                return;
        /* Assert: TSN is in range.  */
        if (!TSN_lt(tsn, map->base_tsn + map->len))
                return;

        gap = tsn - map->base_tsn;

        /* Pretend we never saw the TSN.  */
        clear_bit(gap, map->tsn_map);
}

/* How many gap ack blocks do we have recorded? */
__u16 sctp_tsnmap_num_gabs(struct sctp_tsnmap *map,
                           struct sctp_gap_ack_block *gabs)
{
        struct sctp_tsnmap_iter iter;
        int ngaps = 0;

        /* Refresh the gap ack information. */
        if (sctp_tsnmap_has_gap(map)) {
                __u16 start = 0, end = 0;
                sctp_tsnmap_iter_init(map, &iter);
                while (sctp_tsnmap_next_gap_ack(map, &iter,
                                                &start,
                                                &end)) {

                        gabs[ngaps].start = htons(start);
                        gabs[ngaps].end = htons(end);
                        ngaps++;
                        if (ngaps >= SCTP_MAX_GABS)
                                break;
                }
        }
        return ngaps;
}

static int sctp_tsnmap_grow(struct sctp_tsnmap *map, u16 size)
{
        unsigned long *new;
        unsigned long inc;
        u16  len;

        if (size > SCTP_TSN_MAP_SIZE)
                return 0;

        inc = ALIGN((size - map->len), BITS_PER_LONG) + SCTP_TSN_MAP_INCREMENT;
        len = min_t(u16, map->len + inc, SCTP_TSN_MAP_SIZE);

        new = kzalloc(len>>3, GFP_ATOMIC);
        if (!new)
                return 0;

        bitmap_copy(new, map->tsn_map,
                map->max_tsn_seen - map->cumulative_tsn_ack_point);
        kfree(map->tsn_map);
        map->tsn_map = new;
        map->len = len;

        return 1;
}


















































































































































































































































































































































































































































































































































































































































































































































































    9 








    9 













    9 




    9 

    9 
    7 


    9 

    9 
    5 





    9 
    1 





    9 



    9 






    9 



    9 






1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * xfrm algorithm interface
 *
 * Copyright (c) 2002 James Morris <jmorris@intercode.com.au>
 */

#include <crypto/hash.h>
#include <crypto/skcipher.h>
#include <linux/module.h>
#include <linux/kernel.h>
#include <linux/pfkeyv2.h>
#include <linux/crypto.h>
#include <linux/scatterlist.h>
#include <net/xfrm.h>
#if IS_ENABLED(CONFIG_INET_ESP) || IS_ENABLED(CONFIG_INET6_ESP)
#include <net/esp.h>
#endif

/*
 * Algorithms supported by IPsec.  These entries contain properties which
 * are used in key negotiation and xfrm processing, and are used to verify
 * that instantiated crypto transforms have correct parameters for IPsec
 * purposes.
 */
static struct xfrm_algo_desc aead_list[] = {
{
        .name = "rfc4106(gcm(aes))",

        .uinfo = {
                .aead = {
                        .geniv = "seqiv",
                        .icv_truncbits = 64,
                }
        },

        .pfkey_supported = 1,

        .desc = {
                .sadb_alg_id = SADB_X_EALG_AES_GCM_ICV8,
                .sadb_alg_ivlen = 8,
                .sadb_alg_minbits = 128,
                .sadb_alg_maxbits = 256
        }
},
{
        .name = "rfc4106(gcm(aes))",

        .uinfo = {
                .aead = {
                        .geniv = "seqiv",
                        .icv_truncbits = 96,
                }
        },

        .pfkey_supported = 1,

        .desc = {
                .sadb_alg_id = SADB_X_EALG_AES_GCM_ICV12,
                .sadb_alg_ivlen = 8,
                .sadb_alg_minbits = 128,
                .sadb_alg_maxbits = 256
        }
},
{
        .name = "rfc4106(gcm(aes))",

        .uinfo = {
                .aead = {
                        .geniv = "seqiv",
                        .icv_truncbits = 128,
                }
        },

        .pfkey_supported = 1,

        .desc = {
                .sadb_alg_id = SADB_X_EALG_AES_GCM_ICV16,
                .sadb_alg_ivlen = 8,
                .sadb_alg_minbits = 128,
                .sadb_alg_maxbits = 256
        }
},
{
        .name = "rfc4309(ccm(aes))",

        .uinfo = {
                .aead = {
                        .geniv = "seqiv",
                        .icv_truncbits = 64,
                }
        },

        .pfkey_supported = 1,

        .desc = {
                .sadb_alg_id = SADB_X_EALG_AES_CCM_ICV8,
                .sadb_alg_ivlen = 8,
                .sadb_alg_minbits = 128,
                .sadb_alg_maxbits = 256
        }
},
{
        .name = "rfc4309(ccm(aes))",

        .uinfo = {
                .aead = {
                        .geniv = "seqiv",
                        .icv_truncbits = 96,
                }
        },

        .pfkey_supported = 1,

        .desc = {
                .sadb_alg_id = SADB_X_EALG_AES_CCM_ICV12,
                .sadb_alg_ivlen = 8,
                .sadb_alg_minbits = 128,
                .sadb_alg_maxbits = 256
        }
},
{
        .name = "rfc4309(ccm(aes))",

        .uinfo = {
                .aead = {
                        .geniv = "seqiv",
                        .icv_truncbits = 128,
                }
        },

        .pfkey_supported = 1,

        .desc = {
                .sadb_alg_id = SADB_X_EALG_AES_CCM_ICV16,
                .sadb_alg_ivlen = 8,
                .sadb_alg_minbits = 128,
                .sadb_alg_maxbits = 256
        }
},
{
        .name = "rfc4543(gcm(aes))",

        .uinfo = {
                .aead = {
                        .geniv = "seqiv",
                        .icv_truncbits = 128,
                }
        },

        .pfkey_supported = 1,

        .desc = {
                .sadb_alg_id = SADB_X_EALG_NULL_AES_GMAC,
                .sadb_alg_ivlen = 8,
                .sadb_alg_minbits = 128,
                .sadb_alg_maxbits = 256
        }
},
{
        .name = "rfc7539esp(chacha20,poly1305)",

        .uinfo = {
                .aead = {
                        .geniv = "seqiv",
                        .icv_truncbits = 128,
                }
        },

        .pfkey_supported = 0,
},
};

static struct xfrm_algo_desc aalg_list[] = {
{
        .name = "digest_null",

        .uinfo = {
                .auth = {
                        .icv_truncbits = 0,
                        .icv_fullbits = 0,
                }
        },

        .pfkey_supported = 1,

        .desc = {
                .sadb_alg_id = SADB_X_AALG_NULL,
                .sadb_alg_ivlen = 0,
                .sadb_alg_minbits = 0,
                .sadb_alg_maxbits = 0
        }
},
{
        .name = "hmac(md5)",
        .compat = "md5",

        .uinfo = {
                .auth = {
                        .icv_truncbits = 96,
                        .icv_fullbits = 128,
                }
        },

        .pfkey_supported = 1,

        .desc = {
                .sadb_alg_id = SADB_AALG_MD5HMAC,
                .sadb_alg_ivlen = 0,
                .sadb_alg_minbits = 128,
                .sadb_alg_maxbits = 128
        }
},
{
        .name = "hmac(sha1)",
        .compat = "sha1",

        .uinfo = {
                .auth = {
                        .icv_truncbits = 96,
                        .icv_fullbits = 160,
                }
        },

        .pfkey_supported = 1,

        .desc = {
                .sadb_alg_id = SADB_AALG_SHA1HMAC,
                .sadb_alg_ivlen = 0,
                .sadb_alg_minbits = 160,
                .sadb_alg_maxbits = 160
        }
},
{
        .name = "hmac(sha256)",
        .compat = "sha256",

        .uinfo = {
                .auth = {
                        .icv_truncbits = 96,
                        .icv_fullbits = 256,
                }
        },

        .pfkey_supported = 1,

        .desc = {
                .sadb_alg_id = SADB_X_AALG_SHA2_256HMAC,
                .sadb_alg_ivlen = 0,
                .sadb_alg_minbits = 256,
                .sadb_alg_maxbits = 256
        }
},
{
        .name = "hmac(sha384)",

        .uinfo = {
                .auth = {
                        .icv_truncbits = 192,
                        .icv_fullbits = 384,
                }
        },

        .pfkey_supported = 1,

        .desc = {
                .sadb_alg_id = SADB_X_AALG_SHA2_384HMAC,
                .sadb_alg_ivlen = 0,
                .sadb_alg_minbits = 384,
                .sadb_alg_maxbits = 384
        }
},
{
        .name = "hmac(sha512)",

        .uinfo = {
                .auth = {
                        .icv_truncbits = 256,
                        .icv_fullbits = 512,
                }
        },

        .pfkey_supported = 1,

        .desc = {
                .sadb_alg_id = SADB_X_AALG_SHA2_512HMAC,
                .sadb_alg_ivlen = 0,
                .sadb_alg_minbits = 512,
                .sadb_alg_maxbits = 512
        }
},
{
        .name = "hmac(rmd160)",
        .compat = "rmd160",

        .uinfo = {
                .auth = {
                        .icv_truncbits = 96,
                        .icv_fullbits = 160,
                }
        },

        .pfkey_supported = 1,

        .desc = {
                .sadb_alg_id = SADB_X_AALG_RIPEMD160HMAC,
                .sadb_alg_ivlen = 0,
                .sadb_alg_minbits = 160,
                .sadb_alg_maxbits = 160
        }
},
{
        .name = "xcbc(aes)",

        .uinfo = {
                .auth = {
                        .icv_truncbits = 96,
                        .icv_fullbits = 128,
                }
        },

        .pfkey_supported = 1,

        .desc = {
                .sadb_alg_id = SADB_X_AALG_AES_XCBC_MAC,
                .sadb_alg_ivlen = 0,
                .sadb_alg_minbits = 128,
                .sadb_alg_maxbits = 128
        }
},
{
        /* rfc4494 */
        .name = "cmac(aes)",

        .uinfo = {
                .auth = {
                        .icv_truncbits = 96,
                        .icv_fullbits = 128,
                }
        },

        .pfkey_supported = 0,
},
};

static struct xfrm_algo_desc ealg_list[] = {
{
        .name = "ecb(cipher_null)",
        .compat = "cipher_null",

        .uinfo = {
                .encr = {
                        .blockbits = 8,
                        .defkeybits = 0,
                }
        },

        .pfkey_supported = 1,

        .desc = {
                .sadb_alg_id =        SADB_EALG_NULL,
                .sadb_alg_ivlen = 0,
                .sadb_alg_minbits = 0,
                .sadb_alg_maxbits = 0
        }
},
{
        .name = "cbc(des)",
        .compat = "des",

        .uinfo = {
                .encr = {
                        .geniv = "echainiv",
                        .blockbits = 64,
                        .defkeybits = 64,
                }
        },

        .pfkey_supported = 1,

        .desc = {
                .sadb_alg_id = SADB_EALG_DESCBC,
                .sadb_alg_ivlen = 8,
                .sadb_alg_minbits = 64,
                .sadb_alg_maxbits = 64
        }
},
{
        .name = "cbc(des3_ede)",
        .compat = "des3_ede",

        .uinfo = {
                .encr = {
                        .geniv = "echainiv",
                        .blockbits = 64,
                        .defkeybits = 192,
                }
        },

        .pfkey_supported = 1,

        .desc = {
                .sadb_alg_id = SADB_EALG_3DESCBC,
                .sadb_alg_ivlen = 8,
                .sadb_alg_minbits = 192,
                .sadb_alg_maxbits = 192
        }
},
{
        .name = "cbc(cast5)",
        .compat = "cast5",

        .uinfo = {
                .encr = {
                        .geniv = "echainiv",
                        .blockbits = 64,
                        .defkeybits = 128,
                }
        },

        .pfkey_supported = 1,

        .desc = {
                .sadb_alg_id = SADB_X_EALG_CASTCBC,
                .sadb_alg_ivlen = 8,
                .sadb_alg_minbits = 40,
                .sadb_alg_maxbits = 128
        }
},
{
        .name = "cbc(blowfish)",
        .compat = "blowfish",

        .uinfo = {
                .encr = {
                        .geniv = "echainiv",
                        .blockbits = 64,
                        .defkeybits = 128,
                }
        },

        .pfkey_supported = 1,

        .desc = {
                .sadb_alg_id = SADB_X_EALG_BLOWFISHCBC,
                .sadb_alg_ivlen = 8,
                .sadb_alg_minbits = 40,
                .sadb_alg_maxbits = 448
        }
},
{
        .name = "cbc(aes)",
        .compat = "aes",

        .uinfo = {
                .encr = {
                        .geniv = "echainiv",
                        .blockbits = 128,
                        .defkeybits = 128,
                }
        },

        .pfkey_supported = 1,

        .desc = {
                .sadb_alg_id = SADB_X_EALG_AESCBC,
                .sadb_alg_ivlen = 8,
                .sadb_alg_minbits = 128,
                .sadb_alg_maxbits = 256
        }
},
{
        .name = "cbc(serpent)",
        .compat = "serpent",

        .uinfo = {
                .encr = {
                        .geniv = "echainiv",
                        .blockbits = 128,
                        .defkeybits = 128,
                }
        },

        .pfkey_supported = 1,

        .desc = {
                .sadb_alg_id = SADB_X_EALG_SERPENTCBC,
                .sadb_alg_ivlen = 8,
                .sadb_alg_minbits = 128,
                .sadb_alg_maxbits = 256,
        }
},
{
        .name = "cbc(camellia)",
        .compat = "camellia",

        .uinfo = {
                .encr = {
                        .geniv = "echainiv",
                        .blockbits = 128,
                        .defkeybits = 128,
                }
        },

        .pfkey_supported = 1,

        .desc = {
                .sadb_alg_id = SADB_X_EALG_CAMELLIACBC,
                .sadb_alg_ivlen = 8,
                .sadb_alg_minbits = 128,
                .sadb_alg_maxbits = 256
        }
},
{
        .name = "cbc(twofish)",
        .compat = "twofish",

        .uinfo = {
                .encr = {
                        .geniv = "echainiv",
                        .blockbits = 128,
                        .defkeybits = 128,
                }
        },

        .pfkey_supported = 1,

        .desc = {
                .sadb_alg_id = SADB_X_EALG_TWOFISHCBC,
                .sadb_alg_ivlen = 8,
                .sadb_alg_minbits = 128,
                .sadb_alg_maxbits = 256
        }
},
{
        .name = "rfc3686(ctr(aes))",

        .uinfo = {
                .encr = {
                        .geniv = "seqiv",
                        .blockbits = 128,
                        .defkeybits = 160, /* 128-bit key + 32-bit nonce */
                }
        },

        .pfkey_supported = 1,

        .desc = {
                .sadb_alg_id = SADB_X_EALG_AESCTR,
                .sadb_alg_ivlen        = 8,
                .sadb_alg_minbits = 160,
                .sadb_alg_maxbits = 288
        }
},
};

static struct xfrm_algo_desc calg_list[] = {
{
        .name = "deflate",
        .uinfo = {
                .comp = {
                        .threshold = 90,
                }
        },
        .pfkey_supported = 1,
        .desc = { .sadb_alg_id = SADB_X_CALG_DEFLATE }
},
{
        .name = "lzs",
        .uinfo = {
                .comp = {
                        .threshold = 90,
                }
        },
        .pfkey_supported = 1,
        .desc = { .sadb_alg_id = SADB_X_CALG_LZS }
},
{
        .name = "lzjh",
        .uinfo = {
                .comp = {
                        .threshold = 50,
                }
        },
        .pfkey_supported = 1,
        .desc = { .sadb_alg_id = SADB_X_CALG_LZJH }
},
};

static inline int aalg_entries(void)
{
        return ARRAY_SIZE(aalg_list);
}

static inline int ealg_entries(void)
{
        return ARRAY_SIZE(ealg_list);
}

static inline int calg_entries(void)
{
        return ARRAY_SIZE(calg_list);
}

struct xfrm_algo_list {
        struct xfrm_algo_desc *algs;
        int entries;
        u32 type;
        u32 mask;
};

static const struct xfrm_algo_list xfrm_aead_list = {
        .algs = aead_list,
        .entries = ARRAY_SIZE(aead_list),
        .type = CRYPTO_ALG_TYPE_AEAD,
        .mask = CRYPTO_ALG_TYPE_MASK,
};

static const struct xfrm_algo_list xfrm_aalg_list = {
        .algs = aalg_list,
        .entries = ARRAY_SIZE(aalg_list),
        .type = CRYPTO_ALG_TYPE_HASH,
        .mask = CRYPTO_ALG_TYPE_HASH_MASK,
};

static const struct xfrm_algo_list xfrm_ealg_list = {
        .algs = ealg_list,
        .entries = ARRAY_SIZE(ealg_list),
        .type = CRYPTO_ALG_TYPE_SKCIPHER,
        .mask = CRYPTO_ALG_TYPE_MASK,
};

static const struct xfrm_algo_list xfrm_calg_list = {
        .algs = calg_list,
        .entries = ARRAY_SIZE(calg_list),
        .type = CRYPTO_ALG_TYPE_COMPRESS,
        .mask = CRYPTO_ALG_TYPE_MASK,
};

static struct xfrm_algo_desc *xfrm_find_algo(
        const struct xfrm_algo_list *algo_list,
        int match(const struct xfrm_algo_desc *entry, const void *data),
        const void *data, int probe)
{
        struct xfrm_algo_desc *list = algo_list->algs;
        int i, status;

        for (i = 0; i < algo_list->entries; i++) {
                if (!match(list + i, data))
                        continue;

                if (list[i].available)
                        return &list[i];

                if (!probe)
                        break;

                status = crypto_has_alg(list[i].name, algo_list->type,
                                        algo_list->mask);
                if (!status)
                        break;

                list[i].available = status;
                return &list[i];
        }
        return NULL;
}

static int xfrm_alg_id_match(const struct xfrm_algo_desc *entry,
                             const void *data)
{
        return entry->desc.sadb_alg_id == (unsigned long)data;
}

struct xfrm_algo_desc *xfrm_aalg_get_byid(int alg_id)
{
        return xfrm_find_algo(&xfrm_aalg_list, xfrm_alg_id_match,
                              (void *)(unsigned long)alg_id, 1);
}
EXPORT_SYMBOL_GPL(xfrm_aalg_get_byid);

struct xfrm_algo_desc *xfrm_ealg_get_byid(int alg_id)
{
        return xfrm_find_algo(&xfrm_ealg_list, xfrm_alg_id_match,
                              (void *)(unsigned long)alg_id, 1);
}
EXPORT_SYMBOL_GPL(xfrm_ealg_get_byid);

struct xfrm_algo_desc *xfrm_calg_get_byid(int alg_id)
{
        return xfrm_find_algo(&xfrm_calg_list, xfrm_alg_id_match,
                              (void *)(unsigned long)alg_id, 1);
}
EXPORT_SYMBOL_GPL(xfrm_calg_get_byid);

static int xfrm_alg_name_match(const struct xfrm_algo_desc *entry,
                               const void *data)
{
        const char *name = data;

        return name && (!strcmp(name, entry->name) ||
                        (entry->compat && !strcmp(name, entry->compat)));
}

struct xfrm_algo_desc *xfrm_aalg_get_byname(const char *name, int probe)
{
        return xfrm_find_algo(&xfrm_aalg_list, xfrm_alg_name_match, name,
                              probe);
}
EXPORT_SYMBOL_GPL(xfrm_aalg_get_byname);

struct xfrm_algo_desc *xfrm_ealg_get_byname(const char *name, int probe)
{
        return xfrm_find_algo(&xfrm_ealg_list, xfrm_alg_name_match, name,
                              probe);
}
EXPORT_SYMBOL_GPL(xfrm_ealg_get_byname);

struct xfrm_algo_desc *xfrm_calg_get_byname(const char *name, int probe)
{
        return xfrm_find_algo(&xfrm_calg_list, xfrm_alg_name_match, name,
                              probe);
}
EXPORT_SYMBOL_GPL(xfrm_calg_get_byname);

struct xfrm_aead_name {
        const char *name;
        int icvbits;
};

static int xfrm_aead_name_match(const struct xfrm_algo_desc *entry,
                                const void *data)
{
        const struct xfrm_aead_name *aead = data;
        const char *name = aead->name;

        return aead->icvbits == entry->uinfo.aead.icv_truncbits && name &&
               !strcmp(name, entry->name);
}

struct xfrm_algo_desc *xfrm_aead_get_byname(const char *name, int icv_len, int probe)
{
        struct xfrm_aead_name data = {
                .name = name,
                .icvbits = icv_len,
        };

        return xfrm_find_algo(&xfrm_aead_list, xfrm_aead_name_match, &data,
                              probe);
}
EXPORT_SYMBOL_GPL(xfrm_aead_get_byname);

struct xfrm_algo_desc *xfrm_aalg_get_byidx(unsigned int idx)
{
        if (idx >= aalg_entries())
                return NULL;

        return &aalg_list[idx];
}
EXPORT_SYMBOL_GPL(xfrm_aalg_get_byidx);

struct xfrm_algo_desc *xfrm_ealg_get_byidx(unsigned int idx)
{
        if (idx >= ealg_entries())
                return NULL;

        return &ealg_list[idx];
}
EXPORT_SYMBOL_GPL(xfrm_ealg_get_byidx);

/*
 * Probe for the availability of crypto algorithms, and set the available
 * flag for any algorithms found on the system.  This is typically called by
 * pfkey during userspace SA add, update or register.
 */
void xfrm_probe_algs(void)
{
        int i, status;

        BUG_ON(in_softirq());

        for (i = 0; i < aalg_entries(); i++) {
                status = crypto_has_ahash(aalg_list[i].name, 0, 0);
                if (aalg_list[i].available != status)
                        aalg_list[i].available = status;
        }

        for (i = 0; i < ealg_entries(); i++) {
                status = crypto_has_skcipher(ealg_list[i].name, 0, 0);
                if (ealg_list[i].available != status)
                        ealg_list[i].available = status;
        }

        for (i = 0; i < calg_entries(); i++) {
                status = crypto_has_comp(calg_list[i].name, 0,
                                         CRYPTO_ALG_ASYNC);
                if (calg_list[i].available != status)
                        calg_list[i].available = status;
        }
}
EXPORT_SYMBOL_GPL(xfrm_probe_algs);

int xfrm_count_pfkey_auth_supported(void)
{
        int i, n;

        for (i = 0, n = 0; i < aalg_entries(); i++)
                if (aalg_list[i].available && aalg_list[i].pfkey_supported)
                        n++;
        return n;
}
EXPORT_SYMBOL_GPL(xfrm_count_pfkey_auth_supported);

int xfrm_count_pfkey_enc_supported(void)
{
        int i, n;

        for (i = 0, n = 0; i < ealg_entries(); i++)
                if (ealg_list[i].available && ealg_list[i].pfkey_supported)
                        n++;
        return n;
}
EXPORT_SYMBOL_GPL(xfrm_count_pfkey_enc_supported);

MODULE_LICENSE("GPL");




















































    2 


   22 
   17 
   14 
   11 
    9 















    2 


    2 







   59 




    1 








   42 




    6 










    1 
   53 





   69 





  101 
   97 







  112 



   97 










    4 


   24 


    3 


    1 

   63 
   51 







    2 


   27 




   25 



    4 

   21 









   21 








   90 







   24 





   63 





   46 



   13 
   13 








   23 





   18 



























































   37 
















   33 
   35 
    8 



    4 








   32 







   37 






   37 




   51 









   51 
   51 

   51 




1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
// SPDX-License-Identifier: GPL-2.0-or-later
/* scm.c - Socket level control messages processing.
 *
 * Author:        Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
 *              Alignment and value checking mods by Craig Metz
 */

#include <linux/module.h>
#include <linux/signal.h>
#include <linux/capability.h>
#include <linux/errno.h>
#include <linux/sched.h>
#include <linux/sched/user.h>
#include <linux/mm.h>
#include <linux/kernel.h>
#include <linux/stat.h>
#include <linux/socket.h>
#include <linux/file.h>
#include <linux/fcntl.h>
#include <linux/net.h>
#include <linux/interrupt.h>
#include <linux/netdevice.h>
#include <linux/security.h>
#include <linux/pid_namespace.h>
#include <linux/pid.h>
#include <linux/nsproxy.h>
#include <linux/slab.h>
#include <linux/errqueue.h>
#include <linux/io_uring.h>

#include <linux/uaccess.h>

#include <net/protocol.h>
#include <linux/skbuff.h>
#include <net/sock.h>
#include <net/compat.h>
#include <net/scm.h>
#include <net/cls_cgroup.h>


/*
 *        Only allow a user to send credentials, that they could set with
 *        setu(g)id.
 */

static __inline__ int scm_check_creds(struct ucred *creds)
{
        const struct cred *cred = current_cred();
        kuid_t uid = make_kuid(cred->user_ns, creds->uid);
        kgid_t gid = make_kgid(cred->user_ns, creds->gid);

        if (!uid_valid(uid) || !gid_valid(gid))
                return -EINVAL;

        if ((creds->pid == task_tgid_vnr(current) ||
             ns_capable(task_active_pid_ns(current)->user_ns, CAP_SYS_ADMIN)) &&
            ((uid_eq(uid, cred->uid)   || uid_eq(uid, cred->euid) ||
              uid_eq(uid, cred->suid)) || ns_capable(cred->user_ns, CAP_SETUID)) &&
            ((gid_eq(gid, cred->gid)   || gid_eq(gid, cred->egid) ||
              gid_eq(gid, cred->sgid)) || ns_capable(cred->user_ns, CAP_SETGID))) {
               return 0;
        }
        return -EPERM;
}

static int scm_fp_copy(struct cmsghdr *cmsg, struct scm_fp_list **fplp)
{
        int *fdp = (int*)CMSG_DATA(cmsg);
        struct scm_fp_list *fpl = *fplp;
        struct file **fpp;
        int i, num;

        num = (cmsg->cmsg_len - sizeof(struct cmsghdr))/sizeof(int);

        if (num <= 0)
                return 0;

        if (num > SCM_MAX_FD)
                return -EINVAL;

        if (!fpl)
        {
                fpl = kmalloc(sizeof(struct scm_fp_list), GFP_KERNEL_ACCOUNT);
                if (!fpl)
                        return -ENOMEM;
                *fplp = fpl;
                fpl->count = 0;
                fpl->max = SCM_MAX_FD;
                fpl->user = NULL;
        }
        fpp = &fpl->fp[fpl->count];

        if (fpl->count + num > fpl->max)
                return -EINVAL;

        /*
         *        Verify the descriptors and increment the usage count.
         */

        for (i=0; i< num; i++)
        {
                int fd = fdp[i];
                struct file *file;

                if (fd < 0 || !(file = fget_raw(fd)))
                        return -EBADF;
                /* don't allow io_uring files */
                if (io_is_uring_fops(file)) {
                        fput(file);
                        return -EINVAL;
                }
                *fpp++ = file;
                fpl->count++;
        }

        if (!fpl->user)
                fpl->user = get_uid(current_user());

        return num;
}

void __scm_destroy(struct scm_cookie *scm)
{
        struct scm_fp_list *fpl = scm->fp;
        int i;

        if (fpl) {
                scm->fp = NULL;
                for (i=fpl->count-1; i>=0; i--)
                        fput(fpl->fp[i]);
                free_uid(fpl->user);
                kfree(fpl);
        }
}
EXPORT_SYMBOL(__scm_destroy);

int __scm_send(struct socket *sock, struct msghdr *msg, struct scm_cookie *p)
{
        struct cmsghdr *cmsg;
        int err;

        for_each_cmsghdr(cmsg, msg) {
                err = -EINVAL;

                /* Verify that cmsg_len is at least sizeof(struct cmsghdr) */
                /* The first check was omitted in <= 2.2.5. The reasoning was
                   that parser checks cmsg_len in any case, so that
                   additional check would be work duplication.
                   But if cmsg_level is not SOL_SOCKET, we do not check
                   for too short ancillary data object at all! Oops.
                   OK, let's add it...
                 */
                if (!CMSG_OK(msg, cmsg))
                        goto error;

                if (cmsg->cmsg_level != SOL_SOCKET)
                        continue;

                switch (cmsg->cmsg_type)
                {
                case SCM_RIGHTS:
                        if (!sock->ops || sock->ops->family != PF_UNIX)
                                goto error;
                        err=scm_fp_copy(cmsg, &p->fp);
                        if (err<0)
                                goto error;
                        break;
                case SCM_CREDENTIALS:
                {
                        struct ucred creds;
                        kuid_t uid;
                        kgid_t gid;
                        if (cmsg->cmsg_len != CMSG_LEN(sizeof(struct ucred)))
                                goto error;
                        memcpy(&creds, CMSG_DATA(cmsg), sizeof(struct ucred));
                        err = scm_check_creds(&creds);
                        if (err)
                                goto error;

                        p->creds.pid = creds.pid;
                        if (!p->pid || pid_vnr(p->pid) != creds.pid) {
                                struct pid *pid;
                                err = -ESRCH;
                                pid = find_get_pid(creds.pid);
                                if (!pid)
                                        goto error;
                                put_pid(p->pid);
                                p->pid = pid;
                        }

                        err = -EINVAL;
                        uid = make_kuid(current_user_ns(), creds.uid);
                        gid = make_kgid(current_user_ns(), creds.gid);
                        if (!uid_valid(uid) || !gid_valid(gid))
                                goto error;

                        p->creds.uid = uid;
                        p->creds.gid = gid;
                        break;
                }
                default:
                        goto error;
                }
        }

        if (p->fp && !p->fp->count)
        {
                kfree(p->fp);
                p->fp = NULL;
        }
        return 0;

error:
        scm_destroy(p);
        return err;
}
EXPORT_SYMBOL(__scm_send);

int put_cmsg(struct msghdr * msg, int level, int type, int len, void *data)
{
        int cmlen = CMSG_LEN(len);

        if (msg->msg_flags & MSG_CMSG_COMPAT)
                return put_cmsg_compat(msg, level, type, len, data);

        if (!msg->msg_control || msg->msg_controllen < sizeof(struct cmsghdr)) {
                msg->msg_flags |= MSG_CTRUNC;
                return 0; /* XXX: return error? check spec. */
        }
        if (msg->msg_controllen < cmlen) {
                msg->msg_flags |= MSG_CTRUNC;
                cmlen = msg->msg_controllen;
        }

        if (msg->msg_control_is_user) {
                struct cmsghdr __user *cm = msg->msg_control_user;

                check_object_size(data, cmlen - sizeof(*cm), true);

                if (!user_write_access_begin(cm, cmlen))
                        goto efault;

                unsafe_put_user(cmlen, &cm->cmsg_len, efault_end);
                unsafe_put_user(level, &cm->cmsg_level, efault_end);
                unsafe_put_user(type, &cm->cmsg_type, efault_end);
                unsafe_copy_to_user(CMSG_USER_DATA(cm), data,
                                    cmlen - sizeof(*cm), efault_end);
                user_write_access_end();
        } else {
                struct cmsghdr *cm = msg->msg_control;

                cm->cmsg_level = level;
                cm->cmsg_type = type;
                cm->cmsg_len = cmlen;
                memcpy(CMSG_DATA(cm), data, cmlen - sizeof(*cm));
        }

        cmlen = min(CMSG_SPACE(len), msg->msg_controllen);
        msg->msg_control += cmlen;
        msg->msg_controllen -= cmlen;
        return 0;

efault_end:
        user_write_access_end();
efault:
        return -EFAULT;
}
EXPORT_SYMBOL(put_cmsg);

void put_cmsg_scm_timestamping64(struct msghdr *msg, struct scm_timestamping_internal *tss_internal)
{
        struct scm_timestamping64 tss;
        int i;

        for (i = 0; i < ARRAY_SIZE(tss.ts); i++) {
                tss.ts[i].tv_sec = tss_internal->ts[i].tv_sec;
                tss.ts[i].tv_nsec = tss_internal->ts[i].tv_nsec;
        }

        put_cmsg(msg, SOL_SOCKET, SO_TIMESTAMPING_NEW, sizeof(tss), &tss);
}
EXPORT_SYMBOL(put_cmsg_scm_timestamping64);

void put_cmsg_scm_timestamping(struct msghdr *msg, struct scm_timestamping_internal *tss_internal)
{
        struct scm_timestamping tss;
        int i;

        for (i = 0; i < ARRAY_SIZE(tss.ts); i++) {
                tss.ts[i].tv_sec = tss_internal->ts[i].tv_sec;
                tss.ts[i].tv_nsec = tss_internal->ts[i].tv_nsec;
        }

        put_cmsg(msg, SOL_SOCKET, SO_TIMESTAMPING_OLD, sizeof(tss), &tss);
}
EXPORT_SYMBOL(put_cmsg_scm_timestamping);

static int scm_max_fds(struct msghdr *msg)
{
        if (msg->msg_controllen <= sizeof(struct cmsghdr))
                return 0;
        return (msg->msg_controllen - sizeof(struct cmsghdr)) / sizeof(int);
}

void scm_detach_fds(struct msghdr *msg, struct scm_cookie *scm)
{
        struct cmsghdr __user *cm =
                (__force struct cmsghdr __user *)msg->msg_control;
        unsigned int o_flags = (msg->msg_flags & MSG_CMSG_CLOEXEC) ? O_CLOEXEC : 0;
        int fdmax = min_t(int, scm_max_fds(msg), scm->fp->count);
        int __user *cmsg_data = CMSG_USER_DATA(cm);
        int err = 0, i;

        /* no use for FD passing from kernel space callers */
        if (WARN_ON_ONCE(!msg->msg_control_is_user))
                return;

        if (msg->msg_flags & MSG_CMSG_COMPAT) {
                scm_detach_fds_compat(msg, scm);
                return;
        }

        for (i = 0; i < fdmax; i++) {
                err = receive_fd_user(scm->fp->fp[i], cmsg_data + i, o_flags);
                if (err < 0)
                        break;
        }

        if (i > 0) {
                int cmlen = CMSG_LEN(i * sizeof(int));

                err = put_user(SOL_SOCKET, &cm->cmsg_level);
                if (!err)
                        err = put_user(SCM_RIGHTS, &cm->cmsg_type);
                if (!err)
                        err = put_user(cmlen, &cm->cmsg_len);
                if (!err) {
                        cmlen = CMSG_SPACE(i * sizeof(int));
                        if (msg->msg_controllen < cmlen)
                                cmlen = msg->msg_controllen;
                        msg->msg_control += cmlen;
                        msg->msg_controllen -= cmlen;
                }
        }

        if (i < scm->fp->count || (scm->fp->count && fdmax <= 0))
                msg->msg_flags |= MSG_CTRUNC;

        /*
         * All of the files that fit in the message have had their usage counts
         * incremented, so we just free the list.
         */
        __scm_destroy(scm);
}
EXPORT_SYMBOL(scm_detach_fds);

struct scm_fp_list *scm_fp_dup(struct scm_fp_list *fpl)
{
        struct scm_fp_list *new_fpl;
        int i;

        if (!fpl)
                return NULL;

        new_fpl = kmemdup(fpl, offsetof(struct scm_fp_list, fp[fpl->count]),
                          GFP_KERNEL_ACCOUNT);
        if (new_fpl) {
                for (i = 0; i < fpl->count; i++)
                        get_file(fpl->fp[i]);
                new_fpl->max = new_fpl->count;
                new_fpl->user = get_uid(fpl->user);
        }
        return new_fpl;
}
EXPORT_SYMBOL(scm_fp_dup);









































































































































  651 





  276 


  636 

  638 






  634 

  635 














   12 



















   12 


   12 















  658 




   22 




















































   49 



   49 

   49 






































   49 





























1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
// SPDX-License-Identifier: GPL-2.0-only
/* Event cache for netfilter. */

/*
 * (C) 2005 Harald Welte <laforge@gnumonks.org>
 * (C) 2005 Patrick McHardy <kaber@trash.net>
 * (C) 2005-2006 Netfilter Core Team <coreteam@netfilter.org>
 * (C) 2005 USAGI/WIDE Project <http://www.linux-ipv6.org>
 */

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#include <linux/types.h>
#include <linux/netfilter.h>
#include <linux/skbuff.h>
#include <linux/vmalloc.h>
#include <linux/stddef.h>
#include <linux/err.h>
#include <linux/percpu.h>
#include <linux/kernel.h>
#include <linux/netdevice.h>
#include <linux/slab.h>
#include <linux/export.h>

#include <net/netfilter/nf_conntrack.h>
#include <net/netfilter/nf_conntrack_core.h>
#include <net/netfilter/nf_conntrack_ecache.h>
#include <net/netfilter/nf_conntrack_extend.h>

static DEFINE_MUTEX(nf_ct_ecache_mutex);

#define ECACHE_RETRY_WAIT (HZ/10)
#define ECACHE_STACK_ALLOC (256 / sizeof(void *))

enum retry_state {
        STATE_CONGESTED,
        STATE_RESTART,
        STATE_DONE,
};

static enum retry_state ecache_work_evict_list(struct ct_pcpu *pcpu)
{
        struct nf_conn *refs[ECACHE_STACK_ALLOC];
        enum retry_state ret = STATE_DONE;
        struct nf_conntrack_tuple_hash *h;
        struct hlist_nulls_node *n;
        unsigned int evicted = 0;

        spin_lock(&pcpu->lock);

        hlist_nulls_for_each_entry(h, n, &pcpu->dying, hnnode) {
                struct nf_conn *ct = nf_ct_tuplehash_to_ctrack(h);
                struct nf_conntrack_ecache *e;

                if (!nf_ct_is_confirmed(ct))
                        continue;

                /* This ecache access is safe because the ct is on the
                 * pcpu dying list and we hold the spinlock -- the entry
                 * cannot be free'd until after the lock is released.
                 *
                 * This is true even if ct has a refcount of 0: the
                 * cpu that is about to free the entry must remove it
                 * from the dying list and needs the lock to do so.
                 */
                e = nf_ct_ecache_find(ct);
                if (!e || e->state != NFCT_ECACHE_DESTROY_FAIL)
                        continue;

                /* ct is in NFCT_ECACHE_DESTROY_FAIL state, this means
                 * the worker owns this entry: the ct will remain valid
                 * until the worker puts its ct reference.
                 */
                if (nf_conntrack_event(IPCT_DESTROY, ct)) {
                        ret = STATE_CONGESTED;
                        break;
                }

                e->state = NFCT_ECACHE_DESTROY_SENT;
                refs[evicted] = ct;

                if (++evicted >= ARRAY_SIZE(refs)) {
                        ret = STATE_RESTART;
                        break;
                }
        }

        spin_unlock(&pcpu->lock);

        /* can't _put while holding lock */
        while (evicted)
                nf_ct_put(refs[--evicted]);

        return ret;
}

static void ecache_work(struct work_struct *work)
{
        struct nf_conntrack_net *cnet = container_of(work, struct nf_conntrack_net, ecache_dwork.work);
        struct netns_ct *ctnet = cnet->ct_net;
        int cpu, delay = -1;
        struct ct_pcpu *pcpu;

        local_bh_disable();

        for_each_possible_cpu(cpu) {
                enum retry_state ret;

                pcpu = per_cpu_ptr(ctnet->pcpu_lists, cpu);

                ret = ecache_work_evict_list(pcpu);

                switch (ret) {
                case STATE_CONGESTED:
                        delay = ECACHE_RETRY_WAIT;
                        goto out;
                case STATE_RESTART:
                        delay = 0;
                        break;
                case STATE_DONE:
                        break;
                }
        }

 out:
        local_bh_enable();

        ctnet->ecache_dwork_pending = delay > 0;
        if (delay >= 0)
                schedule_delayed_work(&cnet->ecache_dwork, delay);
}

static int __nf_conntrack_eventmask_report(struct nf_conntrack_ecache *e,
                                           const unsigned int events,
                                           const unsigned long missed,
                                           const struct nf_ct_event *item)
{
        struct nf_conn *ct = item->ct;
        struct net *net = nf_ct_net(item->ct);
        struct nf_ct_event_notifier *notify;
        int ret;

        if (!((events | missed) & e->ctmask))
                return 0;

        rcu_read_lock();

        notify = rcu_dereference(net->ct.nf_conntrack_event_cb);
        if (!notify) {
                rcu_read_unlock();
                return 0;
        }

        ret = notify->ct_event(events | missed, item);
        rcu_read_unlock();

        if (likely(ret >= 0 && missed == 0))
                return 0;

        spin_lock_bh(&ct->lock);
        if (ret < 0)
                e->missed |= events;
        else
                e->missed &= ~missed;
        spin_unlock_bh(&ct->lock);

        return ret;
}

int nf_conntrack_eventmask_report(unsigned int events, struct nf_conn *ct,
                                  u32 portid, int report)
{
        struct nf_conntrack_ecache *e;
        struct nf_ct_event item;
        unsigned long missed;
        int ret;

        if (!nf_ct_is_confirmed(ct))
                return 0;

        e = nf_ct_ecache_find(ct);
        if (!e)
                return 0;

        memset(&item, 0, sizeof(item));

        item.ct = ct;
        item.portid = e->portid ? e->portid : portid;
        item.report = report;

        /* This is a resent of a destroy event? If so, skip missed */
        missed = e->portid ? 0 : e->missed;

        ret = __nf_conntrack_eventmask_report(e, events, missed, &item);
        if (unlikely(ret < 0 && (events & (1 << IPCT_DESTROY)))) {
                /* This is a destroy event that has been triggered by a process,
                 * we store the PORTID to include it in the retransmission.
                 */
                if (e->portid == 0 && portid != 0)
                        e->portid = portid;
                e->state = NFCT_ECACHE_DESTROY_FAIL;
        }

        return ret;
}
EXPORT_SYMBOL_GPL(nf_conntrack_eventmask_report);

/* deliver cached events and clear cache entry - must be called with locally
 * disabled softirqs */
void nf_ct_deliver_cached_events(struct nf_conn *ct)
{
        struct nf_conntrack_ecache *e;
        struct nf_ct_event item;
        unsigned long events;

        if (!nf_ct_is_confirmed(ct) || nf_ct_is_dying(ct))
                return;

        e = nf_ct_ecache_find(ct);
        if (e == NULL)
                return;

        events = xchg(&e->cache, 0);

        item.ct = ct;
        item.portid = 0;
        item.report = 0;

        /* We make a copy of the missed event cache without taking
         * the lock, thus we may send missed events twice. However,
         * this does not harm and it happens very rarely.
         */
        __nf_conntrack_eventmask_report(e, events, e->missed, &item);
}
EXPORT_SYMBOL_GPL(nf_ct_deliver_cached_events);

void nf_ct_expect_event_report(enum ip_conntrack_expect_events event,
                               struct nf_conntrack_expect *exp,
                               u32 portid, int report)

{
        struct net *net = nf_ct_exp_net(exp);
        struct nf_ct_event_notifier *notify;
        struct nf_conntrack_ecache *e;

        rcu_read_lock();
        notify = rcu_dereference(net->ct.nf_conntrack_event_cb);
        if (!notify)
                goto out_unlock;

        e = nf_ct_ecache_find(exp->master);
        if (!e)
                goto out_unlock;

        if (e->expmask & (1 << event)) {
                struct nf_exp_event item = {
                        .exp        = exp,
                        .portid        = portid,
                        .report = report
                };
                notify->exp_event(1 << event, &item);
        }
out_unlock:
        rcu_read_unlock();
}

void nf_conntrack_register_notifier(struct net *net,
                                    const struct nf_ct_event_notifier *new)
{
        struct nf_ct_event_notifier *notify;

        mutex_lock(&nf_ct_ecache_mutex);
        notify = rcu_dereference_protected(net->ct.nf_conntrack_event_cb,
                                           lockdep_is_held(&nf_ct_ecache_mutex));
        WARN_ON_ONCE(notify);
        rcu_assign_pointer(net->ct.nf_conntrack_event_cb, new);
        mutex_unlock(&nf_ct_ecache_mutex);
}
EXPORT_SYMBOL_GPL(nf_conntrack_register_notifier);

void nf_conntrack_unregister_notifier(struct net *net)
{
        mutex_lock(&nf_ct_ecache_mutex);
        RCU_INIT_POINTER(net->ct.nf_conntrack_event_cb, NULL);
        mutex_unlock(&nf_ct_ecache_mutex);
        /* synchronize_rcu() is called after netns pre_exit */
}
EXPORT_SYMBOL_GPL(nf_conntrack_unregister_notifier);

void nf_conntrack_ecache_work(struct net *net, enum nf_ct_ecache_state state)
{
        struct nf_conntrack_net *cnet = nf_ct_pernet(net);

        if (state == NFCT_ECACHE_DESTROY_FAIL &&
            !delayed_work_pending(&cnet->ecache_dwork)) {
                schedule_delayed_work(&cnet->ecache_dwork, HZ);
                net->ct.ecache_dwork_pending = true;
        } else if (state == NFCT_ECACHE_DESTROY_SENT) {
                net->ct.ecache_dwork_pending = false;
                mod_delayed_work(system_wq, &cnet->ecache_dwork, 0);
        }
}

#define NF_CT_EVENTS_DEFAULT 1
static int nf_ct_events __read_mostly = NF_CT_EVENTS_DEFAULT;

static const struct nf_ct_ext_type event_extend = {
        .len        = sizeof(struct nf_conntrack_ecache),
        .align        = __alignof__(struct nf_conntrack_ecache),
        .id        = NF_CT_EXT_ECACHE,
};

void nf_conntrack_ecache_pernet_init(struct net *net)
{
        struct nf_conntrack_net *cnet = nf_ct_pernet(net);

        net->ct.sysctl_events = nf_ct_events;
        cnet->ct_net = &net->ct;
        INIT_DELAYED_WORK(&cnet->ecache_dwork, ecache_work);
}

void nf_conntrack_ecache_pernet_fini(struct net *net)
{
        struct nf_conntrack_net *cnet = nf_ct_pernet(net);

        cancel_delayed_work_sync(&cnet->ecache_dwork);
}

int nf_conntrack_ecache_init(void)
{
        int ret = nf_ct_extend_register(&event_extend);
        if (ret < 0)
                pr_err("Unable to register event extension\n");

        BUILD_BUG_ON(__IPCT_MAX >= 16);        /* ctmask, missed use u16 */

        return ret;
}

void nf_conntrack_ecache_fini(void)
{
        nf_ct_extend_unregister(&event_extend);
}















































































































































































































































































































































































































































































































 1824 

 1825 



















































  475 





   77 



1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_VMSTAT_H
#define _LINUX_VMSTAT_H

#include <linux/types.h>
#include <linux/percpu.h>
#include <linux/mmzone.h>
#include <linux/vm_event_item.h>
#include <linux/atomic.h>
#include <linux/static_key.h>
#include <linux/mmdebug.h>

extern int sysctl_stat_interval;

#ifdef CONFIG_NUMA
#define ENABLE_NUMA_STAT   1
#define DISABLE_NUMA_STAT   0
extern int sysctl_vm_numa_stat;
DECLARE_STATIC_KEY_TRUE(vm_numa_stat_key);
int sysctl_vm_numa_stat_handler(struct ctl_table *table, int write,
                void *buffer, size_t *length, loff_t *ppos);
#endif

struct reclaim_stat {
        unsigned nr_dirty;
        unsigned nr_unqueued_dirty;
        unsigned nr_congested;
        unsigned nr_writeback;
        unsigned nr_immediate;
        unsigned nr_pageout;
        unsigned nr_activate[ANON_AND_FILE];
        unsigned nr_ref_keep;
        unsigned nr_unmap_fail;
        unsigned nr_lazyfree_fail;
};

enum writeback_stat_item {
        NR_DIRTY_THRESHOLD,
        NR_DIRTY_BG_THRESHOLD,
        NR_VM_WRITEBACK_STAT_ITEMS,
};

#ifdef CONFIG_VM_EVENT_COUNTERS
/*
 * Light weight per cpu counter implementation.
 *
 * Counters should only be incremented and no critical kernel component
 * should rely on the counter values.
 *
 * Counters are handled completely inline. On many platforms the code
 * generated will simply be the increment of a global address.
 */

struct vm_event_state {
        unsigned long event[NR_VM_EVENT_ITEMS];
};

DECLARE_PER_CPU(struct vm_event_state, vm_event_states);

/*
 * vm counters are allowed to be racy. Use raw_cpu_ops to avoid the
 * local_irq_disable overhead.
 */
static inline void __count_vm_event(enum vm_event_item item)
{
        raw_cpu_inc(vm_event_states.event[item]);
}

static inline void count_vm_event(enum vm_event_item item)
{
        this_cpu_inc(vm_event_states.event[item]);
}

static inline void __count_vm_events(enum vm_event_item item, long delta)
{
        raw_cpu_add(vm_event_states.event[item], delta);
}

static inline void count_vm_events(enum vm_event_item item, long delta)
{
        this_cpu_add(vm_event_states.event[item], delta);
}

extern void all_vm_events(unsigned long *);

extern void vm_events_fold_cpu(int cpu);

#else

/* Disable counters */
static inline void count_vm_event(enum vm_event_item item)
{
}
static inline void count_vm_events(enum vm_event_item item, long delta)
{
}
static inline void __count_vm_event(enum vm_event_item item)
{
}
static inline void __count_vm_events(enum vm_event_item item, long delta)
{
}
static inline void all_vm_events(unsigned long *ret)
{
}
static inline void vm_events_fold_cpu(int cpu)
{
}

#endif /* CONFIG_VM_EVENT_COUNTERS */

#ifdef CONFIG_NUMA_BALANCING
#define count_vm_numa_event(x)     count_vm_event(x)
#define count_vm_numa_events(x, y) count_vm_events(x, y)
#else
#define count_vm_numa_event(x) do {} while (0)
#define count_vm_numa_events(x, y) do { (void)(y); } while (0)
#endif /* CONFIG_NUMA_BALANCING */

#ifdef CONFIG_DEBUG_TLBFLUSH
#define count_vm_tlb_event(x)           count_vm_event(x)
#define count_vm_tlb_events(x, y)  count_vm_events(x, y)
#else
#define count_vm_tlb_event(x)     do {} while (0)
#define count_vm_tlb_events(x, y) do { (void)(y); } while (0)
#endif

#ifdef CONFIG_DEBUG_VM_VMACACHE
#define count_vm_vmacache_event(x) count_vm_event(x)
#else
#define count_vm_vmacache_event(x) do {} while (0)
#endif

#define __count_zid_vm_events(item, zid, delta) \
        __count_vm_events(item##_NORMAL - ZONE_NORMAL + zid, delta)

/*
 * Zone and node-based page accounting with per cpu differentials.
 */
extern atomic_long_t vm_zone_stat[NR_VM_ZONE_STAT_ITEMS];
extern atomic_long_t vm_node_stat[NR_VM_NODE_STAT_ITEMS];
extern atomic_long_t vm_numa_event[NR_VM_NUMA_EVENT_ITEMS];

#ifdef CONFIG_NUMA
static inline void zone_numa_event_add(long x, struct zone *zone,
                                enum numa_stat_item item)
{
        atomic_long_add(x, &zone->vm_numa_event[item]);
        atomic_long_add(x, &vm_numa_event[item]);
}

static inline unsigned long zone_numa_event_state(struct zone *zone,
                                        enum numa_stat_item item)
{
        return atomic_long_read(&zone->vm_numa_event[item]);
}

static inline unsigned long
global_numa_event_state(enum numa_stat_item item)
{
        return atomic_long_read(&vm_numa_event[item]);
}
#endif /* CONFIG_NUMA */

static inline void zone_page_state_add(long x, struct zone *zone,
                                 enum zone_stat_item item)
{
        atomic_long_add(x, &zone->vm_stat[item]);
        atomic_long_add(x, &vm_zone_stat[item]);
}

static inline void node_page_state_add(long x, struct pglist_data *pgdat,
                                 enum node_stat_item item)
{
        atomic_long_add(x, &pgdat->vm_stat[item]);
        atomic_long_add(x, &vm_node_stat[item]);
}

static inline unsigned long global_zone_page_state(enum zone_stat_item item)
{
        long x = atomic_long_read(&vm_zone_stat[item]);
#ifdef CONFIG_SMP
        if (x < 0)
                x = 0;
#endif
        return x;
}

static inline
unsigned long global_node_page_state_pages(enum node_stat_item item)
{
        long x = atomic_long_read(&vm_node_stat[item]);
#ifdef CONFIG_SMP
        if (x < 0)
                x = 0;
#endif
        return x;
}

static inline unsigned long global_node_page_state(enum node_stat_item item)
{
        VM_WARN_ON_ONCE(vmstat_item_in_bytes(item));

        return global_node_page_state_pages(item);
}

static inline unsigned long zone_page_state(struct zone *zone,
                                        enum zone_stat_item item)
{
        long x = atomic_long_read(&zone->vm_stat[item]);
#ifdef CONFIG_SMP
        if (x < 0)
                x = 0;
#endif
        return x;
}

/*
 * More accurate version that also considers the currently pending
 * deltas. For that we need to loop over all cpus to find the current
 * deltas. There is no synchronization so the result cannot be
 * exactly accurate either.
 */
static inline unsigned long zone_page_state_snapshot(struct zone *zone,
                                        enum zone_stat_item item)
{
        long x = atomic_long_read(&zone->vm_stat[item]);

#ifdef CONFIG_SMP
        int cpu;
        for_each_online_cpu(cpu)
                x += per_cpu_ptr(zone->per_cpu_zonestats, cpu)->vm_stat_diff[item];

        if (x < 0)
                x = 0;
#endif
        return x;
}

#ifdef CONFIG_NUMA
/* See __count_vm_event comment on why raw_cpu_inc is used. */
static inline void
__count_numa_event(struct zone *zone, enum numa_stat_item item)
{
        struct per_cpu_zonestat __percpu *pzstats = zone->per_cpu_zonestats;

        raw_cpu_inc(pzstats->vm_numa_event[item]);
}

static inline void
__count_numa_events(struct zone *zone, enum numa_stat_item item, long delta)
{
        struct per_cpu_zonestat __percpu *pzstats = zone->per_cpu_zonestats;

        raw_cpu_add(pzstats->vm_numa_event[item], delta);
}

extern unsigned long sum_zone_node_page_state(int node,
                                              enum zone_stat_item item);
extern unsigned long sum_zone_numa_event_state(int node, enum numa_stat_item item);
extern unsigned long node_page_state(struct pglist_data *pgdat,
                                                enum node_stat_item item);
extern unsigned long node_page_state_pages(struct pglist_data *pgdat,
                                           enum node_stat_item item);
extern void fold_vm_numa_events(void);
#else
#define sum_zone_node_page_state(node, item) global_zone_page_state(item)
#define node_page_state(node, item) global_node_page_state(item)
#define node_page_state_pages(node, item) global_node_page_state_pages(item)
static inline void fold_vm_numa_events(void)
{
}
#endif /* CONFIG_NUMA */

#ifdef CONFIG_SMP
void __mod_zone_page_state(struct zone *, enum zone_stat_item item, long);
void __inc_zone_page_state(struct page *, enum zone_stat_item);
void __dec_zone_page_state(struct page *, enum zone_stat_item);

void __mod_node_page_state(struct pglist_data *, enum node_stat_item item, long);
void __inc_node_page_state(struct page *, enum node_stat_item);
void __dec_node_page_state(struct page *, enum node_stat_item);

void mod_zone_page_state(struct zone *, enum zone_stat_item, long);
void inc_zone_page_state(struct page *, enum zone_stat_item);
void dec_zone_page_state(struct page *, enum zone_stat_item);

void mod_node_page_state(struct pglist_data *, enum node_stat_item, long);
void inc_node_page_state(struct page *, enum node_stat_item);
void dec_node_page_state(struct page *, enum node_stat_item);

extern void inc_node_state(struct pglist_data *, enum node_stat_item);
extern void __inc_zone_state(struct zone *, enum zone_stat_item);
extern void __inc_node_state(struct pglist_data *, enum node_stat_item);
extern void dec_zone_state(struct zone *, enum zone_stat_item);
extern void __dec_zone_state(struct zone *, enum zone_stat_item);
extern void __dec_node_state(struct pglist_data *, enum node_stat_item);

void quiet_vmstat(void);
void cpu_vm_stats_fold(int cpu);
void refresh_zone_stat_thresholds(void);

struct ctl_table;
int vmstat_refresh(struct ctl_table *, int write, void *buffer, size_t *lenp,
                loff_t *ppos);

void drain_zonestat(struct zone *zone, struct per_cpu_zonestat *);

int calculate_pressure_threshold(struct zone *zone);
int calculate_normal_threshold(struct zone *zone);
void set_pgdat_percpu_threshold(pg_data_t *pgdat,
                                int (*calculate_pressure)(struct zone *));
#else /* CONFIG_SMP */

/*
 * We do not maintain differentials in a single processor configuration.
 * The functions directly modify the zone and global counters.
 */
static inline void __mod_zone_page_state(struct zone *zone,
                        enum zone_stat_item item, long delta)
{
        zone_page_state_add(delta, zone, item);
}

static inline void __mod_node_page_state(struct pglist_data *pgdat,
                        enum node_stat_item item, int delta)
{
        if (vmstat_item_in_bytes(item)) {
                /*
                 * Only cgroups use subpage accounting right now; at
                 * the global level, these items still change in
                 * multiples of whole pages. Store them as pages
                 * internally to keep the per-cpu counters compact.
                 */
                VM_WARN_ON_ONCE(delta & (PAGE_SIZE - 1));
                delta >>= PAGE_SHIFT;
        }

        node_page_state_add(delta, pgdat, item);
}

static inline void __inc_zone_state(struct zone *zone, enum zone_stat_item item)
{
        atomic_long_inc(&zone->vm_stat[item]);
        atomic_long_inc(&vm_zone_stat[item]);
}

static inline void __inc_node_state(struct pglist_data *pgdat, enum node_stat_item item)
{
        atomic_long_inc(&pgdat->vm_stat[item]);
        atomic_long_inc(&vm_node_stat[item]);
}

static inline void __dec_zone_state(struct zone *zone, enum zone_stat_item item)
{
        atomic_long_dec(&zone->vm_stat[item]);
        atomic_long_dec(&vm_zone_stat[item]);
}

static inline void __dec_node_state(struct pglist_data *pgdat, enum node_stat_item item)
{
        atomic_long_dec(&pgdat->vm_stat[item]);
        atomic_long_dec(&vm_node_stat[item]);
}

static inline void __inc_zone_page_state(struct page *page,
                        enum zone_stat_item item)
{
        __inc_zone_state(page_zone(page), item);
}

static inline void __inc_node_page_state(struct page *page,
                        enum node_stat_item item)
{
        __inc_node_state(page_pgdat(page), item);
}


static inline void __dec_zone_page_state(struct page *page,
                        enum zone_stat_item item)
{
        __dec_zone_state(page_zone(page), item);
}

static inline void __dec_node_page_state(struct page *page,
                        enum node_stat_item item)
{
        __dec_node_state(page_pgdat(page), item);
}


/*
 * We only use atomic operations to update counters. So there is no need to
 * disable interrupts.
 */
#define inc_zone_page_state __inc_zone_page_state
#define dec_zone_page_state __dec_zone_page_state
#define mod_zone_page_state __mod_zone_page_state

#define inc_node_page_state __inc_node_page_state
#define dec_node_page_state __dec_node_page_state
#define mod_node_page_state __mod_node_page_state

#define inc_zone_state __inc_zone_state
#define inc_node_state __inc_node_state
#define dec_zone_state __dec_zone_state

#define set_pgdat_percpu_threshold(pgdat, callback) { }

static inline void refresh_zone_stat_thresholds(void) { }
static inline void cpu_vm_stats_fold(int cpu) { }
static inline void quiet_vmstat(void) { }

static inline void drain_zonestat(struct zone *zone,
                        struct per_cpu_zonestat *pzstats) { }
#endif                /* CONFIG_SMP */

static inline void __mod_zone_freepage_state(struct zone *zone, int nr_pages,
                                             int migratetype)
{
        __mod_zone_page_state(zone, NR_FREE_PAGES, nr_pages);
        if (is_migrate_cma(migratetype))
                __mod_zone_page_state(zone, NR_FREE_CMA_PAGES, nr_pages);
}

extern const char * const vmstat_text[];

static inline const char *zone_stat_name(enum zone_stat_item item)
{
        return vmstat_text[item];
}

#ifdef CONFIG_NUMA
static inline const char *numa_stat_name(enum numa_stat_item item)
{
        return vmstat_text[NR_VM_ZONE_STAT_ITEMS +
                           item];
}
#endif /* CONFIG_NUMA */

static inline const char *node_stat_name(enum node_stat_item item)
{
        return vmstat_text[NR_VM_ZONE_STAT_ITEMS +
                           NR_VM_NUMA_EVENT_ITEMS +
                           item];
}

static inline const char *lru_list_name(enum lru_list lru)
{
        return node_stat_name(NR_LRU_BASE + (enum node_stat_item)lru) + 3; // skip "nr_"
}

static inline const char *writeback_stat_name(enum writeback_stat_item item)
{
        return vmstat_text[NR_VM_ZONE_STAT_ITEMS +
                           NR_VM_NUMA_EVENT_ITEMS +
                           NR_VM_NODE_STAT_ITEMS +
                           item];
}

#if defined(CONFIG_VM_EVENT_COUNTERS) || defined(CONFIG_MEMCG)
static inline const char *vm_event_name(enum vm_event_item item)
{
        return vmstat_text[NR_VM_ZONE_STAT_ITEMS +
                           NR_VM_NUMA_EVENT_ITEMS +
                           NR_VM_NODE_STAT_ITEMS +
                           NR_VM_WRITEBACK_STAT_ITEMS +
                           item];
}
#endif /* CONFIG_VM_EVENT_COUNTERS || CONFIG_MEMCG */

#ifdef CONFIG_MEMCG

void __mod_lruvec_state(struct lruvec *lruvec, enum node_stat_item idx,
                        int val);

static inline void mod_lruvec_state(struct lruvec *lruvec,
                                    enum node_stat_item idx, int val)
{
        unsigned long flags;

        local_irq_save(flags);
        __mod_lruvec_state(lruvec, idx, val);
        local_irq_restore(flags);
}

void __mod_lruvec_page_state(struct page *page,
                             enum node_stat_item idx, int val);

static inline void mod_lruvec_page_state(struct page *page,
                                         enum node_stat_item idx, int val)
{
        unsigned long flags;

        local_irq_save(flags);
        __mod_lruvec_page_state(page, idx, val);
        local_irq_restore(flags);
}

#else

static inline void __mod_lruvec_state(struct lruvec *lruvec,
                                      enum node_stat_item idx, int val)
{
        __mod_node_page_state(lruvec_pgdat(lruvec), idx, val);
}

static inline void mod_lruvec_state(struct lruvec *lruvec,
                                    enum node_stat_item idx, int val)
{
        mod_node_page_state(lruvec_pgdat(lruvec), idx, val);
}

static inline void __mod_lruvec_page_state(struct page *page,
                                           enum node_stat_item idx, int val)
{
        __mod_node_page_state(page_pgdat(page), idx, val);
}

static inline void mod_lruvec_page_state(struct page *page,
                                         enum node_stat_item idx, int val)
{
        mod_node_page_state(page_pgdat(page), idx, val);
}

#endif /* CONFIG_MEMCG */

static inline void inc_lruvec_state(struct lruvec *lruvec,
                                    enum node_stat_item idx)
{
        mod_lruvec_state(lruvec, idx, 1);
}

static inline void __inc_lruvec_page_state(struct page *page,
                                           enum node_stat_item idx)
{
        __mod_lruvec_page_state(page, idx, 1);
}

static inline void __dec_lruvec_page_state(struct page *page,
                                           enum node_stat_item idx)
{
        __mod_lruvec_page_state(page, idx, -1);
}

static inline void inc_lruvec_page_state(struct page *page,
                                         enum node_stat_item idx)
{
        mod_lruvec_page_state(page, idx, 1);
}

static inline void dec_lruvec_page_state(struct page *page,
                                         enum node_stat_item idx)
{
        mod_lruvec_page_state(page, idx, -1);
}

#endif /* _LINUX_VMSTAT_H */






























































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
/* SPDX-License-Identifier: GPL-2.0-or-later */
/* SCTP kernel implementation
 * (C) Copyright IBM Corp. 2001, 2004
 * Copyright (c) 1999-2000 Cisco, Inc.
 * Copyright (c) 1999-2001 Motorola, Inc.
 * Copyright (c) 2001 Intel Corp.
 *
 * This file is part of the SCTP kernel implementation
 *
 * These are the definitions needed for the tsnmap type.  The tsnmap is used
 * to track out of order TSNs received.
 *
 * Please send any bug reports or fixes you make to the
 * email address(es):
 *    lksctp developers <linux-sctp@vger.kernel.org>
 *
 * Written or modified by:
 *   Jon Grimm             <jgrimm@us.ibm.com>
 *   La Monte H.P. Yarroll <piggy@acm.org>
 *   Karl Knutson          <karl@athena.chicago.il.us>
 *   Sridhar Samudrala     <sri@us.ibm.com>
 */
#include <net/sctp/constants.h>

#ifndef __sctp_tsnmap_h__
#define __sctp_tsnmap_h__

/* RFC 2960 12.2 Parameters necessary per association (i.e. the TCB)
 * Mapping  An array of bits or bytes indicating which out of
 * Array    order TSN's have been received (relative to the
 *          Last Rcvd TSN). If no gaps exist, i.e. no out of
 *          order packets have been received, this array
 *          will be set to all zero. This structure may be
 *          in the form of a circular buffer or bit array.
 */
struct sctp_tsnmap {
        /* This array counts the number of chunks with each TSN.
         * It points at one of the two buffers with which we will
         * ping-pong between.
         */
        unsigned long *tsn_map;

        /* This is the TSN at tsn_map[0].  */
        __u32 base_tsn;

        /* Last Rcvd   : This is the last TSN received in
         * TSN               : sequence. This value is set initially by
         *             : taking the peer's Initial TSN, received in
         *             : the INIT or INIT ACK chunk, and subtracting
         *             : one from it.
         *
         * Throughout most of the specification this is called the
         * "Cumulative TSN ACK Point".  In this case, we
         * ignore the advice in 12.2 in favour of the term
         * used in the bulk of the text.
         */
        __u32 cumulative_tsn_ack_point;

        /* This is the highest TSN we've marked.  */
        __u32 max_tsn_seen;

        /* This is the minimum number of TSNs we can track.  This corresponds
         * to the size of tsn_map.   Note: the overflow_map allows us to
         * potentially track more than this quantity.
         */
        __u16 len;

        /* Data chunks pending receipt. used by SCTP_STATUS sockopt */
        __u16 pending_data;

        /* Record duplicate TSNs here.  We clear this after
         * every SACK.  Store up to SCTP_MAX_DUP_TSNS worth of
         * information.
         */
        __u16 num_dup_tsns;
        __be32 dup_tsns[SCTP_MAX_DUP_TSNS];
};

struct sctp_tsnmap_iter {
        __u32 start;
};

/* Initialize a block of memory as a tsnmap.  */
struct sctp_tsnmap *sctp_tsnmap_init(struct sctp_tsnmap *, __u16 len,
                                     __u32 initial_tsn, gfp_t gfp);

void sctp_tsnmap_free(struct sctp_tsnmap *map);

/* Test the tracking state of this TSN.
 * Returns:
 *   0 if the TSN has not yet been seen
 *  >0 if the TSN has been seen (duplicate)
 *  <0 if the TSN is invalid (too large to track)
 */
int sctp_tsnmap_check(const struct sctp_tsnmap *, __u32 tsn);

/* Mark this TSN as seen.  */
int sctp_tsnmap_mark(struct sctp_tsnmap *, __u32 tsn,
                     struct sctp_transport *trans);

/* Mark this TSN and all lower as seen. */
void sctp_tsnmap_skip(struct sctp_tsnmap *map, __u32 tsn);

/* Retrieve the Cumulative TSN ACK Point.  */
static inline __u32 sctp_tsnmap_get_ctsn(const struct sctp_tsnmap *map)
{
        return map->cumulative_tsn_ack_point;
}

/* Retrieve the highest TSN we've seen.  */
static inline __u32 sctp_tsnmap_get_max_tsn_seen(const struct sctp_tsnmap *map)
{
        return map->max_tsn_seen;
}

/* How many duplicate TSNs are stored? */
static inline __u16 sctp_tsnmap_num_dups(struct sctp_tsnmap *map)
{
        return map->num_dup_tsns;
}

/* Return pointer to duplicate tsn array as needed by SACK. */
static inline __be32 *sctp_tsnmap_get_dups(struct sctp_tsnmap *map)
{
        map->num_dup_tsns = 0;
        return map->dup_tsns;
}

/* How many gap ack blocks do we have recorded? */
__u16 sctp_tsnmap_num_gabs(struct sctp_tsnmap *map,
                           struct sctp_gap_ack_block *gabs);

/* Refresh the count on pending data. */
__u16 sctp_tsnmap_pending(struct sctp_tsnmap *map);

/* Is there a gap in the TSN map?  */
static inline int sctp_tsnmap_has_gap(const struct sctp_tsnmap *map)
{
        return map->cumulative_tsn_ack_point != map->max_tsn_seen;
}

/* Mark a duplicate TSN.  Note:  limit the storage of duplicate TSN
 * information.
 */
static inline void sctp_tsnmap_mark_dup(struct sctp_tsnmap *map, __u32 tsn)
{
        if (map->num_dup_tsns < SCTP_MAX_DUP_TSNS)
                map->dup_tsns[map->num_dup_tsns++] = htonl(tsn);
}

/* Renege a TSN that was seen.  */
void sctp_tsnmap_renege(struct sctp_tsnmap *, __u32 tsn);

/* Is there a gap in the TSN map? */
int sctp_tsnmap_has_gap(const struct sctp_tsnmap *);

#endif /* __sctp_tsnmap_h__ */





















































































































































































































































































































































































































































































































































































































































































































































































































































    4 




    4 
    4 

    4 





    4 





    4 
    4 


    4 





    5 






   13 



   13 

   13 



1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
// SPDX-License-Identifier: GPL-2.0-or-later

#include <linux/cfm_bridge.h>
#include <uapi/linux/cfm_bridge.h>
#include "br_private_cfm.h"

static struct br_cfm_mep *br_mep_find(struct net_bridge *br, u32 instance)
{
        struct br_cfm_mep *mep;

        hlist_for_each_entry(mep, &br->mep_list, head)
                if (mep->instance == instance)
                        return mep;

        return NULL;
}

static struct br_cfm_mep *br_mep_find_ifindex(struct net_bridge *br,
                                              u32 ifindex)
{
        struct br_cfm_mep *mep;

        hlist_for_each_entry_rcu(mep, &br->mep_list, head,
                                 lockdep_rtnl_is_held())
                if (mep->create.ifindex == ifindex)
                        return mep;

        return NULL;
}

static struct br_cfm_peer_mep *br_peer_mep_find(struct br_cfm_mep *mep,
                                                u32 mepid)
{
        struct br_cfm_peer_mep *peer_mep;

        hlist_for_each_entry_rcu(peer_mep, &mep->peer_mep_list, head,
                                 lockdep_rtnl_is_held())
                if (peer_mep->mepid == mepid)
                        return peer_mep;

        return NULL;
}

static struct net_bridge_port *br_mep_get_port(struct net_bridge *br,
                                               u32 ifindex)
{
        struct net_bridge_port *port;

        list_for_each_entry(port, &br->port_list, list)
                if (port->dev->ifindex == ifindex)
                        return port;

        return NULL;
}

/* Calculate the CCM interval in us. */
static u32 interval_to_us(enum br_cfm_ccm_interval interval)
{
        switch (interval) {
        case BR_CFM_CCM_INTERVAL_NONE:
                return 0;
        case BR_CFM_CCM_INTERVAL_3_3_MS:
                return 3300;
        case BR_CFM_CCM_INTERVAL_10_MS:
                return 10 * 1000;
        case BR_CFM_CCM_INTERVAL_100_MS:
                return 100 * 1000;
        case BR_CFM_CCM_INTERVAL_1_SEC:
                return 1000 * 1000;
        case BR_CFM_CCM_INTERVAL_10_SEC:
                return 10 * 1000 * 1000;
        case BR_CFM_CCM_INTERVAL_1_MIN:
                return 60 * 1000 * 1000;
        case BR_CFM_CCM_INTERVAL_10_MIN:
                return 10 * 60 * 1000 * 1000;
        }
        return 0;
}

/* Convert the interface interval to CCM PDU value. */
static u32 interval_to_pdu(enum br_cfm_ccm_interval interval)
{
        switch (interval) {
        case BR_CFM_CCM_INTERVAL_NONE:
                return 0;
        case BR_CFM_CCM_INTERVAL_3_3_MS:
                return 1;
        case BR_CFM_CCM_INTERVAL_10_MS:
                return 2;
        case BR_CFM_CCM_INTERVAL_100_MS:
                return 3;
        case BR_CFM_CCM_INTERVAL_1_SEC:
                return 4;
        case BR_CFM_CCM_INTERVAL_10_SEC:
                return 5;
        case BR_CFM_CCM_INTERVAL_1_MIN:
                return 6;
        case BR_CFM_CCM_INTERVAL_10_MIN:
                return 7;
        }
        return 0;
}

/* Convert the CCM PDU value to interval on interface. */
static u32 pdu_to_interval(u32 value)
{
        switch (value) {
        case 0:
                return BR_CFM_CCM_INTERVAL_NONE;
        case 1:
                return BR_CFM_CCM_INTERVAL_3_3_MS;
        case 2:
                return BR_CFM_CCM_INTERVAL_10_MS;
        case 3:
                return BR_CFM_CCM_INTERVAL_100_MS;
        case 4:
                return BR_CFM_CCM_INTERVAL_1_SEC;
        case 5:
                return BR_CFM_CCM_INTERVAL_10_SEC;
        case 6:
                return BR_CFM_CCM_INTERVAL_1_MIN;
        case 7:
                return BR_CFM_CCM_INTERVAL_10_MIN;
        }
        return BR_CFM_CCM_INTERVAL_NONE;
}

static void ccm_rx_timer_start(struct br_cfm_peer_mep *peer_mep)
{
        u32 interval_us;

        interval_us = interval_to_us(peer_mep->mep->cc_config.exp_interval);
        /* Function ccm_rx_dwork must be called with 1/4
         * of the configured CC 'expected_interval'
         * in order to detect CCM defect after 3.25 interval.
         */
        queue_delayed_work(system_wq, &peer_mep->ccm_rx_dwork,
                           usecs_to_jiffies(interval_us / 4));
}

static void br_cfm_notify(int event, const struct net_bridge_port *port)
{
        u32 filter = RTEXT_FILTER_CFM_STATUS;

        br_info_notify(event, port->br, NULL, filter);
}

static void cc_peer_enable(struct br_cfm_peer_mep *peer_mep)
{
        memset(&peer_mep->cc_status, 0, sizeof(peer_mep->cc_status));
        peer_mep->ccm_rx_count_miss = 0;

        ccm_rx_timer_start(peer_mep);
}

static void cc_peer_disable(struct br_cfm_peer_mep *peer_mep)
{
        cancel_delayed_work_sync(&peer_mep->ccm_rx_dwork);
}

static struct sk_buff *ccm_frame_build(struct br_cfm_mep *mep,
                                       const struct br_cfm_cc_ccm_tx_info *const tx_info)

{
        struct br_cfm_common_hdr *common_hdr;
        struct net_bridge_port *b_port;
        struct br_cfm_maid *maid;
        u8 *itu_reserved, *e_tlv;
        struct ethhdr *eth_hdr;
        struct sk_buff *skb;
        __be32 *status_tlv;
        __be32 *snumber;
        __be16 *mepid;

        skb = dev_alloc_skb(CFM_CCM_MAX_FRAME_LENGTH);
        if (!skb)
                return NULL;

        rcu_read_lock();
        b_port = rcu_dereference(mep->b_port);
        if (!b_port) {
                kfree_skb(skb);
                rcu_read_unlock();
                return NULL;
        }
        skb->dev = b_port->dev;
        rcu_read_unlock();
        /* The device cannot be deleted until the work_queue functions has
         * completed. This function is called from ccm_tx_work_expired()
         * that is a work_queue functions.
         */

        skb->protocol = htons(ETH_P_CFM);
        skb->priority = CFM_FRAME_PRIO;

        /* Ethernet header */
        eth_hdr = skb_put(skb, sizeof(*eth_hdr));
        ether_addr_copy(eth_hdr->h_dest, tx_info->dmac.addr);
        ether_addr_copy(eth_hdr->h_source, mep->config.unicast_mac.addr);
        eth_hdr->h_proto = htons(ETH_P_CFM);

        /* Common CFM Header */
        common_hdr = skb_put(skb, sizeof(*common_hdr));
        common_hdr->mdlevel_version = mep->config.mdlevel << 5;
        common_hdr->opcode = BR_CFM_OPCODE_CCM;
        common_hdr->flags = (mep->rdi << 7) |
                            interval_to_pdu(mep->cc_config.exp_interval);
        common_hdr->tlv_offset = CFM_CCM_TLV_OFFSET;

        /* Sequence number */
        snumber = skb_put(skb, sizeof(*snumber));
        if (tx_info->seq_no_update) {
                *snumber = cpu_to_be32(mep->ccm_tx_snumber);
                mep->ccm_tx_snumber += 1;
        } else {
                *snumber = 0;
        }

        mepid = skb_put(skb, sizeof(*mepid));
        *mepid = cpu_to_be16((u16)mep->config.mepid);

        maid = skb_put(skb, sizeof(*maid));
        memcpy(maid->data, mep->cc_config.exp_maid.data, sizeof(maid->data));

        /* ITU reserved (CFM_CCM_ITU_RESERVED_SIZE octets) */
        itu_reserved = skb_put(skb, CFM_CCM_ITU_RESERVED_SIZE);
        memset(itu_reserved, 0, CFM_CCM_ITU_RESERVED_SIZE);

        /* Generel CFM TLV format:
         * TLV type:                one byte
         * TLV value length:        two bytes
         * TLV value:                'TLV value length' bytes
         */

        /* Port status TLV. The value length is 1. Total of 4 bytes. */
        if (tx_info->port_tlv) {
                status_tlv = skb_put(skb, sizeof(*status_tlv));
                *status_tlv = cpu_to_be32((CFM_PORT_STATUS_TLV_TYPE << 24) |
                                          (1 << 8) |        /* Value length */
                                          (tx_info->port_tlv_value & 0xFF));
        }

        /* Interface status TLV. The value length is 1. Total of 4 bytes. */
        if (tx_info->if_tlv) {
                status_tlv = skb_put(skb, sizeof(*status_tlv));
                *status_tlv = cpu_to_be32((CFM_IF_STATUS_TLV_TYPE << 24) |
                                          (1 << 8) |        /* Value length */
                                          (tx_info->if_tlv_value & 0xFF));
        }

        /* End TLV */
        e_tlv = skb_put(skb, sizeof(*e_tlv));
        *e_tlv = CFM_ENDE_TLV_TYPE;

        return skb;
}

static void ccm_frame_tx(struct sk_buff *skb)
{
        skb_reset_network_header(skb);
        dev_queue_xmit(skb);
}

/* This function is called with the configured CC 'expected_interval'
 * in order to drive CCM transmission when enabled.
 */
static void ccm_tx_work_expired(struct work_struct *work)
{
        struct delayed_work *del_work;
        struct br_cfm_mep *mep;
        struct sk_buff *skb;
        u32 interval_us;

        del_work = to_delayed_work(work);
        mep = container_of(del_work, struct br_cfm_mep, ccm_tx_dwork);

        if (time_before_eq(mep->ccm_tx_end, jiffies)) {
                /* Transmission period has ended */
                mep->cc_ccm_tx_info.period = 0;
                return;
        }

        skb = ccm_frame_build(mep, &mep->cc_ccm_tx_info);
        if (skb)
                ccm_frame_tx(skb);

        interval_us = interval_to_us(mep->cc_config.exp_interval);
        queue_delayed_work(system_wq, &mep->ccm_tx_dwork,
                           usecs_to_jiffies(interval_us));
}

/* This function is called with 1/4 of the configured CC 'expected_interval'
 * in order to detect CCM defect after 3.25 interval.
 */
static void ccm_rx_work_expired(struct work_struct *work)
{
        struct br_cfm_peer_mep *peer_mep;
        struct net_bridge_port *b_port;
        struct delayed_work *del_work;

        del_work = to_delayed_work(work);
        peer_mep = container_of(del_work, struct br_cfm_peer_mep, ccm_rx_dwork);

        /* After 13 counts (4 * 3,25) then 3.25 intervals are expired */
        if (peer_mep->ccm_rx_count_miss < 13) {
                /* 3.25 intervals are NOT expired without CCM reception */
                peer_mep->ccm_rx_count_miss++;

                /* Start timer again */
                ccm_rx_timer_start(peer_mep);
        } else {
                /* 3.25 intervals are expired without CCM reception.
                 * CCM defect detected
                 */
                peer_mep->cc_status.ccm_defect = true;

                /* Change in CCM defect status - notify */
                rcu_read_lock();
                b_port = rcu_dereference(peer_mep->mep->b_port);
                if (b_port)
                        br_cfm_notify(RTM_NEWLINK, b_port);
                rcu_read_unlock();
        }
}

static u32 ccm_tlv_extract(struct sk_buff *skb, u32 index,
                           struct br_cfm_peer_mep *peer_mep)
{
        __be32 *s_tlv;
        __be32 _s_tlv;
        u32 h_s_tlv;
        u8 *e_tlv;
        u8 _e_tlv;

        e_tlv = skb_header_pointer(skb, index, sizeof(_e_tlv), &_e_tlv);
        if (!e_tlv)
                return 0;

        /* TLV is present - get the status TLV */
        s_tlv = skb_header_pointer(skb,
                                   index,
                                   sizeof(_s_tlv), &_s_tlv);
        if (!s_tlv)
                return 0;

        h_s_tlv = ntohl(*s_tlv);
        if ((h_s_tlv >> 24) == CFM_IF_STATUS_TLV_TYPE) {
                /* Interface status TLV */
                peer_mep->cc_status.tlv_seen = true;
                peer_mep->cc_status.if_tlv_value = (h_s_tlv & 0xFF);
        }

        if ((h_s_tlv >> 24) == CFM_PORT_STATUS_TLV_TYPE) {
                /* Port status TLV */
                peer_mep->cc_status.tlv_seen = true;
                peer_mep->cc_status.port_tlv_value = (h_s_tlv & 0xFF);
        }

        /* The Sender ID TLV is not handled */
        /* The Organization-Specific TLV is not handled */

        /* Return the length of this tlv.
         * This is the length of the value field plus 3 bytes for size of type
         * field and length field
         */
        return ((h_s_tlv >> 8) & 0xFFFF) + 3;
}

/* note: already called with rcu_read_lock */
static int br_cfm_frame_rx(struct net_bridge_port *port, struct sk_buff *skb)
{
        u32 mdlevel, interval, size, index, max;
        const struct br_cfm_common_hdr *hdr;
        struct br_cfm_peer_mep *peer_mep;
        const struct br_cfm_maid *maid;
        struct br_cfm_common_hdr _hdr;
        struct br_cfm_maid _maid;
        struct br_cfm_mep *mep;
        struct net_bridge *br;
        __be32 *snumber;
        __be32 _snumber;
        __be16 *mepid;
        __be16 _mepid;

        if (port->state == BR_STATE_DISABLED)
                return 0;

        hdr = skb_header_pointer(skb, 0, sizeof(_hdr), &_hdr);
        if (!hdr)
                return 1;

        br = port->br;
        mep = br_mep_find_ifindex(br, port->dev->ifindex);
        if (unlikely(!mep))
                /* No MEP on this port - must be forwarded */
                return 0;

        mdlevel = hdr->mdlevel_version >> 5;
        if (mdlevel > mep->config.mdlevel)
                /* The level is above this MEP level - must be forwarded */
                return 0;

        if ((hdr->mdlevel_version & 0x1F) != 0) {
                /* Invalid version */
                mep->status.version_unexp_seen = true;
                return 1;
        }

        if (mdlevel < mep->config.mdlevel) {
                /* The level is below this MEP level */
                mep->status.rx_level_low_seen = true;
                return 1;
        }

        if (hdr->opcode == BR_CFM_OPCODE_CCM) {
                /* CCM PDU received. */
                /* MA ID is after common header + sequence number + MEP ID */
                maid = skb_header_pointer(skb,
                                          CFM_CCM_PDU_MAID_OFFSET,
                                          sizeof(_maid), &_maid);
                if (!maid)
                        return 1;
                if (memcmp(maid->data, mep->cc_config.exp_maid.data,
                           sizeof(maid->data)))
                        /* MA ID not as expected */
                        return 1;

                /* MEP ID is after common header + sequence number */
                mepid = skb_header_pointer(skb,
                                           CFM_CCM_PDU_MEPID_OFFSET,
                                           sizeof(_mepid), &_mepid);
                if (!mepid)
                        return 1;
                peer_mep = br_peer_mep_find(mep, (u32)ntohs(*mepid));
                if (!peer_mep)
                        return 1;

                /* Interval is in common header flags */
                interval = hdr->flags & 0x07;
                if (mep->cc_config.exp_interval != pdu_to_interval(interval))
                        /* Interval not as expected */
                        return 1;

                /* A valid CCM frame is received */
                if (peer_mep->cc_status.ccm_defect) {
                        peer_mep->cc_status.ccm_defect = false;

                        /* Change in CCM defect status - notify */
                        br_cfm_notify(RTM_NEWLINK, port);

                        /* Start CCM RX timer */
                        ccm_rx_timer_start(peer_mep);
                }

                peer_mep->cc_status.seen = true;
                peer_mep->ccm_rx_count_miss = 0;

                /* RDI is in common header flags */
                peer_mep->cc_status.rdi = (hdr->flags & 0x80) ? true : false;

                /* Sequence number is after common header */
                snumber = skb_header_pointer(skb,
                                             CFM_CCM_PDU_SEQNR_OFFSET,
                                             sizeof(_snumber), &_snumber);
                if (!snumber)
                        return 1;
                if (ntohl(*snumber) != (mep->ccm_rx_snumber + 1))
                        /* Unexpected sequence number */
                        peer_mep->cc_status.seq_unexp_seen = true;

                mep->ccm_rx_snumber = ntohl(*snumber);

                /* TLV end is after common header + sequence number + MEP ID +
                 * MA ID + ITU reserved
                 */
                index = CFM_CCM_PDU_TLV_OFFSET;
                max = 0;
                do { /* Handle all TLVs */
                        size = ccm_tlv_extract(skb, index, peer_mep);
                        index += size;
                        max += 1;
                } while (size != 0 && max < 4); /* Max four TLVs possible */

                return 1;
        }

        mep->status.opcode_unexp_seen = true;

        return 1;
}

static struct br_frame_type cfm_frame_type __read_mostly = {
        .type = cpu_to_be16(ETH_P_CFM),
        .frame_handler = br_cfm_frame_rx,
};

int br_cfm_mep_create(struct net_bridge *br,
                      const u32 instance,
                      struct br_cfm_mep_create *const create,
                      struct netlink_ext_ack *extack)
{
        struct net_bridge_port *p;
        struct br_cfm_mep *mep;

        ASSERT_RTNL();

        if (create->domain == BR_CFM_VLAN) {
                NL_SET_ERR_MSG_MOD(extack,
                                   "VLAN domain not supported");
                return -EINVAL;
        }
        if (create->domain != BR_CFM_PORT) {
                NL_SET_ERR_MSG_MOD(extack,
                                   "Invalid domain value");
                return -EINVAL;
        }
        if (create->direction == BR_CFM_MEP_DIRECTION_UP) {
                NL_SET_ERR_MSG_MOD(extack,
                                   "Up-MEP not supported");
                return -EINVAL;
        }
        if (create->direction != BR_CFM_MEP_DIRECTION_DOWN) {
                NL_SET_ERR_MSG_MOD(extack,
                                   "Invalid direction value");
                return -EINVAL;
        }
        p = br_mep_get_port(br, create->ifindex);
        if (!p) {
                NL_SET_ERR_MSG_MOD(extack,
                                   "Port is not related to bridge");
                return -EINVAL;
        }
        mep = br_mep_find(br, instance);
        if (mep) {
                NL_SET_ERR_MSG_MOD(extack,
                                   "MEP instance already exists");
                return -EEXIST;
        }

        /* In PORT domain only one instance can be created per port */
        if (create->domain == BR_CFM_PORT) {
                mep = br_mep_find_ifindex(br, create->ifindex);
                if (mep) {
                        NL_SET_ERR_MSG_MOD(extack,
                                           "Only one Port MEP on a port allowed");
                        return -EINVAL;
                }
        }

        mep = kzalloc(sizeof(*mep), GFP_KERNEL);
        if (!mep)
                return -ENOMEM;

        mep->create = *create;
        mep->instance = instance;
        rcu_assign_pointer(mep->b_port, p);

        INIT_HLIST_HEAD(&mep->peer_mep_list);
        INIT_DELAYED_WORK(&mep->ccm_tx_dwork, ccm_tx_work_expired);

        if (hlist_empty(&br->mep_list))
                br_add_frame(br, &cfm_frame_type);

        hlist_add_tail_rcu(&mep->head, &br->mep_list);

        return 0;
}

static void mep_delete_implementation(struct net_bridge *br,
                                      struct br_cfm_mep *mep)
{
        struct br_cfm_peer_mep *peer_mep;
        struct hlist_node *n_store;

        ASSERT_RTNL();

        /* Empty and free peer MEP list */
        hlist_for_each_entry_safe(peer_mep, n_store, &mep->peer_mep_list, head) {
                cancel_delayed_work_sync(&peer_mep->ccm_rx_dwork);
                hlist_del_rcu(&peer_mep->head);
                kfree_rcu(peer_mep, rcu);
        }

        cancel_delayed_work_sync(&mep->ccm_tx_dwork);

        RCU_INIT_POINTER(mep->b_port, NULL);
        hlist_del_rcu(&mep->head);
        kfree_rcu(mep, rcu);

        if (hlist_empty(&br->mep_list))
                br_del_frame(br, &cfm_frame_type);
}

int br_cfm_mep_delete(struct net_bridge *br,
                      const u32 instance,
                      struct netlink_ext_ack *extack)
{
        struct br_cfm_mep *mep;

        ASSERT_RTNL();

        mep = br_mep_find(br, instance);
        if (!mep) {
                NL_SET_ERR_MSG_MOD(extack,
                                   "MEP instance does not exists");
                return -ENOENT;
        }

        mep_delete_implementation(br, mep);

        return 0;
}

int br_cfm_mep_config_set(struct net_bridge *br,
                          const u32 instance,
                          const struct br_cfm_mep_config *const config,
                          struct netlink_ext_ack *extack)
{
        struct br_cfm_mep *mep;

        ASSERT_RTNL();

        mep = br_mep_find(br, instance);
        if (!mep) {
                NL_SET_ERR_MSG_MOD(extack,
                                   "MEP instance does not exists");
                return -ENOENT;
        }

        mep->config = *config;

        return 0;
}

int br_cfm_cc_config_set(struct net_bridge *br,
                         const u32 instance,
                         const struct br_cfm_cc_config *const config,
                         struct netlink_ext_ack *extack)
{
        struct br_cfm_peer_mep *peer_mep;
        struct br_cfm_mep *mep;

        ASSERT_RTNL();

        mep = br_mep_find(br, instance);
        if (!mep) {
                NL_SET_ERR_MSG_MOD(extack,
                                   "MEP instance does not exists");
                return -ENOENT;
        }

        /* Check for no change in configuration */
        if (memcmp(config, &mep->cc_config, sizeof(*config)) == 0)
                return 0;

        if (config->enable && !mep->cc_config.enable)
                /* CC is enabled */
                hlist_for_each_entry(peer_mep, &mep->peer_mep_list, head)
                        cc_peer_enable(peer_mep);

        if (!config->enable && mep->cc_config.enable)
                /* CC is disabled */
                hlist_for_each_entry(peer_mep, &mep->peer_mep_list, head)
                        cc_peer_disable(peer_mep);

        mep->cc_config = *config;
        mep->ccm_rx_snumber = 0;
        mep->ccm_tx_snumber = 1;

        return 0;
}

int br_cfm_cc_peer_mep_add(struct net_bridge *br, const u32 instance,
                           u32 mepid,
                           struct netlink_ext_ack *extack)
{
        struct br_cfm_peer_mep *peer_mep;
        struct br_cfm_mep *mep;

        ASSERT_RTNL();

        mep = br_mep_find(br, instance);
        if (!mep) {
                NL_SET_ERR_MSG_MOD(extack,
                                   "MEP instance does not exists");
                return -ENOENT;
        }

        peer_mep = br_peer_mep_find(mep, mepid);
        if (peer_mep) {
                NL_SET_ERR_MSG_MOD(extack,
                                   "Peer MEP-ID already exists");
                return -EEXIST;
        }

        peer_mep = kzalloc(sizeof(*peer_mep), GFP_KERNEL);
        if (!peer_mep)
                return -ENOMEM;

        peer_mep->mepid = mepid;
        peer_mep->mep = mep;
        INIT_DELAYED_WORK(&peer_mep->ccm_rx_dwork, ccm_rx_work_expired);

        if (mep->cc_config.enable)
                cc_peer_enable(peer_mep);

        hlist_add_tail_rcu(&peer_mep->head, &mep->peer_mep_list);

        return 0;
}

int br_cfm_cc_peer_mep_remove(struct net_bridge *br, const u32 instance,
                              u32 mepid,
                              struct netlink_ext_ack *extack)
{
        struct br_cfm_peer_mep *peer_mep;
        struct br_cfm_mep *mep;

        ASSERT_RTNL();

        mep = br_mep_find(br, instance);
        if (!mep) {
                NL_SET_ERR_MSG_MOD(extack,
                                   "MEP instance does not exists");
                return -ENOENT;
        }

        peer_mep = br_peer_mep_find(mep, mepid);
        if (!peer_mep) {
                NL_SET_ERR_MSG_MOD(extack,
                                   "Peer MEP-ID does not exists");
                return -ENOENT;
        }

        cc_peer_disable(peer_mep);

        hlist_del_rcu(&peer_mep->head);
        kfree_rcu(peer_mep, rcu);

        return 0;
}

int br_cfm_cc_rdi_set(struct net_bridge *br, const u32 instance,
                      const bool rdi, struct netlink_ext_ack *extack)
{
        struct br_cfm_mep *mep;

        ASSERT_RTNL();

        mep = br_mep_find(br, instance);
        if (!mep) {
                NL_SET_ERR_MSG_MOD(extack,
                                   "MEP instance does not exists");
                return -ENOENT;
        }

        mep->rdi = rdi;

        return 0;
}

int br_cfm_cc_ccm_tx(struct net_bridge *br, const u32 instance,
                     const struct br_cfm_cc_ccm_tx_info *const tx_info,
                     struct netlink_ext_ack *extack)
{
        struct br_cfm_mep *mep;

        ASSERT_RTNL();

        mep = br_mep_find(br, instance);
        if (!mep) {
                NL_SET_ERR_MSG_MOD(extack,
                                   "MEP instance does not exists");
                return -ENOENT;
        }

        if (memcmp(tx_info, &mep->cc_ccm_tx_info, sizeof(*tx_info)) == 0) {
                /* No change in tx_info. */
                if (mep->cc_ccm_tx_info.period == 0)
                        /* Transmission is not enabled - just return */
                        return 0;

                /* Transmission is ongoing, the end time is recalculated */
                mep->ccm_tx_end = jiffies +
                                  usecs_to_jiffies(tx_info->period * 1000000);
                return 0;
        }

        if (tx_info->period == 0 && mep->cc_ccm_tx_info.period == 0)
                /* Some change in info and transmission is not ongoing */
                goto save;

        if (tx_info->period != 0 && mep->cc_ccm_tx_info.period != 0) {
                /* Some change in info and transmission is ongoing
                 * The end time is recalculated
                 */
                mep->ccm_tx_end = jiffies +
                                  usecs_to_jiffies(tx_info->period * 1000000);

                goto save;
        }

        if (tx_info->period == 0 && mep->cc_ccm_tx_info.period != 0) {
                cancel_delayed_work_sync(&mep->ccm_tx_dwork);
                goto save;
        }

        /* Start delayed work to transmit CCM frames. It is done with zero delay
         * to send first frame immediately
         */
        mep->ccm_tx_end = jiffies + usecs_to_jiffies(tx_info->period * 1000000);
        queue_delayed_work(system_wq, &mep->ccm_tx_dwork, 0);

save:
        mep->cc_ccm_tx_info = *tx_info;

        return 0;
}

int br_cfm_mep_count(struct net_bridge *br, u32 *count)
{
        struct br_cfm_mep *mep;

        *count = 0;

        rcu_read_lock();
        hlist_for_each_entry_rcu(mep, &br->mep_list, head)
                *count += 1;
        rcu_read_unlock();

        return 0;
}

int br_cfm_peer_mep_count(struct net_bridge *br, u32 *count)
{
        struct br_cfm_peer_mep *peer_mep;
        struct br_cfm_mep *mep;

        *count = 0;

        rcu_read_lock();
        hlist_for_each_entry_rcu(mep, &br->mep_list, head)
                hlist_for_each_entry_rcu(peer_mep, &mep->peer_mep_list, head)
                        *count += 1;
        rcu_read_unlock();

        return 0;
}

bool br_cfm_created(struct net_bridge *br)
{
        return !hlist_empty(&br->mep_list);
}

/* Deletes the CFM instances on a specific bridge port
 */
void br_cfm_port_del(struct net_bridge *br, struct net_bridge_port *port)
{
        struct hlist_node *n_store;
        struct br_cfm_mep *mep;

        ASSERT_RTNL();

        hlist_for_each_entry_safe(mep, n_store, &br->mep_list, head)
                if (mep->create.ifindex == port->dev->ifindex)
                        mep_delete_implementation(br, mep);
}




































































































 1638 



 1639 





































































































































































































































































































































































































































 1618 


   46 
 1618 
























































































































































































































































































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
#include <linux/gfp.h>
#include <linux/initrd.h>
#include <linux/ioport.h>
#include <linux/swap.h>
#include <linux/memblock.h>
#include <linux/swapfile.h>
#include <linux/swapops.h>
#include <linux/kmemleak.h>
#include <linux/sched/task.h>

#include <asm/set_memory.h>
#include <asm/cpu_device_id.h>
#include <asm/e820/api.h>
#include <asm/init.h>
#include <asm/page.h>
#include <asm/page_types.h>
#include <asm/sections.h>
#include <asm/setup.h>
#include <asm/tlbflush.h>
#include <asm/tlb.h>
#include <asm/proto.h>
#include <asm/dma.h>                /* for MAX_DMA_PFN */
#include <asm/microcode.h>
#include <asm/kaslr.h>
#include <asm/hypervisor.h>
#include <asm/cpufeature.h>
#include <asm/pti.h>
#include <asm/text-patching.h>
#include <asm/memtype.h>
#include <asm/paravirt.h>

/*
 * We need to define the tracepoints somewhere, and tlb.c
 * is only compiled when SMP=y.
 */
#define CREATE_TRACE_POINTS
#include <trace/events/tlb.h>

#include "mm_internal.h"

/*
 * Tables translating between page_cache_type_t and pte encoding.
 *
 * The default values are defined statically as minimal supported mode;
 * WC and WT fall back to UC-.  pat_init() updates these values to support
 * more cache modes, WC and WT, when it is safe to do so.  See pat_init()
 * for the details.  Note, __early_ioremap() used during early boot-time
 * takes pgprot_t (pte encoding) and does not use these tables.
 *
 *   Index into __cachemode2pte_tbl[] is the cachemode.
 *
 *   Index into __pte2cachemode_tbl[] are the caching attribute bits of the pte
 *   (_PAGE_PWT, _PAGE_PCD, _PAGE_PAT) at index bit positions 0, 1, 2.
 */
static uint16_t __cachemode2pte_tbl[_PAGE_CACHE_MODE_NUM] = {
        [_PAGE_CACHE_MODE_WB      ]        = 0         | 0        ,
        [_PAGE_CACHE_MODE_WC      ]        = 0         | _PAGE_PCD,
        [_PAGE_CACHE_MODE_UC_MINUS]        = 0         | _PAGE_PCD,
        [_PAGE_CACHE_MODE_UC      ]        = _PAGE_PWT | _PAGE_PCD,
        [_PAGE_CACHE_MODE_WT      ]        = 0         | _PAGE_PCD,
        [_PAGE_CACHE_MODE_WP      ]        = 0         | _PAGE_PCD,
};

unsigned long cachemode2protval(enum page_cache_mode pcm)
{
        if (likely(pcm == 0))
                return 0;
        return __cachemode2pte_tbl[pcm];
}
EXPORT_SYMBOL(cachemode2protval);

static uint8_t __pte2cachemode_tbl[8] = {
        [__pte2cm_idx( 0        | 0         | 0        )] = _PAGE_CACHE_MODE_WB,
        [__pte2cm_idx(_PAGE_PWT | 0         | 0        )] = _PAGE_CACHE_MODE_UC_MINUS,
        [__pte2cm_idx( 0        | _PAGE_PCD | 0        )] = _PAGE_CACHE_MODE_UC_MINUS,
        [__pte2cm_idx(_PAGE_PWT | _PAGE_PCD | 0        )] = _PAGE_CACHE_MODE_UC,
        [__pte2cm_idx( 0        | 0         | _PAGE_PAT)] = _PAGE_CACHE_MODE_WB,
        [__pte2cm_idx(_PAGE_PWT | 0         | _PAGE_PAT)] = _PAGE_CACHE_MODE_UC_MINUS,
        [__pte2cm_idx(0         | _PAGE_PCD | _PAGE_PAT)] = _PAGE_CACHE_MODE_UC_MINUS,
        [__pte2cm_idx(_PAGE_PWT | _PAGE_PCD | _PAGE_PAT)] = _PAGE_CACHE_MODE_UC,
};

/*
 * Check that the write-protect PAT entry is set for write-protect.
 * To do this without making assumptions how PAT has been set up (Xen has
 * another layout than the kernel), translate the _PAGE_CACHE_MODE_WP cache
 * mode via the __cachemode2pte_tbl[] into protection bits (those protection
 * bits will select a cache mode of WP or better), and then translate the
 * protection bits back into the cache mode using __pte2cm_idx() and the
 * __pte2cachemode_tbl[] array. This will return the really used cache mode.
 */
bool x86_has_pat_wp(void)
{
        uint16_t prot = __cachemode2pte_tbl[_PAGE_CACHE_MODE_WP];

        return __pte2cachemode_tbl[__pte2cm_idx(prot)] == _PAGE_CACHE_MODE_WP;
}

enum page_cache_mode pgprot2cachemode(pgprot_t pgprot)
{
        unsigned long masked;

        masked = pgprot_val(pgprot) & _PAGE_CACHE_MASK;
        if (likely(masked == 0))
                return 0;
        return __pte2cachemode_tbl[__pte2cm_idx(masked)];
}

static unsigned long __initdata pgt_buf_start;
static unsigned long __initdata pgt_buf_end;
static unsigned long __initdata pgt_buf_top;

static unsigned long min_pfn_mapped;

static bool __initdata can_use_brk_pgt = true;

/*
 * Pages returned are already directly mapped.
 *
 * Changing that is likely to break Xen, see commit:
 *
 *    279b706 x86,xen: introduce x86_init.mapping.pagetable_reserve
 *
 * for detailed information.
 */
__ref void *alloc_low_pages(unsigned int num)
{
        unsigned long pfn;
        int i;

        if (after_bootmem) {
                unsigned int order;

                order = get_order((unsigned long)num << PAGE_SHIFT);
                return (void *)__get_free_pages(GFP_ATOMIC | __GFP_ZERO, order);
        }

        if ((pgt_buf_end + num) > pgt_buf_top || !can_use_brk_pgt) {
                unsigned long ret = 0;

                if (min_pfn_mapped < max_pfn_mapped) {
                        ret = memblock_phys_alloc_range(
                                        PAGE_SIZE * num, PAGE_SIZE,
                                        min_pfn_mapped << PAGE_SHIFT,
                                        max_pfn_mapped << PAGE_SHIFT);
                }
                if (!ret && can_use_brk_pgt)
                        ret = __pa(extend_brk(PAGE_SIZE * num, PAGE_SIZE));

                if (!ret)
                        panic("alloc_low_pages: can not alloc memory");

                pfn = ret >> PAGE_SHIFT;
        } else {
                pfn = pgt_buf_end;
                pgt_buf_end += num;
        }

        for (i = 0; i < num; i++) {
                void *adr;

                adr = __va((pfn + i) << PAGE_SHIFT);
                clear_page(adr);
        }

        return __va(pfn << PAGE_SHIFT);
}

/*
 * By default need to be able to allocate page tables below PGD firstly for
 * the 0-ISA_END_ADDRESS range and secondly for the initial PMD_SIZE mapping.
 * With KASLR memory randomization, depending on the machine e820 memory and the
 * PUD alignment, twice that many pages may be needed when KASLR memory
 * randomization is enabled.
 */

#ifndef CONFIG_X86_5LEVEL
#define INIT_PGD_PAGE_TABLES    3
#else
#define INIT_PGD_PAGE_TABLES    4
#endif

#ifndef CONFIG_RANDOMIZE_MEMORY
#define INIT_PGD_PAGE_COUNT      (2 * INIT_PGD_PAGE_TABLES)
#else
#define INIT_PGD_PAGE_COUNT      (4 * INIT_PGD_PAGE_TABLES)
#endif

#define INIT_PGT_BUF_SIZE        (INIT_PGD_PAGE_COUNT * PAGE_SIZE)
RESERVE_BRK(early_pgt_alloc, INIT_PGT_BUF_SIZE);
void  __init early_alloc_pgt_buf(void)
{
        unsigned long tables = INIT_PGT_BUF_SIZE;
        phys_addr_t base;

        base = __pa(extend_brk(tables, PAGE_SIZE));

        pgt_buf_start = base >> PAGE_SHIFT;
        pgt_buf_end = pgt_buf_start;
        pgt_buf_top = pgt_buf_start + (tables >> PAGE_SHIFT);
}

int after_bootmem;

early_param_on_off("gbpages", "nogbpages", direct_gbpages, CONFIG_X86_DIRECT_GBPAGES);

struct map_range {
        unsigned long start;
        unsigned long end;
        unsigned page_size_mask;
};

static int page_size_mask;

/*
 * Save some of cr4 feature set we're using (e.g.  Pentium 4MB
 * enable and PPro Global page enable), so that any CPU's that boot
 * up after us can get the correct flags. Invoked on the boot CPU.
 */
static inline void cr4_set_bits_and_update_boot(unsigned long mask)
{
        mmu_cr4_features |= mask;
        if (trampoline_cr4_features)
                *trampoline_cr4_features = mmu_cr4_features;
        cr4_set_bits(mask);
}

static void __init probe_page_size_mask(void)
{
        /*
         * For pagealloc debugging, identity mapping will use small pages.
         * This will simplify cpa(), which otherwise needs to support splitting
         * large pages into small in interrupt context, etc.
         */
        if (boot_cpu_has(X86_FEATURE_PSE) && !debug_pagealloc_enabled())
                page_size_mask |= 1 << PG_LEVEL_2M;
        else
                direct_gbpages = 0;

        /* Enable PSE if available */
        if (boot_cpu_has(X86_FEATURE_PSE))
                cr4_set_bits_and_update_boot(X86_CR4_PSE);

        /* Enable PGE if available */
        __supported_pte_mask &= ~_PAGE_GLOBAL;
        if (boot_cpu_has(X86_FEATURE_PGE)) {
                cr4_set_bits_and_update_boot(X86_CR4_PGE);
                __supported_pte_mask |= _PAGE_GLOBAL;
        }

        /* By the default is everything supported: */
        __default_kernel_pte_mask = __supported_pte_mask;
        /* Except when with PTI where the kernel is mostly non-Global: */
        if (cpu_feature_enabled(X86_FEATURE_PTI))
                __default_kernel_pte_mask &= ~_PAGE_GLOBAL;

        /* Enable 1 GB linear kernel mappings if available: */
        if (direct_gbpages && boot_cpu_has(X86_FEATURE_GBPAGES)) {
                printk(KERN_INFO "Using GB pages for direct mapping\n");
                page_size_mask |= 1 << PG_LEVEL_1G;
        } else {
                direct_gbpages = 0;
        }
}

/*
 * INVLPG may not properly flush Global entries
 * on these CPUs when PCIDs are enabled.
 */
static const struct x86_cpu_id invlpg_miss_ids[] = {
        X86_MATCH_INTEL_FAM6_MODEL(ALDERLAKE,      0),
        X86_MATCH_INTEL_FAM6_MODEL(ALDERLAKE_L,    0),
        X86_MATCH_INTEL_FAM6_MODEL(ALDERLAKE_N,    0),
        X86_MATCH_INTEL_FAM6_MODEL(RAPTORLAKE,     0),
        X86_MATCH_INTEL_FAM6_MODEL(RAPTORLAKE_P,   0),
        X86_MATCH_INTEL_FAM6_MODEL(RAPTORLAKE_S,   0),
        {}
};

static void setup_pcid(void)
{
        if (!IS_ENABLED(CONFIG_X86_64))
                return;

        if (!boot_cpu_has(X86_FEATURE_PCID))
                return;

        if (x86_match_cpu(invlpg_miss_ids)) {
                pr_info("Incomplete global flushes, disabling PCID");
                setup_clear_cpu_cap(X86_FEATURE_PCID);
                return;
        }

        if (boot_cpu_has(X86_FEATURE_PGE)) {
                /*
                 * This can't be cr4_set_bits_and_update_boot() -- the
                 * trampoline code can't handle CR4.PCIDE and it wouldn't
                 * do any good anyway.  Despite the name,
                 * cr4_set_bits_and_update_boot() doesn't actually cause
                 * the bits in question to remain set all the way through
                 * the secondary boot asm.
                 *
                 * Instead, we brute-force it and set CR4.PCIDE manually in
                 * start_secondary().
                 */
                cr4_set_bits(X86_CR4_PCIDE);

                /*
                 * INVPCID's single-context modes (2/3) only work if we set
                 * X86_CR4_PCIDE, *and* we INVPCID support.  It's unusable
                 * on systems that have X86_CR4_PCIDE clear, or that have
                 * no INVPCID support at all.
                 */
                if (boot_cpu_has(X86_FEATURE_INVPCID))
                        setup_force_cpu_cap(X86_FEATURE_INVPCID_SINGLE);
        } else {
                /*
                 * flush_tlb_all(), as currently implemented, won't work if
                 * PCID is on but PGE is not.  Since that combination
                 * doesn't exist on real hardware, there's no reason to try
                 * to fully support it, but it's polite to avoid corrupting
                 * data if we're on an improperly configured VM.
                 */
                setup_clear_cpu_cap(X86_FEATURE_PCID);
        }
}

#ifdef CONFIG_X86_32
#define NR_RANGE_MR 3
#else /* CONFIG_X86_64 */
#define NR_RANGE_MR 5
#endif

static int __meminit save_mr(struct map_range *mr, int nr_range,
                             unsigned long start_pfn, unsigned long end_pfn,
                             unsigned long page_size_mask)
{
        if (start_pfn < end_pfn) {
                if (nr_range >= NR_RANGE_MR)
                        panic("run out of range for init_memory_mapping\n");
                mr[nr_range].start = start_pfn<<PAGE_SHIFT;
                mr[nr_range].end   = end_pfn<<PAGE_SHIFT;
                mr[nr_range].page_size_mask = page_size_mask;
                nr_range++;
        }

        return nr_range;
}

/*
 * adjust the page_size_mask for small range to go with
 *        big page size instead small one if nearby are ram too.
 */
static void __ref adjust_range_page_size_mask(struct map_range *mr,
                                                         int nr_range)
{
        int i;

        for (i = 0; i < nr_range; i++) {
                if ((page_size_mask & (1<<PG_LEVEL_2M)) &&
                    !(mr[i].page_size_mask & (1<<PG_LEVEL_2M))) {
                        unsigned long start = round_down(mr[i].start, PMD_SIZE);
                        unsigned long end = round_up(mr[i].end, PMD_SIZE);

#ifdef CONFIG_X86_32
                        if ((end >> PAGE_SHIFT) > max_low_pfn)
                                continue;
#endif

                        if (memblock_is_region_memory(start, end - start))
                                mr[i].page_size_mask |= 1<<PG_LEVEL_2M;
                }
                if ((page_size_mask & (1<<PG_LEVEL_1G)) &&
                    !(mr[i].page_size_mask & (1<<PG_LEVEL_1G))) {
                        unsigned long start = round_down(mr[i].start, PUD_SIZE);
                        unsigned long end = round_up(mr[i].end, PUD_SIZE);

                        if (memblock_is_region_memory(start, end - start))
                                mr[i].page_size_mask |= 1<<PG_LEVEL_1G;
                }
        }
}

static const char *page_size_string(struct map_range *mr)
{
        static const char str_1g[] = "1G";
        static const char str_2m[] = "2M";
        static const char str_4m[] = "4M";
        static const char str_4k[] = "4k";

        if (mr->page_size_mask & (1<<PG_LEVEL_1G))
                return str_1g;
        /*
         * 32-bit without PAE has a 4M large page size.
         * PG_LEVEL_2M is misnamed, but we can at least
         * print out the right size in the string.
         */
        if (IS_ENABLED(CONFIG_X86_32) &&
            !IS_ENABLED(CONFIG_X86_PAE) &&
            mr->page_size_mask & (1<<PG_LEVEL_2M))
                return str_4m;

        if (mr->page_size_mask & (1<<PG_LEVEL_2M))
                return str_2m;

        return str_4k;
}

static int __meminit split_mem_range(struct map_range *mr, int nr_range,
                                     unsigned long start,
                                     unsigned long end)
{
        unsigned long start_pfn, end_pfn, limit_pfn;
        unsigned long pfn;
        int i;

        limit_pfn = PFN_DOWN(end);

        /* head if not big page alignment ? */
        pfn = start_pfn = PFN_DOWN(start);
#ifdef CONFIG_X86_32
        /*
         * Don't use a large page for the first 2/4MB of memory
         * because there are often fixed size MTRRs in there
         * and overlapping MTRRs into large pages can cause
         * slowdowns.
         */
        if (pfn == 0)
                end_pfn = PFN_DOWN(PMD_SIZE);
        else
                end_pfn = round_up(pfn, PFN_DOWN(PMD_SIZE));
#else /* CONFIG_X86_64 */
        end_pfn = round_up(pfn, PFN_DOWN(PMD_SIZE));
#endif
        if (end_pfn > limit_pfn)
                end_pfn = limit_pfn;
        if (start_pfn < end_pfn) {
                nr_range = save_mr(mr, nr_range, start_pfn, end_pfn, 0);
                pfn = end_pfn;
        }

        /* big page (2M) range */
        start_pfn = round_up(pfn, PFN_DOWN(PMD_SIZE));
#ifdef CONFIG_X86_32
        end_pfn = round_down(limit_pfn, PFN_DOWN(PMD_SIZE));
#else /* CONFIG_X86_64 */
        end_pfn = round_up(pfn, PFN_DOWN(PUD_SIZE));
        if (end_pfn > round_down(limit_pfn, PFN_DOWN(PMD_SIZE)))
                end_pfn = round_down(limit_pfn, PFN_DOWN(PMD_SIZE));
#endif

        if (start_pfn < end_pfn) {
                nr_range = save_mr(mr, nr_range, start_pfn, end_pfn,
                                page_size_mask & (1<<PG_LEVEL_2M));
                pfn = end_pfn;
        }

#ifdef CONFIG_X86_64
        /* big page (1G) range */
        start_pfn = round_up(pfn, PFN_DOWN(PUD_SIZE));
        end_pfn = round_down(limit_pfn, PFN_DOWN(PUD_SIZE));
        if (start_pfn < end_pfn) {
                nr_range = save_mr(mr, nr_range, start_pfn, end_pfn,
                                page_size_mask &
                                 ((1<<PG_LEVEL_2M)|(1<<PG_LEVEL_1G)));
                pfn = end_pfn;
        }

        /* tail is not big page (1G) alignment */
        start_pfn = round_up(pfn, PFN_DOWN(PMD_SIZE));
        end_pfn = round_down(limit_pfn, PFN_DOWN(PMD_SIZE));
        if (start_pfn < end_pfn) {
                nr_range = save_mr(mr, nr_range, start_pfn, end_pfn,
                                page_size_mask & (1<<PG_LEVEL_2M));
                pfn = end_pfn;
        }
#endif

        /* tail is not big page (2M) alignment */
        start_pfn = pfn;
        end_pfn = limit_pfn;
        nr_range = save_mr(mr, nr_range, start_pfn, end_pfn, 0);

        if (!after_bootmem)
                adjust_range_page_size_mask(mr, nr_range);

        /* try to merge same page size and continuous */
        for (i = 0; nr_range > 1 && i < nr_range - 1; i++) {
                unsigned long old_start;
                if (mr[i].end != mr[i+1].start ||
                    mr[i].page_size_mask != mr[i+1].page_size_mask)
                        continue;
                /* move it */
                old_start = mr[i].start;
                memmove(&mr[i], &mr[i+1],
                        (nr_range - 1 - i) * sizeof(struct map_range));
                mr[i--].start = old_start;
                nr_range--;
        }

        for (i = 0; i < nr_range; i++)
                pr_debug(" [mem %#010lx-%#010lx] page %s\n",
                                mr[i].start, mr[i].end - 1,
                                page_size_string(&mr[i]));

        return nr_range;
}

struct range pfn_mapped[E820_MAX_ENTRIES];
int nr_pfn_mapped;

static void add_pfn_range_mapped(unsigned long start_pfn, unsigned long end_pfn)
{
        nr_pfn_mapped = add_range_with_merge(pfn_mapped, E820_MAX_ENTRIES,
                                             nr_pfn_mapped, start_pfn, end_pfn);
        nr_pfn_mapped = clean_sort_range(pfn_mapped, E820_MAX_ENTRIES);

        max_pfn_mapped = max(max_pfn_mapped, end_pfn);

        if (start_pfn < (1UL<<(32-PAGE_SHIFT)))
                max_low_pfn_mapped = max(max_low_pfn_mapped,
                                         min(end_pfn, 1UL<<(32-PAGE_SHIFT)));
}

bool pfn_range_is_mapped(unsigned long start_pfn, unsigned long end_pfn)
{
        int i;

        for (i = 0; i < nr_pfn_mapped; i++)
                if ((start_pfn >= pfn_mapped[i].start) &&
                    (end_pfn <= pfn_mapped[i].end))
                        return true;

        return false;
}

/*
 * Setup the direct mapping of the physical memory at PAGE_OFFSET.
 * This runs before bootmem is initialized and gets pages directly from
 * the physical memory. To access them they are temporarily mapped.
 */
unsigned long __ref init_memory_mapping(unsigned long start,
                                        unsigned long end, pgprot_t prot)
{
        struct map_range mr[NR_RANGE_MR];
        unsigned long ret = 0;
        int nr_range, i;

        pr_debug("init_memory_mapping: [mem %#010lx-%#010lx]\n",
               start, end - 1);

        memset(mr, 0, sizeof(mr));
        nr_range = split_mem_range(mr, 0, start, end);

        for (i = 0; i < nr_range; i++)
                ret = kernel_physical_mapping_init(mr[i].start, mr[i].end,
                                                   mr[i].page_size_mask,
                                                   prot);

        add_pfn_range_mapped(start >> PAGE_SHIFT, ret >> PAGE_SHIFT);

        return ret >> PAGE_SHIFT;
}

/*
 * We need to iterate through the E820 memory map and create direct mappings
 * for only E820_TYPE_RAM and E820_KERN_RESERVED regions. We cannot simply
 * create direct mappings for all pfns from [0 to max_low_pfn) and
 * [4GB to max_pfn) because of possible memory holes in high addresses
 * that cannot be marked as UC by fixed/variable range MTRRs.
 * Depending on the alignment of E820 ranges, this may possibly result
 * in using smaller size (i.e. 4K instead of 2M or 1G) page tables.
 *
 * init_mem_mapping() calls init_range_memory_mapping() with big range.
 * That range would have hole in the middle or ends, and only ram parts
 * will be mapped in init_range_memory_mapping().
 */
static unsigned long __init init_range_memory_mapping(
                                           unsigned long r_start,
                                           unsigned long r_end)
{
        unsigned long start_pfn, end_pfn;
        unsigned long mapped_ram_size = 0;
        int i;

        for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, NULL) {
                u64 start = clamp_val(PFN_PHYS(start_pfn), r_start, r_end);
                u64 end = clamp_val(PFN_PHYS(end_pfn), r_start, r_end);
                if (start >= end)
                        continue;

                /*
                 * if it is overlapping with brk pgt, we need to
                 * alloc pgt buf from memblock instead.
                 */
                can_use_brk_pgt = max(start, (u64)pgt_buf_end<<PAGE_SHIFT) >=
                                    min(end, (u64)pgt_buf_top<<PAGE_SHIFT);
                init_memory_mapping(start, end, PAGE_KERNEL);
                mapped_ram_size += end - start;
                can_use_brk_pgt = true;
        }

        return mapped_ram_size;
}

static unsigned long __init get_new_step_size(unsigned long step_size)
{
        /*
         * Initial mapped size is PMD_SIZE (2M).
         * We can not set step_size to be PUD_SIZE (1G) yet.
         * In worse case, when we cross the 1G boundary, and
         * PG_LEVEL_2M is not set, we will need 1+1+512 pages (2M + 8k)
         * to map 1G range with PTE. Hence we use one less than the
         * difference of page table level shifts.
         *
         * Don't need to worry about overflow in the top-down case, on 32bit,
         * when step_size is 0, round_down() returns 0 for start, and that
         * turns it into 0x100000000ULL.
         * In the bottom-up case, round_up(x, 0) returns 0 though too, which
         * needs to be taken into consideration by the code below.
         */
        return step_size << (PMD_SHIFT - PAGE_SHIFT - 1);
}

/**
 * memory_map_top_down - Map [map_start, map_end) top down
 * @map_start: start address of the target memory range
 * @map_end: end address of the target memory range
 *
 * This function will setup direct mapping for memory range
 * [map_start, map_end) in top-down. That said, the page tables
 * will be allocated at the end of the memory, and we map the
 * memory in top-down.
 */
static void __init memory_map_top_down(unsigned long map_start,
                                       unsigned long map_end)
{
        unsigned long real_end, last_start;
        unsigned long step_size;
        unsigned long addr;
        unsigned long mapped_ram_size = 0;

        /*
         * Systems that have many reserved areas near top of the memory,
         * e.g. QEMU with less than 1G RAM and EFI enabled, or Xen, will
         * require lots of 4K mappings which may exhaust pgt_buf.
         * Start with top-most PMD_SIZE range aligned at PMD_SIZE to ensure
         * there is enough mapped memory that can be allocated from
         * memblock.
         */
        addr = memblock_phys_alloc_range(PMD_SIZE, PMD_SIZE, map_start,
                                         map_end);
        memblock_free(addr, PMD_SIZE);
        real_end = addr + PMD_SIZE;

        /* step_size need to be small so pgt_buf from BRK could cover it */
        step_size = PMD_SIZE;
        max_pfn_mapped = 0; /* will get exact value next */
        min_pfn_mapped = real_end >> PAGE_SHIFT;
        last_start = real_end;

        /*
         * We start from the top (end of memory) and go to the bottom.
         * The memblock_find_in_range() gets us a block of RAM from the
         * end of RAM in [min_pfn_mapped, max_pfn_mapped) used as new pages
         * for page table.
         */
        while (last_start > map_start) {
                unsigned long start;

                if (last_start > step_size) {
                        start = round_down(last_start - 1, step_size);
                        if (start < map_start)
                                start = map_start;
                } else
                        start = map_start;
                mapped_ram_size += init_range_memory_mapping(start,
                                                        last_start);
                last_start = start;
                min_pfn_mapped = last_start >> PAGE_SHIFT;
                if (mapped_ram_size >= step_size)
                        step_size = get_new_step_size(step_size);
        }

        if (real_end < map_end)
                init_range_memory_mapping(real_end, map_end);
}

/**
 * memory_map_bottom_up - Map [map_start, map_end) bottom up
 * @map_start: start address of the target memory range
 * @map_end: end address of the target memory range
 *
 * This function will setup direct mapping for memory range
 * [map_start, map_end) in bottom-up. Since we have limited the
 * bottom-up allocation above the kernel, the page tables will
 * be allocated just above the kernel and we map the memory
 * in [map_start, map_end) in bottom-up.
 */
static void __init memory_map_bottom_up(unsigned long map_start,
                                        unsigned long map_end)
{
        unsigned long next, start;
        unsigned long mapped_ram_size = 0;
        /* step_size need to be small so pgt_buf from BRK could cover it */
        unsigned long step_size = PMD_SIZE;

        start = map_start;
        min_pfn_mapped = start >> PAGE_SHIFT;

        /*
         * We start from the bottom (@map_start) and go to the top (@map_end).
         * The memblock_find_in_range() gets us a block of RAM from the
         * end of RAM in [min_pfn_mapped, max_pfn_mapped) used as new pages
         * for page table.
         */
        while (start < map_end) {
                if (step_size && map_end - start > step_size) {
                        next = round_up(start + 1, step_size);
                        if (next > map_end)
                                next = map_end;
                } else {
                        next = map_end;
                }

                mapped_ram_size += init_range_memory_mapping(start, next);
                start = next;

                if (mapped_ram_size >= step_size)
                        step_size = get_new_step_size(step_size);
        }
}

/*
 * The real mode trampoline, which is required for bootstrapping CPUs
 * occupies only a small area under the low 1MB.  See reserve_real_mode()
 * for details.
 *
 * If KASLR is disabled the first PGD entry of the direct mapping is copied
 * to map the real mode trampoline.
 *
 * If KASLR is enabled, copy only the PUD which covers the low 1MB
 * area. This limits the randomization granularity to 1GB for both 4-level
 * and 5-level paging.
 */
static void __init init_trampoline(void)
{
#ifdef CONFIG_X86_64
        if (!kaslr_memory_enabled())
                trampoline_pgd_entry = init_top_pgt[pgd_index(__PAGE_OFFSET)];
        else
                init_trampoline_kaslr();
#endif
}

void __init init_mem_mapping(void)
{
        unsigned long end;

        pti_check_boottime_disable();
        probe_page_size_mask();
        setup_pcid();

#ifdef CONFIG_X86_64
        end = max_pfn << PAGE_SHIFT;
#else
        end = max_low_pfn << PAGE_SHIFT;
#endif

        /* the ISA range is always mapped regardless of memory holes */
        init_memory_mapping(0, ISA_END_ADDRESS, PAGE_KERNEL);

        /* Init the trampoline, possibly with KASLR memory offset */
        init_trampoline();

        /*
         * If the allocation is in bottom-up direction, we setup direct mapping
         * in bottom-up, otherwise we setup direct mapping in top-down.
         */
        if (memblock_bottom_up()) {
                unsigned long kernel_end = __pa_symbol(_end);

                /*
                 * we need two separate calls here. This is because we want to
                 * allocate page tables above the kernel. So we first map
                 * [kernel_end, end) to make memory above the kernel be mapped
                 * as soon as possible. And then use page tables allocated above
                 * the kernel to map [ISA_END_ADDRESS, kernel_end).
                 */
                memory_map_bottom_up(kernel_end, end);
                memory_map_bottom_up(ISA_END_ADDRESS, kernel_end);
        } else {
                memory_map_top_down(ISA_END_ADDRESS, end);
        }

#ifdef CONFIG_X86_64
        if (max_pfn > max_low_pfn) {
                /* can we preserve max_low_pfn ?*/
                max_low_pfn = max_pfn;
        }
#else
        early_ioremap_page_table_range_init();
#endif

        load_cr3(swapper_pg_dir);
        __flush_tlb_all();

        x86_init.hyper.init_mem_mapping();

        early_memtest(0, max_pfn_mapped << PAGE_SHIFT);
}

/*
 * Initialize an mm_struct to be used during poking and a pointer to be used
 * during patching.
 */
void __init poking_init(void)
{
        spinlock_t *ptl;
        pte_t *ptep;

        poking_mm = mm_alloc();
        BUG_ON(!poking_mm);

        /* Xen PV guests need the PGD to be pinned. */
        paravirt_arch_dup_mmap(NULL, poking_mm);

        /*
         * Randomize the poking address, but make sure that the following page
         * will be mapped at the same PMD. We need 2 pages, so find space for 3,
         * and adjust the address if the PMD ends after the first one.
         */
        poking_addr = TASK_UNMAPPED_BASE;
        if (IS_ENABLED(CONFIG_RANDOMIZE_BASE))
                poking_addr += (kaslr_get_random_long("Poking") & PAGE_MASK) %
                        (TASK_SIZE - TASK_UNMAPPED_BASE - 3 * PAGE_SIZE);

        if (((poking_addr + PAGE_SIZE) & ~PMD_MASK) == 0)
                poking_addr += PAGE_SIZE;

        /*
         * We need to trigger the allocation of the page-tables that will be
         * needed for poking now. Later, poking may be performed in an atomic
         * section, which might cause allocation to fail.
         */
        ptep = get_locked_pte(poking_mm, poking_addr, &ptl);
        BUG_ON(!ptep);
        pte_unmap_unlock(ptep, ptl);
}

/*
 * devmem_is_allowed() checks to see if /dev/mem access to a certain address
 * is valid. The argument is a physical page number.
 *
 * On x86, access has to be given to the first megabyte of RAM because that
 * area traditionally contains BIOS code and data regions used by X, dosemu,
 * and similar apps. Since they map the entire memory range, the whole range
 * must be allowed (for mapping), but any areas that would otherwise be
 * disallowed are flagged as being "zero filled" instead of rejected.
 * Access has to be given to non-kernel-ram areas as well, these contain the
 * PCI mmio resources as well as potential bios/acpi data regions.
 */
int devmem_is_allowed(unsigned long pagenr)
{
        if (region_intersects(PFN_PHYS(pagenr), PAGE_SIZE,
                                IORESOURCE_SYSTEM_RAM, IORES_DESC_NONE)
                        != REGION_DISJOINT) {
                /*
                 * For disallowed memory regions in the low 1MB range,
                 * request that the page be shown as all zeros.
                 */
                if (pagenr < 256)
                        return 2;

                return 0;
        }

        /*
         * This must follow RAM test, since System RAM is considered a
         * restricted resource under CONFIG_STRICT_IOMEM.
         */
        if (iomem_is_exclusive(pagenr << PAGE_SHIFT)) {
                /* Low 1MB bypasses iomem restrictions. */
                if (pagenr < 256)
                        return 1;

                return 0;
        }

        return 1;
}

void free_init_pages(const char *what, unsigned long begin, unsigned long end)
{
        unsigned long begin_aligned, end_aligned;

        /* Make sure boundaries are page aligned */
        begin_aligned = PAGE_ALIGN(begin);
        end_aligned   = end & PAGE_MASK;

        if (WARN_ON(begin_aligned != begin || end_aligned != end)) {
                begin = begin_aligned;
                end   = end_aligned;
        }

        if (begin >= end)
                return;

        /*
         * If debugging page accesses then do not free this memory but
         * mark them not present - any buggy init-section access will
         * create a kernel page fault:
         */
        if (debug_pagealloc_enabled()) {
                pr_info("debug: unmapping init [mem %#010lx-%#010lx]\n",
                        begin, end - 1);
                /*
                 * Inform kmemleak about the hole in the memory since the
                 * corresponding pages will be unmapped.
                 */
                kmemleak_free_part((void *)begin, end - begin);
                set_memory_np(begin, (end - begin) >> PAGE_SHIFT);
        } else {
                /*
                 * We just marked the kernel text read only above, now that
                 * we are going to free part of that, we need to make that
                 * writeable and non-executable first.
                 */
                set_memory_nx(begin, (end - begin) >> PAGE_SHIFT);
                set_memory_rw(begin, (end - begin) >> PAGE_SHIFT);

                free_reserved_area((void *)begin, (void *)end,
                                   POISON_FREE_INITMEM, what);
        }
}

/*
 * begin/end can be in the direct map or the "high kernel mapping"
 * used for the kernel image only.  free_init_pages() will do the
 * right thing for either kind of address.
 */
void free_kernel_image_pages(const char *what, void *begin, void *end)
{
        unsigned long begin_ul = (unsigned long)begin;
        unsigned long end_ul = (unsigned long)end;
        unsigned long len_pages = (end_ul - begin_ul) >> PAGE_SHIFT;

        free_init_pages(what, begin_ul, end_ul);

        /*
         * PTI maps some of the kernel into userspace.  For performance,
         * this includes some kernel areas that do not contain secrets.
         * Those areas might be adjacent to the parts of the kernel image
         * being freed, which may contain secrets.  Remove the "high kernel
         * image mapping" for these freed areas, ensuring they are not even
         * potentially vulnerable to Meltdown regardless of the specific
         * optimizations PTI is currently using.
         *
         * The "noalias" prevents unmapping the direct map alias which is
         * needed to access the freed pages.
         *
         * This is only valid for 64bit kernels. 32bit has only one mapping
         * which can't be treated in this way for obvious reasons.
         */
        if (IS_ENABLED(CONFIG_X86_64) && cpu_feature_enabled(X86_FEATURE_PTI))
                set_memory_np_noalias(begin_ul, len_pages);
}

void __ref free_initmem(void)
{
        e820__reallocate_tables();

        mem_encrypt_free_decrypted_mem();

        free_kernel_image_pages("unused kernel image (initmem)",
                                &__init_begin, &__init_end);
}

#ifdef CONFIG_BLK_DEV_INITRD
void __init free_initrd_mem(unsigned long start, unsigned long end)
{
        /*
         * end could be not aligned, and We can not align that,
         * decompressor could be confused by aligned initrd_end
         * We already reserve the end partial page before in
         *   - i386_start_kernel()
         *   - x86_64_start_kernel()
         *   - relocate_initrd()
         * So here We can do PAGE_ALIGN() safely to get partial page to be freed
         */
        free_init_pages("initrd", start, PAGE_ALIGN(end));
}
#endif

/*
 * Calculate the precise size of the DMA zone (first 16 MB of RAM),
 * and pass it to the MM layer - to help it set zone watermarks more
 * accurately.
 *
 * Done on 64-bit systems only for the time being, although 32-bit systems
 * might benefit from this as well.
 */
void __init memblock_find_dma_reserve(void)
{
#ifdef CONFIG_X86_64
        u64 nr_pages = 0, nr_free_pages = 0;
        unsigned long start_pfn, end_pfn;
        phys_addr_t start_addr, end_addr;
        int i;
        u64 u;

        /*
         * Iterate over all memory ranges (free and reserved ones alike),
         * to calculate the total number of pages in the first 16 MB of RAM:
         */
        nr_pages = 0;
        for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, NULL) {
                start_pfn = min(start_pfn, MAX_DMA_PFN);
                end_pfn   = min(end_pfn,   MAX_DMA_PFN);

                nr_pages += end_pfn - start_pfn;
        }

        /*
         * Iterate over free memory ranges to calculate the number of free
         * pages in the DMA zone, while not counting potential partial
         * pages at the beginning or the end of the range:
         */
        nr_free_pages = 0;
        for_each_free_mem_range(u, NUMA_NO_NODE, MEMBLOCK_NONE, &start_addr, &end_addr, NULL) {
                start_pfn = min_t(unsigned long, PFN_UP(start_addr), MAX_DMA_PFN);
                end_pfn   = min_t(unsigned long, PFN_DOWN(end_addr), MAX_DMA_PFN);

                if (start_pfn < end_pfn)
                        nr_free_pages += end_pfn - start_pfn;
        }

        set_dma_reserve(nr_pages - nr_free_pages);
#endif
}

void __init zone_sizes_init(void)
{
        unsigned long max_zone_pfns[MAX_NR_ZONES];

        memset(max_zone_pfns, 0, sizeof(max_zone_pfns));

#ifdef CONFIG_ZONE_DMA
        max_zone_pfns[ZONE_DMA]                = min(MAX_DMA_PFN, max_low_pfn);
#endif
#ifdef CONFIG_ZONE_DMA32
        max_zone_pfns[ZONE_DMA32]        = min(MAX_DMA32_PFN, max_low_pfn);
#endif
        max_zone_pfns[ZONE_NORMAL]        = max_low_pfn;
#ifdef CONFIG_HIGHMEM
        max_zone_pfns[ZONE_HIGHMEM]        = max_pfn;
#endif

        free_area_init(max_zone_pfns);
}

__visible DEFINE_PER_CPU_ALIGNED(struct tlb_state, cpu_tlbstate) = {
        .loaded_mm = &init_mm,
        .next_asid = 1,
        .cr4 = ~0UL,        /* fail hard if we screw up cr4 shadow initialization */
};

void update_cache_mode_entry(unsigned entry, enum page_cache_mode cache)
{
        /* entry 0 MUST be WB (hardwired to speed up translations) */
        BUG_ON(!entry && cache != _PAGE_CACHE_MODE_WB);

        __cachemode2pte_tbl[cache] = __cm_idx2pte(entry);
        __pte2cachemode_tbl[entry] = cache;
}

#ifdef CONFIG_SWAP
unsigned long max_swapfile_size(void)
{
        unsigned long pages;

        pages = generic_max_swapfile_size();

        if (boot_cpu_has_bug(X86_BUG_L1TF) && l1tf_mitigation != L1TF_MITIGATION_OFF) {
                /* Limit the swap file size to MAX_PA/2 for L1TF workaround */
                unsigned long long l1tf_limit = l1tf_pfn_limit();
                /*
                 * We encode swap offsets also with 3 bits below those for pfn
                 * which makes the usable limit higher.
                 */
#if CONFIG_PGTABLE_LEVELS > 2
                l1tf_limit <<= PAGE_SHIFT - SWP_OFFSET_FIRST_BIT;
#endif
                pages = min_t(unsigned long long, l1tf_limit, pages);
        }
        return pages;
}
#endif
























































































































   35 


























































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
 * Copyright (c) 2014 Mahesh Bandewar <maheshb@google.com>
 */
#ifndef __IPVLAN_H
#define __IPVLAN_H

#include <linux/kernel.h>
#include <linux/types.h>
#include <linux/module.h>
#include <linux/init.h>
#include <linux/rculist.h>
#include <linux/notifier.h>
#include <linux/netdevice.h>
#include <linux/etherdevice.h>
#include <linux/if_arp.h>
#include <linux/if_link.h>
#include <linux/if_vlan.h>
#include <linux/ip.h>
#include <linux/inetdevice.h>
#include <linux/netfilter.h>
#include <net/ip.h>
#include <net/ip6_route.h>
#include <net/netns/generic.h>
#include <net/rtnetlink.h>
#include <net/route.h>
#include <net/addrconf.h>
#include <net/l3mdev.h>

#define IPVLAN_DRV        "ipvlan"
#define IPV_DRV_VER        "0.1"

#define IPVLAN_HASH_SIZE        (1 << BITS_PER_BYTE)
#define IPVLAN_HASH_MASK        (IPVLAN_HASH_SIZE - 1)

#define IPVLAN_MAC_FILTER_BITS        8
#define IPVLAN_MAC_FILTER_SIZE        (1 << IPVLAN_MAC_FILTER_BITS)
#define IPVLAN_MAC_FILTER_MASK        (IPVLAN_MAC_FILTER_SIZE - 1)

#define IPVLAN_QBACKLOG_LIMIT        1000

typedef enum {
        IPVL_IPV6 = 0,
        IPVL_ICMPV6,
        IPVL_IPV4,
        IPVL_ARP,
} ipvl_hdr_type;

struct ipvl_pcpu_stats {
        u64                        rx_pkts;
        u64                        rx_bytes;
        u64                        rx_mcast;
        u64                        tx_pkts;
        u64                        tx_bytes;
        struct u64_stats_sync        syncp;
        u32                        rx_errs;
        u32                        tx_drps;
};

struct ipvl_port;

struct ipvl_dev {
        struct net_device        *dev;
        struct list_head        pnode;
        struct ipvl_port        *port;
        struct net_device        *phy_dev;
        struct list_head        addrs;
        struct ipvl_pcpu_stats        __percpu *pcpu_stats;
        DECLARE_BITMAP(mac_filters, IPVLAN_MAC_FILTER_SIZE);
        netdev_features_t        sfeatures;
        u32                        msg_enable;
        spinlock_t                addrs_lock;
};

struct ipvl_addr {
        struct ipvl_dev                *master; /* Back pointer to master */
        union {
                struct in6_addr        ip6;         /* IPv6 address on logical interface */
                struct in_addr        ip4;         /* IPv4 address on logical interface */
        } ipu;
#define ip6addr        ipu.ip6
#define ip4addr ipu.ip4
        struct hlist_node        hlnode;  /* Hash-table linkage */
        struct list_head        anode;   /* logical-interface linkage */
        ipvl_hdr_type                atype;
        struct rcu_head                rcu;
};

struct ipvl_port {
        struct net_device        *dev;
        possible_net_t                pnet;
        struct hlist_head        hlhead[IPVLAN_HASH_SIZE];
        struct list_head        ipvlans;
        u16                        mode;
        u16                        flags;
        u16                        dev_id_start;
        struct work_struct        wq;
        struct sk_buff_head        backlog;
        int                        count;
        struct ida                ida;
};

struct ipvl_skb_cb {
        bool tx_pkt;
};
#define IPVL_SKB_CB(_skb) ((struct ipvl_skb_cb *)&((_skb)->cb[0]))

static inline struct ipvl_port *ipvlan_port_get_rcu(const struct net_device *d)
{
        return rcu_dereference(d->rx_handler_data);
}

static inline struct ipvl_port *ipvlan_port_get_rcu_bh(const struct net_device *d)
{
        return rcu_dereference_bh(d->rx_handler_data);
}

static inline struct ipvl_port *ipvlan_port_get_rtnl(const struct net_device *d)
{
        return rtnl_dereference(d->rx_handler_data);
}

static inline bool ipvlan_is_private(const struct ipvl_port *port)
{
        return !!(port->flags & IPVLAN_F_PRIVATE);
}

static inline void ipvlan_mark_private(struct ipvl_port *port)
{
        port->flags |= IPVLAN_F_PRIVATE;
}

static inline void ipvlan_clear_private(struct ipvl_port *port)
{
        port->flags &= ~IPVLAN_F_PRIVATE;
}

static inline bool ipvlan_is_vepa(const struct ipvl_port *port)
{
        return !!(port->flags & IPVLAN_F_VEPA);
}

static inline void ipvlan_mark_vepa(struct ipvl_port *port)
{
        port->flags |= IPVLAN_F_VEPA;
}

static inline void ipvlan_clear_vepa(struct ipvl_port *port)
{
        port->flags &= ~IPVLAN_F_VEPA;
}

void ipvlan_init_secret(void);
unsigned int ipvlan_mac_hash(const unsigned char *addr);
rx_handler_result_t ipvlan_handle_frame(struct sk_buff **pskb);
void ipvlan_process_multicast(struct work_struct *work);
int ipvlan_queue_xmit(struct sk_buff *skb, struct net_device *dev);
void ipvlan_ht_addr_add(struct ipvl_dev *ipvlan, struct ipvl_addr *addr);
struct ipvl_addr *ipvlan_find_addr(const struct ipvl_dev *ipvlan,
                                   const void *iaddr, bool is_v6);
bool ipvlan_addr_busy(struct ipvl_port *port, void *iaddr, bool is_v6);
void ipvlan_ht_addr_del(struct ipvl_addr *addr);
struct ipvl_addr *ipvlan_addr_lookup(struct ipvl_port *port, void *lyr3h,
                                     int addr_type, bool use_dest);
void *ipvlan_get_L3_hdr(struct ipvl_port *port, struct sk_buff *skb, int *type);
void ipvlan_count_rx(const struct ipvl_dev *ipvlan,
                     unsigned int len, bool success, bool mcast);
int ipvlan_link_new(struct net *src_net, struct net_device *dev,
                    struct nlattr *tb[], struct nlattr *data[],
                    struct netlink_ext_ack *extack);
void ipvlan_link_delete(struct net_device *dev, struct list_head *head);
void ipvlan_link_setup(struct net_device *dev);
int ipvlan_link_register(struct rtnl_link_ops *ops);
#ifdef CONFIG_IPVLAN_L3S
int ipvlan_l3s_register(struct ipvl_port *port);
void ipvlan_l3s_unregister(struct ipvl_port *port);
void ipvlan_migrate_l3s_hook(struct net *oldnet, struct net *newnet);
int ipvlan_l3s_init(void);
void ipvlan_l3s_cleanup(void);
#else
static inline int ipvlan_l3s_register(struct ipvl_port *port)
{
        return -ENOTSUPP;
}

static inline void ipvlan_l3s_unregister(struct ipvl_port *port)
{
}

static inline void ipvlan_migrate_l3s_hook(struct net *oldnet,
                                           struct net *newnet)
{
}

static inline int ipvlan_l3s_init(void)
{
        return 0;
}

static inline void ipvlan_l3s_cleanup(void)
{
}
#endif /* CONFIG_IPVLAN_L3S */

static inline bool netif_is_ipvlan_port(const struct net_device *dev)
{
        return rcu_access_pointer(dev->rx_handler) == ipvlan_handle_frame;
}

#endif /* __IPVLAN_H */


















































































































































































































































































































































































































































































































































































































































































































































    6 









    6 















    6 
































































































































































































































































































































































   35 










   35 




   25 
   10 












    3 






    1 



    2 

    7 





    2 







    2 




    8 


    4 








    4 






    7 
    2 


















    2 







   12 



























































































































































































   35 






   34 


   35 






































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
/*
 * Copyright (c) 2006, 2018 Oracle and/or its affiliates. All rights reserved.
 *
 * This software is available to you under a choice of one of two
 * licenses.  You may choose to be licensed under the terms of the GNU
 * General Public License (GPL) Version 2, available from the file
 * COPYING in the main directory of this source tree, or the
 * OpenIB.org BSD license below:
 *
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
 *
 *      - Redistributions of source code must retain the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer.
 *
 *      - Redistributions in binary form must reproduce the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer in the documentation and/or other materials
 *        provided with the distribution.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 */
#include <linux/kernel.h>
#include <linux/moduleparam.h>
#include <linux/gfp.h>
#include <net/sock.h>
#include <linux/in.h>
#include <linux/list.h>
#include <linux/ratelimit.h>
#include <linux/export.h>
#include <linux/sizes.h>

#include "rds.h"

/* When transmitting messages in rds_send_xmit, we need to emerge from
 * time to time and briefly release the CPU. Otherwise the softlock watchdog
 * will kick our shin.
 * Also, it seems fairer to not let one busy connection stall all the
 * others.
 *
 * send_batch_count is the number of times we'll loop in send_xmit. Setting
 * it to 0 will restore the old behavior (where we looped until we had
 * drained the queue).
 */
static int send_batch_count = SZ_1K;
module_param(send_batch_count, int, 0444);
MODULE_PARM_DESC(send_batch_count, " batch factor when working the send queue");

static void rds_send_remove_from_sock(struct list_head *messages, int status);

/*
 * Reset the send state.  Callers must ensure that this doesn't race with
 * rds_send_xmit().
 */
void rds_send_path_reset(struct rds_conn_path *cp)
{
        struct rds_message *rm, *tmp;
        unsigned long flags;

        if (cp->cp_xmit_rm) {
                rm = cp->cp_xmit_rm;
                cp->cp_xmit_rm = NULL;
                /* Tell the user the RDMA op is no longer mapped by the
                 * transport. This isn't entirely true (it's flushed out
                 * independently) but as the connection is down, there's
                 * no ongoing RDMA to/from that memory */
                rds_message_unmapped(rm);
                rds_message_put(rm);
        }

        cp->cp_xmit_sg = 0;
        cp->cp_xmit_hdr_off = 0;
        cp->cp_xmit_data_off = 0;
        cp->cp_xmit_atomic_sent = 0;
        cp->cp_xmit_rdma_sent = 0;
        cp->cp_xmit_data_sent = 0;

        cp->cp_conn->c_map_queued = 0;

        cp->cp_unacked_packets = rds_sysctl_max_unacked_packets;
        cp->cp_unacked_bytes = rds_sysctl_max_unacked_bytes;

        /* Mark messages as retransmissions, and move them to the send q */
        spin_lock_irqsave(&cp->cp_lock, flags);
        list_for_each_entry_safe(rm, tmp, &cp->cp_retrans, m_conn_item) {
                set_bit(RDS_MSG_ACK_REQUIRED, &rm->m_flags);
                set_bit(RDS_MSG_RETRANSMITTED, &rm->m_flags);
        }
        list_splice_init(&cp->cp_retrans, &cp->cp_send_queue);
        spin_unlock_irqrestore(&cp->cp_lock, flags);
}
EXPORT_SYMBOL_GPL(rds_send_path_reset);

static int acquire_in_xmit(struct rds_conn_path *cp)
{
        return test_and_set_bit_lock(RDS_IN_XMIT, &cp->cp_flags) == 0;
}

static void release_in_xmit(struct rds_conn_path *cp)
{
        clear_bit_unlock(RDS_IN_XMIT, &cp->cp_flags);
        /*
         * We don't use wait_on_bit()/wake_up_bit() because our waking is in a
         * hot path and finding waiters is very rare.  We don't want to walk
         * the system-wide hashed waitqueue buckets in the fast path only to
         * almost never find waiters.
         */
        if (waitqueue_active(&cp->cp_waitq))
                wake_up_all(&cp->cp_waitq);
}

/*
 * We're making the conscious trade-off here to only send one message
 * down the connection at a time.
 *   Pro:
 *      - tx queueing is a simple fifo list
 *           - reassembly is optional and easily done by transports per conn
 *      - no per flow rx lookup at all, straight to the socket
 *           - less per-frag memory and wire overhead
 *   Con:
 *      - queued acks can be delayed behind large messages
 *   Depends:
 *      - small message latency is higher behind queued large messages
 *      - large message latency isn't starved by intervening small sends
 */
int rds_send_xmit(struct rds_conn_path *cp)
{
        struct rds_connection *conn = cp->cp_conn;
        struct rds_message *rm;
        unsigned long flags;
        unsigned int tmp;
        struct scatterlist *sg;
        int ret = 0;
        LIST_HEAD(to_be_dropped);
        int batch_count;
        unsigned long send_gen = 0;
        int same_rm = 0;

restart:
        batch_count = 0;

        /*
         * sendmsg calls here after having queued its message on the send
         * queue.  We only have one task feeding the connection at a time.  If
         * another thread is already feeding the queue then we back off.  This
         * avoids blocking the caller and trading per-connection data between
         * caches per message.
         */
        if (!acquire_in_xmit(cp)) {
                rds_stats_inc(s_send_lock_contention);
                ret = -ENOMEM;
                goto out;
        }

        if (rds_destroy_pending(cp->cp_conn)) {
                release_in_xmit(cp);
                ret = -ENETUNREACH; /* dont requeue send work */
                goto out;
        }

        /*
         * we record the send generation after doing the xmit acquire.
         * if someone else manages to jump in and do some work, we'll use
         * this to avoid a goto restart farther down.
         *
         * The acquire_in_xmit() check above ensures that only one
         * caller can increment c_send_gen at any time.
         */
        send_gen = READ_ONCE(cp->cp_send_gen) + 1;
        WRITE_ONCE(cp->cp_send_gen, send_gen);

        /*
         * rds_conn_shutdown() sets the conn state and then tests RDS_IN_XMIT,
         * we do the opposite to avoid races.
         */
        if (!rds_conn_path_up(cp)) {
                release_in_xmit(cp);
                ret = 0;
                goto out;
        }

        if (conn->c_trans->xmit_path_prepare)
                conn->c_trans->xmit_path_prepare(cp);

        /*
         * spin trying to push headers and data down the connection until
         * the connection doesn't make forward progress.
         */
        while (1) {

                rm = cp->cp_xmit_rm;

                if (!rm) {
                        same_rm = 0;
                } else {
                        same_rm++;
                        if (same_rm >= 4096) {
                                rds_stats_inc(s_send_stuck_rm);
                                ret = -EAGAIN;
                                break;
                        }
                }

                /*
                 * If between sending messages, we can send a pending congestion
                 * map update.
                 */
                if (!rm && test_and_clear_bit(0, &conn->c_map_queued)) {
                        rm = rds_cong_update_alloc(conn);
                        if (IS_ERR(rm)) {
                                ret = PTR_ERR(rm);
                                break;
                        }
                        rm->data.op_active = 1;
                        rm->m_inc.i_conn_path = cp;
                        rm->m_inc.i_conn = cp->cp_conn;

                        cp->cp_xmit_rm = rm;
                }

                /*
                 * If not already working on one, grab the next message.
                 *
                 * cp_xmit_rm holds a ref while we're sending this message down
                 * the connction.  We can use this ref while holding the
                 * send_sem.. rds_send_reset() is serialized with it.
                 */
                if (!rm) {
                        unsigned int len;

                        batch_count++;

                        /* we want to process as big a batch as we can, but
                         * we also want to avoid softlockups.  If we've been
                         * through a lot of messages, lets back off and see
                         * if anyone else jumps in
                         */
                        if (batch_count >= send_batch_count)
                                goto over_batch;

                        spin_lock_irqsave(&cp->cp_lock, flags);

                        if (!list_empty(&cp->cp_send_queue)) {
                                rm = list_entry(cp->cp_send_queue.next,
                                                struct rds_message,
                                                m_conn_item);
                                rds_message_addref(rm);

                                /*
                                 * Move the message from the send queue to the retransmit
                                 * list right away.
                                 */
                                list_move_tail(&rm->m_conn_item,
                                               &cp->cp_retrans);
                        }

                        spin_unlock_irqrestore(&cp->cp_lock, flags);

                        if (!rm)
                                break;

                        /* Unfortunately, the way Infiniband deals with
                         * RDMA to a bad MR key is by moving the entire
                         * queue pair to error state. We cold possibly
                         * recover from that, but right now we drop the
                         * connection.
                         * Therefore, we never retransmit messages with RDMA ops.
                         */
                        if (test_bit(RDS_MSG_FLUSH, &rm->m_flags) ||
                            (rm->rdma.op_active &&
                            test_bit(RDS_MSG_RETRANSMITTED, &rm->m_flags))) {
                                spin_lock_irqsave(&cp->cp_lock, flags);
                                if (test_and_clear_bit(RDS_MSG_ON_CONN, &rm->m_flags))
                                        list_move(&rm->m_conn_item, &to_be_dropped);
                                spin_unlock_irqrestore(&cp->cp_lock, flags);
                                continue;
                        }

                        /* Require an ACK every once in a while */
                        len = ntohl(rm->m_inc.i_hdr.h_len);
                        if (cp->cp_unacked_packets == 0 ||
                            cp->cp_unacked_bytes < len) {
                                set_bit(RDS_MSG_ACK_REQUIRED, &rm->m_flags);

                                cp->cp_unacked_packets =
                                        rds_sysctl_max_unacked_packets;
                                cp->cp_unacked_bytes =
                                        rds_sysctl_max_unacked_bytes;
                                rds_stats_inc(s_send_ack_required);
                        } else {
                                cp->cp_unacked_bytes -= len;
                                cp->cp_unacked_packets--;
                        }

                        cp->cp_xmit_rm = rm;
                }

                /* The transport either sends the whole rdma or none of it */
                if (rm->rdma.op_active && !cp->cp_xmit_rdma_sent) {
                        rm->m_final_op = &rm->rdma;
                        /* The transport owns the mapped memory for now.
                         * You can't unmap it while it's on the send queue
                         */
                        set_bit(RDS_MSG_MAPPED, &rm->m_flags);
                        ret = conn->c_trans->xmit_rdma(conn, &rm->rdma);
                        if (ret) {
                                clear_bit(RDS_MSG_MAPPED, &rm->m_flags);
                                wake_up_interruptible(&rm->m_flush_wait);
                                break;
                        }
                        cp->cp_xmit_rdma_sent = 1;

                }

                if (rm->atomic.op_active && !cp->cp_xmit_atomic_sent) {
                        rm->m_final_op = &rm->atomic;
                        /* The transport owns the mapped memory for now.
                         * You can't unmap it while it's on the send queue
                         */
                        set_bit(RDS_MSG_MAPPED, &rm->m_flags);
                        ret = conn->c_trans->xmit_atomic(conn, &rm->atomic);
                        if (ret) {
                                clear_bit(RDS_MSG_MAPPED, &rm->m_flags);
                                wake_up_interruptible(&rm->m_flush_wait);
                                break;
                        }
                        cp->cp_xmit_atomic_sent = 1;

                }

                /*
                 * A number of cases require an RDS header to be sent
                 * even if there is no data.
                 * We permit 0-byte sends; rds-ping depends on this.
                 * However, if there are exclusively attached silent ops,
                 * we skip the hdr/data send, to enable silent operation.
                 */
                if (rm->data.op_nents == 0) {
                        int ops_present;
                        int all_ops_are_silent = 1;

                        ops_present = (rm->atomic.op_active || rm->rdma.op_active);
                        if (rm->atomic.op_active && !rm->atomic.op_silent)
                                all_ops_are_silent = 0;
                        if (rm->rdma.op_active && !rm->rdma.op_silent)
                                all_ops_are_silent = 0;

                        if (ops_present && all_ops_are_silent
                            && !rm->m_rdma_cookie)
                                rm->data.op_active = 0;
                }

                if (rm->data.op_active && !cp->cp_xmit_data_sent) {
                        rm->m_final_op = &rm->data;

                        ret = conn->c_trans->xmit(conn, rm,
                                                  cp->cp_xmit_hdr_off,
                                                  cp->cp_xmit_sg,
                                                  cp->cp_xmit_data_off);
                        if (ret <= 0)
                                break;

                        if (cp->cp_xmit_hdr_off < sizeof(struct rds_header)) {
                                tmp = min_t(int, ret,
                                            sizeof(struct rds_header) -
                                            cp->cp_xmit_hdr_off);
                                cp->cp_xmit_hdr_off += tmp;
                                ret -= tmp;
                        }

                        sg = &rm->data.op_sg[cp->cp_xmit_sg];
                        while (ret) {
                                tmp = min_t(int, ret, sg->length -
                                                      cp->cp_xmit_data_off);
                                cp->cp_xmit_data_off += tmp;
                                ret -= tmp;
                                if (cp->cp_xmit_data_off == sg->length) {
                                        cp->cp_xmit_data_off = 0;
                                        sg++;
                                        cp->cp_xmit_sg++;
                                        BUG_ON(ret != 0 && cp->cp_xmit_sg ==
                                               rm->data.op_nents);
                                }
                        }

                        if (cp->cp_xmit_hdr_off == sizeof(struct rds_header) &&
                            (cp->cp_xmit_sg == rm->data.op_nents))
                                cp->cp_xmit_data_sent = 1;
                }

                /*
                 * A rm will only take multiple times through this loop
                 * if there is a data op. Thus, if the data is sent (or there was
                 * none), then we're done with the rm.
                 */
                if (!rm->data.op_active || cp->cp_xmit_data_sent) {
                        cp->cp_xmit_rm = NULL;
                        cp->cp_xmit_sg = 0;
                        cp->cp_xmit_hdr_off = 0;
                        cp->cp_xmit_data_off = 0;
                        cp->cp_xmit_rdma_sent = 0;
                        cp->cp_xmit_atomic_sent = 0;
                        cp->cp_xmit_data_sent = 0;

                        rds_message_put(rm);
                }
        }

over_batch:
        if (conn->c_trans->xmit_path_complete)
                conn->c_trans->xmit_path_complete(cp);
        release_in_xmit(cp);

        /* Nuke any messages we decided not to retransmit. */
        if (!list_empty(&to_be_dropped)) {
                /* irqs on here, so we can put(), unlike above */
                list_for_each_entry(rm, &to_be_dropped, m_conn_item)
                        rds_message_put(rm);
                rds_send_remove_from_sock(&to_be_dropped, RDS_RDMA_DROPPED);
        }

        /*
         * Other senders can queue a message after we last test the send queue
         * but before we clear RDS_IN_XMIT.  In that case they'd back off and
         * not try and send their newly queued message.  We need to check the
         * send queue after having cleared RDS_IN_XMIT so that their message
         * doesn't get stuck on the send queue.
         *
         * If the transport cannot continue (i.e ret != 0), then it must
         * call us when more room is available, such as from the tx
         * completion handler.
         *
         * We have an extra generation check here so that if someone manages
         * to jump in after our release_in_xmit, we'll see that they have done
         * some work and we will skip our goto
         */
        if (ret == 0) {
                bool raced;

                smp_mb();
                raced = send_gen != READ_ONCE(cp->cp_send_gen);

                if ((test_bit(0, &conn->c_map_queued) ||
                    !list_empty(&cp->cp_send_queue)) && !raced) {
                        if (batch_count < send_batch_count)
                                goto restart;
                        rcu_read_lock();
                        if (rds_destroy_pending(cp->cp_conn))
                                ret = -ENETUNREACH;
                        else
                                queue_delayed_work(rds_wq, &cp->cp_send_w, 1);
                        rcu_read_unlock();
                } else if (raced) {
                        rds_stats_inc(s_send_lock_queue_raced);
                }
        }
out:
        return ret;
}
EXPORT_SYMBOL_GPL(rds_send_xmit);

static void rds_send_sndbuf_remove(struct rds_sock *rs, struct rds_message *rm)
{
        u32 len = be32_to_cpu(rm->m_inc.i_hdr.h_len);

        assert_spin_locked(&rs->rs_lock);

        BUG_ON(rs->rs_snd_bytes < len);
        rs->rs_snd_bytes -= len;

        if (rs->rs_snd_bytes == 0)
                rds_stats_inc(s_send_queue_empty);
}

static inline int rds_send_is_acked(struct rds_message *rm, u64 ack,
                                    is_acked_func is_acked)
{
        if (is_acked)
                return is_acked(rm, ack);
        return be64_to_cpu(rm->m_inc.i_hdr.h_sequence) <= ack;
}

/*
 * This is pretty similar to what happens below in the ACK
 * handling code - except that we call here as soon as we get
 * the IB send completion on the RDMA op and the accompanying
 * message.
 */
void rds_rdma_send_complete(struct rds_message *rm, int status)
{
        struct rds_sock *rs = NULL;
        struct rm_rdma_op *ro;
        struct rds_notifier *notifier;
        unsigned long flags;

        spin_lock_irqsave(&rm->m_rs_lock, flags);

        ro = &rm->rdma;
        if (test_bit(RDS_MSG_ON_SOCK, &rm->m_flags) &&
            ro->op_active && ro->op_notify && ro->op_notifier) {
                notifier = ro->op_notifier;
                rs = rm->m_rs;
                sock_hold(rds_rs_to_sk(rs));

                notifier->n_status = status;
                spin_lock(&rs->rs_lock);
                list_add_tail(&notifier->n_list, &rs->rs_notify_queue);
                spin_unlock(&rs->rs_lock);

                ro->op_notifier = NULL;
        }

        spin_unlock_irqrestore(&rm->m_rs_lock, flags);

        if (rs) {
                rds_wake_sk_sleep(rs);
                sock_put(rds_rs_to_sk(rs));
        }
}
EXPORT_SYMBOL_GPL(rds_rdma_send_complete);

/*
 * Just like above, except looks at atomic op
 */
void rds_atomic_send_complete(struct rds_message *rm, int status)
{
        struct rds_sock *rs = NULL;
        struct rm_atomic_op *ao;
        struct rds_notifier *notifier;
        unsigned long flags;

        spin_lock_irqsave(&rm->m_rs_lock, flags);

        ao = &rm->atomic;
        if (test_bit(RDS_MSG_ON_SOCK, &rm->m_flags)
            && ao->op_active && ao->op_notify && ao->op_notifier) {
                notifier = ao->op_notifier;
                rs = rm->m_rs;
                sock_hold(rds_rs_to_sk(rs));

                notifier->n_status = status;
                spin_lock(&rs->rs_lock);
                list_add_tail(&notifier->n_list, &rs->rs_notify_queue);
                spin_unlock(&rs->rs_lock);

                ao->op_notifier = NULL;
        }

        spin_unlock_irqrestore(&rm->m_rs_lock, flags);

        if (rs) {
                rds_wake_sk_sleep(rs);
                sock_put(rds_rs_to_sk(rs));
        }
}
EXPORT_SYMBOL_GPL(rds_atomic_send_complete);

/*
 * This is the same as rds_rdma_send_complete except we
 * don't do any locking - we have all the ingredients (message,
 * socket, socket lock) and can just move the notifier.
 */
static inline void
__rds_send_complete(struct rds_sock *rs, struct rds_message *rm, int status)
{
        struct rm_rdma_op *ro;
        struct rm_atomic_op *ao;

        ro = &rm->rdma;
        if (ro->op_active && ro->op_notify && ro->op_notifier) {
                ro->op_notifier->n_status = status;
                list_add_tail(&ro->op_notifier->n_list, &rs->rs_notify_queue);
                ro->op_notifier = NULL;
        }

        ao = &rm->atomic;
        if (ao->op_active && ao->op_notify && ao->op_notifier) {
                ao->op_notifier->n_status = status;
                list_add_tail(&ao->op_notifier->n_list, &rs->rs_notify_queue);
                ao->op_notifier = NULL;
        }

        /* No need to wake the app - caller does this */
}

/*
 * This removes messages from the socket's list if they're on it.  The list
 * argument must be private to the caller, we must be able to modify it
 * without locks.  The messages must have a reference held for their
 * position on the list.  This function will drop that reference after
 * removing the messages from the 'messages' list regardless of if it found
 * the messages on the socket list or not.
 */
static void rds_send_remove_from_sock(struct list_head *messages, int status)
{
        unsigned long flags;
        struct rds_sock *rs = NULL;
        struct rds_message *rm;

        while (!list_empty(messages)) {
                int was_on_sock = 0;

                rm = list_entry(messages->next, struct rds_message,
                                m_conn_item);
                list_del_init(&rm->m_conn_item);

                /*
                 * If we see this flag cleared then we're *sure* that someone
                 * else beat us to removing it from the sock.  If we race
                 * with their flag update we'll get the lock and then really
                 * see that the flag has been cleared.
                 *
                 * The message spinlock makes sure nobody clears rm->m_rs
                 * while we're messing with it. It does not prevent the
                 * message from being removed from the socket, though.
                 */
                spin_lock_irqsave(&rm->m_rs_lock, flags);
                if (!test_bit(RDS_MSG_ON_SOCK, &rm->m_flags))
                        goto unlock_and_drop;

                if (rs != rm->m_rs) {
                        if (rs) {
                                rds_wake_sk_sleep(rs);
                                sock_put(rds_rs_to_sk(rs));
                        }
                        rs = rm->m_rs;
                        if (rs)
                                sock_hold(rds_rs_to_sk(rs));
                }
                if (!rs)
                        goto unlock_and_drop;
                spin_lock(&rs->rs_lock);

                if (test_and_clear_bit(RDS_MSG_ON_SOCK, &rm->m_flags)) {
                        struct rm_rdma_op *ro = &rm->rdma;
                        struct rds_notifier *notifier;

                        list_del_init(&rm->m_sock_item);
                        rds_send_sndbuf_remove(rs, rm);

                        if (ro->op_active && ro->op_notifier &&
                               (ro->op_notify || (ro->op_recverr && status))) {
                                notifier = ro->op_notifier;
                                list_add_tail(&notifier->n_list,
                                                &rs->rs_notify_queue);
                                if (!notifier->n_status)
                                        notifier->n_status = status;
                                rm->rdma.op_notifier = NULL;
                        }
                        was_on_sock = 1;
                }
                spin_unlock(&rs->rs_lock);

unlock_and_drop:
                spin_unlock_irqrestore(&rm->m_rs_lock, flags);
                rds_message_put(rm);
                if (was_on_sock)
                        rds_message_put(rm);
        }

        if (rs) {
                rds_wake_sk_sleep(rs);
                sock_put(rds_rs_to_sk(rs));
        }
}

/*
 * Transports call here when they've determined that the receiver queued
 * messages up to, and including, the given sequence number.  Messages are
 * moved to the retrans queue when rds_send_xmit picks them off the send
 * queue. This means that in the TCP case, the message may not have been
 * assigned the m_ack_seq yet - but that's fine as long as tcp_is_acked
 * checks the RDS_MSG_HAS_ACK_SEQ bit.
 */
void rds_send_path_drop_acked(struct rds_conn_path *cp, u64 ack,
                              is_acked_func is_acked)
{
        struct rds_message *rm, *tmp;
        unsigned long flags;
        LIST_HEAD(list);

        spin_lock_irqsave(&cp->cp_lock, flags);

        list_for_each_entry_safe(rm, tmp, &cp->cp_retrans, m_conn_item) {
                if (!rds_send_is_acked(rm, ack, is_acked))
                        break;

                list_move(&rm->m_conn_item, &list);
                clear_bit(RDS_MSG_ON_CONN, &rm->m_flags);
        }

        /* order flag updates with spin locks */
        if (!list_empty(&list))
                smp_mb__after_atomic();

        spin_unlock_irqrestore(&cp->cp_lock, flags);

        /* now remove the messages from the sock list as needed */
        rds_send_remove_from_sock(&list, RDS_RDMA_SUCCESS);
}
EXPORT_SYMBOL_GPL(rds_send_path_drop_acked);

void rds_send_drop_acked(struct rds_connection *conn, u64 ack,
                         is_acked_func is_acked)
{
        WARN_ON(conn->c_trans->t_mp_capable);
        rds_send_path_drop_acked(&conn->c_path[0], ack, is_acked);
}
EXPORT_SYMBOL_GPL(rds_send_drop_acked);

void rds_send_drop_to(struct rds_sock *rs, struct sockaddr_in6 *dest)
{
        struct rds_message *rm, *tmp;
        struct rds_connection *conn;
        struct rds_conn_path *cp;
        unsigned long flags;
        LIST_HEAD(list);

        /* get all the messages we're dropping under the rs lock */
        spin_lock_irqsave(&rs->rs_lock, flags);

        list_for_each_entry_safe(rm, tmp, &rs->rs_send_queue, m_sock_item) {
                if (dest &&
                    (!ipv6_addr_equal(&dest->sin6_addr, &rm->m_daddr) ||
                     dest->sin6_port != rm->m_inc.i_hdr.h_dport))
                        continue;

                list_move(&rm->m_sock_item, &list);
                rds_send_sndbuf_remove(rs, rm);
                clear_bit(RDS_MSG_ON_SOCK, &rm->m_flags);
        }

        /* order flag updates with the rs lock */
        smp_mb__after_atomic();

        spin_unlock_irqrestore(&rs->rs_lock, flags);

        if (list_empty(&list))
                return;

        /* Remove the messages from the conn */
        list_for_each_entry(rm, &list, m_sock_item) {

                conn = rm->m_inc.i_conn;
                if (conn->c_trans->t_mp_capable)
                        cp = rm->m_inc.i_conn_path;
                else
                        cp = &conn->c_path[0];

                spin_lock_irqsave(&cp->cp_lock, flags);
                /*
                 * Maybe someone else beat us to removing rm from the conn.
                 * If we race with their flag update we'll get the lock and
                 * then really see that the flag has been cleared.
                 */
                if (!test_and_clear_bit(RDS_MSG_ON_CONN, &rm->m_flags)) {
                        spin_unlock_irqrestore(&cp->cp_lock, flags);
                        continue;
                }
                list_del_init(&rm->m_conn_item);
                spin_unlock_irqrestore(&cp->cp_lock, flags);

                /*
                 * Couldn't grab m_rs_lock in top loop (lock ordering),
                 * but we can now.
                 */
                spin_lock_irqsave(&rm->m_rs_lock, flags);

                spin_lock(&rs->rs_lock);
                __rds_send_complete(rs, rm, RDS_RDMA_CANCELED);
                spin_unlock(&rs->rs_lock);

                spin_unlock_irqrestore(&rm->m_rs_lock, flags);

                rds_message_put(rm);
        }

        rds_wake_sk_sleep(rs);

        while (!list_empty(&list)) {
                rm = list_entry(list.next, struct rds_message, m_sock_item);
                list_del_init(&rm->m_sock_item);
                rds_message_wait(rm);

                /* just in case the code above skipped this message
                 * because RDS_MSG_ON_CONN wasn't set, run it again here
                 * taking m_rs_lock is the only thing that keeps us
                 * from racing with ack processing.
                 */
                spin_lock_irqsave(&rm->m_rs_lock, flags);

                spin_lock(&rs->rs_lock);
                __rds_send_complete(rs, rm, RDS_RDMA_CANCELED);
                spin_unlock(&rs->rs_lock);

                spin_unlock_irqrestore(&rm->m_rs_lock, flags);

                rds_message_put(rm);
        }
}

/*
 * we only want this to fire once so we use the callers 'queued'.  It's
 * possible that another thread can race with us and remove the
 * message from the flow with RDS_CANCEL_SENT_TO.
 */
static int rds_send_queue_rm(struct rds_sock *rs, struct rds_connection *conn,
                             struct rds_conn_path *cp,
                             struct rds_message *rm, __be16 sport,
                             __be16 dport, int *queued)
{
        unsigned long flags;
        u32 len;

        if (*queued)
                goto out;

        len = be32_to_cpu(rm->m_inc.i_hdr.h_len);

        /* this is the only place which holds both the socket's rs_lock
         * and the connection's c_lock */
        spin_lock_irqsave(&rs->rs_lock, flags);

        /*
         * If there is a little space in sndbuf, we don't queue anything,
         * and userspace gets -EAGAIN. But poll() indicates there's send
         * room. This can lead to bad behavior (spinning) if snd_bytes isn't
         * freed up by incoming acks. So we check the *old* value of
         * rs_snd_bytes here to allow the last msg to exceed the buffer,
         * and poll() now knows no more data can be sent.
         */
        if (rs->rs_snd_bytes < rds_sk_sndbuf(rs)) {
                rs->rs_snd_bytes += len;

                /* let recv side know we are close to send space exhaustion.
                 * This is probably not the optimal way to do it, as this
                 * means we set the flag on *all* messages as soon as our
                 * throughput hits a certain threshold.
                 */
                if (rs->rs_snd_bytes >= rds_sk_sndbuf(rs) / 2)
                        set_bit(RDS_MSG_ACK_REQUIRED, &rm->m_flags);

                list_add_tail(&rm->m_sock_item, &rs->rs_send_queue);
                set_bit(RDS_MSG_ON_SOCK, &rm->m_flags);
                rds_message_addref(rm);
                sock_hold(rds_rs_to_sk(rs));
                rm->m_rs = rs;

                /* The code ordering is a little weird, but we're
                   trying to minimize the time we hold c_lock */
                rds_message_populate_header(&rm->m_inc.i_hdr, sport, dport, 0);
                rm->m_inc.i_conn = conn;
                rm->m_inc.i_conn_path = cp;
                rds_message_addref(rm);

                spin_lock(&cp->cp_lock);
                rm->m_inc.i_hdr.h_sequence = cpu_to_be64(cp->cp_next_tx_seq++);
                list_add_tail(&rm->m_conn_item, &cp->cp_send_queue);
                set_bit(RDS_MSG_ON_CONN, &rm->m_flags);
                spin_unlock(&cp->cp_lock);

                rdsdebug("queued msg %p len %d, rs %p bytes %d seq %llu\n",
                         rm, len, rs, rs->rs_snd_bytes,
                         (unsigned long long)be64_to_cpu(rm->m_inc.i_hdr.h_sequence));

                *queued = 1;
        }

        spin_unlock_irqrestore(&rs->rs_lock, flags);
out:
        return *queued;
}

/*
 * rds_message is getting to be quite complicated, and we'd like to allocate
 * it all in one go. This figures out how big it needs to be up front.
 */
static int rds_rm_size(struct msghdr *msg, int num_sgs,
                       struct rds_iov_vector_arr *vct)
{
        struct cmsghdr *cmsg;
        int size = 0;
        int cmsg_groups = 0;
        int retval;
        bool zcopy_cookie = false;
        struct rds_iov_vector *iov, *tmp_iov;

        if (num_sgs < 0)
                return -EINVAL;

        for_each_cmsghdr(cmsg, msg) {
                if (!CMSG_OK(msg, cmsg))
                        return -EINVAL;

                if (cmsg->cmsg_level != SOL_RDS)
                        continue;

                switch (cmsg->cmsg_type) {
                case RDS_CMSG_RDMA_ARGS:
                        if (vct->indx >= vct->len) {
                                vct->len += vct->incr;
                                tmp_iov =
                                        krealloc(vct->vec,
                                                 vct->len *
                                                 sizeof(struct rds_iov_vector),
                                                 GFP_KERNEL);
                                if (!tmp_iov) {
                                        vct->len -= vct->incr;
                                        return -ENOMEM;
                                }
                                vct->vec = tmp_iov;
                        }
                        iov = &vct->vec[vct->indx];
                        memset(iov, 0, sizeof(struct rds_iov_vector));
                        vct->indx++;
                        cmsg_groups |= 1;
                        retval = rds_rdma_extra_size(CMSG_DATA(cmsg), iov);
                        if (retval < 0)
                                return retval;
                        size += retval;

                        break;

                case RDS_CMSG_ZCOPY_COOKIE:
                        zcopy_cookie = true;
                        fallthrough;

                case RDS_CMSG_RDMA_DEST:
                case RDS_CMSG_RDMA_MAP:
                        cmsg_groups |= 2;
                        /* these are valid but do no add any size */
                        break;

                case RDS_CMSG_ATOMIC_CSWP:
                case RDS_CMSG_ATOMIC_FADD:
                case RDS_CMSG_MASKED_ATOMIC_CSWP:
                case RDS_CMSG_MASKED_ATOMIC_FADD:
                        cmsg_groups |= 1;
                        size += sizeof(struct scatterlist);
                        break;

                default:
                        return -EINVAL;
                }

        }

        if ((msg->msg_flags & MSG_ZEROCOPY) && !zcopy_cookie)
                return -EINVAL;

        size += num_sgs * sizeof(struct scatterlist);

        /* Ensure (DEST, MAP) are never used with (ARGS, ATOMIC) */
        if (cmsg_groups == 3)
                return -EINVAL;

        return size;
}

static int rds_cmsg_zcopy(struct rds_sock *rs, struct rds_message *rm,
                          struct cmsghdr *cmsg)
{
        u32 *cookie;

        if (cmsg->cmsg_len < CMSG_LEN(sizeof(*cookie)) ||
            !rm->data.op_mmp_znotifier)
                return -EINVAL;
        cookie = CMSG_DATA(cmsg);
        rm->data.op_mmp_znotifier->z_cookie = *cookie;
        return 0;
}

static int rds_cmsg_send(struct rds_sock *rs, struct rds_message *rm,
                         struct msghdr *msg, int *allocated_mr,
                         struct rds_iov_vector_arr *vct)
{
        struct cmsghdr *cmsg;
        int ret = 0, ind = 0;

        for_each_cmsghdr(cmsg, msg) {
                if (!CMSG_OK(msg, cmsg))
                        return -EINVAL;

                if (cmsg->cmsg_level != SOL_RDS)
                        continue;

                /* As a side effect, RDMA_DEST and RDMA_MAP will set
                 * rm->rdma.m_rdma_cookie and rm->rdma.m_rdma_mr.
                 */
                switch (cmsg->cmsg_type) {
                case RDS_CMSG_RDMA_ARGS:
                        if (ind >= vct->indx)
                                return -ENOMEM;
                        ret = rds_cmsg_rdma_args(rs, rm, cmsg, &vct->vec[ind]);
                        ind++;
                        break;

                case RDS_CMSG_RDMA_DEST:
                        ret = rds_cmsg_rdma_dest(rs, rm, cmsg);
                        break;

                case RDS_CMSG_RDMA_MAP:
                        ret = rds_cmsg_rdma_map(rs, rm, cmsg);
                        if (!ret)
                                *allocated_mr = 1;
                        else if (ret == -ENODEV)
                                /* Accommodate the get_mr() case which can fail
                                 * if connection isn't established yet.
                                 */
                                ret = -EAGAIN;
                        break;
                case RDS_CMSG_ATOMIC_CSWP:
                case RDS_CMSG_ATOMIC_FADD:
                case RDS_CMSG_MASKED_ATOMIC_CSWP:
                case RDS_CMSG_MASKED_ATOMIC_FADD:
                        ret = rds_cmsg_atomic(rs, rm, cmsg);
                        break;

                case RDS_CMSG_ZCOPY_COOKIE:
                        ret = rds_cmsg_zcopy(rs, rm, cmsg);
                        break;

                default:
                        return -EINVAL;
                }

                if (ret)
                        break;
        }

        return ret;
}

static int rds_send_mprds_hash(struct rds_sock *rs,
                               struct rds_connection *conn, int nonblock)
{
        int hash;

        if (conn->c_npaths == 0)
                hash = RDS_MPATH_HASH(rs, RDS_MPATH_WORKERS);
        else
                hash = RDS_MPATH_HASH(rs, conn->c_npaths);
        if (conn->c_npaths == 0 && hash != 0) {
                rds_send_ping(conn, 0);

                /* The underlying connection is not up yet.  Need to wait
                 * until it is up to be sure that the non-zero c_path can be
                 * used.  But if we are interrupted, we have to use the zero
                 * c_path in case the connection ends up being non-MP capable.
                 */
                if (conn->c_npaths == 0) {
                        /* Cannot wait for the connection be made, so just use
                         * the base c_path.
                         */
                        if (nonblock)
                                return 0;
                        if (wait_event_interruptible(conn->c_hs_waitq,
                                                     conn->c_npaths != 0))
                                hash = 0;
                }
                if (conn->c_npaths == 1)
                        hash = 0;
        }
        return hash;
}

static int rds_rdma_bytes(struct msghdr *msg, size_t *rdma_bytes)
{
        struct rds_rdma_args *args;
        struct cmsghdr *cmsg;

        for_each_cmsghdr(cmsg, msg) {
                if (!CMSG_OK(msg, cmsg))
                        return -EINVAL;

                if (cmsg->cmsg_level != SOL_RDS)
                        continue;

                if (cmsg->cmsg_type == RDS_CMSG_RDMA_ARGS) {
                        if (cmsg->cmsg_len <
                            CMSG_LEN(sizeof(struct rds_rdma_args)))
                                return -EINVAL;
                        args = CMSG_DATA(cmsg);
                        *rdma_bytes += args->remote_vec.bytes;
                }
        }
        return 0;
}

int rds_sendmsg(struct socket *sock, struct msghdr *msg, size_t payload_len)
{
        struct sock *sk = sock->sk;
        struct rds_sock *rs = rds_sk_to_rs(sk);
        DECLARE_SOCKADDR(struct sockaddr_in6 *, sin6, msg->msg_name);
        DECLARE_SOCKADDR(struct sockaddr_in *, usin, msg->msg_name);
        __be16 dport;
        struct rds_message *rm = NULL;
        struct rds_connection *conn;
        int ret = 0;
        int queued = 0, allocated_mr = 0;
        int nonblock = msg->msg_flags & MSG_DONTWAIT;
        long timeo = sock_sndtimeo(sk, nonblock);
        struct rds_conn_path *cpath;
        struct in6_addr daddr;
        __u32 scope_id = 0;
        size_t total_payload_len = payload_len, rdma_payload_len = 0;
        bool zcopy = ((msg->msg_flags & MSG_ZEROCOPY) &&
                      sock_flag(rds_rs_to_sk(rs), SOCK_ZEROCOPY));
        int num_sgs = DIV_ROUND_UP(payload_len, PAGE_SIZE);
        int namelen;
        struct rds_iov_vector_arr vct;
        int ind;

        memset(&vct, 0, sizeof(vct));

        /* expect 1 RDMA CMSG per rds_sendmsg. can still grow if more needed. */
        vct.incr = 1;

        /* Mirror Linux UDP mirror of BSD error message compatibility */
        /* XXX: Perhaps MSG_MORE someday */
        if (msg->msg_flags & ~(MSG_DONTWAIT | MSG_CMSG_COMPAT | MSG_ZEROCOPY)) {
                ret = -EOPNOTSUPP;
                goto out;
        }

        namelen = msg->msg_namelen;
        if (namelen != 0) {
                if (namelen < sizeof(*usin)) {
                        ret = -EINVAL;
                        goto out;
                }
                switch (usin->sin_family) {
                case AF_INET:
                        if (usin->sin_addr.s_addr == htonl(INADDR_ANY) ||
                            usin->sin_addr.s_addr == htonl(INADDR_BROADCAST) ||
                            ipv4_is_multicast(usin->sin_addr.s_addr)) {
                                ret = -EINVAL;
                                goto out;
                        }
                        ipv6_addr_set_v4mapped(usin->sin_addr.s_addr, &daddr);
                        dport = usin->sin_port;
                        break;

#if IS_ENABLED(CONFIG_IPV6)
                case AF_INET6: {
                        int addr_type;

                        if (namelen < sizeof(*sin6)) {
                                ret = -EINVAL;
                                goto out;
                        }
                        addr_type = ipv6_addr_type(&sin6->sin6_addr);
                        if (!(addr_type & IPV6_ADDR_UNICAST)) {
                                __be32 addr4;

                                if (!(addr_type & IPV6_ADDR_MAPPED)) {
                                        ret = -EINVAL;
                                        goto out;
                                }

                                /* It is a mapped address.  Need to do some
                                 * sanity checks.
                                 */
                                addr4 = sin6->sin6_addr.s6_addr32[3];
                                if (addr4 == htonl(INADDR_ANY) ||
                                    addr4 == htonl(INADDR_BROADCAST) ||
                                    ipv4_is_multicast(addr4)) {
                                        ret = -EINVAL;
                                        goto out;
                                }
                        }
                        if (addr_type & IPV6_ADDR_LINKLOCAL) {
                                if (sin6->sin6_scope_id == 0) {
                                        ret = -EINVAL;
                                        goto out;
                                }
                                scope_id = sin6->sin6_scope_id;
                        }

                        daddr = sin6->sin6_addr;
                        dport = sin6->sin6_port;
                        break;
                }
#endif

                default:
                        ret = -EINVAL;
                        goto out;
                }
        } else {
                /* We only care about consistency with ->connect() */
                lock_sock(sk);
                daddr = rs->rs_conn_addr;
                dport = rs->rs_conn_port;
                scope_id = rs->rs_bound_scope_id;
                release_sock(sk);
        }

        lock_sock(sk);
        if (ipv6_addr_any(&rs->rs_bound_addr) || ipv6_addr_any(&daddr)) {
                release_sock(sk);
                ret = -ENOTCONN;
                goto out;
        } else if (namelen != 0) {
                /* Cannot send to an IPv4 address using an IPv6 source
                 * address and cannot send to an IPv6 address using an
                 * IPv4 source address.
                 */
                if (ipv6_addr_v4mapped(&daddr) ^
                    ipv6_addr_v4mapped(&rs->rs_bound_addr)) {
                        release_sock(sk);
                        ret = -EOPNOTSUPP;
                        goto out;
                }
                /* If the socket is already bound to a link local address,
                 * it can only send to peers on the same link.  But allow
                 * communicating between link local and non-link local address.
                 */
                if (scope_id != rs->rs_bound_scope_id) {
                        if (!scope_id) {
                                scope_id = rs->rs_bound_scope_id;
                        } else if (rs->rs_bound_scope_id) {
                                release_sock(sk);
                                ret = -EINVAL;
                                goto out;
                        }
                }
        }
        release_sock(sk);

        ret = rds_rdma_bytes(msg, &rdma_payload_len);
        if (ret)
                goto out;

        total_payload_len += rdma_payload_len;
        if (max_t(size_t, payload_len, rdma_payload_len) > RDS_MAX_MSG_SIZE) {
                ret = -EMSGSIZE;
                goto out;
        }

        if (payload_len > rds_sk_sndbuf(rs)) {
                ret = -EMSGSIZE;
                goto out;
        }

        if (zcopy) {
                if (rs->rs_transport->t_type != RDS_TRANS_TCP) {
                        ret = -EOPNOTSUPP;
                        goto out;
                }
                num_sgs = iov_iter_npages(&msg->msg_iter, INT_MAX);
        }
        /* size of rm including all sgs */
        ret = rds_rm_size(msg, num_sgs, &vct);
        if (ret < 0)
                goto out;

        rm = rds_message_alloc(ret, GFP_KERNEL);
        if (!rm) {
                ret = -ENOMEM;
                goto out;
        }

        /* Attach data to the rm */
        if (payload_len) {
                rm->data.op_sg = rds_message_alloc_sgs(rm, num_sgs);
                if (IS_ERR(rm->data.op_sg)) {
                        ret = PTR_ERR(rm->data.op_sg);
                        goto out;
                }
                ret = rds_message_copy_from_user(rm, &msg->msg_iter, zcopy);
                if (ret)
                        goto out;
        }
        rm->data.op_active = 1;

        rm->m_daddr = daddr;

        /* rds_conn_create has a spinlock that runs with IRQ off.
         * Caching the conn in the socket helps a lot. */
        if (rs->rs_conn && ipv6_addr_equal(&rs->rs_conn->c_faddr, &daddr) &&
            rs->rs_tos == rs->rs_conn->c_tos) {
                conn = rs->rs_conn;
        } else {
                conn = rds_conn_create_outgoing(sock_net(sock->sk),
                                                &rs->rs_bound_addr, &daddr,
                                                rs->rs_transport, rs->rs_tos,
                                                sock->sk->sk_allocation,
                                                scope_id);
                if (IS_ERR(conn)) {
                        ret = PTR_ERR(conn);
                        goto out;
                }
                rs->rs_conn = conn;
        }

        if (conn->c_trans->t_mp_capable)
                cpath = &conn->c_path[rds_send_mprds_hash(rs, conn, nonblock)];
        else
                cpath = &conn->c_path[0];

        rm->m_conn_path = cpath;

        /* Parse any control messages the user may have included. */
        ret = rds_cmsg_send(rs, rm, msg, &allocated_mr, &vct);
        if (ret)
                goto out;

        if (rm->rdma.op_active && !conn->c_trans->xmit_rdma) {
                printk_ratelimited(KERN_NOTICE "rdma_op %p conn xmit_rdma %p\n",
                               &rm->rdma, conn->c_trans->xmit_rdma);
                ret = -EOPNOTSUPP;
                goto out;
        }

        if (rm->atomic.op_active && !conn->c_trans->xmit_atomic) {
                printk_ratelimited(KERN_NOTICE "atomic_op %p conn xmit_atomic %p\n",
                               &rm->atomic, conn->c_trans->xmit_atomic);
                ret = -EOPNOTSUPP;
                goto out;
        }

        if (rds_destroy_pending(conn)) {
                ret = -EAGAIN;
                goto out;
        }

        if (rds_conn_path_down(cpath))
                rds_check_all_paths(conn);

        ret = rds_cong_wait(conn->c_fcong, dport, nonblock, rs);
        if (ret) {
                rs->rs_seen_congestion = 1;
                goto out;
        }
        while (!rds_send_queue_rm(rs, conn, cpath, rm, rs->rs_bound_port,
                                  dport, &queued)) {
                rds_stats_inc(s_send_queue_full);

                if (nonblock) {
                        ret = -EAGAIN;
                        goto out;
                }

                timeo = wait_event_interruptible_timeout(*sk_sleep(sk),
                                        rds_send_queue_rm(rs, conn, cpath, rm,
                                                          rs->rs_bound_port,
                                                          dport,
                                                          &queued),
                                        timeo);
                rdsdebug("sendmsg woke queued %d timeo %ld\n", queued, timeo);
                if (timeo > 0 || timeo == MAX_SCHEDULE_TIMEOUT)
                        continue;

                ret = timeo;
                if (ret == 0)
                        ret = -ETIMEDOUT;
                goto out;
        }

        /*
         * By now we've committed to the send.  We reuse rds_send_worker()
         * to retry sends in the rds thread if the transport asks us to.
         */
        rds_stats_inc(s_send_queued);

        ret = rds_send_xmit(cpath);
        if (ret == -ENOMEM || ret == -EAGAIN) {
                ret = 0;
                rcu_read_lock();
                if (rds_destroy_pending(cpath->cp_conn))
                        ret = -ENETUNREACH;
                else
                        queue_delayed_work(rds_wq, &cpath->cp_send_w, 1);
                rcu_read_unlock();
        }
        if (ret)
                goto out;
        rds_message_put(rm);

        for (ind = 0; ind < vct.indx; ind++)
                kfree(vct.vec[ind].iov);
        kfree(vct.vec);

        return payload_len;

out:
        for (ind = 0; ind < vct.indx; ind++)
                kfree(vct.vec[ind].iov);
        kfree(vct.vec);

        /* If the user included a RDMA_MAP cmsg, we allocated a MR on the fly.
         * If the sendmsg goes through, we keep the MR. If it fails with EAGAIN
         * or in any other way, we need to destroy the MR again */
        if (allocated_mr)
                rds_rdma_unuse(rs, rds_rdma_cookie_key(rm->m_rdma_cookie), 1);

        if (rm)
                rds_message_put(rm);
        return ret;
}

/*
 * send out a probe. Can be shared by rds_send_ping,
 * rds_send_pong, rds_send_hb.
 * rds_send_hb should use h_flags
 *   RDS_FLAG_HB_PING|RDS_FLAG_ACK_REQUIRED
 * or
 *   RDS_FLAG_HB_PONG|RDS_FLAG_ACK_REQUIRED
 */
static int
rds_send_probe(struct rds_conn_path *cp, __be16 sport,
               __be16 dport, u8 h_flags)
{
        struct rds_message *rm;
        unsigned long flags;
        int ret = 0;

        rm = rds_message_alloc(0, GFP_ATOMIC);
        if (!rm) {
                ret = -ENOMEM;
                goto out;
        }

        rm->m_daddr = cp->cp_conn->c_faddr;
        rm->data.op_active = 1;

        rds_conn_path_connect_if_down(cp);

        ret = rds_cong_wait(cp->cp_conn->c_fcong, dport, 1, NULL);
        if (ret)
                goto out;

        spin_lock_irqsave(&cp->cp_lock, flags);
        list_add_tail(&rm->m_conn_item, &cp->cp_send_queue);
        set_bit(RDS_MSG_ON_CONN, &rm->m_flags);
        rds_message_addref(rm);
        rm->m_inc.i_conn = cp->cp_conn;
        rm->m_inc.i_conn_path = cp;

        rds_message_populate_header(&rm->m_inc.i_hdr, sport, dport,
                                    cp->cp_next_tx_seq);
        rm->m_inc.i_hdr.h_flags |= h_flags;
        cp->cp_next_tx_seq++;

        if (RDS_HS_PROBE(be16_to_cpu(sport), be16_to_cpu(dport)) &&
            cp->cp_conn->c_trans->t_mp_capable) {
                u16 npaths = cpu_to_be16(RDS_MPATH_WORKERS);
                u32 my_gen_num = cpu_to_be32(cp->cp_conn->c_my_gen_num);

                rds_message_add_extension(&rm->m_inc.i_hdr,
                                          RDS_EXTHDR_NPATHS, &npaths,
                                          sizeof(npaths));
                rds_message_add_extension(&rm->m_inc.i_hdr,
                                          RDS_EXTHDR_GEN_NUM,
                                          &my_gen_num,
                                          sizeof(u32));
        }
        spin_unlock_irqrestore(&cp->cp_lock, flags);

        rds_stats_inc(s_send_queued);
        rds_stats_inc(s_send_pong);

        /* schedule the send work on rds_wq */
        rcu_read_lock();
        if (!rds_destroy_pending(cp->cp_conn))
                queue_delayed_work(rds_wq, &cp->cp_send_w, 1);
        rcu_read_unlock();

        rds_message_put(rm);
        return 0;

out:
        if (rm)
                rds_message_put(rm);
        return ret;
}

int
rds_send_pong(struct rds_conn_path *cp, __be16 dport)
{
        return rds_send_probe(cp, 0, dport, 0);
}

void
rds_send_ping(struct rds_connection *conn, int cp_index)
{
        unsigned long flags;
        struct rds_conn_path *cp = &conn->c_path[cp_index];

        spin_lock_irqsave(&cp->cp_lock, flags);
        if (conn->c_ping_triggered) {
                spin_unlock_irqrestore(&cp->cp_lock, flags);
                return;
        }
        conn->c_ping_triggered = 1;
        spin_unlock_irqrestore(&cp->cp_lock, flags);
        rds_send_probe(cp, cpu_to_be16(RDS_FLAG_PROBE_PORT), 0, 0);
}
EXPORT_SYMBOL_GPL(rds_send_ping);


























































































   49 



   49 

































  416 






  132 



























































































































































    8 

























    3 


    5 






































    3 












    3 
    3 

































    5 











    5 
    5 
    4 
    5 










































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
// SPDX-License-Identifier: GPL-2.0
/*
 * class.c - basic device class management
 *
 * Copyright (c) 2002-3 Patrick Mochel
 * Copyright (c) 2002-3 Open Source Development Labs
 * Copyright (c) 2003-2004 Greg Kroah-Hartman
 * Copyright (c) 2003-2004 IBM Corp.
 */

#include <linux/device/class.h>
#include <linux/device.h>
#include <linux/module.h>
#include <linux/init.h>
#include <linux/string.h>
#include <linux/kdev_t.h>
#include <linux/err.h>
#include <linux/slab.h>
#include <linux/genhd.h>
#include <linux/mutex.h>
#include "base.h"

#define to_class_attr(_attr) container_of(_attr, struct class_attribute, attr)

static ssize_t class_attr_show(struct kobject *kobj, struct attribute *attr,
                               char *buf)
{
        struct class_attribute *class_attr = to_class_attr(attr);
        struct subsys_private *cp = to_subsys_private(kobj);
        ssize_t ret = -EIO;

        if (class_attr->show)
                ret = class_attr->show(cp->class, class_attr, buf);
        return ret;
}

static ssize_t class_attr_store(struct kobject *kobj, struct attribute *attr,
                                const char *buf, size_t count)
{
        struct class_attribute *class_attr = to_class_attr(attr);
        struct subsys_private *cp = to_subsys_private(kobj);
        ssize_t ret = -EIO;

        if (class_attr->store)
                ret = class_attr->store(cp->class, class_attr, buf, count);
        return ret;
}

static void class_release(struct kobject *kobj)
{
        struct subsys_private *cp = to_subsys_private(kobj);
        struct class *class = cp->class;

        pr_debug("class '%s': release.\n", class->name);

        if (class->class_release)
                class->class_release(class);
        else
                pr_debug("class '%s' does not have a release() function, "
                         "be careful\n", class->name);

        kfree(cp);
}

static const struct kobj_ns_type_operations *class_child_ns_type(struct kobject *kobj)
{
        struct subsys_private *cp = to_subsys_private(kobj);
        struct class *class = cp->class;

        return class->ns_type;
}

static const struct sysfs_ops class_sysfs_ops = {
        .show           = class_attr_show,
        .store           = class_attr_store,
};

static struct kobj_type class_ktype = {
        .sysfs_ops        = &class_sysfs_ops,
        .release        = class_release,
        .child_ns_type        = class_child_ns_type,
};

/* Hotplug events for classes go to the class subsys */
static struct kset *class_kset;


int class_create_file_ns(struct class *cls, const struct class_attribute *attr,
                         const void *ns)
{
        int error;

        if (cls)
                error = sysfs_create_file_ns(&cls->p->subsys.kobj,
                                             &attr->attr, ns);
        else
                error = -EINVAL;
        return error;
}

void class_remove_file_ns(struct class *cls, const struct class_attribute *attr,
                          const void *ns)
{
        if (cls)
                sysfs_remove_file_ns(&cls->p->subsys.kobj, &attr->attr, ns);
}

static struct class *class_get(struct class *cls)
{
        if (cls)
                kset_get(&cls->p->subsys);
        return cls;
}

static void class_put(struct class *cls)
{
        if (cls)
                kset_put(&cls->p->subsys);
}

static struct device *klist_class_to_dev(struct klist_node *n)
{
        struct device_private *p = to_device_private_class(n);
        return p->device;
}

static void klist_class_dev_get(struct klist_node *n)
{
        struct device *dev = klist_class_to_dev(n);

        get_device(dev);
}

static void klist_class_dev_put(struct klist_node *n)
{
        struct device *dev = klist_class_to_dev(n);

        put_device(dev);
}

static int class_add_groups(struct class *cls,
                            const struct attribute_group **groups)
{
        return sysfs_create_groups(&cls->p->subsys.kobj, groups);
}

static void class_remove_groups(struct class *cls,
                                const struct attribute_group **groups)
{
        return sysfs_remove_groups(&cls->p->subsys.kobj, groups);
}

int __class_register(struct class *cls, struct lock_class_key *key)
{
        struct subsys_private *cp;
        int error;

        pr_debug("device class '%s': registering\n", cls->name);

        cp = kzalloc(sizeof(*cp), GFP_KERNEL);
        if (!cp)
                return -ENOMEM;
        klist_init(&cp->klist_devices, klist_class_dev_get, klist_class_dev_put);
        INIT_LIST_HEAD(&cp->interfaces);
        kset_init(&cp->glue_dirs);
        __mutex_init(&cp->mutex, "subsys mutex", key);
        error = kobject_set_name(&cp->subsys.kobj, "%s", cls->name);
        if (error) {
                kfree(cp);
                return error;
        }

        /* set the default /sys/dev directory for devices of this class */
        if (!cls->dev_kobj)
                cls->dev_kobj = sysfs_dev_char_kobj;

#if defined(CONFIG_BLOCK)
        /* let the block class directory show up in the root of sysfs */
        if (!sysfs_deprecated || cls != &block_class)
                cp->subsys.kobj.kset = class_kset;
#else
        cp->subsys.kobj.kset = class_kset;
#endif
        cp->subsys.kobj.ktype = &class_ktype;
        cp->class = cls;
        cls->p = cp;

        error = kset_register(&cp->subsys);
        if (error) {
                kfree(cp);
                return error;
        }
        error = class_add_groups(class_get(cls), cls->class_groups);
        class_put(cls);
        if (error) {
                kobject_del(&cp->subsys.kobj);
                kfree_const(cp->subsys.kobj.name);
                kfree(cp);
        }
        return error;
}
EXPORT_SYMBOL_GPL(__class_register);

void class_unregister(struct class *cls)
{
        pr_debug("device class '%s': unregistering\n", cls->name);
        class_remove_groups(cls, cls->class_groups);
        kset_unregister(&cls->p->subsys);
}

static void class_create_release(struct class *cls)
{
        pr_debug("%s called for %s\n", __func__, cls->name);
        kfree(cls);
}

/**
 * __class_create - create a struct class structure
 * @owner: pointer to the module that is to "own" this struct class
 * @name: pointer to a string for the name of this class.
 * @key: the lock_class_key for this class; used by mutex lock debugging
 *
 * This is used to create a struct class pointer that can then be used
 * in calls to device_create().
 *
 * Returns &struct class pointer on success, or ERR_PTR() on error.
 *
 * Note, the pointer created here is to be destroyed when finished by
 * making a call to class_destroy().
 */
struct class *__class_create(struct module *owner, const char *name,
                             struct lock_class_key *key)
{
        struct class *cls;
        int retval;

        cls = kzalloc(sizeof(*cls), GFP_KERNEL);
        if (!cls) {
                retval = -ENOMEM;
                goto error;
        }

        cls->name = name;
        cls->owner = owner;
        cls->class_release = class_create_release;

        retval = __class_register(cls, key);
        if (retval)
                goto error;

        return cls;

error:
        kfree(cls);
        return ERR_PTR(retval);
}
EXPORT_SYMBOL_GPL(__class_create);

/**
 * class_destroy - destroys a struct class structure
 * @cls: pointer to the struct class that is to be destroyed
 *
 * Note, the pointer to be destroyed must have been created with a call
 * to class_create().
 */
void class_destroy(struct class *cls)
{
        if ((cls == NULL) || (IS_ERR(cls)))
                return;

        class_unregister(cls);
}

/**
 * class_dev_iter_init - initialize class device iterator
 * @iter: class iterator to initialize
 * @class: the class we wanna iterate over
 * @start: the device to start iterating from, if any
 * @type: device_type of the devices to iterate over, NULL for all
 *
 * Initialize class iterator @iter such that it iterates over devices
 * of @class.  If @start is set, the list iteration will start there,
 * otherwise if it is NULL, the iteration starts at the beginning of
 * the list.
 */
void class_dev_iter_init(struct class_dev_iter *iter, struct class *class,
                         struct device *start, const struct device_type *type)
{
        struct klist_node *start_knode = NULL;

        if (start)
                start_knode = &start->p->knode_class;
        klist_iter_init_node(&class->p->klist_devices, &iter->ki, start_knode);
        iter->type = type;
}
EXPORT_SYMBOL_GPL(class_dev_iter_init);

/**
 * class_dev_iter_next - iterate to the next device
 * @iter: class iterator to proceed
 *
 * Proceed @iter to the next device and return it.  Returns NULL if
 * iteration is complete.
 *
 * The returned device is referenced and won't be released till
 * iterator is proceed to the next device or exited.  The caller is
 * free to do whatever it wants to do with the device including
 * calling back into class code.
 */
struct device *class_dev_iter_next(struct class_dev_iter *iter)
{
        struct klist_node *knode;
        struct device *dev;

        while (1) {
                knode = klist_next(&iter->ki);
                if (!knode)
                        return NULL;
                dev = klist_class_to_dev(knode);
                if (!iter->type || iter->type == dev->type)
                        return dev;
        }
}
EXPORT_SYMBOL_GPL(class_dev_iter_next);

/**
 * class_dev_iter_exit - finish iteration
 * @iter: class iterator to finish
 *
 * Finish an iteration.  Always call this function after iteration is
 * complete whether the iteration ran till the end or not.
 */
void class_dev_iter_exit(struct class_dev_iter *iter)
{
        klist_iter_exit(&iter->ki);
}
EXPORT_SYMBOL_GPL(class_dev_iter_exit);

/**
 * class_for_each_device - device iterator
 * @class: the class we're iterating
 * @start: the device to start with in the list, if any.
 * @data: data for the callback
 * @fn: function to be called for each device
 *
 * Iterate over @class's list of devices, and call @fn for each,
 * passing it @data.  If @start is set, the list iteration will start
 * there, otherwise if it is NULL, the iteration starts at the
 * beginning of the list.
 *
 * We check the return of @fn each time. If it returns anything
 * other than 0, we break out and return that value.
 *
 * @fn is allowed to do anything including calling back into class
 * code.  There's no locking restriction.
 */
int class_for_each_device(struct class *class, struct device *start,
                          void *data, int (*fn)(struct device *, void *))
{
        struct class_dev_iter iter;
        struct device *dev;
        int error = 0;

        if (!class)
                return -EINVAL;
        if (!class->p) {
                WARN(1, "%s called for class '%s' before it was initialized",
                     __func__, class->name);
                return -EINVAL;
        }

        class_dev_iter_init(&iter, class, start, NULL);
        while ((dev = class_dev_iter_next(&iter))) {
                error = fn(dev, data);
                if (error)
                        break;
        }
        class_dev_iter_exit(&iter);

        return error;
}
EXPORT_SYMBOL_GPL(class_for_each_device);

/**
 * class_find_device - device iterator for locating a particular device
 * @class: the class we're iterating
 * @start: Device to begin with
 * @data: data for the match function
 * @match: function to check device
 *
 * This is similar to the class_for_each_dev() function above, but it
 * returns a reference to a device that is 'found' for later use, as
 * determined by the @match callback.
 *
 * The callback should return 0 if the device doesn't match and non-zero
 * if it does.  If the callback returns non-zero, this function will
 * return to the caller and not iterate over any more devices.
 *
 * Note, you will need to drop the reference with put_device() after use.
 *
 * @match is allowed to do anything including calling back into class
 * code.  There's no locking restriction.
 */
struct device *class_find_device(struct class *class, struct device *start,
                                 const void *data,
                                 int (*match)(struct device *, const void *))
{
        struct class_dev_iter iter;
        struct device *dev;

        if (!class)
                return NULL;
        if (!class->p) {
                WARN(1, "%s called for class '%s' before it was initialized",
                     __func__, class->name);
                return NULL;
        }

        class_dev_iter_init(&iter, class, start, NULL);
        while ((dev = class_dev_iter_next(&iter))) {
                if (match(dev, data)) {
                        get_device(dev);
                        break;
                }
        }
        class_dev_iter_exit(&iter);

        return dev;
}
EXPORT_SYMBOL_GPL(class_find_device);

int class_interface_register(struct class_interface *class_intf)
{
        struct class *parent;
        struct class_dev_iter iter;
        struct device *dev;

        if (!class_intf || !class_intf->class)
                return -ENODEV;

        parent = class_get(class_intf->class);
        if (!parent)
                return -EINVAL;

        mutex_lock(&parent->p->mutex);
        list_add_tail(&class_intf->node, &parent->p->interfaces);
        if (class_intf->add_dev) {
                class_dev_iter_init(&iter, parent, NULL, NULL);
                while ((dev = class_dev_iter_next(&iter)))
                        class_intf->add_dev(dev, class_intf);
                class_dev_iter_exit(&iter);
        }
        mutex_unlock(&parent->p->mutex);

        return 0;
}

void class_interface_unregister(struct class_interface *class_intf)
{
        struct class *parent = class_intf->class;
        struct class_dev_iter iter;
        struct device *dev;

        if (!parent)
                return;

        mutex_lock(&parent->p->mutex);
        list_del_init(&class_intf->node);
        if (class_intf->remove_dev) {
                class_dev_iter_init(&iter, parent, NULL, NULL);
                while ((dev = class_dev_iter_next(&iter)))
                        class_intf->remove_dev(dev, class_intf);
                class_dev_iter_exit(&iter);
        }
        mutex_unlock(&parent->p->mutex);

        class_put(parent);
}

ssize_t show_class_attr_string(struct class *class,
                               struct class_attribute *attr, char *buf)
{
        struct class_attribute_string *cs;

        cs = container_of(attr, struct class_attribute_string, attr);
        return sysfs_emit(buf, "%s\n", cs->str);
}

EXPORT_SYMBOL_GPL(show_class_attr_string);

struct class_compat {
        struct kobject *kobj;
};

/**
 * class_compat_register - register a compatibility class
 * @name: the name of the class
 *
 * Compatibility class are meant as a temporary user-space compatibility
 * workaround when converting a family of class devices to a bus devices.
 */
struct class_compat *class_compat_register(const char *name)
{
        struct class_compat *cls;

        cls = kmalloc(sizeof(struct class_compat), GFP_KERNEL);
        if (!cls)
                return NULL;
        cls->kobj = kobject_create_and_add(name, &class_kset->kobj);
        if (!cls->kobj) {
                kfree(cls);
                return NULL;
        }
        return cls;
}
EXPORT_SYMBOL_GPL(class_compat_register);

/**
 * class_compat_unregister - unregister a compatibility class
 * @cls: the class to unregister
 */
void class_compat_unregister(struct class_compat *cls)
{
        kobject_put(cls->kobj);
        kfree(cls);
}
EXPORT_SYMBOL_GPL(class_compat_unregister);

/**
 * class_compat_create_link - create a compatibility class device link to
 *                              a bus device
 * @cls: the compatibility class
 * @dev: the target bus device
 * @device_link: an optional device to which a "device" link should be created
 */
int class_compat_create_link(struct class_compat *cls, struct device *dev,
                             struct device *device_link)
{
        int error;

        error = sysfs_create_link(cls->kobj, &dev->kobj, dev_name(dev));
        if (error)
                return error;

        /*
         * Optionally add a "device" link (typically to the parent), as a
         * class device would have one and we want to provide as much
         * backwards compatibility as possible.
         */
        if (device_link) {
                error = sysfs_create_link(&dev->kobj, &device_link->kobj,
                                          "device");
                if (error)
                        sysfs_remove_link(cls->kobj, dev_name(dev));
        }

        return error;
}
EXPORT_SYMBOL_GPL(class_compat_create_link);

/**
 * class_compat_remove_link - remove a compatibility class device link to
 *                              a bus device
 * @cls: the compatibility class
 * @dev: the target bus device
 * @device_link: an optional device to which a "device" link was previously
 *                  created
 */
void class_compat_remove_link(struct class_compat *cls, struct device *dev,
                              struct device *device_link)
{
        if (device_link)
                sysfs_remove_link(&dev->kobj, "device");
        sysfs_remove_link(cls->kobj, dev_name(dev));
}
EXPORT_SYMBOL_GPL(class_compat_remove_link);

int __init classes_init(void)
{
        class_kset = kset_create_and_add("class", NULL, NULL);
        if (!class_kset)
                return -ENOMEM;
        return 0;
}

EXPORT_SYMBOL_GPL(class_create_file_ns);
EXPORT_SYMBOL_GPL(class_remove_file_ns);
EXPORT_SYMBOL_GPL(class_unregister);
EXPORT_SYMBOL_GPL(class_destroy);

EXPORT_SYMBOL_GPL(class_interface_register);
EXPORT_SYMBOL_GPL(class_interface_unregister);
















































































































































































































































































































































































































































































































































   47 


















   47 















   47 

















































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
/*
 * Copyright (c) 2006, 2018 Oracle and/or its affiliates. All rights reserved.
 *
 * This software is available to you under a choice of one of two
 * licenses.  You may choose to be licensed under the terms of the GNU
 * General Public License (GPL) Version 2, available from the file
 * COPYING in the main directory of this source tree, or the
 * OpenIB.org BSD license below:
 *
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
 *
 *      - Redistributions of source code must retain the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer.
 *
 *      - Redistributions in binary form must reproduce the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer in the documentation and/or other materials
 *        provided with the distribution.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 */
#include <linux/kernel.h>
#include <linux/slab.h>
#include <linux/in.h>
#include <linux/module.h>
#include <net/tcp.h>
#include <net/net_namespace.h>
#include <net/netns/generic.h>
#include <net/addrconf.h>

#include "rds.h"
#include "tcp.h"

/* only for info exporting */
static DEFINE_SPINLOCK(rds_tcp_tc_list_lock);
static LIST_HEAD(rds_tcp_tc_list);

/* rds_tcp_tc_count counts only IPv4 connections.
 * rds6_tcp_tc_count counts both IPv4 and IPv6 connections.
 */
static unsigned int rds_tcp_tc_count;
#if IS_ENABLED(CONFIG_IPV6)
static unsigned int rds6_tcp_tc_count;
#endif

/* Track rds_tcp_connection structs so they can be cleaned up */
static DEFINE_SPINLOCK(rds_tcp_conn_lock);
static LIST_HEAD(rds_tcp_conn_list);
static atomic_t rds_tcp_unloading = ATOMIC_INIT(0);

static struct kmem_cache *rds_tcp_conn_slab;

static int rds_tcp_skbuf_handler(struct ctl_table *ctl, int write,
                                 void *buffer, size_t *lenp, loff_t *fpos);

static int rds_tcp_min_sndbuf = SOCK_MIN_SNDBUF;
static int rds_tcp_min_rcvbuf = SOCK_MIN_RCVBUF;

static struct ctl_table rds_tcp_sysctl_table[] = {
#define        RDS_TCP_SNDBUF        0
        {
                .procname       = "rds_tcp_sndbuf",
                /* data is per-net pointer */
                .maxlen         = sizeof(int),
                .mode           = 0644,
                .proc_handler   = rds_tcp_skbuf_handler,
                .extra1                = &rds_tcp_min_sndbuf,
        },
#define        RDS_TCP_RCVBUF        1
        {
                .procname       = "rds_tcp_rcvbuf",
                /* data is per-net pointer */
                .maxlen         = sizeof(int),
                .mode           = 0644,
                .proc_handler   = rds_tcp_skbuf_handler,
                .extra1                = &rds_tcp_min_rcvbuf,
        },
        { }
};

u32 rds_tcp_write_seq(struct rds_tcp_connection *tc)
{
        /* seq# of the last byte of data in tcp send buffer */
        return tcp_sk(tc->t_sock->sk)->write_seq;
}

u32 rds_tcp_snd_una(struct rds_tcp_connection *tc)
{
        return tcp_sk(tc->t_sock->sk)->snd_una;
}

void rds_tcp_restore_callbacks(struct socket *sock,
                               struct rds_tcp_connection *tc)
{
        rdsdebug("restoring sock %p callbacks from tc %p\n", sock, tc);
        write_lock_bh(&sock->sk->sk_callback_lock);

        /* done under the callback_lock to serialize with write_space */
        spin_lock(&rds_tcp_tc_list_lock);
        list_del_init(&tc->t_list_item);
#if IS_ENABLED(CONFIG_IPV6)
        rds6_tcp_tc_count--;
#endif
        if (!tc->t_cpath->cp_conn->c_isv6)
                rds_tcp_tc_count--;
        spin_unlock(&rds_tcp_tc_list_lock);

        tc->t_sock = NULL;

        sock->sk->sk_write_space = tc->t_orig_write_space;
        sock->sk->sk_data_ready = tc->t_orig_data_ready;
        sock->sk->sk_state_change = tc->t_orig_state_change;
        sock->sk->sk_user_data = NULL;

        write_unlock_bh(&sock->sk->sk_callback_lock);
}

/*
 * rds_tcp_reset_callbacks() switches the to the new sock and
 * returns the existing tc->t_sock.
 *
 * The only functions that set tc->t_sock are rds_tcp_set_callbacks
 * and rds_tcp_reset_callbacks.  Send and receive trust that
 * it is set.  The absence of RDS_CONN_UP bit protects those paths
 * from being called while it isn't set.
 */
void rds_tcp_reset_callbacks(struct socket *sock,
                             struct rds_conn_path *cp)
{
        struct rds_tcp_connection *tc = cp->cp_transport_data;
        struct socket *osock = tc->t_sock;

        if (!osock)
                goto newsock;

        /* Need to resolve a duelling SYN between peers.
         * We have an outstanding SYN to this peer, which may
         * potentially have transitioned to the RDS_CONN_UP state,
         * so we must quiesce any send threads before resetting
         * cp_transport_data. We quiesce these threads by setting
         * cp_state to something other than RDS_CONN_UP, and then
         * waiting for any existing threads in rds_send_xmit to
         * complete release_in_xmit(). (Subsequent threads entering
         * rds_send_xmit() will bail on !rds_conn_up().
         *
         * However an incoming syn-ack at this point would end up
         * marking the conn as RDS_CONN_UP, and would again permit
         * rds_send_xmi() threads through, so ideally we would
         * synchronize on RDS_CONN_UP after lock_sock(), but cannot
         * do that: waiting on !RDS_IN_XMIT after lock_sock() may
         * end up deadlocking with tcp_sendmsg(), and the RDS_IN_XMIT
         * would not get set. As a result, we set c_state to
         * RDS_CONN_RESETTTING, to ensure that rds_tcp_state_change
         * cannot mark rds_conn_path_up() in the window before lock_sock()
         */
        atomic_set(&cp->cp_state, RDS_CONN_RESETTING);
        wait_event(cp->cp_waitq, !test_bit(RDS_IN_XMIT, &cp->cp_flags));
        /* reset receive side state for rds_tcp_data_recv() for osock  */
        cancel_delayed_work_sync(&cp->cp_send_w);
        cancel_delayed_work_sync(&cp->cp_recv_w);
        lock_sock(osock->sk);
        if (tc->t_tinc) {
                rds_inc_put(&tc->t_tinc->ti_inc);
                tc->t_tinc = NULL;
        }
        tc->t_tinc_hdr_rem = sizeof(struct rds_header);
        tc->t_tinc_data_rem = 0;
        rds_tcp_restore_callbacks(osock, tc);
        release_sock(osock->sk);
        sock_release(osock);
newsock:
        rds_send_path_reset(cp);
        lock_sock(sock->sk);
        rds_tcp_set_callbacks(sock, cp);
        release_sock(sock->sk);
}

/* Add tc to rds_tcp_tc_list and set tc->t_sock. See comments
 * above rds_tcp_reset_callbacks for notes about synchronization
 * with data path
 */
void rds_tcp_set_callbacks(struct socket *sock, struct rds_conn_path *cp)
{
        struct rds_tcp_connection *tc = cp->cp_transport_data;

        rdsdebug("setting sock %p callbacks to tc %p\n", sock, tc);
        write_lock_bh(&sock->sk->sk_callback_lock);

        /* done under the callback_lock to serialize with write_space */
        spin_lock(&rds_tcp_tc_list_lock);
        list_add_tail(&tc->t_list_item, &rds_tcp_tc_list);
#if IS_ENABLED(CONFIG_IPV6)
        rds6_tcp_tc_count++;
#endif
        if (!tc->t_cpath->cp_conn->c_isv6)
                rds_tcp_tc_count++;
        spin_unlock(&rds_tcp_tc_list_lock);

        /* accepted sockets need our listen data ready undone */
        if (sock->sk->sk_data_ready == rds_tcp_listen_data_ready)
                sock->sk->sk_data_ready = sock->sk->sk_user_data;

        tc->t_sock = sock;
        tc->t_cpath = cp;
        tc->t_orig_data_ready = sock->sk->sk_data_ready;
        tc->t_orig_write_space = sock->sk->sk_write_space;
        tc->t_orig_state_change = sock->sk->sk_state_change;

        sock->sk->sk_user_data = cp;
        sock->sk->sk_data_ready = rds_tcp_data_ready;
        sock->sk->sk_write_space = rds_tcp_write_space;
        sock->sk->sk_state_change = rds_tcp_state_change;

        write_unlock_bh(&sock->sk->sk_callback_lock);
}

/* Handle RDS_INFO_TCP_SOCKETS socket option.  It only returns IPv4
 * connections for backward compatibility.
 */
static void rds_tcp_tc_info(struct socket *rds_sock, unsigned int len,
                            struct rds_info_iterator *iter,
                            struct rds_info_lengths *lens)
{
        struct rds_info_tcp_socket tsinfo;
        struct rds_tcp_connection *tc;
        unsigned long flags;

        spin_lock_irqsave(&rds_tcp_tc_list_lock, flags);

        if (len / sizeof(tsinfo) < rds_tcp_tc_count)
                goto out;

        list_for_each_entry(tc, &rds_tcp_tc_list, t_list_item) {
                struct inet_sock *inet = inet_sk(tc->t_sock->sk);

                if (tc->t_cpath->cp_conn->c_isv6)
                        continue;

                tsinfo.local_addr = inet->inet_saddr;
                tsinfo.local_port = inet->inet_sport;
                tsinfo.peer_addr = inet->inet_daddr;
                tsinfo.peer_port = inet->inet_dport;

                tsinfo.hdr_rem = tc->t_tinc_hdr_rem;
                tsinfo.data_rem = tc->t_tinc_data_rem;
                tsinfo.last_sent_nxt = tc->t_last_sent_nxt;
                tsinfo.last_expected_una = tc->t_last_expected_una;
                tsinfo.last_seen_una = tc->t_last_seen_una;
                tsinfo.tos = tc->t_cpath->cp_conn->c_tos;

                rds_info_copy(iter, &tsinfo, sizeof(tsinfo));
        }

out:
        lens->nr = rds_tcp_tc_count;
        lens->each = sizeof(tsinfo);

        spin_unlock_irqrestore(&rds_tcp_tc_list_lock, flags);
}

#if IS_ENABLED(CONFIG_IPV6)
/* Handle RDS6_INFO_TCP_SOCKETS socket option. It returns both IPv4 and
 * IPv6 connections. IPv4 connection address is returned in an IPv4 mapped
 * address.
 */
static void rds6_tcp_tc_info(struct socket *sock, unsigned int len,
                             struct rds_info_iterator *iter,
                             struct rds_info_lengths *lens)
{
        struct rds6_info_tcp_socket tsinfo6;
        struct rds_tcp_connection *tc;
        unsigned long flags;

        spin_lock_irqsave(&rds_tcp_tc_list_lock, flags);

        if (len / sizeof(tsinfo6) < rds6_tcp_tc_count)
                goto out;

        list_for_each_entry(tc, &rds_tcp_tc_list, t_list_item) {
                struct sock *sk = tc->t_sock->sk;
                struct inet_sock *inet = inet_sk(sk);

                tsinfo6.local_addr = sk->sk_v6_rcv_saddr;
                tsinfo6.local_port = inet->inet_sport;
                tsinfo6.peer_addr = sk->sk_v6_daddr;
                tsinfo6.peer_port = inet->inet_dport;

                tsinfo6.hdr_rem = tc->t_tinc_hdr_rem;
                tsinfo6.data_rem = tc->t_tinc_data_rem;
                tsinfo6.last_sent_nxt = tc->t_last_sent_nxt;
                tsinfo6.last_expected_una = tc->t_last_expected_una;
                tsinfo6.last_seen_una = tc->t_last_seen_una;

                rds_info_copy(iter, &tsinfo6, sizeof(tsinfo6));
        }

out:
        lens->nr = rds6_tcp_tc_count;
        lens->each = sizeof(tsinfo6);

        spin_unlock_irqrestore(&rds_tcp_tc_list_lock, flags);
}
#endif

int rds_tcp_laddr_check(struct net *net, const struct in6_addr *addr,
                        __u32 scope_id)
{
        struct net_device *dev = NULL;
#if IS_ENABLED(CONFIG_IPV6)
        int ret;
#endif

        if (ipv6_addr_v4mapped(addr)) {
                if (inet_addr_type(net, addr->s6_addr32[3]) == RTN_LOCAL)
                        return 0;
                return -EADDRNOTAVAIL;
        }

        /* If the scope_id is specified, check only those addresses
         * hosted on the specified interface.
         */
        if (scope_id != 0) {
                rcu_read_lock();
                dev = dev_get_by_index_rcu(net, scope_id);
                /* scope_id is not valid... */
                if (!dev) {
                        rcu_read_unlock();
                        return -EADDRNOTAVAIL;
                }
                rcu_read_unlock();
        }
#if IS_ENABLED(CONFIG_IPV6)
        ret = ipv6_chk_addr(net, addr, dev, 0);
        if (ret)
                return 0;
#endif
        return -EADDRNOTAVAIL;
}

static void rds_tcp_conn_free(void *arg)
{
        struct rds_tcp_connection *tc = arg;
        unsigned long flags;

        rdsdebug("freeing tc %p\n", tc);

        spin_lock_irqsave(&rds_tcp_conn_lock, flags);
        if (!tc->t_tcp_node_detached)
                list_del(&tc->t_tcp_node);
        spin_unlock_irqrestore(&rds_tcp_conn_lock, flags);

        kmem_cache_free(rds_tcp_conn_slab, tc);
}

static int rds_tcp_conn_alloc(struct rds_connection *conn, gfp_t gfp)
{
        struct rds_tcp_connection *tc;
        int i, j;
        int ret = 0;

        for (i = 0; i < RDS_MPATH_WORKERS; i++) {
                tc = kmem_cache_alloc(rds_tcp_conn_slab, gfp);
                if (!tc) {
                        ret = -ENOMEM;
                        goto fail;
                }
                mutex_init(&tc->t_conn_path_lock);
                tc->t_sock = NULL;
                tc->t_tinc = NULL;
                tc->t_tinc_hdr_rem = sizeof(struct rds_header);
                tc->t_tinc_data_rem = 0;

                conn->c_path[i].cp_transport_data = tc;
                tc->t_cpath = &conn->c_path[i];
                tc->t_tcp_node_detached = true;

                rdsdebug("rds_conn_path [%d] tc %p\n", i,
                         conn->c_path[i].cp_transport_data);
        }
        spin_lock_irq(&rds_tcp_conn_lock);
        for (i = 0; i < RDS_MPATH_WORKERS; i++) {
                tc = conn->c_path[i].cp_transport_data;
                tc->t_tcp_node_detached = false;
                list_add_tail(&tc->t_tcp_node, &rds_tcp_conn_list);
        }
        spin_unlock_irq(&rds_tcp_conn_lock);
fail:
        if (ret) {
                for (j = 0; j < i; j++)
                        rds_tcp_conn_free(conn->c_path[j].cp_transport_data);
        }
        return ret;
}

static bool list_has_conn(struct list_head *list, struct rds_connection *conn)
{
        struct rds_tcp_connection *tc, *_tc;

        list_for_each_entry_safe(tc, _tc, list, t_tcp_node) {
                if (tc->t_cpath->cp_conn == conn)
                        return true;
        }
        return false;
}

static void rds_tcp_set_unloading(void)
{
        atomic_set(&rds_tcp_unloading, 1);
}

static bool rds_tcp_is_unloading(struct rds_connection *conn)
{
        return atomic_read(&rds_tcp_unloading) != 0;
}

static void rds_tcp_destroy_conns(void)
{
        struct rds_tcp_connection *tc, *_tc;
        LIST_HEAD(tmp_list);

        /* avoid calling conn_destroy with irqs off */
        spin_lock_irq(&rds_tcp_conn_lock);
        list_for_each_entry_safe(tc, _tc, &rds_tcp_conn_list, t_tcp_node) {
                if (!list_has_conn(&tmp_list, tc->t_cpath->cp_conn))
                        list_move_tail(&tc->t_tcp_node, &tmp_list);
        }
        spin_unlock_irq(&rds_tcp_conn_lock);

        list_for_each_entry_safe(tc, _tc, &tmp_list, t_tcp_node)
                rds_conn_destroy(tc->t_cpath->cp_conn);
}

static void rds_tcp_exit(void);

static u8 rds_tcp_get_tos_map(u8 tos)
{
        /* all user tos mapped to default 0 for TCP transport */
        return 0;
}

struct rds_transport rds_tcp_transport = {
        .laddr_check                = rds_tcp_laddr_check,
        .xmit_path_prepare        = rds_tcp_xmit_path_prepare,
        .xmit_path_complete        = rds_tcp_xmit_path_complete,
        .xmit                        = rds_tcp_xmit,
        .recv_path                = rds_tcp_recv_path,
        .conn_alloc                = rds_tcp_conn_alloc,
        .conn_free                = rds_tcp_conn_free,
        .conn_path_connect        = rds_tcp_conn_path_connect,
        .conn_path_shutdown        = rds_tcp_conn_path_shutdown,
        .inc_copy_to_user        = rds_tcp_inc_copy_to_user,
        .inc_free                = rds_tcp_inc_free,
        .stats_info_copy        = rds_tcp_stats_info_copy,
        .exit                        = rds_tcp_exit,
        .get_tos_map                = rds_tcp_get_tos_map,
        .t_owner                = THIS_MODULE,
        .t_name                        = "tcp",
        .t_type                        = RDS_TRANS_TCP,
        .t_prefer_loopback        = 1,
        .t_mp_capable                = 1,
        .t_unloading                = rds_tcp_is_unloading,
};

static unsigned int rds_tcp_netid;

/* per-network namespace private data for this module */
struct rds_tcp_net {
        struct socket *rds_tcp_listen_sock;
        struct work_struct rds_tcp_accept_w;
        struct ctl_table_header *rds_tcp_sysctl;
        struct ctl_table *ctl_table;
        int sndbuf_size;
        int rcvbuf_size;
};

/* All module specific customizations to the RDS-TCP socket should be done in
 * rds_tcp_tune() and applied after socket creation.
 */
void rds_tcp_tune(struct socket *sock)
{
        struct sock *sk = sock->sk;
        struct net *net = sock_net(sk);
        struct rds_tcp_net *rtn = net_generic(net, rds_tcp_netid);

        tcp_sock_set_nodelay(sock->sk);
        lock_sock(sk);
        if (rtn->sndbuf_size > 0) {
                sk->sk_sndbuf = rtn->sndbuf_size;
                sk->sk_userlocks |= SOCK_SNDBUF_LOCK;
        }
        if (rtn->rcvbuf_size > 0) {
                sk->sk_rcvbuf = rtn->rcvbuf_size;
                sk->sk_userlocks |= SOCK_RCVBUF_LOCK;
        }
        release_sock(sk);
}

static void rds_tcp_accept_worker(struct work_struct *work)
{
        struct rds_tcp_net *rtn = container_of(work,
                                               struct rds_tcp_net,
                                               rds_tcp_accept_w);

        while (rds_tcp_accept_one(rtn->rds_tcp_listen_sock) == 0)
                cond_resched();
}

void rds_tcp_accept_work(struct sock *sk)
{
        struct net *net = sock_net(sk);
        struct rds_tcp_net *rtn = net_generic(net, rds_tcp_netid);

        queue_work(rds_wq, &rtn->rds_tcp_accept_w);
}

static __net_init int rds_tcp_init_net(struct net *net)
{
        struct rds_tcp_net *rtn = net_generic(net, rds_tcp_netid);
        struct ctl_table *tbl;
        int err = 0;

        memset(rtn, 0, sizeof(*rtn));

        /* {snd, rcv}buf_size default to 0, which implies we let the
         * stack pick the value, and permit auto-tuning of buffer size.
         */
        if (net == &init_net) {
                tbl = rds_tcp_sysctl_table;
        } else {
                tbl = kmemdup(rds_tcp_sysctl_table,
                              sizeof(rds_tcp_sysctl_table), GFP_KERNEL);
                if (!tbl) {
                        pr_warn("could not set allocate sysctl table\n");
                        return -ENOMEM;
                }
                rtn->ctl_table = tbl;
        }
        tbl[RDS_TCP_SNDBUF].data = &rtn->sndbuf_size;
        tbl[RDS_TCP_RCVBUF].data = &rtn->rcvbuf_size;
        rtn->rds_tcp_sysctl = register_net_sysctl(net, "net/rds/tcp", tbl);
        if (!rtn->rds_tcp_sysctl) {
                pr_warn("could not register sysctl\n");
                err = -ENOMEM;
                goto fail;
        }

#if IS_ENABLED(CONFIG_IPV6)
        rtn->rds_tcp_listen_sock = rds_tcp_listen_init(net, true);
#else
        rtn->rds_tcp_listen_sock = rds_tcp_listen_init(net, false);
#endif
        if (!rtn->rds_tcp_listen_sock) {
                pr_warn("could not set up IPv6 listen sock\n");

#if IS_ENABLED(CONFIG_IPV6)
                /* Try IPv4 as some systems disable IPv6 */
                rtn->rds_tcp_listen_sock = rds_tcp_listen_init(net, false);
                if (!rtn->rds_tcp_listen_sock) {
#endif
                        unregister_net_sysctl_table(rtn->rds_tcp_sysctl);
                        rtn->rds_tcp_sysctl = NULL;
                        err = -EAFNOSUPPORT;
                        goto fail;
#if IS_ENABLED(CONFIG_IPV6)
                }
#endif
        }
        INIT_WORK(&rtn->rds_tcp_accept_w, rds_tcp_accept_worker);
        return 0;

fail:
        if (net != &init_net)
                kfree(tbl);
        return err;
}

static void rds_tcp_kill_sock(struct net *net)
{
        struct rds_tcp_connection *tc, *_tc;
        LIST_HEAD(tmp_list);
        struct rds_tcp_net *rtn = net_generic(net, rds_tcp_netid);
        struct socket *lsock = rtn->rds_tcp_listen_sock;

        rtn->rds_tcp_listen_sock = NULL;
        rds_tcp_listen_stop(lsock, &rtn->rds_tcp_accept_w);
        spin_lock_irq(&rds_tcp_conn_lock);
        list_for_each_entry_safe(tc, _tc, &rds_tcp_conn_list, t_tcp_node) {
                struct net *c_net = read_pnet(&tc->t_cpath->cp_conn->c_net);

                if (net != c_net)
                        continue;
                if (!list_has_conn(&tmp_list, tc->t_cpath->cp_conn)) {
                        list_move_tail(&tc->t_tcp_node, &tmp_list);
                } else {
                        list_del(&tc->t_tcp_node);
                        tc->t_tcp_node_detached = true;
                }
        }
        spin_unlock_irq(&rds_tcp_conn_lock);
        list_for_each_entry_safe(tc, _tc, &tmp_list, t_tcp_node)
                rds_conn_destroy(tc->t_cpath->cp_conn);
}

static void __net_exit rds_tcp_exit_net(struct net *net)
{
        struct rds_tcp_net *rtn = net_generic(net, rds_tcp_netid);

        rds_tcp_kill_sock(net);

        if (rtn->rds_tcp_sysctl)
                unregister_net_sysctl_table(rtn->rds_tcp_sysctl);

        if (net != &init_net)
                kfree(rtn->ctl_table);
}

static struct pernet_operations rds_tcp_net_ops = {
        .init = rds_tcp_init_net,
        .exit = rds_tcp_exit_net,
        .id = &rds_tcp_netid,
        .size = sizeof(struct rds_tcp_net),
};

void *rds_tcp_listen_sock_def_readable(struct net *net)
{
        struct rds_tcp_net *rtn = net_generic(net, rds_tcp_netid);
        struct socket *lsock = rtn->rds_tcp_listen_sock;

        if (!lsock)
                return NULL;

        return lsock->sk->sk_user_data;
}

/* when sysctl is used to modify some kernel socket parameters,this
 * function  resets the RDS connections in that netns  so that we can
 * restart with new parameters.  The assumption is that such reset
 * events are few and far-between.
 */
static void rds_tcp_sysctl_reset(struct net *net)
{
        struct rds_tcp_connection *tc, *_tc;

        spin_lock_irq(&rds_tcp_conn_lock);
        list_for_each_entry_safe(tc, _tc, &rds_tcp_conn_list, t_tcp_node) {
                struct net *c_net = read_pnet(&tc->t_cpath->cp_conn->c_net);

                if (net != c_net || !tc->t_sock)
                        continue;

                /* reconnect with new parameters */
                rds_conn_path_drop(tc->t_cpath, false);
        }
        spin_unlock_irq(&rds_tcp_conn_lock);
}

static int rds_tcp_skbuf_handler(struct ctl_table *ctl, int write,
                                 void *buffer, size_t *lenp, loff_t *fpos)
{
        struct net *net = current->nsproxy->net_ns;
        int err;

        err = proc_dointvec_minmax(ctl, write, buffer, lenp, fpos);
        if (err < 0) {
                pr_warn("Invalid input. Must be >= %d\n",
                        *(int *)(ctl->extra1));
                return err;
        }
        if (write)
                rds_tcp_sysctl_reset(net);
        return 0;
}

static void rds_tcp_exit(void)
{
        rds_tcp_set_unloading();
        synchronize_rcu();
        rds_info_deregister_func(RDS_INFO_TCP_SOCKETS, rds_tcp_tc_info);
#if IS_ENABLED(CONFIG_IPV6)
        rds_info_deregister_func(RDS6_INFO_TCP_SOCKETS, rds6_tcp_tc_info);
#endif
        unregister_pernet_device(&rds_tcp_net_ops);
        rds_tcp_destroy_conns();
        rds_trans_unregister(&rds_tcp_transport);
        rds_tcp_recv_exit();
        kmem_cache_destroy(rds_tcp_conn_slab);
}
module_exit(rds_tcp_exit);

static int rds_tcp_init(void)
{
        int ret;

        rds_tcp_conn_slab = kmem_cache_create("rds_tcp_connection",
                                              sizeof(struct rds_tcp_connection),
                                              0, 0, NULL);
        if (!rds_tcp_conn_slab) {
                ret = -ENOMEM;
                goto out;
        }

        ret = rds_tcp_recv_init();
        if (ret)
                goto out_slab;

        ret = register_pernet_device(&rds_tcp_net_ops);
        if (ret)
                goto out_recv;

        rds_trans_register(&rds_tcp_transport);

        rds_info_register_func(RDS_INFO_TCP_SOCKETS, rds_tcp_tc_info);
#if IS_ENABLED(CONFIG_IPV6)
        rds_info_register_func(RDS6_INFO_TCP_SOCKETS, rds6_tcp_tc_info);
#endif

        goto out;
out_recv:
        rds_tcp_recv_exit();
out_slab:
        kmem_cache_destroy(rds_tcp_conn_slab);
out:
        return ret;
}
module_init(rds_tcp_init);

MODULE_AUTHOR("Oracle Corporation <rds-devel@oss.oracle.com>");
MODULE_DESCRIPTION("RDS: TCP transport");
MODULE_LICENSE("Dual BSD/GPL");















































































































































































































































































































































































  547 





  285 

  455 



  457 



  456 


  175 
  143 
   68 





  457 





















    1 









































    1 






































































































    1 











  176 




  100 
  188 


























1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
 * INET                An implementation of the TCP/IP protocol suite for the LINUX
 *                operating system.  INET  is implemented using the  BSD Socket
 *                interface as the means of communication with the user level.
 *
 *                Definitions for the Forwarding Information Base.
 *
 * Authors:        A.N.Kuznetsov, <kuznet@ms2.inr.ac.ru>
 */

#ifndef _NET_IP_FIB_H
#define _NET_IP_FIB_H

#include <net/flow.h>
#include <linux/seq_file.h>
#include <linux/rcupdate.h>
#include <net/fib_notifier.h>
#include <net/fib_rules.h>
#include <net/inetpeer.h>
#include <linux/percpu.h>
#include <linux/notifier.h>
#include <linux/refcount.h>

struct fib_config {
        u8                        fc_dst_len;
        u8                        fc_tos;
        u8                        fc_protocol;
        u8                        fc_scope;
        u8                        fc_type;
        u8                        fc_gw_family;
        /* 2 bytes unused */
        u32                        fc_table;
        __be32                        fc_dst;
        union {
                __be32                fc_gw4;
                struct in6_addr        fc_gw6;
        };
        int                        fc_oif;
        u32                        fc_flags;
        u32                        fc_priority;
        __be32                        fc_prefsrc;
        u32                        fc_nh_id;
        struct nlattr                *fc_mx;
        struct rtnexthop        *fc_mp;
        int                        fc_mx_len;
        int                        fc_mp_len;
        u32                        fc_flow;
        u32                        fc_nlflags;
        struct nl_info                fc_nlinfo;
        struct nlattr                *fc_encap;
        u16                        fc_encap_type;
};

struct fib_info;
struct rtable;

struct fib_nh_exception {
        struct fib_nh_exception __rcu        *fnhe_next;
        int                                fnhe_genid;
        __be32                                fnhe_daddr;
        u32                                fnhe_pmtu;
        bool                                fnhe_mtu_locked;
        __be32                                fnhe_gw;
        unsigned long                        fnhe_expires;
        struct rtable __rcu                *fnhe_rth_input;
        struct rtable __rcu                *fnhe_rth_output;
        unsigned long                        fnhe_stamp;
        struct rcu_head                        rcu;
};

struct fnhe_hash_bucket {
        struct fib_nh_exception __rcu        *chain;
};

#define FNHE_HASH_SHIFT                11
#define FNHE_HASH_SIZE                (1 << FNHE_HASH_SHIFT)
#define FNHE_RECLAIM_DEPTH        5

struct fib_nh_common {
        struct net_device        *nhc_dev;
        int                        nhc_oif;
        unsigned char                nhc_scope;
        u8                        nhc_family;
        u8                        nhc_gw_family;
        unsigned char                nhc_flags;
        struct lwtunnel_state        *nhc_lwtstate;

        union {
                __be32          ipv4;
                struct in6_addr ipv6;
        } nhc_gw;

        int                        nhc_weight;
        atomic_t                nhc_upper_bound;

        /* v4 specific, but allows fib6_nh with v4 routes */
        struct rtable __rcu * __percpu *nhc_pcpu_rth_output;
        struct rtable __rcu     *nhc_rth_input;
        struct fnhe_hash_bucket        __rcu *nhc_exceptions;
};

struct fib_nh {
        struct fib_nh_common        nh_common;
        struct hlist_node        nh_hash;
        struct fib_info                *nh_parent;
#ifdef CONFIG_IP_ROUTE_CLASSID
        __u32                        nh_tclassid;
#endif
        __be32                        nh_saddr;
        int                        nh_saddr_genid;
#define fib_nh_family                nh_common.nhc_family
#define fib_nh_dev                nh_common.nhc_dev
#define fib_nh_oif                nh_common.nhc_oif
#define fib_nh_flags                nh_common.nhc_flags
#define fib_nh_lws                nh_common.nhc_lwtstate
#define fib_nh_scope                nh_common.nhc_scope
#define fib_nh_gw_family        nh_common.nhc_gw_family
#define fib_nh_gw4                nh_common.nhc_gw.ipv4
#define fib_nh_gw6                nh_common.nhc_gw.ipv6
#define fib_nh_weight                nh_common.nhc_weight
#define fib_nh_upper_bound        nh_common.nhc_upper_bound
};

/*
 * This structure contains data shared by many of routes.
 */

struct nexthop;

struct fib_info {
        struct hlist_node        fib_hash;
        struct hlist_node        fib_lhash;
        struct list_head        nh_list;
        struct net                *fib_net;
        refcount_t                fib_treeref;
        refcount_t                fib_clntref;
        unsigned int                fib_flags;
        unsigned char                fib_dead;
        unsigned char                fib_protocol;
        unsigned char                fib_scope;
        unsigned char                fib_type;
        __be32                        fib_prefsrc;
        u32                        fib_tb_id;
        u32                        fib_priority;
        struct dst_metrics        *fib_metrics;
#define fib_mtu fib_metrics->metrics[RTAX_MTU-1]
#define fib_window fib_metrics->metrics[RTAX_WINDOW-1]
#define fib_rtt fib_metrics->metrics[RTAX_RTT-1]
#define fib_advmss fib_metrics->metrics[RTAX_ADVMSS-1]
        int                        fib_nhs;
        bool                        fib_nh_is_v6;
        bool                        nh_updated;
        bool                        pfsrc_removed;
        struct nexthop                *nh;
        struct rcu_head                rcu;
        struct fib_nh                fib_nh[];
};


#ifdef CONFIG_IP_MULTIPLE_TABLES
struct fib_rule;
#endif

struct fib_table;
struct fib_result {
        __be32                        prefix;
        unsigned char                prefixlen;
        unsigned char                nh_sel;
        unsigned char                type;
        unsigned char                scope;
        u32                        tclassid;
        struct fib_nh_common        *nhc;
        struct fib_info                *fi;
        struct fib_table        *table;
        struct hlist_head        *fa_head;
};

struct fib_result_nl {
        __be32                fl_addr;   /* To be looked up*/
        u32                fl_mark;
        unsigned char        fl_tos;
        unsigned char   fl_scope;
        unsigned char   tb_id_in;

        unsigned char   tb_id;      /* Results */
        unsigned char        prefixlen;
        unsigned char        nh_sel;
        unsigned char        type;
        unsigned char        scope;
        int             err;
};

#ifdef CONFIG_IP_MULTIPLE_TABLES
#define FIB_TABLE_HASHSZ 256
#else
#define FIB_TABLE_HASHSZ 2
#endif

__be32 fib_info_update_nhc_saddr(struct net *net, struct fib_nh_common *nhc,
                                 unsigned char scope);
__be32 fib_result_prefsrc(struct net *net, struct fib_result *res);

#define FIB_RES_NHC(res)                ((res).nhc)
#define FIB_RES_DEV(res)        (FIB_RES_NHC(res)->nhc_dev)
#define FIB_RES_OIF(res)        (FIB_RES_NHC(res)->nhc_oif)

struct fib_rt_info {
        struct fib_info                *fi;
        u32                        tb_id;
        __be32                        dst;
        int                        dst_len;
        u8                        tos;
        u8                        type;
        u8                        offload:1,
                                trap:1,
                                offload_failed:1,
                                unused:5;
};

struct fib_entry_notifier_info {
        struct fib_notifier_info info; /* must be first */
        u32 dst;
        int dst_len;
        struct fib_info *fi;
        u8 tos;
        u8 type;
        u32 tb_id;
};

struct fib_nh_notifier_info {
        struct fib_notifier_info info; /* must be first */
        struct fib_nh *fib_nh;
};

int call_fib4_notifier(struct notifier_block *nb,
                       enum fib_event_type event_type,
                       struct fib_notifier_info *info);
int call_fib4_notifiers(struct net *net, enum fib_event_type event_type,
                        struct fib_notifier_info *info);

int __net_init fib4_notifier_init(struct net *net);
void __net_exit fib4_notifier_exit(struct net *net);

void fib_info_notify_update(struct net *net, struct nl_info *info);
int fib_notify(struct net *net, struct notifier_block *nb,
               struct netlink_ext_ack *extack);

struct fib_table {
        struct hlist_node        tb_hlist;
        u32                        tb_id;
        int                        tb_num_default;
        struct rcu_head                rcu;
        unsigned long                 *tb_data;
        unsigned long                __data[];
};

struct fib_dump_filter {
        u32                        table_id;
        /* filter_set is an optimization that an entry is set */
        bool                        filter_set;
        bool                        dump_routes;
        bool                        dump_exceptions;
        unsigned char                protocol;
        unsigned char                rt_type;
        unsigned int                flags;
        struct net_device        *dev;
};

int fib_table_lookup(struct fib_table *tb, const struct flowi4 *flp,
                     struct fib_result *res, int fib_flags);
int fib_table_insert(struct net *, struct fib_table *, struct fib_config *,
                     struct netlink_ext_ack *extack);
int fib_table_delete(struct net *, struct fib_table *, struct fib_config *,
                     struct netlink_ext_ack *extack);
int fib_table_dump(struct fib_table *table, struct sk_buff *skb,
                   struct netlink_callback *cb, struct fib_dump_filter *filter);
int fib_table_flush(struct net *net, struct fib_table *table, bool flush_all);
struct fib_table *fib_trie_unmerge(struct fib_table *main_tb);
void fib_table_flush_external(struct fib_table *table);
void fib_free_table(struct fib_table *tb);

#ifndef CONFIG_IP_MULTIPLE_TABLES

#define TABLE_LOCAL_INDEX        (RT_TABLE_LOCAL & (FIB_TABLE_HASHSZ - 1))
#define TABLE_MAIN_INDEX        (RT_TABLE_MAIN  & (FIB_TABLE_HASHSZ - 1))

static inline struct fib_table *fib_get_table(struct net *net, u32 id)
{
        struct hlist_node *tb_hlist;
        struct hlist_head *ptr;

        ptr = id == RT_TABLE_LOCAL ?
                &net->ipv4.fib_table_hash[TABLE_LOCAL_INDEX] :
                &net->ipv4.fib_table_hash[TABLE_MAIN_INDEX];

        tb_hlist = rcu_dereference_rtnl(hlist_first_rcu(ptr));

        return hlist_entry(tb_hlist, struct fib_table, tb_hlist);
}

static inline struct fib_table *fib_new_table(struct net *net, u32 id)
{
        return fib_get_table(net, id);
}

static inline int fib_lookup(struct net *net, const struct flowi4 *flp,
                             struct fib_result *res, unsigned int flags)
{
        struct fib_table *tb;
        int err = -ENETUNREACH;

        rcu_read_lock();

        tb = fib_get_table(net, RT_TABLE_MAIN);
        if (tb)
                err = fib_table_lookup(tb, flp, res, flags | FIB_LOOKUP_NOREF);

        if (err == -EAGAIN)
                err = -ENETUNREACH;

        rcu_read_unlock();

        return err;
}

static inline bool fib4_has_custom_rules(const struct net *net)
{
        return false;
}

static inline bool fib4_rule_default(const struct fib_rule *rule)
{
        return true;
}

static inline int fib4_rules_dump(struct net *net, struct notifier_block *nb,
                                  struct netlink_ext_ack *extack)
{
        return 0;
}

static inline unsigned int fib4_rules_seq_read(struct net *net)
{
        return 0;
}

static inline bool fib4_rules_early_flow_dissect(struct net *net,
                                                 struct sk_buff *skb,
                                                 struct flowi4 *fl4,
                                                 struct flow_keys *flkeys)
{
        return false;
}
#else /* CONFIG_IP_MULTIPLE_TABLES */
int __net_init fib4_rules_init(struct net *net);
void __net_exit fib4_rules_exit(struct net *net);

struct fib_table *fib_new_table(struct net *net, u32 id);
struct fib_table *fib_get_table(struct net *net, u32 id);

int __fib_lookup(struct net *net, struct flowi4 *flp,
                 struct fib_result *res, unsigned int flags);

static inline int fib_lookup(struct net *net, struct flowi4 *flp,
                             struct fib_result *res, unsigned int flags)
{
        struct fib_table *tb;
        int err = -ENETUNREACH;

        flags |= FIB_LOOKUP_NOREF;
        if (net->ipv4.fib_has_custom_rules)
                return __fib_lookup(net, flp, res, flags);

        rcu_read_lock();

        res->tclassid = 0;

        tb = rcu_dereference_rtnl(net->ipv4.fib_main);
        if (tb)
                err = fib_table_lookup(tb, flp, res, flags);

        if (!err)
                goto out;

        tb = rcu_dereference_rtnl(net->ipv4.fib_default);
        if (tb)
                err = fib_table_lookup(tb, flp, res, flags);

out:
        if (err == -EAGAIN)
                err = -ENETUNREACH;

        rcu_read_unlock();

        return err;
}

static inline bool fib4_has_custom_rules(const struct net *net)
{
        return net->ipv4.fib_has_custom_rules;
}

bool fib4_rule_default(const struct fib_rule *rule);
int fib4_rules_dump(struct net *net, struct notifier_block *nb,
                    struct netlink_ext_ack *extack);
unsigned int fib4_rules_seq_read(struct net *net);

static inline bool fib4_rules_early_flow_dissect(struct net *net,
                                                 struct sk_buff *skb,
                                                 struct flowi4 *fl4,
                                                 struct flow_keys *flkeys)
{
        unsigned int flag = FLOW_DISSECTOR_F_STOP_AT_ENCAP;

        if (!net->ipv4.fib_rules_require_fldissect)
                return false;

        memset(flkeys, 0, sizeof(*flkeys));
        __skb_flow_dissect(net, skb, &flow_keys_dissector,
                           flkeys, NULL, 0, 0, 0, flag);

        fl4->fl4_sport = flkeys->ports.src;
        fl4->fl4_dport = flkeys->ports.dst;
        fl4->flowi4_proto = flkeys->basic.ip_proto;

        return true;
}

#endif /* CONFIG_IP_MULTIPLE_TABLES */

/* Exported by fib_frontend.c */
extern const struct nla_policy rtm_ipv4_policy[];
void ip_fib_init(void);
int fib_gw_from_via(struct fib_config *cfg, struct nlattr *nla,
                    struct netlink_ext_ack *extack);
__be32 fib_compute_spec_dst(struct sk_buff *skb);
bool fib_info_nh_uses_dev(struct fib_info *fi, const struct net_device *dev);
int fib_validate_source(struct sk_buff *skb, __be32 src, __be32 dst,
                        u8 tos, int oif, struct net_device *dev,
                        struct in_device *idev, u32 *itag);
#ifdef CONFIG_IP_ROUTE_CLASSID
static inline int fib_num_tclassid_users(struct net *net)
{
        return atomic_read(&net->ipv4.fib_num_tclassid_users);
}
#else
static inline int fib_num_tclassid_users(struct net *net)
{
        return 0;
}
#endif
int fib_unmerge(struct net *net);

static inline bool nhc_l3mdev_matches_dev(const struct fib_nh_common *nhc,
const struct net_device *dev)
{
        if (nhc->nhc_dev == dev ||
            l3mdev_master_ifindex_rcu(nhc->nhc_dev) == dev->ifindex)
                return true;

        return false;
}

/* Exported by fib_semantics.c */
int ip_fib_check_default(__be32 gw, struct net_device *dev);
int fib_sync_down_dev(struct net_device *dev, unsigned long event, bool force);
int fib_sync_down_addr(struct net_device *dev, __be32 local);
int fib_sync_up(struct net_device *dev, unsigned char nh_flags);
void fib_sync_mtu(struct net_device *dev, u32 orig_mtu);
void fib_nhc_update_mtu(struct fib_nh_common *nhc, u32 new, u32 orig);

/* Fields used for sysctl_fib_multipath_hash_fields.
 * Common to IPv4 and IPv6.
 *
 * Add new fields at the end. This is user API.
 */
#define FIB_MULTIPATH_HASH_FIELD_SRC_IP                        BIT(0)
#define FIB_MULTIPATH_HASH_FIELD_DST_IP                        BIT(1)
#define FIB_MULTIPATH_HASH_FIELD_IP_PROTO                BIT(2)
#define FIB_MULTIPATH_HASH_FIELD_FLOWLABEL                BIT(3)
#define FIB_MULTIPATH_HASH_FIELD_SRC_PORT                BIT(4)
#define FIB_MULTIPATH_HASH_FIELD_DST_PORT                BIT(5)
#define FIB_MULTIPATH_HASH_FIELD_INNER_SRC_IP                BIT(6)
#define FIB_MULTIPATH_HASH_FIELD_INNER_DST_IP                BIT(7)
#define FIB_MULTIPATH_HASH_FIELD_INNER_IP_PROTO                BIT(8)
#define FIB_MULTIPATH_HASH_FIELD_INNER_FLOWLABEL        BIT(9)
#define FIB_MULTIPATH_HASH_FIELD_INNER_SRC_PORT                BIT(10)
#define FIB_MULTIPATH_HASH_FIELD_INNER_DST_PORT                BIT(11)

#define FIB_MULTIPATH_HASH_FIELD_OUTER_MASK                \
        (FIB_MULTIPATH_HASH_FIELD_SRC_IP |                \
         FIB_MULTIPATH_HASH_FIELD_DST_IP |                \
         FIB_MULTIPATH_HASH_FIELD_IP_PROTO |                \
         FIB_MULTIPATH_HASH_FIELD_FLOWLABEL |                \
         FIB_MULTIPATH_HASH_FIELD_SRC_PORT |                \
         FIB_MULTIPATH_HASH_FIELD_DST_PORT)

#define FIB_MULTIPATH_HASH_FIELD_INNER_MASK                \
        (FIB_MULTIPATH_HASH_FIELD_INNER_SRC_IP |        \
         FIB_MULTIPATH_HASH_FIELD_INNER_DST_IP |        \
         FIB_MULTIPATH_HASH_FIELD_INNER_IP_PROTO |        \
         FIB_MULTIPATH_HASH_FIELD_INNER_FLOWLABEL |        \
         FIB_MULTIPATH_HASH_FIELD_INNER_SRC_PORT |        \
         FIB_MULTIPATH_HASH_FIELD_INNER_DST_PORT)

#define FIB_MULTIPATH_HASH_FIELD_ALL_MASK                \
        (FIB_MULTIPATH_HASH_FIELD_OUTER_MASK |                \
         FIB_MULTIPATH_HASH_FIELD_INNER_MASK)

#define FIB_MULTIPATH_HASH_FIELD_DEFAULT_MASK                \
        (FIB_MULTIPATH_HASH_FIELD_SRC_IP |                \
         FIB_MULTIPATH_HASH_FIELD_DST_IP |                \
         FIB_MULTIPATH_HASH_FIELD_IP_PROTO)

#ifdef CONFIG_IP_ROUTE_MULTIPATH
int fib_multipath_hash(const struct net *net, const struct flowi4 *fl4,
                       const struct sk_buff *skb, struct flow_keys *flkeys);
#endif
int fib_check_nh(struct net *net, struct fib_nh *nh, u32 table, u8 scope,
                 struct netlink_ext_ack *extack);
void fib_select_multipath(struct fib_result *res, int hash);
void fib_select_path(struct net *net, struct fib_result *res,
                     struct flowi4 *fl4, const struct sk_buff *skb);

int fib_nh_init(struct net *net, struct fib_nh *fib_nh,
                struct fib_config *cfg, int nh_weight,
                struct netlink_ext_ack *extack);
void fib_nh_release(struct net *net, struct fib_nh *fib_nh);
int fib_nh_common_init(struct net *net, struct fib_nh_common *nhc,
                       struct nlattr *fc_encap, u16 fc_encap_type,
                       void *cfg, gfp_t gfp_flags,
                       struct netlink_ext_ack *extack);
void fib_nh_common_release(struct fib_nh_common *nhc);

/* Exported by fib_trie.c */
void fib_alias_hw_flags_set(struct net *net, const struct fib_rt_info *fri);
void fib_trie_init(void);
struct fib_table *fib_trie_table(u32 id, struct fib_table *alias);
bool fib_lookup_good_nhc(const struct fib_nh_common *nhc, int fib_flags,
                         const struct flowi4 *flp);

static inline void fib_combine_itag(u32 *itag, const struct fib_result *res)
{
#ifdef CONFIG_IP_ROUTE_CLASSID
        struct fib_nh_common *nhc = res->nhc;
#ifdef CONFIG_IP_MULTIPLE_TABLES
        u32 rtag;
#endif
        if (nhc->nhc_family == AF_INET) {
                struct fib_nh *nh;

                nh = container_of(nhc, struct fib_nh, nh_common);
                *itag = nh->nh_tclassid << 16;
        } else {
                *itag = 0;
        }

#ifdef CONFIG_IP_MULTIPLE_TABLES
        rtag = res->tclassid;
        if (*itag == 0)
                *itag = (rtag<<16);
        *itag |= (rtag>>16);
#endif
#endif
}

void fib_flush(struct net *net);
void free_fib_info(struct fib_info *fi);

static inline void fib_info_hold(struct fib_info *fi)
{
        refcount_inc(&fi->fib_clntref);
}

static inline void fib_info_put(struct fib_info *fi)
{
        if (refcount_dec_and_test(&fi->fib_clntref))
                free_fib_info(fi);
}

#ifdef CONFIG_PROC_FS
int __net_init fib_proc_init(struct net *net);
void __net_exit fib_proc_exit(struct net *net);
#else
static inline int fib_proc_init(struct net *net)
{
        return 0;
}
static inline void fib_proc_exit(struct net *net)
{
}
#endif

u32 ip_mtu_from_fib_result(struct fib_result *res, __be32 daddr);

int ip_valid_fib_dump_req(struct net *net, const struct nlmsghdr *nlh,
                          struct fib_dump_filter *filter,
                          struct netlink_callback *cb);

int fib_nexthop_info(struct sk_buff *skb, const struct fib_nh_common *nh,
                     u8 rt_family, unsigned char *flags, bool skip_oif);
int fib_add_nexthop(struct sk_buff *skb, const struct fib_nh_common *nh,
                    int nh_weight, u8 rt_family, u32 nh_tclassid);
#endif  /* _NET_FIB_H */






































   13 






   13 






   13 
















    5 































   14 
    5 


   16 
    3 


   13 
    6 



   16 
    3 




   17 
    2 













   12 

















    5 




































   12 
   12 


















    7 


    3 








    7 























    7 





    2 






























    7 









    6 










    6 















































    7 




    7 



    7 



    6 




    6 



    6 











    1 


















    1 











































    3 









    3 








    6 







    6 







    6 


    5 







    6 




    3 

    5 













































































































   48 







   48 





































   12 

   12 



   12 



1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
// SPDX-License-Identifier: GPL-2.0
#include <linux/jhash.h>
#include <linux/netfilter.h>
#include <linux/rcupdate.h>
#include <linux/rhashtable.h>
#include <linux/vmalloc.h>
#include <net/genetlink.h>
#include <net/ila.h>
#include <net/netns/generic.h>
#include <uapi/linux/genetlink.h>
#include "ila.h"

struct ila_xlat_params {
        struct ila_params ip;
        int ifindex;
};

struct ila_map {
        struct ila_xlat_params xp;
        struct rhash_head node;
        struct ila_map __rcu *next;
        struct rcu_head rcu;
};

#define MAX_LOCKS 1024
#define        LOCKS_PER_CPU 10

static int alloc_ila_locks(struct ila_net *ilan)
{
        return alloc_bucket_spinlocks(&ilan->xlat.locks, &ilan->xlat.locks_mask,
                                      MAX_LOCKS, LOCKS_PER_CPU,
                                      GFP_KERNEL);
}

static u32 hashrnd __read_mostly;
static __always_inline void __ila_hash_secret_init(void)
{
        net_get_random_once(&hashrnd, sizeof(hashrnd));
}

static inline u32 ila_locator_hash(struct ila_locator loc)
{
        u32 *v = (u32 *)loc.v32;

        __ila_hash_secret_init();
        return jhash_2words(v[0], v[1], hashrnd);
}

static inline spinlock_t *ila_get_lock(struct ila_net *ilan,
                                       struct ila_locator loc)
{
        return &ilan->xlat.locks[ila_locator_hash(loc) & ilan->xlat.locks_mask];
}

static inline int ila_cmp_wildcards(struct ila_map *ila,
                                    struct ila_addr *iaddr, int ifindex)
{
        return (ila->xp.ifindex && ila->xp.ifindex != ifindex);
}

static inline int ila_cmp_params(struct ila_map *ila,
                                 struct ila_xlat_params *xp)
{
        return (ila->xp.ifindex != xp->ifindex);
}

static int ila_cmpfn(struct rhashtable_compare_arg *arg,
                     const void *obj)
{
        const struct ila_map *ila = obj;

        return (ila->xp.ip.locator_match.v64 != *(__be64 *)arg->key);
}

static inline int ila_order(struct ila_map *ila)
{
        int score = 0;

        if (ila->xp.ifindex)
                score += 1 << 1;

        return score;
}

static const struct rhashtable_params rht_params = {
        .nelem_hint = 1024,
        .head_offset = offsetof(struct ila_map, node),
        .key_offset = offsetof(struct ila_map, xp.ip.locator_match),
        .key_len = sizeof(u64), /* identifier */
        .max_size = 1048576,
        .min_size = 256,
        .automatic_shrinking = true,
        .obj_cmpfn = ila_cmpfn,
};

static int parse_nl_config(struct genl_info *info,
                           struct ila_xlat_params *xp)
{
        memset(xp, 0, sizeof(*xp));

        if (info->attrs[ILA_ATTR_LOCATOR])
                xp->ip.locator.v64 = (__force __be64)nla_get_u64(
                        info->attrs[ILA_ATTR_LOCATOR]);

        if (info->attrs[ILA_ATTR_LOCATOR_MATCH])
                xp->ip.locator_match.v64 = (__force __be64)nla_get_u64(
                        info->attrs[ILA_ATTR_LOCATOR_MATCH]);

        if (info->attrs[ILA_ATTR_CSUM_MODE])
                xp->ip.csum_mode = nla_get_u8(info->attrs[ILA_ATTR_CSUM_MODE]);
        else
                xp->ip.csum_mode = ILA_CSUM_NO_ACTION;

        if (info->attrs[ILA_ATTR_IDENT_TYPE])
                xp->ip.ident_type = nla_get_u8(
                                info->attrs[ILA_ATTR_IDENT_TYPE]);
        else
                xp->ip.ident_type = ILA_ATYPE_USE_FORMAT;

        if (info->attrs[ILA_ATTR_IFINDEX])
                xp->ifindex = nla_get_s32(info->attrs[ILA_ATTR_IFINDEX]);

        return 0;
}

/* Must be called with rcu readlock */
static inline struct ila_map *ila_lookup_wildcards(struct ila_addr *iaddr,
                                                   int ifindex,
                                                   struct ila_net *ilan)
{
        struct ila_map *ila;

        ila = rhashtable_lookup_fast(&ilan->xlat.rhash_table, &iaddr->loc,
                                     rht_params);
        while (ila) {
                if (!ila_cmp_wildcards(ila, iaddr, ifindex))
                        return ila;
                ila = rcu_access_pointer(ila->next);
        }

        return NULL;
}

/* Must be called with rcu readlock */
static inline struct ila_map *ila_lookup_by_params(struct ila_xlat_params *xp,
                                                   struct ila_net *ilan)
{
        struct ila_map *ila;

        ila = rhashtable_lookup_fast(&ilan->xlat.rhash_table,
                                     &xp->ip.locator_match,
                                     rht_params);
        while (ila) {
                if (!ila_cmp_params(ila, xp))
                        return ila;
                ila = rcu_access_pointer(ila->next);
        }

        return NULL;
}

static inline void ila_release(struct ila_map *ila)
{
        kfree_rcu(ila, rcu);
}

static void ila_free_node(struct ila_map *ila)
{
        struct ila_map *next;

        /* Assume rcu_readlock held */
        while (ila) {
                next = rcu_access_pointer(ila->next);
                ila_release(ila);
                ila = next;
        }
}

static void ila_free_cb(void *ptr, void *arg)
{
        ila_free_node((struct ila_map *)ptr);
}

static int ila_xlat_addr(struct sk_buff *skb, bool sir2ila);

static unsigned int
ila_nf_input(void *priv,
             struct sk_buff *skb,
             const struct nf_hook_state *state)
{
        ila_xlat_addr(skb, false);
        return NF_ACCEPT;
}

static const struct nf_hook_ops ila_nf_hook_ops[] = {
        {
                .hook = ila_nf_input,
                .pf = NFPROTO_IPV6,
                .hooknum = NF_INET_PRE_ROUTING,
                .priority = -1,
        },
};

static DEFINE_MUTEX(ila_mutex);

static int ila_add_mapping(struct net *net, struct ila_xlat_params *xp)
{
        struct ila_net *ilan = net_generic(net, ila_net_id);
        struct ila_map *ila, *head;
        spinlock_t *lock = ila_get_lock(ilan, xp->ip.locator_match);
        int err = 0, order;

        if (!READ_ONCE(ilan->xlat.hooks_registered)) {
                /* We defer registering net hooks in the namespace until the
                 * first mapping is added.
                 */
                mutex_lock(&ila_mutex);
                if (!ilan->xlat.hooks_registered) {
                        err = nf_register_net_hooks(net, ila_nf_hook_ops,
                                                ARRAY_SIZE(ila_nf_hook_ops));
                        if (!err)
                                WRITE_ONCE(ilan->xlat.hooks_registered, true);
                }
                mutex_unlock(&ila_mutex);
                if (err)
                        return err;
        }

        ila = kzalloc(sizeof(*ila), GFP_KERNEL);
        if (!ila)
                return -ENOMEM;

        ila_init_saved_csum(&xp->ip);

        ila->xp = *xp;

        order = ila_order(ila);

        spin_lock(lock);

        head = rhashtable_lookup_fast(&ilan->xlat.rhash_table,
                                      &xp->ip.locator_match,
                                      rht_params);
        if (!head) {
                /* New entry for the rhash_table */
                err = rhashtable_lookup_insert_fast(&ilan->xlat.rhash_table,
                                                    &ila->node, rht_params);
        } else {
                struct ila_map *tila = head, *prev = NULL;

                do {
                        if (!ila_cmp_params(tila, xp)) {
                                err = -EEXIST;
                                goto out;
                        }

                        if (order > ila_order(tila))
                                break;

                        prev = tila;
                        tila = rcu_dereference_protected(tila->next,
                                lockdep_is_held(lock));
                } while (tila);

                if (prev) {
                        /* Insert in sub list of head */
                        RCU_INIT_POINTER(ila->next, tila);
                        rcu_assign_pointer(prev->next, ila);
                } else {
                        /* Make this ila new head */
                        RCU_INIT_POINTER(ila->next, head);
                        err = rhashtable_replace_fast(&ilan->xlat.rhash_table,
                                                      &head->node,
                                                      &ila->node, rht_params);
                        if (err)
                                goto out;
                }
        }

out:
        spin_unlock(lock);

        if (err)
                kfree(ila);

        return err;
}

static int ila_del_mapping(struct net *net, struct ila_xlat_params *xp)
{
        struct ila_net *ilan = net_generic(net, ila_net_id);
        struct ila_map *ila, *head, *prev;
        spinlock_t *lock = ila_get_lock(ilan, xp->ip.locator_match);
        int err = -ENOENT;

        spin_lock(lock);

        head = rhashtable_lookup_fast(&ilan->xlat.rhash_table,
                                      &xp->ip.locator_match, rht_params);
        ila = head;

        prev = NULL;

        while (ila) {
                if (ila_cmp_params(ila, xp)) {
                        prev = ila;
                        ila = rcu_dereference_protected(ila->next,
                                                        lockdep_is_held(lock));
                        continue;
                }

                err = 0;

                if (prev) {
                        /* Not head, just delete from list */
                        rcu_assign_pointer(prev->next, ila->next);
                } else {
                        /* It is the head. If there is something in the
                         * sublist we need to make a new head.
                         */
                        head = rcu_dereference_protected(ila->next,
                                                         lockdep_is_held(lock));
                        if (head) {
                                /* Put first entry in the sublist into the
                                 * table
                                 */
                                err = rhashtable_replace_fast(
                                        &ilan->xlat.rhash_table, &ila->node,
                                        &head->node, rht_params);
                                if (err)
                                        goto out;
                        } else {
                                /* Entry no longer used */
                                err = rhashtable_remove_fast(
                                                &ilan->xlat.rhash_table,
                                                &ila->node, rht_params);
                        }
                }

                ila_release(ila);

                break;
        }

out:
        spin_unlock(lock);

        return err;
}

int ila_xlat_nl_cmd_add_mapping(struct sk_buff *skb, struct genl_info *info)
{
        struct net *net = genl_info_net(info);
        struct ila_xlat_params p;
        int err;

        err = parse_nl_config(info, &p);
        if (err)
                return err;

        return ila_add_mapping(net, &p);
}

int ila_xlat_nl_cmd_del_mapping(struct sk_buff *skb, struct genl_info *info)
{
        struct net *net = genl_info_net(info);
        struct ila_xlat_params xp;
        int err;

        err = parse_nl_config(info, &xp);
        if (err)
                return err;

        ila_del_mapping(net, &xp);

        return 0;
}

static inline spinlock_t *lock_from_ila_map(struct ila_net *ilan,
                                            struct ila_map *ila)
{
        return ila_get_lock(ilan, ila->xp.ip.locator_match);
}

int ila_xlat_nl_cmd_flush(struct sk_buff *skb, struct genl_info *info)
{
        struct net *net = genl_info_net(info);
        struct ila_net *ilan = net_generic(net, ila_net_id);
        struct rhashtable_iter iter;
        struct ila_map *ila;
        spinlock_t *lock;
        int ret = 0;

        rhashtable_walk_enter(&ilan->xlat.rhash_table, &iter);
        rhashtable_walk_start(&iter);

        for (;;) {
                ila = rhashtable_walk_next(&iter);

                if (IS_ERR(ila)) {
                        if (PTR_ERR(ila) == -EAGAIN)
                                continue;
                        ret = PTR_ERR(ila);
                        goto done;
                } else if (!ila) {
                        break;
                }

                lock = lock_from_ila_map(ilan, ila);

                spin_lock(lock);

                ret = rhashtable_remove_fast(&ilan->xlat.rhash_table,
                                             &ila->node, rht_params);
                if (!ret)
                        ila_free_node(ila);

                spin_unlock(lock);

                if (ret)
                        break;
        }

done:
        rhashtable_walk_stop(&iter);
        rhashtable_walk_exit(&iter);
        return ret;
}

static int ila_fill_info(struct ila_map *ila, struct sk_buff *msg)
{
        if (nla_put_u64_64bit(msg, ILA_ATTR_LOCATOR,
                              (__force u64)ila->xp.ip.locator.v64,
                              ILA_ATTR_PAD) ||
            nla_put_u64_64bit(msg, ILA_ATTR_LOCATOR_MATCH,
                              (__force u64)ila->xp.ip.locator_match.v64,
                              ILA_ATTR_PAD) ||
            nla_put_s32(msg, ILA_ATTR_IFINDEX, ila->xp.ifindex) ||
            nla_put_u8(msg, ILA_ATTR_CSUM_MODE, ila->xp.ip.csum_mode) ||
            nla_put_u8(msg, ILA_ATTR_IDENT_TYPE, ila->xp.ip.ident_type))
                return -1;

        return 0;
}

static int ila_dump_info(struct ila_map *ila,
                         u32 portid, u32 seq, u32 flags,
                         struct sk_buff *skb, u8 cmd)
{
        void *hdr;

        hdr = genlmsg_put(skb, portid, seq, &ila_nl_family, flags, cmd);
        if (!hdr)
                return -ENOMEM;

        if (ila_fill_info(ila, skb) < 0)
                goto nla_put_failure;

        genlmsg_end(skb, hdr);
        return 0;

nla_put_failure:
        genlmsg_cancel(skb, hdr);
        return -EMSGSIZE;
}

int ila_xlat_nl_cmd_get_mapping(struct sk_buff *skb, struct genl_info *info)
{
        struct net *net = genl_info_net(info);
        struct ila_net *ilan = net_generic(net, ila_net_id);
        struct sk_buff *msg;
        struct ila_xlat_params xp;
        struct ila_map *ila;
        int ret;

        ret = parse_nl_config(info, &xp);
        if (ret)
                return ret;

        msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
        if (!msg)
                return -ENOMEM;

        rcu_read_lock();

        ret = -ESRCH;
        ila = ila_lookup_by_params(&xp, ilan);
        if (ila) {
                ret = ila_dump_info(ila,
                                    info->snd_portid,
                                    info->snd_seq, 0, msg,
                                    info->genlhdr->cmd);
        }

        rcu_read_unlock();

        if (ret < 0)
                goto out_free;

        return genlmsg_reply(msg, info);

out_free:
        nlmsg_free(msg);
        return ret;
}

struct ila_dump_iter {
        struct rhashtable_iter rhiter;
        int skip;
};

int ila_xlat_nl_dump_start(struct netlink_callback *cb)
{
        struct net *net = sock_net(cb->skb->sk);
        struct ila_net *ilan = net_generic(net, ila_net_id);
        struct ila_dump_iter *iter;

        iter = kmalloc(sizeof(*iter), GFP_KERNEL);
        if (!iter)
                return -ENOMEM;

        rhashtable_walk_enter(&ilan->xlat.rhash_table, &iter->rhiter);

        iter->skip = 0;
        cb->args[0] = (long)iter;

        return 0;
}

int ila_xlat_nl_dump_done(struct netlink_callback *cb)
{
        struct ila_dump_iter *iter = (struct ila_dump_iter *)cb->args[0];

        rhashtable_walk_exit(&iter->rhiter);

        kfree(iter);

        return 0;
}

int ila_xlat_nl_dump(struct sk_buff *skb, struct netlink_callback *cb)
{
        struct ila_dump_iter *iter = (struct ila_dump_iter *)cb->args[0];
        struct rhashtable_iter *rhiter = &iter->rhiter;
        int skip = iter->skip;
        struct ila_map *ila;
        int ret;

        rhashtable_walk_start(rhiter);

        /* Get first entry */
        ila = rhashtable_walk_peek(rhiter);

        if (ila && !IS_ERR(ila) && skip) {
                /* Skip over visited entries */

                while (ila && skip) {
                        /* Skip over any ila entries in this list that we
                         * have already dumped.
                         */
                        ila = rcu_access_pointer(ila->next);
                        skip--;
                }
        }

        skip = 0;

        for (;;) {
                if (IS_ERR(ila)) {
                        ret = PTR_ERR(ila);
                        if (ret == -EAGAIN) {
                                /* Table has changed and iter has reset. Return
                                 * -EAGAIN to the application even if we have
                                 * written data to the skb. The application
                                 * needs to deal with this.
                                 */

                                goto out_ret;
                        } else {
                                break;
                        }
                } else if (!ila) {
                        ret = 0;
                        break;
                }

                while (ila) {
                        ret =  ila_dump_info(ila, NETLINK_CB(cb->skb).portid,
                                             cb->nlh->nlmsg_seq, NLM_F_MULTI,
                                             skb, ILA_CMD_GET);
                        if (ret)
                                goto out;

                        skip++;
                        ila = rcu_access_pointer(ila->next);
                }

                skip = 0;
                ila = rhashtable_walk_next(rhiter);
        }

out:
        iter->skip = skip;
        ret = (skb->len ? : ret);

out_ret:
        rhashtable_walk_stop(rhiter);
        return ret;
}

int ila_xlat_init_net(struct net *net)
{
        struct ila_net *ilan = net_generic(net, ila_net_id);
        int err;

        err = alloc_ila_locks(ilan);
        if (err)
                return err;

        rhashtable_init(&ilan->xlat.rhash_table, &rht_params);

        return 0;
}

void ila_xlat_pre_exit_net(struct net *net)
{
        struct ila_net *ilan = net_generic(net, ila_net_id);

        if (ilan->xlat.hooks_registered)
                nf_unregister_net_hooks(net, ila_nf_hook_ops,
                                        ARRAY_SIZE(ila_nf_hook_ops));
}

void ila_xlat_exit_net(struct net *net)
{
        struct ila_net *ilan = net_generic(net, ila_net_id);

        rhashtable_free_and_destroy(&ilan->xlat.rhash_table, ila_free_cb, NULL);

        free_bucket_spinlocks(ilan->xlat.locks);
}

static int ila_xlat_addr(struct sk_buff *skb, bool sir2ila)
{
        struct ila_map *ila;
        struct ipv6hdr *ip6h = ipv6_hdr(skb);
        struct net *net = dev_net(skb->dev);
        struct ila_net *ilan = net_generic(net, ila_net_id);
        struct ila_addr *iaddr = ila_a2i(&ip6h->daddr);

        /* Assumes skb contains a valid IPv6 header that is pulled */

        /* No check here that ILA type in the mapping matches what is in the
         * address. We assume that whatever sender gaves us can be translated.
         * The checksum mode however is relevant.
         */

        rcu_read_lock();

        ila = ila_lookup_wildcards(iaddr, skb->dev->ifindex, ilan);
        if (ila)
                ila_update_ipv6_locator(skb, &ila->xp.ip, sir2ila);

        rcu_read_unlock();

        return 0;
}














   93 










   93 
















































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
// SPDX-License-Identifier: GPL-2.0
#include <linux/cache.h>
#include <linux/sched.h>
#include <linux/slab.h>
#include <linux/pid_namespace.h>
#include "internal.h"

/*
 * /proc/self:
 */
static const char *proc_self_get_link(struct dentry *dentry,
                                      struct inode *inode,
                                      struct delayed_call *done)
{
        struct pid_namespace *ns = proc_pid_ns(inode->i_sb);
        pid_t tgid = task_tgid_nr_ns(current, ns);
        char *name;

        if (!tgid)
                return ERR_PTR(-ENOENT);
        /* max length of unsigned int in decimal + NULL term */
        name = kmalloc(10 + 1, dentry ? GFP_KERNEL : GFP_ATOMIC);
        if (unlikely(!name))
                return dentry ? ERR_PTR(-ENOMEM) : ERR_PTR(-ECHILD);
        sprintf(name, "%u", tgid);
        set_delayed_call(done, kfree_link, name);
        return name;
}

static const struct inode_operations proc_self_inode_operations = {
        .get_link        = proc_self_get_link,
};

static unsigned self_inum __ro_after_init;

int proc_setup_self(struct super_block *s)
{
        struct inode *root_inode = d_inode(s->s_root);
        struct proc_fs_info *fs_info = proc_sb_info(s);
        struct dentry *self;
        int ret = -ENOMEM;

        inode_lock(root_inode);
        self = d_alloc_name(s->s_root, "self");
        if (self) {
                struct inode *inode = new_inode(s);
                if (inode) {
                        inode->i_ino = self_inum;
                        inode->i_mtime = inode->i_atime = inode->i_ctime = current_time(inode);
                        inode->i_mode = S_IFLNK | S_IRWXUGO;
                        inode->i_uid = GLOBAL_ROOT_UID;
                        inode->i_gid = GLOBAL_ROOT_GID;
                        inode->i_op = &proc_self_inode_operations;
                        d_add(self, inode);
                        ret = 0;
                } else {
                        dput(self);
                }
        }
        inode_unlock(root_inode);

        if (ret)
                pr_err("proc_fill_super: can't allocate /proc/self\n");
        else
                fs_info->proc_self = self;

        return ret;
}

void __init proc_self_init(void)
{
        proc_alloc_inum(&self_inum);
}





















































































































































    6 

    6 
















    6 









































































    6 




    6 



























































































































































    5 









    5 

































































   27 




















































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright (C) ST-Ericsson AB 2010
 * Authors:        Sjur Brendeland
 *                Daniel Martensson
 */

#define pr_fmt(fmt) KBUILD_MODNAME ":%s(): " fmt, __func__

#include <linux/fs.h>
#include <linux/init.h>
#include <linux/module.h>
#include <linux/netdevice.h>
#include <linux/if_ether.h>
#include <linux/ip.h>
#include <linux/sched.h>
#include <linux/sockios.h>
#include <linux/caif/if_caif.h>
#include <net/rtnetlink.h>
#include <net/caif/caif_layer.h>
#include <net/caif/cfpkt.h>
#include <net/caif/caif_dev.h>

/* GPRS PDP connection has MTU to 1500 */
#define GPRS_PDP_MTU 1500
/* 5 sec. connect timeout */
#define CONNECT_TIMEOUT (5 * HZ)
#define CAIF_NET_DEFAULT_QUEUE_LEN 500
#define UNDEF_CONNID 0xffffffff

/*This list is protected by the rtnl lock. */
static LIST_HEAD(chnl_net_list);

MODULE_LICENSE("GPL");
MODULE_ALIAS_RTNL_LINK("caif");

enum caif_states {
        CAIF_CONNECTED                = 1,
        CAIF_CONNECTING,
        CAIF_DISCONNECTED,
        CAIF_SHUTDOWN
};

struct chnl_net {
        struct cflayer chnl;
        struct caif_connect_request conn_req;
        struct list_head list_field;
        struct net_device *netdev;
        char name[256];
        wait_queue_head_t netmgmt_wq;
        /* Flow status to remember and control the transmission. */
        bool flowenabled;
        enum caif_states state;
};

static int chnl_recv_cb(struct cflayer *layr, struct cfpkt *pkt)
{
        struct sk_buff *skb;
        struct chnl_net *priv;
        int pktlen;
        const u8 *ip_version;
        u8 buf;

        priv = container_of(layr, struct chnl_net, chnl);

        skb = (struct sk_buff *) cfpkt_tonative(pkt);

        /* Get length of CAIF packet. */
        pktlen = skb->len;

        /* Pass some minimum information and
         * send the packet to the net stack.
         */
        skb->dev = priv->netdev;

        /* check the version of IP */
        ip_version = skb_header_pointer(skb, 0, 1, &buf);
        if (!ip_version) {
                kfree_skb(skb);
                return -EINVAL;
        }

        switch (*ip_version >> 4) {
        case 4:
                skb->protocol = htons(ETH_P_IP);
                break;
        case 6:
                skb->protocol = htons(ETH_P_IPV6);
                break;
        default:
                kfree_skb(skb);
                priv->netdev->stats.rx_errors++;
                return -EINVAL;
        }

        /* If we change the header in loop mode, the checksum is corrupted. */
        if (priv->conn_req.protocol == CAIFPROTO_DATAGRAM_LOOP)
                skb->ip_summed = CHECKSUM_UNNECESSARY;
        else
                skb->ip_summed = CHECKSUM_NONE;

        netif_rx_any_context(skb);

        /* Update statistics. */
        priv->netdev->stats.rx_packets++;
        priv->netdev->stats.rx_bytes += pktlen;

        return 0;
}

static int delete_device(struct chnl_net *dev)
{
        ASSERT_RTNL();
        if (dev->netdev)
                unregister_netdevice(dev->netdev);
        return 0;
}

static void close_work(struct work_struct *work)
{
        struct chnl_net *dev = NULL;
        struct list_head *list_node;
        struct list_head *_tmp;

        rtnl_lock();
        list_for_each_safe(list_node, _tmp, &chnl_net_list) {
                dev = list_entry(list_node, struct chnl_net, list_field);
                if (dev->state == CAIF_SHUTDOWN)
                        dev_close(dev->netdev);
        }
        rtnl_unlock();
}
static DECLARE_WORK(close_worker, close_work);

static void chnl_hold(struct cflayer *lyr)
{
        struct chnl_net *priv = container_of(lyr, struct chnl_net, chnl);
        dev_hold(priv->netdev);
}

static void chnl_put(struct cflayer *lyr)
{
        struct chnl_net *priv = container_of(lyr, struct chnl_net, chnl);
        dev_put(priv->netdev);
}

static void chnl_flowctrl_cb(struct cflayer *layr, enum caif_ctrlcmd flow,
                             int phyid)
{
        struct chnl_net *priv = container_of(layr, struct chnl_net, chnl);
        pr_debug("NET flowctrl func called flow: %s\n",
                flow == CAIF_CTRLCMD_FLOW_ON_IND ? "ON" :
                flow == CAIF_CTRLCMD_INIT_RSP ? "INIT" :
                flow == CAIF_CTRLCMD_FLOW_OFF_IND ? "OFF" :
                flow == CAIF_CTRLCMD_DEINIT_RSP ? "CLOSE/DEINIT" :
                flow == CAIF_CTRLCMD_INIT_FAIL_RSP ? "OPEN_FAIL" :
                flow == CAIF_CTRLCMD_REMOTE_SHUTDOWN_IND ?
                 "REMOTE_SHUTDOWN" : "UNKNOWN CTRL COMMAND");



        switch (flow) {
        case CAIF_CTRLCMD_FLOW_OFF_IND:
                priv->flowenabled = false;
                netif_stop_queue(priv->netdev);
                break;
        case CAIF_CTRLCMD_DEINIT_RSP:
                priv->state = CAIF_DISCONNECTED;
                break;
        case CAIF_CTRLCMD_INIT_FAIL_RSP:
                priv->state = CAIF_DISCONNECTED;
                wake_up_interruptible(&priv->netmgmt_wq);
                break;
        case CAIF_CTRLCMD_REMOTE_SHUTDOWN_IND:
                priv->state = CAIF_SHUTDOWN;
                netif_tx_disable(priv->netdev);
                schedule_work(&close_worker);
                break;
        case CAIF_CTRLCMD_FLOW_ON_IND:
                priv->flowenabled = true;
                netif_wake_queue(priv->netdev);
                break;
        case CAIF_CTRLCMD_INIT_RSP:
                caif_client_register_refcnt(&priv->chnl, chnl_hold, chnl_put);
                priv->state = CAIF_CONNECTED;
                priv->flowenabled = true;
                netif_wake_queue(priv->netdev);
                wake_up_interruptible(&priv->netmgmt_wq);
                break;
        default:
                break;
        }
}

static netdev_tx_t chnl_net_start_xmit(struct sk_buff *skb,
                                       struct net_device *dev)
{
        struct chnl_net *priv;
        struct cfpkt *pkt = NULL;
        int len;
        int result = -1;
        /* Get our private data. */
        priv = netdev_priv(dev);

        if (skb->len > priv->netdev->mtu) {
                pr_warn("Size of skb exceeded MTU\n");
                kfree_skb(skb);
                dev->stats.tx_errors++;
                return NETDEV_TX_OK;
        }

        if (!priv->flowenabled) {
                pr_debug("dropping packets flow off\n");
                kfree_skb(skb);
                dev->stats.tx_dropped++;
                return NETDEV_TX_OK;
        }

        if (priv->conn_req.protocol == CAIFPROTO_DATAGRAM_LOOP)
                swap(ip_hdr(skb)->saddr, ip_hdr(skb)->daddr);

        /* Store original SKB length. */
        len = skb->len;

        pkt = cfpkt_fromnative(CAIF_DIR_OUT, (void *) skb);

        /* Send the packet down the stack. */
        result = priv->chnl.dn->transmit(priv->chnl.dn, pkt);
        if (result) {
                dev->stats.tx_dropped++;
                return NETDEV_TX_OK;
        }

        /* Update statistics. */
        dev->stats.tx_packets++;
        dev->stats.tx_bytes += len;

        return NETDEV_TX_OK;
}

static int chnl_net_open(struct net_device *dev)
{
        struct chnl_net *priv = NULL;
        int result = -1;
        int llifindex, headroom, tailroom, mtu;
        struct net_device *lldev;
        ASSERT_RTNL();
        priv = netdev_priv(dev);
        if (!priv) {
                pr_debug("chnl_net_open: no priv\n");
                return -ENODEV;
        }

        if (priv->state != CAIF_CONNECTING) {
                priv->state = CAIF_CONNECTING;
                result = caif_connect_client(dev_net(dev), &priv->conn_req,
                                                &priv->chnl, &llifindex,
                                                &headroom, &tailroom);
                if (result != 0) {
                                pr_debug("err: "
                                         "Unable to register and open device,"
                                         " Err:%d\n",
                                         result);
                                goto error;
                }

                lldev = __dev_get_by_index(dev_net(dev), llifindex);

                if (lldev == NULL) {
                        pr_debug("no interface?\n");
                        result = -ENODEV;
                        goto error;
                }

                dev->needed_tailroom = tailroom + lldev->needed_tailroom;
                dev->hard_header_len = headroom + lldev->hard_header_len +
                        lldev->needed_tailroom;

                /*
                 * MTU, head-room etc is not know before we have a
                 * CAIF link layer device available. MTU calculation may
                 * override initial RTNL configuration.
                 * MTU is minimum of current mtu, link layer mtu pluss
                 * CAIF head and tail, and PDP GPRS contexts max MTU.
                 */
                mtu = min_t(int, dev->mtu, lldev->mtu - (headroom + tailroom));
                mtu = min_t(int, GPRS_PDP_MTU, mtu);
                dev_set_mtu(dev, mtu);

                if (mtu < 100) {
                        pr_warn("CAIF Interface MTU too small (%d)\n", mtu);
                        result = -ENODEV;
                        goto error;
                }
        }

        rtnl_unlock();  /* Release RTNL lock during connect wait */

        result = wait_event_interruptible_timeout(priv->netmgmt_wq,
                                                priv->state != CAIF_CONNECTING,
                                                CONNECT_TIMEOUT);

        rtnl_lock();

        if (result == -ERESTARTSYS) {
                pr_debug("wait_event_interruptible woken by a signal\n");
                result = -ERESTARTSYS;
                goto error;
        }

        if (result == 0) {
                pr_debug("connect timeout\n");
                result = -ETIMEDOUT;
                goto error;
        }

        if (priv->state != CAIF_CONNECTED) {
                pr_debug("connect failed\n");
                result = -ECONNREFUSED;
                goto error;
        }
        pr_debug("CAIF Netdevice connected\n");
        return 0;

error:
        caif_disconnect_client(dev_net(dev), &priv->chnl);
        priv->state = CAIF_DISCONNECTED;
        pr_debug("state disconnected\n");
        return result;

}

static int chnl_net_stop(struct net_device *dev)
{
        struct chnl_net *priv;

        ASSERT_RTNL();
        priv = netdev_priv(dev);
        priv->state = CAIF_DISCONNECTED;
        caif_disconnect_client(dev_net(dev), &priv->chnl);
        return 0;
}

static int chnl_net_init(struct net_device *dev)
{
        struct chnl_net *priv;
        ASSERT_RTNL();
        priv = netdev_priv(dev);
        strncpy(priv->name, dev->name, sizeof(priv->name));
        INIT_LIST_HEAD(&priv->list_field);
        return 0;
}

static void chnl_net_uninit(struct net_device *dev)
{
        struct chnl_net *priv;
        ASSERT_RTNL();
        priv = netdev_priv(dev);
        list_del_init(&priv->list_field);
}

static const struct net_device_ops netdev_ops = {
        .ndo_open = chnl_net_open,
        .ndo_stop = chnl_net_stop,
        .ndo_init = chnl_net_init,
        .ndo_uninit = chnl_net_uninit,
        .ndo_start_xmit = chnl_net_start_xmit,
};

static void chnl_net_destructor(struct net_device *dev)
{
        struct chnl_net *priv = netdev_priv(dev);
        caif_free_client(&priv->chnl);
}

static void ipcaif_net_setup(struct net_device *dev)
{
        struct chnl_net *priv;
        dev->netdev_ops = &netdev_ops;
        dev->needs_free_netdev = true;
        dev->priv_destructor = chnl_net_destructor;
        dev->flags |= IFF_NOARP;
        dev->flags |= IFF_POINTOPOINT;
        dev->mtu = GPRS_PDP_MTU;
        dev->tx_queue_len = CAIF_NET_DEFAULT_QUEUE_LEN;

        priv = netdev_priv(dev);
        priv->chnl.receive = chnl_recv_cb;
        priv->chnl.ctrlcmd = chnl_flowctrl_cb;
        priv->netdev = dev;
        priv->conn_req.protocol = CAIFPROTO_DATAGRAM;
        priv->conn_req.link_selector = CAIF_LINK_HIGH_BANDW;
        priv->conn_req.priority = CAIF_PRIO_LOW;
        /* Insert illegal value */
        priv->conn_req.sockaddr.u.dgm.connection_id = UNDEF_CONNID;
        priv->flowenabled = false;

        init_waitqueue_head(&priv->netmgmt_wq);
}


static int ipcaif_fill_info(struct sk_buff *skb, const struct net_device *dev)
{
        struct chnl_net *priv;
        u8 loop;
        priv = netdev_priv(dev);
        if (nla_put_u32(skb, IFLA_CAIF_IPV4_CONNID,
                        priv->conn_req.sockaddr.u.dgm.connection_id) ||
            nla_put_u32(skb, IFLA_CAIF_IPV6_CONNID,
                        priv->conn_req.sockaddr.u.dgm.connection_id))
                goto nla_put_failure;
        loop = priv->conn_req.protocol == CAIFPROTO_DATAGRAM_LOOP;
        if (nla_put_u8(skb, IFLA_CAIF_LOOPBACK, loop))
                goto nla_put_failure;
        return 0;
nla_put_failure:
        return -EMSGSIZE;

}

static void caif_netlink_parms(struct nlattr *data[],
                               struct caif_connect_request *conn_req)
{
        if (!data) {
                pr_warn("no params data found\n");
                return;
        }
        if (data[IFLA_CAIF_IPV4_CONNID])
                conn_req->sockaddr.u.dgm.connection_id =
                        nla_get_u32(data[IFLA_CAIF_IPV4_CONNID]);
        if (data[IFLA_CAIF_IPV6_CONNID])
                conn_req->sockaddr.u.dgm.connection_id =
                        nla_get_u32(data[IFLA_CAIF_IPV6_CONNID]);
        if (data[IFLA_CAIF_LOOPBACK]) {
                if (nla_get_u8(data[IFLA_CAIF_LOOPBACK]))
                        conn_req->protocol = CAIFPROTO_DATAGRAM_LOOP;
                else
                        conn_req->protocol = CAIFPROTO_DATAGRAM;
        }
}

static int ipcaif_newlink(struct net *src_net, struct net_device *dev,
                          struct nlattr *tb[], struct nlattr *data[],
                          struct netlink_ext_ack *extack)
{
        int ret;
        struct chnl_net *caifdev;
        ASSERT_RTNL();
        caifdev = netdev_priv(dev);
        caif_netlink_parms(data, &caifdev->conn_req);

        ret = register_netdevice(dev);
        if (ret)
                pr_warn("device rtml registration failed\n");
        else
                list_add(&caifdev->list_field, &chnl_net_list);

        /* Use ifindex as connection id, and use loopback channel default. */
        if (caifdev->conn_req.sockaddr.u.dgm.connection_id == UNDEF_CONNID) {
                caifdev->conn_req.sockaddr.u.dgm.connection_id = dev->ifindex;
                caifdev->conn_req.protocol = CAIFPROTO_DATAGRAM_LOOP;
        }
        return ret;
}

static int ipcaif_changelink(struct net_device *dev, struct nlattr *tb[],
                             struct nlattr *data[],
                             struct netlink_ext_ack *extack)
{
        struct chnl_net *caifdev;
        ASSERT_RTNL();
        caifdev = netdev_priv(dev);
        caif_netlink_parms(data, &caifdev->conn_req);
        netdev_state_change(dev);
        return 0;
}

static size_t ipcaif_get_size(const struct net_device *dev)
{
        return
                /* IFLA_CAIF_IPV4_CONNID */
                nla_total_size(4) +
                /* IFLA_CAIF_IPV6_CONNID */
                nla_total_size(4) +
                /* IFLA_CAIF_LOOPBACK */
                nla_total_size(2) +
                0;
}

static const struct nla_policy ipcaif_policy[IFLA_CAIF_MAX + 1] = {
        [IFLA_CAIF_IPV4_CONNID]              = { .type = NLA_U32 },
        [IFLA_CAIF_IPV6_CONNID]              = { .type = NLA_U32 },
        [IFLA_CAIF_LOOPBACK]              = { .type = NLA_U8 }
};


static struct rtnl_link_ops ipcaif_link_ops __read_mostly = {
        .kind                = "caif",
        .priv_size        = sizeof(struct chnl_net),
        .setup                = ipcaif_net_setup,
        .maxtype        = IFLA_CAIF_MAX,
        .policy                = ipcaif_policy,
        .newlink        = ipcaif_newlink,
        .changelink        = ipcaif_changelink,
        .get_size        = ipcaif_get_size,
        .fill_info        = ipcaif_fill_info,

};

static int __init chnl_init_module(void)
{
        return rtnl_link_register(&ipcaif_link_ops);
}

static void __exit chnl_exit_module(void)
{
        struct chnl_net *dev = NULL;
        struct list_head *list_node;
        struct list_head *_tmp;
        rtnl_link_unregister(&ipcaif_link_ops);
        rtnl_lock();
        list_for_each_safe(list_node, _tmp, &chnl_net_list) {
                dev = list_entry(list_node, struct chnl_net, list_field);
                list_del_init(list_node);
                delete_device(dev);
        }
        rtnl_unlock();
}

module_init(chnl_init_module);
module_exit(chnl_exit_module);

































































































































































































































    8 




    2 



    7 






















































































































































































































































































































































































































































































































































































































































































   16 


   16 

   16 

   16 



























































































































   36 




    9 


   27 




   15 





   10 




    3 





   10 









    9 





    1 










    3 



   10 


    3 

    7 








    9 




















    9 





    9 







































































    9 


    8 


    1 





    1 






    1 













































































































































































































































































































































































































































































































































































































































































































































   15 



























    1 

    1 














    8 



    4 



    1 







    9 


    3 










    3 
    3 




    3 






    3 






























































































































    3 


    3 
















   16 





















    1 

    1 







   14 
    5 






    6 


    3 








    3 
    2 





    3 






    3 

















































































    3 











    9 


















    1 

    1 








    1 






    4 


    2 








    2 
    2 




    2 






    2 



















































    2 





























































































   10 













    1 



    1 


    8 








    3 















    2 



    1 

    2 
    2 


    1 









































































































































































































   12 











   12 





    2 

    1 






    9 


    9 








































































































   10 











    1 



    1 




    4 








    3 


    1 






    1 
































































































































































































































































































































































































































































































































































































































































































































































































































































































































   49 




















































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965
2966
2967
2968
2969
2970
2971
2972
2973
2974
2975
2976
2977
2978
2979
2980
2981
2982
2983
2984
2985
2986
2987
2988
2989
2990
2991
2992
2993
2994
2995
2996
2997
2998
2999
3000
3001
3002
3003
3004
3005
3006
3007
3008
3009
3010
3011
3012
3013
3014
3015
3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029
3030
3031
3032
3033
3034
3035
3036
3037
3038
3039
3040
3041
3042
3043
3044
3045
3046
3047
3048
3049
3050
3051
3052
3053
3054
3055
3056
3057
3058
3059
3060
3061
3062
3063
3064
3065
3066
3067
3068
3069
3070
3071
3072
3073
3074
3075
3076
3077
3078
3079
3080
3081
3082
3083
3084
3085
3086
3087
3088
3089
3090
3091
3092
3093
3094
3095
3096
3097
3098
3099
3100
3101
3102
3103
3104
3105
3106
3107
3108
3109
3110
3111
3112
3113
3114
3115
3116
3117
3118
3119
3120
3121
3122
3123
3124
3125
3126
3127
3128
3129
3130
3131
3132
3133
3134
3135
3136
3137
3138
3139
3140
3141
3142
3143
3144
3145
3146
3147
3148
3149
3150
3151
3152
3153
3154
3155
3156
3157
3158
3159
3160
3161
3162
3163
3164
3165
3166
3167
3168
3169
3170
3171
3172
3173
3174
3175
3176
3177
3178
3179
3180
3181
3182
3183
3184
3185
3186
3187
3188
3189
3190
3191
3192
3193
3194
3195
3196
3197
3198
3199
3200
3201
3202
3203
3204
3205
3206
3207
3208
3209
3210
3211
3212
3213
3214
3215
3216
3217
3218
3219
3220
3221
3222
3223
3224
3225
3226
3227
3228
3229
3230
3231
3232
3233
3234
3235
3236
3237
3238
3239
3240
3241
3242
3243
3244
3245
3246
3247
3248
3249
3250
3251
3252
3253
3254
3255
3256
3257
3258
3259
3260
3261
3262
3263
3264
3265
3266
3267
3268
3269
3270
3271
3272
3273
3274
3275
3276
3277
3278
3279
3280
3281
3282
3283
3284
3285
3286
3287
3288
3289
3290
3291
3292
3293
3294
3295
3296
3297
3298
3299
3300
3301
3302
3303
3304
3305
3306
3307
3308
3309
3310
3311
3312
3313
3314
3315
3316
3317
3318
3319
3320
3321
3322
3323
3324
3325
3326
3327
3328
3329
3330
3331
3332
3333
3334
3335
3336
3337
3338
3339
3340
3341
3342
3343
3344
3345
3346
3347
3348
3349
3350
3351
3352
3353
3354
3355
3356
3357
3358
3359
3360
3361
3362
3363
3364
3365
3366
3367
3368
3369
3370
3371
3372
3373
3374
3375
3376
3377
3378
3379
3380
3381
3382
3383
3384
3385
3386
3387
3388
3389
3390
3391
3392
3393
3394
3395
3396
3397
3398
3399
3400
3401
3402
3403
3404
3405
3406
3407
3408
3409
3410
3411
3412
3413
3414
3415
3416
3417
3418
3419
3420
3421
3422
3423
3424
3425
3426
3427
3428
3429
3430
3431
3432
3433
3434
3435
3436
3437
3438
3439
3440
3441
3442
3443
3444
3445
3446
3447
3448
3449
3450
3451
3452
3453
3454
3455
3456
3457
3458
3459
3460
3461
3462
3463
3464
3465
3466
3467
3468
3469
3470
3471
3472
3473
3474
3475
3476
3477
3478
3479
3480
3481
3482
3483
3484
3485
3486
3487
3488
3489
3490
3491
3492
3493
3494
3495
3496
3497
3498
3499
3500
3501
3502
3503
3504
3505
3506
3507
3508
3509
3510
3511
3512
3513
3514
3515
3516
3517
3518
3519
3520
3521
3522
3523
3524
3525
3526
3527
3528
3529
3530
3531
3532
3533
3534
3535
3536
3537
3538
3539
3540
3541
3542
3543
3544
3545
3546
3547
3548
3549
3550
3551
3552
3553
3554
3555
3556
3557
3558
3559
3560
3561
3562
3563
3564
3565
3566
3567
3568
3569
3570
3571
3572
3573
3574
3575
3576
3577
3578
3579
3580
3581
3582
3583
3584
3585
3586
3587
3588
3589
3590
3591
3592
3593
3594
3595
3596
3597
3598
3599
3600
3601
3602
3603
3604
3605
3606
3607
3608
3609
3610
3611
3612
3613
3614
3615
3616
3617
3618
3619
3620
3621
3622
3623
3624
3625
3626
3627
3628
3629
3630
3631
3632
3633
3634
3635
3636
3637
3638
3639
3640
3641
3642
3643
3644
3645
3646
3647
3648
3649
3650
3651
3652
3653
3654
3655
3656
3657
3658
3659
3660
3661
3662
3663
3664
3665
3666
3667
3668
3669
3670
3671
3672
3673
3674
3675
3676
3677
3678
3679
3680
3681
3682
3683
3684
3685
3686
3687
3688
3689
3690
3691
3692
3693
3694
3695
3696
3697
3698
3699
3700
3701
3702
3703
3704
3705
3706
3707
3708
3709
3710
3711
3712
3713
3714
3715
3716
3717
3718
3719
3720
3721
3722
3723
3724
3725
3726
3727
3728
3729
3730
3731
3732
3733
3734
3735
3736
3737
3738
3739
3740
3741
3742
3743
3744
3745
3746
3747
3748
3749
3750
3751
3752
3753
3754
3755
3756
3757
3758
3759
3760
3761
3762
3763
3764
3765
3766
3767
3768
3769
3770
3771
3772
3773
3774
3775
3776
3777
3778
3779
3780
3781
3782
3783
3784
3785
3786
3787
3788
3789
3790
3791
3792
3793
3794
3795
3796
3797
3798
3799
3800
3801
3802
3803
3804
3805
3806
3807
3808
3809
3810
3811
3812
3813
3814
3815
3816
3817
3818
3819
3820
3821
3822
3823
3824
3825
3826
3827
3828
3829
3830
3831
3832
3833
3834
3835
3836
3837
3838
3839
3840
3841
3842
3843
3844
3845
3846
3847
3848
3849
3850
3851
3852
3853
3854
3855
3856
3857
3858
3859
3860
3861
3862
3863
3864
3865
3866
3867
3868
3869
3870
3871
3872
3873
3874
3875
3876
3877
3878
3879
3880
3881
3882
3883
3884
3885
3886
3887
3888
3889
3890
3891
3892
3893
3894
3895
3896
3897
3898
3899
3900
3901
3902
3903
3904
3905
3906
3907
3908
3909
3910
3911
3912
3913
3914
3915
3916
3917
3918
3919
3920
3921
3922
3923
3924
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * net/sched/cls_api.c        Packet classifier API.
 *
 * Authors:        Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
 *
 * Changes:
 *
 * Eduardo J. Blanco <ejbs@netlabs.com.uy> :990222: kmod support
 */

#include <linux/module.h>
#include <linux/types.h>
#include <linux/kernel.h>
#include <linux/string.h>
#include <linux/errno.h>
#include <linux/err.h>
#include <linux/skbuff.h>
#include <linux/init.h>
#include <linux/kmod.h>
#include <linux/slab.h>
#include <linux/idr.h>
#include <linux/jhash.h>
#include <linux/rculist.h>
#include <net/net_namespace.h>
#include <net/sock.h>
#include <net/netlink.h>
#include <net/pkt_sched.h>
#include <net/pkt_cls.h>
#include <net/tc_act/tc_pedit.h>
#include <net/tc_act/tc_mirred.h>
#include <net/tc_act/tc_vlan.h>
#include <net/tc_act/tc_tunnel_key.h>
#include <net/tc_act/tc_csum.h>
#include <net/tc_act/tc_gact.h>
#include <net/tc_act/tc_police.h>
#include <net/tc_act/tc_sample.h>
#include <net/tc_act/tc_skbedit.h>
#include <net/tc_act/tc_ct.h>
#include <net/tc_act/tc_mpls.h>
#include <net/tc_act/tc_gate.h>
#include <net/flow_offload.h>

/* The list of all installed classifier types */
static LIST_HEAD(tcf_proto_base);

/* Protects list of registered TC modules. It is pure SMP lock. */
static DEFINE_RWLOCK(cls_mod_lock);

static u32 destroy_obj_hashfn(const struct tcf_proto *tp)
{
        return jhash_3words(tp->chain->index, tp->prio,
                            (__force __u32)tp->protocol, 0);
}

static void tcf_proto_signal_destroying(struct tcf_chain *chain,
                                        struct tcf_proto *tp)
{
        struct tcf_block *block = chain->block;

        mutex_lock(&block->proto_destroy_lock);
        hash_add_rcu(block->proto_destroy_ht, &tp->destroy_ht_node,
                     destroy_obj_hashfn(tp));
        mutex_unlock(&block->proto_destroy_lock);
}

static bool tcf_proto_cmp(const struct tcf_proto *tp1,
                          const struct tcf_proto *tp2)
{
        return tp1->chain->index == tp2->chain->index &&
               tp1->prio == tp2->prio &&
               tp1->protocol == tp2->protocol;
}

static bool tcf_proto_exists_destroying(struct tcf_chain *chain,
                                        struct tcf_proto *tp)
{
        u32 hash = destroy_obj_hashfn(tp);
        struct tcf_proto *iter;
        bool found = false;

        rcu_read_lock();
        hash_for_each_possible_rcu(chain->block->proto_destroy_ht, iter,
                                   destroy_ht_node, hash) {
                if (tcf_proto_cmp(tp, iter)) {
                        found = true;
                        break;
                }
        }
        rcu_read_unlock();

        return found;
}

static void
tcf_proto_signal_destroyed(struct tcf_chain *chain, struct tcf_proto *tp)
{
        struct tcf_block *block = chain->block;

        mutex_lock(&block->proto_destroy_lock);
        if (hash_hashed(&tp->destroy_ht_node))
                hash_del_rcu(&tp->destroy_ht_node);
        mutex_unlock(&block->proto_destroy_lock);
}

/* Find classifier type by string name */

static const struct tcf_proto_ops *__tcf_proto_lookup_ops(const char *kind)
{
        const struct tcf_proto_ops *t, *res = NULL;

        if (kind) {
                read_lock(&cls_mod_lock);
                list_for_each_entry(t, &tcf_proto_base, head) {
                        if (strcmp(kind, t->kind) == 0) {
                                if (try_module_get(t->owner))
                                        res = t;
                                break;
                        }
                }
                read_unlock(&cls_mod_lock);
        }
        return res;
}

static const struct tcf_proto_ops *
tcf_proto_lookup_ops(const char *kind, bool rtnl_held,
                     struct netlink_ext_ack *extack)
{
        const struct tcf_proto_ops *ops;

        ops = __tcf_proto_lookup_ops(kind);
        if (ops)
                return ops;
#ifdef CONFIG_MODULES
        if (rtnl_held)
                rtnl_unlock();
        request_module("cls_%s", kind);
        if (rtnl_held)
                rtnl_lock();
        ops = __tcf_proto_lookup_ops(kind);
        /* We dropped the RTNL semaphore in order to perform
         * the module load. So, even if we succeeded in loading
         * the module we have to replay the request. We indicate
         * this using -EAGAIN.
         */
        if (ops) {
                module_put(ops->owner);
                return ERR_PTR(-EAGAIN);
        }
#endif
        NL_SET_ERR_MSG(extack, "TC classifier not found");
        return ERR_PTR(-ENOENT);
}

/* Register(unregister) new classifier type */

int register_tcf_proto_ops(struct tcf_proto_ops *ops)
{
        struct tcf_proto_ops *t;
        int rc = -EEXIST;

        write_lock(&cls_mod_lock);
        list_for_each_entry(t, &tcf_proto_base, head)
                if (!strcmp(ops->kind, t->kind))
                        goto out;

        list_add_tail(&ops->head, &tcf_proto_base);
        rc = 0;
out:
        write_unlock(&cls_mod_lock);
        return rc;
}
EXPORT_SYMBOL(register_tcf_proto_ops);

static struct workqueue_struct *tc_filter_wq;

int unregister_tcf_proto_ops(struct tcf_proto_ops *ops)
{
        struct tcf_proto_ops *t;
        int rc = -ENOENT;

        /* Wait for outstanding call_rcu()s, if any, from a
         * tcf_proto_ops's destroy() handler.
         */
        rcu_barrier();
        flush_workqueue(tc_filter_wq);

        write_lock(&cls_mod_lock);
        list_for_each_entry(t, &tcf_proto_base, head) {
                if (t == ops) {
                        list_del(&t->head);
                        rc = 0;
                        break;
                }
        }
        write_unlock(&cls_mod_lock);
        return rc;
}
EXPORT_SYMBOL(unregister_tcf_proto_ops);

bool tcf_queue_work(struct rcu_work *rwork, work_func_t func)
{
        INIT_RCU_WORK(rwork, func);
        return queue_rcu_work(tc_filter_wq, rwork);
}
EXPORT_SYMBOL(tcf_queue_work);

/* Select new prio value from the range, managed by kernel. */

static inline u32 tcf_auto_prio(struct tcf_proto *tp)
{
        u32 first = TC_H_MAKE(0xC0000000U, 0U);

        if (tp)
                first = tp->prio - 1;

        return TC_H_MAJ(first);
}

static bool tcf_proto_check_kind(struct nlattr *kind, char *name)
{
        if (kind)
                return nla_strscpy(name, kind, IFNAMSIZ) < 0;
        memset(name, 0, IFNAMSIZ);
        return false;
}

static bool tcf_proto_is_unlocked(const char *kind)
{
        const struct tcf_proto_ops *ops;
        bool ret;

        if (strlen(kind) == 0)
                return false;

        ops = tcf_proto_lookup_ops(kind, false, NULL);
        /* On error return false to take rtnl lock. Proto lookup/create
         * functions will perform lookup again and properly handle errors.
         */
        if (IS_ERR(ops))
                return false;

        ret = !!(ops->flags & TCF_PROTO_OPS_DOIT_UNLOCKED);
        module_put(ops->owner);
        return ret;
}

static struct tcf_proto *tcf_proto_create(const char *kind, u32 protocol,
                                          u32 prio, struct tcf_chain *chain,
                                          bool rtnl_held,
                                          struct netlink_ext_ack *extack)
{
        struct tcf_proto *tp;
        int err;

        tp = kzalloc(sizeof(*tp), GFP_KERNEL);
        if (!tp)
                return ERR_PTR(-ENOBUFS);

        tp->ops = tcf_proto_lookup_ops(kind, rtnl_held, extack);
        if (IS_ERR(tp->ops)) {
                err = PTR_ERR(tp->ops);
                goto errout;
        }
        tp->classify = tp->ops->classify;
        tp->protocol = protocol;
        tp->prio = prio;
        tp->chain = chain;
        spin_lock_init(&tp->lock);
        refcount_set(&tp->refcnt, 1);

        err = tp->ops->init(tp);
        if (err) {
                module_put(tp->ops->owner);
                goto errout;
        }
        return tp;

errout:
        kfree(tp);
        return ERR_PTR(err);
}

static void tcf_proto_get(struct tcf_proto *tp)
{
        refcount_inc(&tp->refcnt);
}

static void tcf_chain_put(struct tcf_chain *chain);

static void tcf_proto_destroy(struct tcf_proto *tp, bool rtnl_held,
                              bool sig_destroy, struct netlink_ext_ack *extack)
{
        tp->ops->destroy(tp, rtnl_held, extack);
        if (sig_destroy)
                tcf_proto_signal_destroyed(tp->chain, tp);
        tcf_chain_put(tp->chain);
        module_put(tp->ops->owner);
        kfree_rcu(tp, rcu);
}

static void tcf_proto_put(struct tcf_proto *tp, bool rtnl_held,
                          struct netlink_ext_ack *extack)
{
        if (refcount_dec_and_test(&tp->refcnt))
                tcf_proto_destroy(tp, rtnl_held, true, extack);
}

static bool tcf_proto_check_delete(struct tcf_proto *tp)
{
        if (tp->ops->delete_empty)
                return tp->ops->delete_empty(tp);

        tp->deleting = true;
        return tp->deleting;
}

static void tcf_proto_mark_delete(struct tcf_proto *tp)
{
        spin_lock(&tp->lock);
        tp->deleting = true;
        spin_unlock(&tp->lock);
}

static bool tcf_proto_is_deleting(struct tcf_proto *tp)
{
        bool deleting;

        spin_lock(&tp->lock);
        deleting = tp->deleting;
        spin_unlock(&tp->lock);

        return deleting;
}

#define ASSERT_BLOCK_LOCKED(block)                                        \
        lockdep_assert_held(&(block)->lock)

struct tcf_filter_chain_list_item {
        struct list_head list;
        tcf_chain_head_change_t *chain_head_change;
        void *chain_head_change_priv;
};

static struct tcf_chain *tcf_chain_create(struct tcf_block *block,
                                          u32 chain_index)
{
        struct tcf_chain *chain;

        ASSERT_BLOCK_LOCKED(block);

        chain = kzalloc(sizeof(*chain), GFP_KERNEL);
        if (!chain)
                return NULL;
        list_add_tail_rcu(&chain->list, &block->chain_list);
        mutex_init(&chain->filter_chain_lock);
        chain->block = block;
        chain->index = chain_index;
        chain->refcnt = 1;
        if (!chain->index)
                block->chain0.chain = chain;
        return chain;
}

static void tcf_chain_head_change_item(struct tcf_filter_chain_list_item *item,
                                       struct tcf_proto *tp_head)
{
        if (item->chain_head_change)
                item->chain_head_change(tp_head, item->chain_head_change_priv);
}

static void tcf_chain0_head_change(struct tcf_chain *chain,
                                   struct tcf_proto *tp_head)
{
        struct tcf_filter_chain_list_item *item;
        struct tcf_block *block = chain->block;

        if (chain->index)
                return;

        mutex_lock(&block->lock);
        list_for_each_entry(item, &block->chain0.filter_chain_list, list)
                tcf_chain_head_change_item(item, tp_head);
        mutex_unlock(&block->lock);
}

/* Returns true if block can be safely freed. */

static bool tcf_chain_detach(struct tcf_chain *chain)
{
        struct tcf_block *block = chain->block;

        ASSERT_BLOCK_LOCKED(block);

        list_del_rcu(&chain->list);
        if (!chain->index)
                block->chain0.chain = NULL;

        if (list_empty(&block->chain_list) &&
            refcount_read(&block->refcnt) == 0)
                return true;

        return false;
}

static void tcf_block_destroy(struct tcf_block *block)
{
        mutex_destroy(&block->lock);
        mutex_destroy(&block->proto_destroy_lock);
        kfree_rcu(block, rcu);
}

static void tcf_chain_destroy(struct tcf_chain *chain, bool free_block)
{
        struct tcf_block *block = chain->block;

        mutex_destroy(&chain->filter_chain_lock);
        kfree_rcu(chain, rcu);
        if (free_block)
                tcf_block_destroy(block);
}

static void tcf_chain_hold(struct tcf_chain *chain)
{
        ASSERT_BLOCK_LOCKED(chain->block);

        ++chain->refcnt;
}

static bool tcf_chain_held_by_acts_only(struct tcf_chain *chain)
{
        ASSERT_BLOCK_LOCKED(chain->block);

        /* In case all the references are action references, this
         * chain should not be shown to the user.
         */
        return chain->refcnt == chain->action_refcnt;
}

static struct tcf_chain *tcf_chain_lookup(struct tcf_block *block,
                                          u32 chain_index)
{
        struct tcf_chain *chain;

        ASSERT_BLOCK_LOCKED(block);

        list_for_each_entry(chain, &block->chain_list, list) {
                if (chain->index == chain_index)
                        return chain;
        }
        return NULL;
}

#if IS_ENABLED(CONFIG_NET_TC_SKB_EXT)
static struct tcf_chain *tcf_chain_lookup_rcu(const struct tcf_block *block,
                                              u32 chain_index)
{
        struct tcf_chain *chain;

        list_for_each_entry_rcu(chain, &block->chain_list, list) {
                if (chain->index == chain_index)
                        return chain;
        }
        return NULL;
}
#endif

static int tc_chain_notify(struct tcf_chain *chain, struct sk_buff *oskb,
                           u32 seq, u16 flags, int event, bool unicast);

static struct tcf_chain *__tcf_chain_get(struct tcf_block *block,
                                         u32 chain_index, bool create,
                                         bool by_act)
{
        struct tcf_chain *chain = NULL;
        bool is_first_reference;

        mutex_lock(&block->lock);
        chain = tcf_chain_lookup(block, chain_index);
        if (chain) {
                tcf_chain_hold(chain);
        } else {
                if (!create)
                        goto errout;
                chain = tcf_chain_create(block, chain_index);
                if (!chain)
                        goto errout;
        }

        if (by_act)
                ++chain->action_refcnt;
        is_first_reference = chain->refcnt - chain->action_refcnt == 1;
        mutex_unlock(&block->lock);

        /* Send notification only in case we got the first
         * non-action reference. Until then, the chain acts only as
         * a placeholder for actions pointing to it and user ought
         * not know about them.
         */
        if (is_first_reference && !by_act)
                tc_chain_notify(chain, NULL, 0, NLM_F_CREATE | NLM_F_EXCL,
                                RTM_NEWCHAIN, false);

        return chain;

errout:
        mutex_unlock(&block->lock);
        return chain;
}

static struct tcf_chain *tcf_chain_get(struct tcf_block *block, u32 chain_index,
                                       bool create)
{
        return __tcf_chain_get(block, chain_index, create, false);
}

struct tcf_chain *tcf_chain_get_by_act(struct tcf_block *block, u32 chain_index)
{
        return __tcf_chain_get(block, chain_index, true, true);
}
EXPORT_SYMBOL(tcf_chain_get_by_act);

static void tc_chain_tmplt_del(const struct tcf_proto_ops *tmplt_ops,
                               void *tmplt_priv);
static int tc_chain_notify_delete(const struct tcf_proto_ops *tmplt_ops,
                                  void *tmplt_priv, u32 chain_index,
                                  struct tcf_block *block, struct sk_buff *oskb,
                                  u32 seq, u16 flags, bool unicast);

static void __tcf_chain_put(struct tcf_chain *chain, bool by_act,
                            bool explicitly_created)
{
        struct tcf_block *block = chain->block;
        const struct tcf_proto_ops *tmplt_ops;
        unsigned int refcnt, non_act_refcnt;
        bool free_block = false;
        void *tmplt_priv;

        mutex_lock(&block->lock);
        if (explicitly_created) {
                if (!chain->explicitly_created) {
                        mutex_unlock(&block->lock);
                        return;
                }
                chain->explicitly_created = false;
        }

        if (by_act)
                chain->action_refcnt--;

        /* tc_chain_notify_delete can't be called while holding block lock.
         * However, when block is unlocked chain can be changed concurrently, so
         * save these to temporary variables.
         */
        refcnt = --chain->refcnt;
        non_act_refcnt = refcnt - chain->action_refcnt;
        tmplt_ops = chain->tmplt_ops;
        tmplt_priv = chain->tmplt_priv;

        if (non_act_refcnt == chain->explicitly_created && !by_act) {
                if (non_act_refcnt == 0)
                        tc_chain_notify_delete(tmplt_ops, tmplt_priv,
                                               chain->index, block, NULL, 0, 0,
                                               false);
                /* Last reference to chain, no need to lock. */
                chain->flushing = false;
        }

        if (refcnt == 0)
                free_block = tcf_chain_detach(chain);
        mutex_unlock(&block->lock);

        if (refcnt == 0) {
                tc_chain_tmplt_del(tmplt_ops, tmplt_priv);
                tcf_chain_destroy(chain, free_block);
        }
}

static void tcf_chain_put(struct tcf_chain *chain)
{
        __tcf_chain_put(chain, false, false);
}

void tcf_chain_put_by_act(struct tcf_chain *chain)
{
        __tcf_chain_put(chain, true, false);
}
EXPORT_SYMBOL(tcf_chain_put_by_act);

static void tcf_chain_put_explicitly_created(struct tcf_chain *chain)
{
        __tcf_chain_put(chain, false, true);
}

static void tcf_chain_flush(struct tcf_chain *chain, bool rtnl_held)
{
        struct tcf_proto *tp, *tp_next;

        mutex_lock(&chain->filter_chain_lock);
        tp = tcf_chain_dereference(chain->filter_chain, chain);
        while (tp) {
                tp_next = rcu_dereference_protected(tp->next, 1);
                tcf_proto_signal_destroying(chain, tp);
                tp = tp_next;
        }
        tp = tcf_chain_dereference(chain->filter_chain, chain);
        RCU_INIT_POINTER(chain->filter_chain, NULL);
        tcf_chain0_head_change(chain, NULL);
        chain->flushing = true;
        mutex_unlock(&chain->filter_chain_lock);

        while (tp) {
                tp_next = rcu_dereference_protected(tp->next, 1);
                tcf_proto_put(tp, rtnl_held, NULL);
                tp = tp_next;
        }
}

static int tcf_block_setup(struct tcf_block *block,
                           struct flow_block_offload *bo);

static void tcf_block_offload_init(struct flow_block_offload *bo,
                                   struct net_device *dev, struct Qdisc *sch,
                                   enum flow_block_command command,
                                   enum flow_block_binder_type binder_type,
                                   struct flow_block *flow_block,
                                   bool shared, struct netlink_ext_ack *extack)
{
        bo->net = dev_net(dev);
        bo->command = command;
        bo->binder_type = binder_type;
        bo->block = flow_block;
        bo->block_shared = shared;
        bo->extack = extack;
        bo->sch = sch;
        bo->cb_list_head = &flow_block->cb_list;
        INIT_LIST_HEAD(&bo->cb_list);
}

static void tcf_block_unbind(struct tcf_block *block,
                             struct flow_block_offload *bo);

static void tc_block_indr_cleanup(struct flow_block_cb *block_cb)
{
        struct tcf_block *block = block_cb->indr.data;
        struct net_device *dev = block_cb->indr.dev;
        struct Qdisc *sch = block_cb->indr.sch;
        struct netlink_ext_ack extack = {};
        struct flow_block_offload bo = {};

        tcf_block_offload_init(&bo, dev, sch, FLOW_BLOCK_UNBIND,
                               block_cb->indr.binder_type,
                               &block->flow_block, tcf_block_shared(block),
                               &extack);
        rtnl_lock();
        down_write(&block->cb_lock);
        list_del(&block_cb->driver_list);
        list_move(&block_cb->list, &bo.cb_list);
        tcf_block_unbind(block, &bo);
        up_write(&block->cb_lock);
        rtnl_unlock();
}

static bool tcf_block_offload_in_use(struct tcf_block *block)
{
        return atomic_read(&block->offloadcnt);
}

static int tcf_block_offload_cmd(struct tcf_block *block,
                                 struct net_device *dev, struct Qdisc *sch,
                                 struct tcf_block_ext_info *ei,
                                 enum flow_block_command command,
                                 struct netlink_ext_ack *extack)
{
        struct flow_block_offload bo = {};

        tcf_block_offload_init(&bo, dev, sch, command, ei->binder_type,
                               &block->flow_block, tcf_block_shared(block),
                               extack);

        if (dev->netdev_ops->ndo_setup_tc) {
                int err;

                err = dev->netdev_ops->ndo_setup_tc(dev, TC_SETUP_BLOCK, &bo);
                if (err < 0) {
                        if (err != -EOPNOTSUPP)
                                NL_SET_ERR_MSG(extack, "Driver ndo_setup_tc failed");
                        return err;
                }

                return tcf_block_setup(block, &bo);
        }

        flow_indr_dev_setup_offload(dev, sch, TC_SETUP_BLOCK, block, &bo,
                                    tc_block_indr_cleanup);
        tcf_block_setup(block, &bo);

        return -EOPNOTSUPP;
}

static int tcf_block_offload_bind(struct tcf_block *block, struct Qdisc *q,
                                  struct tcf_block_ext_info *ei,
                                  struct netlink_ext_ack *extack)
{
        struct net_device *dev = q->dev_queue->dev;
        int err;

        down_write(&block->cb_lock);

        /* If tc offload feature is disabled and the block we try to bind
         * to already has some offloaded filters, forbid to bind.
         */
        if (dev->netdev_ops->ndo_setup_tc &&
            !tc_can_offload(dev) &&
            tcf_block_offload_in_use(block)) {
                NL_SET_ERR_MSG(extack, "Bind to offloaded block failed as dev has offload disabled");
                err = -EOPNOTSUPP;
                goto err_unlock;
        }

        err = tcf_block_offload_cmd(block, dev, q, ei, FLOW_BLOCK_BIND, extack);
        if (err == -EOPNOTSUPP)
                goto no_offload_dev_inc;
        if (err)
                goto err_unlock;

        up_write(&block->cb_lock);
        return 0;

no_offload_dev_inc:
        if (tcf_block_offload_in_use(block))
                goto err_unlock;

        err = 0;
        block->nooffloaddevcnt++;
err_unlock:
        up_write(&block->cb_lock);
        return err;
}

static void tcf_block_offload_unbind(struct tcf_block *block, struct Qdisc *q,
                                     struct tcf_block_ext_info *ei)
{
        struct net_device *dev = q->dev_queue->dev;
        int err;

        down_write(&block->cb_lock);
        err = tcf_block_offload_cmd(block, dev, q, ei, FLOW_BLOCK_UNBIND, NULL);
        if (err == -EOPNOTSUPP)
                goto no_offload_dev_dec;
        up_write(&block->cb_lock);
        return;

no_offload_dev_dec:
        WARN_ON(block->nooffloaddevcnt-- == 0);
        up_write(&block->cb_lock);
}

static int
tcf_chain0_head_change_cb_add(struct tcf_block *block,
                              struct tcf_block_ext_info *ei,
                              struct netlink_ext_ack *extack)
{
        struct tcf_filter_chain_list_item *item;
        struct tcf_chain *chain0;

        item = kmalloc(sizeof(*item), GFP_KERNEL);
        if (!item) {
                NL_SET_ERR_MSG(extack, "Memory allocation for head change callback item failed");
                return -ENOMEM;
        }
        item->chain_head_change = ei->chain_head_change;
        item->chain_head_change_priv = ei->chain_head_change_priv;

        mutex_lock(&block->lock);
        chain0 = block->chain0.chain;
        if (chain0)
                tcf_chain_hold(chain0);
        else
                list_add(&item->list, &block->chain0.filter_chain_list);
        mutex_unlock(&block->lock);

        if (chain0) {
                struct tcf_proto *tp_head;

                mutex_lock(&chain0->filter_chain_lock);

                tp_head = tcf_chain_dereference(chain0->filter_chain, chain0);
                if (tp_head)
                        tcf_chain_head_change_item(item, tp_head);

                mutex_lock(&block->lock);
                list_add(&item->list, &block->chain0.filter_chain_list);
                mutex_unlock(&block->lock);

                mutex_unlock(&chain0->filter_chain_lock);
                tcf_chain_put(chain0);
        }

        return 0;
}

static void
tcf_chain0_head_change_cb_del(struct tcf_block *block,
                              struct tcf_block_ext_info *ei)
{
        struct tcf_filter_chain_list_item *item;

        mutex_lock(&block->lock);
        list_for_each_entry(item, &block->chain0.filter_chain_list, list) {
                if ((!ei->chain_head_change && !ei->chain_head_change_priv) ||
                    (item->chain_head_change == ei->chain_head_change &&
                     item->chain_head_change_priv == ei->chain_head_change_priv)) {
                        if (block->chain0.chain)
                                tcf_chain_head_change_item(item, NULL);
                        list_del(&item->list);
                        mutex_unlock(&block->lock);

                        kfree(item);
                        return;
                }
        }
        mutex_unlock(&block->lock);
        WARN_ON(1);
}

struct tcf_net {
        spinlock_t idr_lock; /* Protects idr */
        struct idr idr;
};

static unsigned int tcf_net_id;

static int tcf_block_insert(struct tcf_block *block, struct net *net,
                            struct netlink_ext_ack *extack)
{
        struct tcf_net *tn = net_generic(net, tcf_net_id);
        int err;

        idr_preload(GFP_KERNEL);
        spin_lock(&tn->idr_lock);
        err = idr_alloc_u32(&tn->idr, block, &block->index, block->index,
                            GFP_NOWAIT);
        spin_unlock(&tn->idr_lock);
        idr_preload_end();

        return err;
}

static void tcf_block_remove(struct tcf_block *block, struct net *net)
{
        struct tcf_net *tn = net_generic(net, tcf_net_id);

        spin_lock(&tn->idr_lock);
        idr_remove(&tn->idr, block->index);
        spin_unlock(&tn->idr_lock);
}

static struct tcf_block *tcf_block_create(struct net *net, struct Qdisc *q,
                                          u32 block_index,
                                          struct netlink_ext_ack *extack)
{
        struct tcf_block *block;

        block = kzalloc(sizeof(*block), GFP_KERNEL);
        if (!block) {
                NL_SET_ERR_MSG(extack, "Memory allocation for block failed");
                return ERR_PTR(-ENOMEM);
        }
        mutex_init(&block->lock);
        mutex_init(&block->proto_destroy_lock);
        init_rwsem(&block->cb_lock);
        flow_block_init(&block->flow_block);
        INIT_LIST_HEAD(&block->chain_list);
        INIT_LIST_HEAD(&block->owner_list);
        INIT_LIST_HEAD(&block->chain0.filter_chain_list);

        refcount_set(&block->refcnt, 1);
        block->net = net;
        block->index = block_index;

        /* Don't store q pointer for blocks which are shared */
        if (!tcf_block_shared(block))
                block->q = q;
        return block;
}

static struct tcf_block *tcf_block_lookup(struct net *net, u32 block_index)
{
        struct tcf_net *tn = net_generic(net, tcf_net_id);

        return idr_find(&tn->idr, block_index);
}

static struct tcf_block *tcf_block_refcnt_get(struct net *net, u32 block_index)
{
        struct tcf_block *block;

        rcu_read_lock();
        block = tcf_block_lookup(net, block_index);
        if (block && !refcount_inc_not_zero(&block->refcnt))
                block = NULL;
        rcu_read_unlock();

        return block;
}

static struct tcf_chain *
__tcf_get_next_chain(struct tcf_block *block, struct tcf_chain *chain)
{
        mutex_lock(&block->lock);
        if (chain)
                chain = list_is_last(&chain->list, &block->chain_list) ?
                        NULL : list_next_entry(chain, list);
        else
                chain = list_first_entry_or_null(&block->chain_list,
                                                 struct tcf_chain, list);

        /* skip all action-only chains */
        while (chain && tcf_chain_held_by_acts_only(chain))
                chain = list_is_last(&chain->list, &block->chain_list) ?
                        NULL : list_next_entry(chain, list);

        if (chain)
                tcf_chain_hold(chain);
        mutex_unlock(&block->lock);

        return chain;
}

/* Function to be used by all clients that want to iterate over all chains on
 * block. It properly obtains block->lock and takes reference to chain before
 * returning it. Users of this function must be tolerant to concurrent chain
 * insertion/deletion or ensure that no concurrent chain modification is
 * possible. Note that all netlink dump callbacks cannot guarantee to provide
 * consistent dump because rtnl lock is released each time skb is filled with
 * data and sent to user-space.
 */

struct tcf_chain *
tcf_get_next_chain(struct tcf_block *block, struct tcf_chain *chain)
{
        struct tcf_chain *chain_next = __tcf_get_next_chain(block, chain);

        if (chain)
                tcf_chain_put(chain);

        return chain_next;
}
EXPORT_SYMBOL(tcf_get_next_chain);

static struct tcf_proto *
__tcf_get_next_proto(struct tcf_chain *chain, struct tcf_proto *tp)
{
        u32 prio = 0;

        ASSERT_RTNL();
        mutex_lock(&chain->filter_chain_lock);

        if (!tp) {
                tp = tcf_chain_dereference(chain->filter_chain, chain);
        } else if (tcf_proto_is_deleting(tp)) {
                /* 'deleting' flag is set and chain->filter_chain_lock was
                 * unlocked, which means next pointer could be invalid. Restart
                 * search.
                 */
                prio = tp->prio + 1;
                tp = tcf_chain_dereference(chain->filter_chain, chain);

                for (; tp; tp = tcf_chain_dereference(tp->next, chain))
                        if (!tp->deleting && tp->prio >= prio)
                                break;
        } else {
                tp = tcf_chain_dereference(tp->next, chain);
        }

        if (tp)
                tcf_proto_get(tp);

        mutex_unlock(&chain->filter_chain_lock);

        return tp;
}

/* Function to be used by all clients that want to iterate over all tp's on
 * chain. Users of this function must be tolerant to concurrent tp
 * insertion/deletion or ensure that no concurrent chain modification is
 * possible. Note that all netlink dump callbacks cannot guarantee to provide
 * consistent dump because rtnl lock is released each time skb is filled with
 * data and sent to user-space.
 */

struct tcf_proto *
tcf_get_next_proto(struct tcf_chain *chain, struct tcf_proto *tp)
{
        struct tcf_proto *tp_next = __tcf_get_next_proto(chain, tp);

        if (tp)
                tcf_proto_put(tp, true, NULL);

        return tp_next;
}
EXPORT_SYMBOL(tcf_get_next_proto);

static void tcf_block_flush_all_chains(struct tcf_block *block, bool rtnl_held)
{
        struct tcf_chain *chain;

        /* Last reference to block. At this point chains cannot be added or
         * removed concurrently.
         */
        for (chain = tcf_get_next_chain(block, NULL);
             chain;
             chain = tcf_get_next_chain(block, chain)) {
                tcf_chain_put_explicitly_created(chain);
                tcf_chain_flush(chain, rtnl_held);
        }
}

/* Lookup Qdisc and increments its reference counter.
 * Set parent, if necessary.
 */

static int __tcf_qdisc_find(struct net *net, struct Qdisc **q,
                            u32 *parent, int ifindex, bool rtnl_held,
                            struct netlink_ext_ack *extack)
{
        const struct Qdisc_class_ops *cops;
        struct net_device *dev;
        int err = 0;

        if (ifindex == TCM_IFINDEX_MAGIC_BLOCK)
                return 0;

        rcu_read_lock();

        /* Find link */
        dev = dev_get_by_index_rcu(net, ifindex);
        if (!dev) {
                rcu_read_unlock();
                return -ENODEV;
        }

        /* Find qdisc */
        if (!*parent) {
                *q = rcu_dereference(dev->qdisc);
                *parent = (*q)->handle;
        } else {
                *q = qdisc_lookup_rcu(dev, TC_H_MAJ(*parent));
                if (!*q) {
                        NL_SET_ERR_MSG(extack, "Parent Qdisc doesn't exists");
                        err = -EINVAL;
                        goto errout_rcu;
                }
        }

        *q = qdisc_refcount_inc_nz(*q);
        if (!*q) {
                NL_SET_ERR_MSG(extack, "Parent Qdisc doesn't exists");
                err = -EINVAL;
                goto errout_rcu;
        }

        /* Is it classful? */
        cops = (*q)->ops->cl_ops;
        if (!cops) {
                NL_SET_ERR_MSG(extack, "Qdisc not classful");
                err = -EINVAL;
                goto errout_qdisc;
        }

        if (!cops->tcf_block) {
                NL_SET_ERR_MSG(extack, "Class doesn't support blocks");
                err = -EOPNOTSUPP;
                goto errout_qdisc;
        }

errout_rcu:
        /* At this point we know that qdisc is not noop_qdisc,
         * which means that qdisc holds a reference to net_device
         * and we hold a reference to qdisc, so it is safe to release
         * rcu read lock.
         */
        rcu_read_unlock();
        return err;

errout_qdisc:
        rcu_read_unlock();

        if (rtnl_held)
                qdisc_put(*q);
        else
                qdisc_put_unlocked(*q);
        *q = NULL;

        return err;
}

static int __tcf_qdisc_cl_find(struct Qdisc *q, u32 parent, unsigned long *cl,
                               int ifindex, struct netlink_ext_ack *extack)
{
        if (ifindex == TCM_IFINDEX_MAGIC_BLOCK)
                return 0;

        /* Do we search for filter, attached to class? */
        if (TC_H_MIN(parent)) {
                const struct Qdisc_class_ops *cops = q->ops->cl_ops;

                *cl = cops->find(q, parent);
                if (*cl == 0) {
                        NL_SET_ERR_MSG(extack, "Specified class doesn't exist");
                        return -ENOENT;
                }
        }

        return 0;
}

static struct tcf_block *__tcf_block_find(struct net *net, struct Qdisc *q,
                                          unsigned long cl, int ifindex,
                                          u32 block_index,
                                          struct netlink_ext_ack *extack)
{
        struct tcf_block *block;

        if (ifindex == TCM_IFINDEX_MAGIC_BLOCK) {
                block = tcf_block_refcnt_get(net, block_index);
                if (!block) {
                        NL_SET_ERR_MSG(extack, "Block of given index was not found");
                        return ERR_PTR(-EINVAL);
                }
        } else {
                const struct Qdisc_class_ops *cops = q->ops->cl_ops;

                block = cops->tcf_block(q, cl, extack);
                if (!block)
                        return ERR_PTR(-EINVAL);

                if (tcf_block_shared(block)) {
                        NL_SET_ERR_MSG(extack, "This filter block is shared. Please use the block index to manipulate the filters");
                        return ERR_PTR(-EOPNOTSUPP);
                }

                /* Always take reference to block in order to support execution
                 * of rules update path of cls API without rtnl lock. Caller
                 * must release block when it is finished using it. 'if' block
                 * of this conditional obtain reference to block by calling
                 * tcf_block_refcnt_get().
                 */
                refcount_inc(&block->refcnt);
        }

        return block;
}

static void __tcf_block_put(struct tcf_block *block, struct Qdisc *q,
                            struct tcf_block_ext_info *ei, bool rtnl_held)
{
        if (refcount_dec_and_mutex_lock(&block->refcnt, &block->lock)) {
                /* Flushing/putting all chains will cause the block to be
                 * deallocated when last chain is freed. However, if chain_list
                 * is empty, block has to be manually deallocated. After block
                 * reference counter reached 0, it is no longer possible to
                 * increment it or add new chains to block.
                 */
                bool free_block = list_empty(&block->chain_list);

                mutex_unlock(&block->lock);
                if (tcf_block_shared(block))
                        tcf_block_remove(block, block->net);

                if (q)
                        tcf_block_offload_unbind(block, q, ei);

                if (free_block)
                        tcf_block_destroy(block);
                else
                        tcf_block_flush_all_chains(block, rtnl_held);
        } else if (q) {
                tcf_block_offload_unbind(block, q, ei);
        }
}

static void tcf_block_refcnt_put(struct tcf_block *block, bool rtnl_held)
{
        __tcf_block_put(block, NULL, NULL, rtnl_held);
}

/* Find tcf block.
 * Set q, parent, cl when appropriate.
 */

static struct tcf_block *tcf_block_find(struct net *net, struct Qdisc **q,
                                        u32 *parent, unsigned long *cl,
                                        int ifindex, u32 block_index,
                                        struct netlink_ext_ack *extack)
{
        struct tcf_block *block;
        int err = 0;

        ASSERT_RTNL();

        err = __tcf_qdisc_find(net, q, parent, ifindex, true, extack);
        if (err)
                goto errout;

        err = __tcf_qdisc_cl_find(*q, *parent, cl, ifindex, extack);
        if (err)
                goto errout_qdisc;

        block = __tcf_block_find(net, *q, *cl, ifindex, block_index, extack);
        if (IS_ERR(block)) {
                err = PTR_ERR(block);
                goto errout_qdisc;
        }

        return block;

errout_qdisc:
        if (*q)
                qdisc_put(*q);
errout:
        *q = NULL;
        return ERR_PTR(err);
}

static void tcf_block_release(struct Qdisc *q, struct tcf_block *block,
                              bool rtnl_held)
{
        if (!IS_ERR_OR_NULL(block))
                tcf_block_refcnt_put(block, rtnl_held);

        if (q) {
                if (rtnl_held)
                        qdisc_put(q);
                else
                        qdisc_put_unlocked(q);
        }
}

struct tcf_block_owner_item {
        struct list_head list;
        struct Qdisc *q;
        enum flow_block_binder_type binder_type;
};

static void
tcf_block_owner_netif_keep_dst(struct tcf_block *block,
                               struct Qdisc *q,
                               enum flow_block_binder_type binder_type)
{
        if (block->keep_dst &&
            binder_type != FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS &&
            binder_type != FLOW_BLOCK_BINDER_TYPE_CLSACT_EGRESS)
                netif_keep_dst(qdisc_dev(q));
}

void tcf_block_netif_keep_dst(struct tcf_block *block)
{
        struct tcf_block_owner_item *item;

        block->keep_dst = true;
        list_for_each_entry(item, &block->owner_list, list)
                tcf_block_owner_netif_keep_dst(block, item->q,
                                               item->binder_type);
}
EXPORT_SYMBOL(tcf_block_netif_keep_dst);

static int tcf_block_owner_add(struct tcf_block *block,
                               struct Qdisc *q,
                               enum flow_block_binder_type binder_type)
{
        struct tcf_block_owner_item *item;

        item = kmalloc(sizeof(*item), GFP_KERNEL);
        if (!item)
                return -ENOMEM;
        item->q = q;
        item->binder_type = binder_type;
        list_add(&item->list, &block->owner_list);
        return 0;
}

static void tcf_block_owner_del(struct tcf_block *block,
                                struct Qdisc *q,
                                enum flow_block_binder_type binder_type)
{
        struct tcf_block_owner_item *item;

        list_for_each_entry(item, &block->owner_list, list) {
                if (item->q == q && item->binder_type == binder_type) {
                        list_del(&item->list);
                        kfree(item);
                        return;
                }
        }
        WARN_ON(1);
}

int tcf_block_get_ext(struct tcf_block **p_block, struct Qdisc *q,
                      struct tcf_block_ext_info *ei,
                      struct netlink_ext_ack *extack)
{
        struct net *net = qdisc_net(q);
        struct tcf_block *block = NULL;
        int err;

        if (ei->block_index)
                /* block_index not 0 means the shared block is requested */
                block = tcf_block_refcnt_get(net, ei->block_index);

        if (!block) {
                block = tcf_block_create(net, q, ei->block_index, extack);
                if (IS_ERR(block))
                        return PTR_ERR(block);
                if (tcf_block_shared(block)) {
                        err = tcf_block_insert(block, net, extack);
                        if (err)
                                goto err_block_insert;
                }
        }

        err = tcf_block_owner_add(block, q, ei->binder_type);
        if (err)
                goto err_block_owner_add;

        tcf_block_owner_netif_keep_dst(block, q, ei->binder_type);

        err = tcf_chain0_head_change_cb_add(block, ei, extack);
        if (err)
                goto err_chain0_head_change_cb_add;

        err = tcf_block_offload_bind(block, q, ei, extack);
        if (err)
                goto err_block_offload_bind;

        *p_block = block;
        return 0;

err_block_offload_bind:
        tcf_chain0_head_change_cb_del(block, ei);
err_chain0_head_change_cb_add:
        tcf_block_owner_del(block, q, ei->binder_type);
err_block_owner_add:
err_block_insert:
        tcf_block_refcnt_put(block, true);
        return err;
}
EXPORT_SYMBOL(tcf_block_get_ext);

static void tcf_chain_head_change_dflt(struct tcf_proto *tp_head, void *priv)
{
        struct tcf_proto __rcu **p_filter_chain = priv;

        rcu_assign_pointer(*p_filter_chain, tp_head);
}

int tcf_block_get(struct tcf_block **p_block,
                  struct tcf_proto __rcu **p_filter_chain, struct Qdisc *q,
                  struct netlink_ext_ack *extack)
{
        struct tcf_block_ext_info ei = {
                .chain_head_change = tcf_chain_head_change_dflt,
                .chain_head_change_priv = p_filter_chain,
        };

        WARN_ON(!p_filter_chain);
        return tcf_block_get_ext(p_block, q, &ei, extack);
}
EXPORT_SYMBOL(tcf_block_get);

/* XXX: Standalone actions are not allowed to jump to any chain, and bound
 * actions should be all removed after flushing.
 */
void tcf_block_put_ext(struct tcf_block *block, struct Qdisc *q,
                       struct tcf_block_ext_info *ei)
{
        if (!block)
                return;
        tcf_chain0_head_change_cb_del(block, ei);
        tcf_block_owner_del(block, q, ei->binder_type);

        __tcf_block_put(block, q, ei, true);
}
EXPORT_SYMBOL(tcf_block_put_ext);

void tcf_block_put(struct tcf_block *block)
{
        struct tcf_block_ext_info ei = {0, };

        if (!block)
                return;
        tcf_block_put_ext(block, block->q, &ei);
}

EXPORT_SYMBOL(tcf_block_put);

static int
tcf_block_playback_offloads(struct tcf_block *block, flow_setup_cb_t *cb,
                            void *cb_priv, bool add, bool offload_in_use,
                            struct netlink_ext_ack *extack)
{
        struct tcf_chain *chain, *chain_prev;
        struct tcf_proto *tp, *tp_prev;
        int err;

        lockdep_assert_held(&block->cb_lock);

        for (chain = __tcf_get_next_chain(block, NULL);
             chain;
             chain_prev = chain,
                     chain = __tcf_get_next_chain(block, chain),
                     tcf_chain_put(chain_prev)) {
                for (tp = __tcf_get_next_proto(chain, NULL); tp;
                     tp_prev = tp,
                             tp = __tcf_get_next_proto(chain, tp),
                             tcf_proto_put(tp_prev, true, NULL)) {
                        if (tp->ops->reoffload) {
                                err = tp->ops->reoffload(tp, add, cb, cb_priv,
                                                         extack);
                                if (err && add)
                                        goto err_playback_remove;
                        } else if (add && offload_in_use) {
                                err = -EOPNOTSUPP;
                                NL_SET_ERR_MSG(extack, "Filter HW offload failed - classifier without re-offloading support");
                                goto err_playback_remove;
                        }
                }
        }

        return 0;

err_playback_remove:
        tcf_proto_put(tp, true, NULL);
        tcf_chain_put(chain);
        tcf_block_playback_offloads(block, cb, cb_priv, false, offload_in_use,
                                    extack);
        return err;
}

static int tcf_block_bind(struct tcf_block *block,
                          struct flow_block_offload *bo)
{
        struct flow_block_cb *block_cb, *next;
        int err, i = 0;

        lockdep_assert_held(&block->cb_lock);

        list_for_each_entry(block_cb, &bo->cb_list, list) {
                err = tcf_block_playback_offloads(block, block_cb->cb,
                                                  block_cb->cb_priv, true,
                                                  tcf_block_offload_in_use(block),
                                                  bo->extack);
                if (err)
                        goto err_unroll;
                if (!bo->unlocked_driver_cb)
                        block->lockeddevcnt++;

                i++;
        }
        list_splice(&bo->cb_list, &block->flow_block.cb_list);

        return 0;

err_unroll:
        list_for_each_entry_safe(block_cb, next, &bo->cb_list, list) {
                list_del(&block_cb->driver_list);
                if (i-- > 0) {
                        list_del(&block_cb->list);
                        tcf_block_playback_offloads(block, block_cb->cb,
                                                    block_cb->cb_priv, false,
                                                    tcf_block_offload_in_use(block),
                                                    NULL);
                        if (!bo->unlocked_driver_cb)
                                block->lockeddevcnt--;
                }
                flow_block_cb_free(block_cb);
        }

        return err;
}

static void tcf_block_unbind(struct tcf_block *block,
                             struct flow_block_offload *bo)
{
        struct flow_block_cb *block_cb, *next;

        lockdep_assert_held(&block->cb_lock);

        list_for_each_entry_safe(block_cb, next, &bo->cb_list, list) {
                tcf_block_playback_offloads(block, block_cb->cb,
                                            block_cb->cb_priv, false,
                                            tcf_block_offload_in_use(block),
                                            NULL);
                list_del(&block_cb->list);
                flow_block_cb_free(block_cb);
                if (!bo->unlocked_driver_cb)
                        block->lockeddevcnt--;
        }
}

static int tcf_block_setup(struct tcf_block *block,
                           struct flow_block_offload *bo)
{
        int err;

        switch (bo->command) {
        case FLOW_BLOCK_BIND:
                err = tcf_block_bind(block, bo);
                break;
        case FLOW_BLOCK_UNBIND:
                err = 0;
                tcf_block_unbind(block, bo);
                break;
        default:
                WARN_ON_ONCE(1);
                err = -EOPNOTSUPP;
        }

        return err;
}

/* Main classifier routine: scans classifier chain attached
 * to this qdisc, (optionally) tests for protocol and asks
 * specific classifiers.
 */
static inline int __tcf_classify(struct sk_buff *skb,
                                 const struct tcf_proto *tp,
                                 const struct tcf_proto *orig_tp,
                                 struct tcf_result *res,
                                 bool compat_mode,
                                 u32 *last_executed_chain)
{
#ifdef CONFIG_NET_CLS_ACT
        const int max_reclassify_loop = 16;
        const struct tcf_proto *first_tp;
        int limit = 0;

reclassify:
#endif
        for (; tp; tp = rcu_dereference_bh(tp->next)) {
                __be16 protocol = skb_protocol(skb, false);
                int err;

                if (tp->protocol != protocol &&
                    tp->protocol != htons(ETH_P_ALL))
                        continue;

                err = tp->classify(skb, tp, res);
#ifdef CONFIG_NET_CLS_ACT
                if (unlikely(err == TC_ACT_RECLASSIFY && !compat_mode)) {
                        first_tp = orig_tp;
                        *last_executed_chain = first_tp->chain->index;
                        goto reset;
                } else if (unlikely(TC_ACT_EXT_CMP(err, TC_ACT_GOTO_CHAIN))) {
                        first_tp = res->goto_tp;
                        *last_executed_chain = err & TC_ACT_EXT_VAL_MASK;
                        goto reset;
                }
#endif
                if (err >= 0)
                        return err;
        }

        return TC_ACT_UNSPEC; /* signal: continue lookup */
#ifdef CONFIG_NET_CLS_ACT
reset:
        if (unlikely(limit++ >= max_reclassify_loop)) {
                net_notice_ratelimited("%u: reclassify loop, rule prio %u, protocol %02x\n",
                                       tp->chain->block->index,
                                       tp->prio & 0xffff,
                                       ntohs(tp->protocol));
                return TC_ACT_SHOT;
        }

        tp = first_tp;
        goto reclassify;
#endif
}

int tcf_classify(struct sk_buff *skb,
                 const struct tcf_block *block,
                 const struct tcf_proto *tp,
                 struct tcf_result *res, bool compat_mode)
{
#if !IS_ENABLED(CONFIG_NET_TC_SKB_EXT)
        u32 last_executed_chain = 0;

        return __tcf_classify(skb, tp, tp, res, compat_mode,
                              &last_executed_chain);
#else
        u32 last_executed_chain = tp ? tp->chain->index : 0;
        const struct tcf_proto *orig_tp = tp;
        struct tc_skb_ext *ext;
        int ret;

        if (block) {
                ext = skb_ext_find(skb, TC_SKB_EXT);

                if (ext && ext->chain) {
                        struct tcf_chain *fchain;

                        fchain = tcf_chain_lookup_rcu(block, ext->chain);
                        if (!fchain)
                                return TC_ACT_SHOT;

                        /* Consume, so cloned/redirect skbs won't inherit ext */
                        skb_ext_del(skb, TC_SKB_EXT);

                        tp = rcu_dereference_bh(fchain->filter_chain);
                        last_executed_chain = fchain->index;
                }
        }

        ret = __tcf_classify(skb, tp, orig_tp, res, compat_mode,
                             &last_executed_chain);

        /* If we missed on some chain */
        if (ret == TC_ACT_UNSPEC && last_executed_chain) {
                struct tc_skb_cb *cb = tc_skb_cb(skb);

                ext = tc_skb_ext_alloc(skb);
                if (WARN_ON_ONCE(!ext))
                        return TC_ACT_SHOT;
                ext->chain = last_executed_chain;
                ext->mru = cb->mru;
                ext->post_ct = cb->post_ct;
                ext->post_ct_snat = cb->post_ct_snat;
                ext->post_ct_dnat = cb->post_ct_dnat;
                ext->zone = cb->zone;
        }

        return ret;
#endif
}
EXPORT_SYMBOL(tcf_classify);

struct tcf_chain_info {
        struct tcf_proto __rcu **pprev;
        struct tcf_proto __rcu *next;
};

static struct tcf_proto *tcf_chain_tp_prev(struct tcf_chain *chain,
                                           struct tcf_chain_info *chain_info)
{
        return tcf_chain_dereference(*chain_info->pprev, chain);
}

static int tcf_chain_tp_insert(struct tcf_chain *chain,
                               struct tcf_chain_info *chain_info,
                               struct tcf_proto *tp)
{
        if (chain->flushing)
                return -EAGAIN;

        RCU_INIT_POINTER(tp->next, tcf_chain_tp_prev(chain, chain_info));
        if (*chain_info->pprev == chain->filter_chain)
                tcf_chain0_head_change(chain, tp);
        tcf_proto_get(tp);
        rcu_assign_pointer(*chain_info->pprev, tp);

        return 0;
}

static void tcf_chain_tp_remove(struct tcf_chain *chain,
                                struct tcf_chain_info *chain_info,
                                struct tcf_proto *tp)
{
        struct tcf_proto *next = tcf_chain_dereference(chain_info->next, chain);

        tcf_proto_mark_delete(tp);
        if (tp == chain->filter_chain)
                tcf_chain0_head_change(chain, next);
        RCU_INIT_POINTER(*chain_info->pprev, next);
}

static struct tcf_proto *tcf_chain_tp_find(struct tcf_chain *chain,
                                           struct tcf_chain_info *chain_info,
                                           u32 protocol, u32 prio,
                                           bool prio_allocate);

/* Try to insert new proto.
 * If proto with specified priority already exists, free new proto
 * and return existing one.
 */

static struct tcf_proto *tcf_chain_tp_insert_unique(struct tcf_chain *chain,
                                                    struct tcf_proto *tp_new,
                                                    u32 protocol, u32 prio,
                                                    bool rtnl_held)
{
        struct tcf_chain_info chain_info;
        struct tcf_proto *tp;
        int err = 0;

        mutex_lock(&chain->filter_chain_lock);

        if (tcf_proto_exists_destroying(chain, tp_new)) {
                mutex_unlock(&chain->filter_chain_lock);
                tcf_proto_destroy(tp_new, rtnl_held, false, NULL);
                return ERR_PTR(-EAGAIN);
        }

        tp = tcf_chain_tp_find(chain, &chain_info,
                               protocol, prio, false);
        if (!tp)
                err = tcf_chain_tp_insert(chain, &chain_info, tp_new);
        mutex_unlock(&chain->filter_chain_lock);

        if (tp) {
                tcf_proto_destroy(tp_new, rtnl_held, false, NULL);
                tp_new = tp;
        } else if (err) {
                tcf_proto_destroy(tp_new, rtnl_held, false, NULL);
                tp_new = ERR_PTR(err);
        }

        return tp_new;
}

static void tcf_chain_tp_delete_empty(struct tcf_chain *chain,
                                      struct tcf_proto *tp, bool rtnl_held,
                                      struct netlink_ext_ack *extack)
{
        struct tcf_chain_info chain_info;
        struct tcf_proto *tp_iter;
        struct tcf_proto **pprev;
        struct tcf_proto *next;

        mutex_lock(&chain->filter_chain_lock);

        /* Atomically find and remove tp from chain. */
        for (pprev = &chain->filter_chain;
             (tp_iter = tcf_chain_dereference(*pprev, chain));
             pprev = &tp_iter->next) {
                if (tp_iter == tp) {
                        chain_info.pprev = pprev;
                        chain_info.next = tp_iter->next;
                        WARN_ON(tp_iter->deleting);
                        break;
                }
        }
        /* Verify that tp still exists and no new filters were inserted
         * concurrently.
         * Mark tp for deletion if it is empty.
         */
        if (!tp_iter || !tcf_proto_check_delete(tp)) {
                mutex_unlock(&chain->filter_chain_lock);
                return;
        }

        tcf_proto_signal_destroying(chain, tp);
        next = tcf_chain_dereference(chain_info.next, chain);
        if (tp == chain->filter_chain)
                tcf_chain0_head_change(chain, next);
        RCU_INIT_POINTER(*chain_info.pprev, next);
        mutex_unlock(&chain->filter_chain_lock);

        tcf_proto_put(tp, rtnl_held, extack);
}

static struct tcf_proto *tcf_chain_tp_find(struct tcf_chain *chain,
                                           struct tcf_chain_info *chain_info,
                                           u32 protocol, u32 prio,
                                           bool prio_allocate)
{
        struct tcf_proto **pprev;
        struct tcf_proto *tp;

        /* Check the chain for existence of proto-tcf with this priority */
        for (pprev = &chain->filter_chain;
             (tp = tcf_chain_dereference(*pprev, chain));
             pprev = &tp->next) {
                if (tp->prio >= prio) {
                        if (tp->prio == prio) {
                                if (prio_allocate ||
                                    (tp->protocol != protocol && protocol))
                                        return ERR_PTR(-EINVAL);
                        } else {
                                tp = NULL;
                        }
                        break;
                }
        }
        chain_info->pprev = pprev;
        if (tp) {
                chain_info->next = tp->next;
                tcf_proto_get(tp);
        } else {
                chain_info->next = NULL;
        }
        return tp;
}

static int tcf_fill_node(struct net *net, struct sk_buff *skb,
                         struct tcf_proto *tp, struct tcf_block *block,
                         struct Qdisc *q, u32 parent, void *fh,
                         u32 portid, u32 seq, u16 flags, int event,
                         bool terse_dump, bool rtnl_held)
{
        struct tcmsg *tcm;
        struct nlmsghdr  *nlh;
        unsigned char *b = skb_tail_pointer(skb);

        nlh = nlmsg_put(skb, portid, seq, event, sizeof(*tcm), flags);
        if (!nlh)
                goto out_nlmsg_trim;
        tcm = nlmsg_data(nlh);
        tcm->tcm_family = AF_UNSPEC;
        tcm->tcm__pad1 = 0;
        tcm->tcm__pad2 = 0;
        if (q) {
                tcm->tcm_ifindex = qdisc_dev(q)->ifindex;
                tcm->tcm_parent = parent;
        } else {
                tcm->tcm_ifindex = TCM_IFINDEX_MAGIC_BLOCK;
                tcm->tcm_block_index = block->index;
        }
        tcm->tcm_info = TC_H_MAKE(tp->prio, tp->protocol);
        if (nla_put_string(skb, TCA_KIND, tp->ops->kind))
                goto nla_put_failure;
        if (nla_put_u32(skb, TCA_CHAIN, tp->chain->index))
                goto nla_put_failure;
        if (!fh) {
                tcm->tcm_handle = 0;
        } else if (terse_dump) {
                if (tp->ops->terse_dump) {
                        if (tp->ops->terse_dump(net, tp, fh, skb, tcm,
                                                rtnl_held) < 0)
                                goto nla_put_failure;
                } else {
                        goto cls_op_not_supp;
                }
        } else {
                if (tp->ops->dump &&
                    tp->ops->dump(net, tp, fh, skb, tcm, rtnl_held) < 0)
                        goto nla_put_failure;
        }
        nlh->nlmsg_len = skb_tail_pointer(skb) - b;
        return skb->len;

out_nlmsg_trim:
nla_put_failure:
cls_op_not_supp:
        nlmsg_trim(skb, b);
        return -1;
}

static int tfilter_notify(struct net *net, struct sk_buff *oskb,
                          struct nlmsghdr *n, struct tcf_proto *tp,
                          struct tcf_block *block, struct Qdisc *q,
                          u32 parent, void *fh, int event, bool unicast,
                          bool rtnl_held)
{
        struct sk_buff *skb;
        u32 portid = oskb ? NETLINK_CB(oskb).portid : 0;
        int err = 0;

        skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
        if (!skb)
                return -ENOBUFS;

        if (tcf_fill_node(net, skb, tp, block, q, parent, fh, portid,
                          n->nlmsg_seq, n->nlmsg_flags, event,
                          false, rtnl_held) <= 0) {
                kfree_skb(skb);
                return -EINVAL;
        }

        if (unicast)
                err = rtnl_unicast(skb, net, portid);
        else
                err = rtnetlink_send(skb, net, portid, RTNLGRP_TC,
                                     n->nlmsg_flags & NLM_F_ECHO);
        return err;
}

static int tfilter_del_notify(struct net *net, struct sk_buff *oskb,
                              struct nlmsghdr *n, struct tcf_proto *tp,
                              struct tcf_block *block, struct Qdisc *q,
                              u32 parent, void *fh, bool unicast, bool *last,
                              bool rtnl_held, struct netlink_ext_ack *extack)
{
        struct sk_buff *skb;
        u32 portid = oskb ? NETLINK_CB(oskb).portid : 0;
        int err;

        skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
        if (!skb)
                return -ENOBUFS;

        if (tcf_fill_node(net, skb, tp, block, q, parent, fh, portid,
                          n->nlmsg_seq, n->nlmsg_flags, RTM_DELTFILTER,
                          false, rtnl_held) <= 0) {
                NL_SET_ERR_MSG(extack, "Failed to build del event notification");
                kfree_skb(skb);
                return -EINVAL;
        }

        err = tp->ops->delete(tp, fh, last, rtnl_held, extack);
        if (err) {
                kfree_skb(skb);
                return err;
        }

        if (unicast)
                err = rtnl_unicast(skb, net, portid);
        else
                err = rtnetlink_send(skb, net, portid, RTNLGRP_TC,
                                     n->nlmsg_flags & NLM_F_ECHO);
        if (err < 0)
                NL_SET_ERR_MSG(extack, "Failed to send filter delete notification");

        return err;
}

static void tfilter_notify_chain(struct net *net, struct sk_buff *oskb,
                                 struct tcf_block *block, struct Qdisc *q,
                                 u32 parent, struct nlmsghdr *n,
                                 struct tcf_chain *chain, int event)
{
        struct tcf_proto *tp;

        for (tp = tcf_get_next_proto(chain, NULL);
             tp; tp = tcf_get_next_proto(chain, tp))
                tfilter_notify(net, oskb, n, tp, block,
                               q, parent, NULL, event, false, true);
}

static void tfilter_put(struct tcf_proto *tp, void *fh)
{
        if (tp->ops->put && fh)
                tp->ops->put(tp, fh);
}

static int tc_new_tfilter(struct sk_buff *skb, struct nlmsghdr *n,
                          struct netlink_ext_ack *extack)
{
        struct net *net = sock_net(skb->sk);
        struct nlattr *tca[TCA_MAX + 1];
        char name[IFNAMSIZ];
        struct tcmsg *t;
        u32 protocol;
        u32 prio;
        bool prio_allocate;
        u32 parent;
        u32 chain_index;
        struct Qdisc *q;
        struct tcf_chain_info chain_info;
        struct tcf_chain *chain;
        struct tcf_block *block;
        struct tcf_proto *tp;
        unsigned long cl;
        void *fh;
        int err;
        int tp_created;
        bool rtnl_held = false;
        u32 flags;

        if (!netlink_ns_capable(skb, net->user_ns, CAP_NET_ADMIN))
                return -EPERM;

replay:
        tp_created = 0;

        err = nlmsg_parse_deprecated(n, sizeof(*t), tca, TCA_MAX,
                                     rtm_tca_policy, extack);
        if (err < 0)
                return err;

        t = nlmsg_data(n);
        protocol = TC_H_MIN(t->tcm_info);
        prio = TC_H_MAJ(t->tcm_info);
        prio_allocate = false;
        parent = t->tcm_parent;
        tp = NULL;
        cl = 0;
        block = NULL;
        q = NULL;
        chain = NULL;
        flags = 0;

        if (prio == 0) {
                /* If no priority is provided by the user,
                 * we allocate one.
                 */
                if (n->nlmsg_flags & NLM_F_CREATE) {
                        prio = TC_H_MAKE(0x80000000U, 0U);
                        prio_allocate = true;
                } else {
                        NL_SET_ERR_MSG(extack, "Invalid filter command with priority of zero");
                        return -ENOENT;
                }
        }

        /* Find head of filter chain. */

        err = __tcf_qdisc_find(net, &q, &parent, t->tcm_ifindex, false, extack);
        if (err)
                return err;

        if (tcf_proto_check_kind(tca[TCA_KIND], name)) {
                NL_SET_ERR_MSG(extack, "Specified TC filter name too long");
                err = -EINVAL;
                goto errout;
        }

        /* Take rtnl mutex if rtnl_held was set to true on previous iteration,
         * block is shared (no qdisc found), qdisc is not unlocked, classifier
         * type is not specified, classifier is not unlocked.
         */
        if (rtnl_held ||
            (q && !(q->ops->cl_ops->flags & QDISC_CLASS_OPS_DOIT_UNLOCKED)) ||
            !tcf_proto_is_unlocked(name)) {
                rtnl_held = true;
                rtnl_lock();
        }

        err = __tcf_qdisc_cl_find(q, parent, &cl, t->tcm_ifindex, extack);
        if (err)
                goto errout;

        block = __tcf_block_find(net, q, cl, t->tcm_ifindex, t->tcm_block_index,
                                 extack);
        if (IS_ERR(block)) {
                err = PTR_ERR(block);
                goto errout;
        }
        block->classid = parent;

        chain_index = tca[TCA_CHAIN] ? nla_get_u32(tca[TCA_CHAIN]) : 0;
        if (chain_index > TC_ACT_EXT_VAL_MASK) {
                NL_SET_ERR_MSG(extack, "Specified chain index exceeds upper limit");
                err = -EINVAL;
                goto errout;
        }
        chain = tcf_chain_get(block, chain_index, true);
        if (!chain) {
                NL_SET_ERR_MSG(extack, "Cannot create specified filter chain");
                err = -ENOMEM;
                goto errout;
        }

        mutex_lock(&chain->filter_chain_lock);
        tp = tcf_chain_tp_find(chain, &chain_info, protocol,
                               prio, prio_allocate);
        if (IS_ERR(tp)) {
                NL_SET_ERR_MSG(extack, "Filter with specified priority/protocol not found");
                err = PTR_ERR(tp);
                goto errout_locked;
        }

        if (tp == NULL) {
                struct tcf_proto *tp_new = NULL;

                if (chain->flushing) {
                        err = -EAGAIN;
                        goto errout_locked;
                }

                /* Proto-tcf does not exist, create new one */

                if (tca[TCA_KIND] == NULL || !protocol) {
                        NL_SET_ERR_MSG(extack, "Filter kind and protocol must be specified");
                        err = -EINVAL;
                        goto errout_locked;
                }

                if (!(n->nlmsg_flags & NLM_F_CREATE)) {
                        NL_SET_ERR_MSG(extack, "Need both RTM_NEWTFILTER and NLM_F_CREATE to create a new filter");
                        err = -ENOENT;
                        goto errout_locked;
                }

                if (prio_allocate)
                        prio = tcf_auto_prio(tcf_chain_tp_prev(chain,
                                                               &chain_info));

                mutex_unlock(&chain->filter_chain_lock);
                tp_new = tcf_proto_create(name, protocol, prio, chain,
                                          rtnl_held, extack);
                if (IS_ERR(tp_new)) {
                        err = PTR_ERR(tp_new);
                        goto errout_tp;
                }

                tp_created = 1;
                tp = tcf_chain_tp_insert_unique(chain, tp_new, protocol, prio,
                                                rtnl_held);
                if (IS_ERR(tp)) {
                        err = PTR_ERR(tp);
                        goto errout_tp;
                }
        } else {
                mutex_unlock(&chain->filter_chain_lock);
        }

        if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], tp->ops->kind)) {
                NL_SET_ERR_MSG(extack, "Specified filter kind does not match existing one");
                err = -EINVAL;
                goto errout;
        }

        fh = tp->ops->get(tp, t->tcm_handle);

        if (!fh) {
                if (!(n->nlmsg_flags & NLM_F_CREATE)) {
                        NL_SET_ERR_MSG(extack, "Need both RTM_NEWTFILTER and NLM_F_CREATE to create a new filter");
                        err = -ENOENT;
                        goto errout;
                }
        } else if (n->nlmsg_flags & NLM_F_EXCL) {
                tfilter_put(tp, fh);
                NL_SET_ERR_MSG(extack, "Filter already exists");
                err = -EEXIST;
                goto errout;
        }

        if (chain->tmplt_ops && chain->tmplt_ops != tp->ops) {
                tfilter_put(tp, fh);
                NL_SET_ERR_MSG(extack, "Chain template is set to a different filter kind");
                err = -EINVAL;
                goto errout;
        }

        if (!(n->nlmsg_flags & NLM_F_CREATE))
                flags |= TCA_ACT_FLAGS_REPLACE;
        if (!rtnl_held)
                flags |= TCA_ACT_FLAGS_NO_RTNL;
        err = tp->ops->change(net, skb, tp, cl, t->tcm_handle, tca, &fh,
                              flags, extack);
        if (err == 0) {
                tfilter_notify(net, skb, n, tp, block, q, parent, fh,
                               RTM_NEWTFILTER, false, rtnl_held);
                tfilter_put(tp, fh);
                /* q pointer is NULL for shared blocks */
                if (q)
                        q->flags &= ~TCQ_F_CAN_BYPASS;
        }

errout:
        if (err && tp_created)
                tcf_chain_tp_delete_empty(chain, tp, rtnl_held, NULL);
errout_tp:
        if (chain) {
                if (tp && !IS_ERR(tp))
                        tcf_proto_put(tp, rtnl_held, NULL);
                if (!tp_created)
                        tcf_chain_put(chain);
        }
        tcf_block_release(q, block, rtnl_held);

        if (rtnl_held)
                rtnl_unlock();

        if (err == -EAGAIN) {
                /* Take rtnl lock in case EAGAIN is caused by concurrent flush
                 * of target chain.
                 */
                rtnl_held = true;
                /* Replay the request. */
                goto replay;
        }
        return err;

errout_locked:
        mutex_unlock(&chain->filter_chain_lock);
        goto errout;
}

static int tc_del_tfilter(struct sk_buff *skb, struct nlmsghdr *n,
                          struct netlink_ext_ack *extack)
{
        struct net *net = sock_net(skb->sk);
        struct nlattr *tca[TCA_MAX + 1];
        char name[IFNAMSIZ];
        struct tcmsg *t;
        u32 protocol;
        u32 prio;
        u32 parent;
        u32 chain_index;
        struct Qdisc *q = NULL;
        struct tcf_chain_info chain_info;
        struct tcf_chain *chain = NULL;
        struct tcf_block *block = NULL;
        struct tcf_proto *tp = NULL;
        unsigned long cl = 0;
        void *fh = NULL;
        int err;
        bool rtnl_held = false;

        if (!netlink_ns_capable(skb, net->user_ns, CAP_NET_ADMIN))
                return -EPERM;

        err = nlmsg_parse_deprecated(n, sizeof(*t), tca, TCA_MAX,
                                     rtm_tca_policy, extack);
        if (err < 0)
                return err;

        t = nlmsg_data(n);
        protocol = TC_H_MIN(t->tcm_info);
        prio = TC_H_MAJ(t->tcm_info);
        parent = t->tcm_parent;

        if (prio == 0 && (protocol || t->tcm_handle || tca[TCA_KIND])) {
                NL_SET_ERR_MSG(extack, "Cannot flush filters with protocol, handle or kind set");
                return -ENOENT;
        }

        /* Find head of filter chain. */

        err = __tcf_qdisc_find(net, &q, &parent, t->tcm_ifindex, false, extack);
        if (err)
                return err;

        if (tcf_proto_check_kind(tca[TCA_KIND], name)) {
                NL_SET_ERR_MSG(extack, "Specified TC filter name too long");
                err = -EINVAL;
                goto errout;
        }
        /* Take rtnl mutex if flushing whole chain, block is shared (no qdisc
         * found), qdisc is not unlocked, classifier type is not specified,
         * classifier is not unlocked.
         */
        if (!prio ||
            (q && !(q->ops->cl_ops->flags & QDISC_CLASS_OPS_DOIT_UNLOCKED)) ||
            !tcf_proto_is_unlocked(name)) {
                rtnl_held = true;
                rtnl_lock();
        }

        err = __tcf_qdisc_cl_find(q, parent, &cl, t->tcm_ifindex, extack);
        if (err)
                goto errout;

        block = __tcf_block_find(net, q, cl, t->tcm_ifindex, t->tcm_block_index,
                                 extack);
        if (IS_ERR(block)) {
                err = PTR_ERR(block);
                goto errout;
        }

        chain_index = tca[TCA_CHAIN] ? nla_get_u32(tca[TCA_CHAIN]) : 0;
        if (chain_index > TC_ACT_EXT_VAL_MASK) {
                NL_SET_ERR_MSG(extack, "Specified chain index exceeds upper limit");
                err = -EINVAL;
                goto errout;
        }
        chain = tcf_chain_get(block, chain_index, false);
        if (!chain) {
                /* User requested flush on non-existent chain. Nothing to do,
                 * so just return success.
                 */
                if (prio == 0) {
                        err = 0;
                        goto errout;
                }
                NL_SET_ERR_MSG(extack, "Cannot find specified filter chain");
                err = -ENOENT;
                goto errout;
        }

        if (prio == 0) {
                tfilter_notify_chain(net, skb, block, q, parent, n,
                                     chain, RTM_DELTFILTER);
                tcf_chain_flush(chain, rtnl_held);
                err = 0;
                goto errout;
        }

        mutex_lock(&chain->filter_chain_lock);
        tp = tcf_chain_tp_find(chain, &chain_info, protocol,
                               prio, false);
        if (!tp || IS_ERR(tp)) {
                NL_SET_ERR_MSG(extack, "Filter with specified priority/protocol not found");
                err = tp ? PTR_ERR(tp) : -ENOENT;
                goto errout_locked;
        } else if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], tp->ops->kind)) {
                NL_SET_ERR_MSG(extack, "Specified filter kind does not match existing one");
                err = -EINVAL;
                goto errout_locked;
        } else if (t->tcm_handle == 0) {
                tcf_proto_signal_destroying(chain, tp);
                tcf_chain_tp_remove(chain, &chain_info, tp);
                mutex_unlock(&chain->filter_chain_lock);

                tcf_proto_put(tp, rtnl_held, NULL);
                tfilter_notify(net, skb, n, tp, block, q, parent, fh,
                               RTM_DELTFILTER, false, rtnl_held);
                err = 0;
                goto errout;
        }
        mutex_unlock(&chain->filter_chain_lock);

        fh = tp->ops->get(tp, t->tcm_handle);

        if (!fh) {
                NL_SET_ERR_MSG(extack, "Specified filter handle not found");
                err = -ENOENT;
        } else {
                bool last;

                err = tfilter_del_notify(net, skb, n, tp, block,
                                         q, parent, fh, false, &last,
                                         rtnl_held, extack);

                if (err)
                        goto errout;
                if (last)
                        tcf_chain_tp_delete_empty(chain, tp, rtnl_held, extack);
        }

errout:
        if (chain) {
                if (tp && !IS_ERR(tp))
                        tcf_proto_put(tp, rtnl_held, NULL);
                tcf_chain_put(chain);
        }
        tcf_block_release(q, block, rtnl_held);

        if (rtnl_held)
                rtnl_unlock();

        return err;

errout_locked:
        mutex_unlock(&chain->filter_chain_lock);
        goto errout;
}

static int tc_get_tfilter(struct sk_buff *skb, struct nlmsghdr *n,
                          struct netlink_ext_ack *extack)
{
        struct net *net = sock_net(skb->sk);
        struct nlattr *tca[TCA_MAX + 1];
        char name[IFNAMSIZ];
        struct tcmsg *t;
        u32 protocol;
        u32 prio;
        u32 parent;
        u32 chain_index;
        struct Qdisc *q = NULL;
        struct tcf_chain_info chain_info;
        struct tcf_chain *chain = NULL;
        struct tcf_block *block = NULL;
        struct tcf_proto *tp = NULL;
        unsigned long cl = 0;
        void *fh = NULL;
        int err;
        bool rtnl_held = false;

        err = nlmsg_parse_deprecated(n, sizeof(*t), tca, TCA_MAX,
                                     rtm_tca_policy, extack);
        if (err < 0)
                return err;

        t = nlmsg_data(n);
        protocol = TC_H_MIN(t->tcm_info);
        prio = TC_H_MAJ(t->tcm_info);
        parent = t->tcm_parent;

        if (prio == 0) {
                NL_SET_ERR_MSG(extack, "Invalid filter command with priority of zero");
                return -ENOENT;
        }

        /* Find head of filter chain. */

        err = __tcf_qdisc_find(net, &q, &parent, t->tcm_ifindex, false, extack);
        if (err)
                return err;

        if (tcf_proto_check_kind(tca[TCA_KIND], name)) {
                NL_SET_ERR_MSG(extack, "Specified TC filter name too long");
                err = -EINVAL;
                goto errout;
        }
        /* Take rtnl mutex if block is shared (no qdisc found), qdisc is not
         * unlocked, classifier type is not specified, classifier is not
         * unlocked.
         */
        if ((q && !(q->ops->cl_ops->flags & QDISC_CLASS_OPS_DOIT_UNLOCKED)) ||
            !tcf_proto_is_unlocked(name)) {
                rtnl_held = true;
                rtnl_lock();
        }

        err = __tcf_qdisc_cl_find(q, parent, &cl, t->tcm_ifindex, extack);
        if (err)
                goto errout;

        block = __tcf_block_find(net, q, cl, t->tcm_ifindex, t->tcm_block_index,
                                 extack);
        if (IS_ERR(block)) {
                err = PTR_ERR(block);
                goto errout;
        }

        chain_index = tca[TCA_CHAIN] ? nla_get_u32(tca[TCA_CHAIN]) : 0;
        if (chain_index > TC_ACT_EXT_VAL_MASK) {
                NL_SET_ERR_MSG(extack, "Specified chain index exceeds upper limit");
                err = -EINVAL;
                goto errout;
        }
        chain = tcf_chain_get(block, chain_index, false);
        if (!chain) {
                NL_SET_ERR_MSG(extack, "Cannot find specified filter chain");
                err = -EINVAL;
                goto errout;
        }

        mutex_lock(&chain->filter_chain_lock);
        tp = tcf_chain_tp_find(chain, &chain_info, protocol,
                               prio, false);
        mutex_unlock(&chain->filter_chain_lock);
        if (!tp || IS_ERR(tp)) {
                NL_SET_ERR_MSG(extack, "Filter with specified priority/protocol not found");
                err = tp ? PTR_ERR(tp) : -ENOENT;
                goto errout;
        } else if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], tp->ops->kind)) {
                NL_SET_ERR_MSG(extack, "Specified filter kind does not match existing one");
                err = -EINVAL;
                goto errout;
        }

        fh = tp->ops->get(tp, t->tcm_handle);

        if (!fh) {
                NL_SET_ERR_MSG(extack, "Specified filter handle not found");
                err = -ENOENT;
        } else {
                err = tfilter_notify(net, skb, n, tp, block, q, parent,
                                     fh, RTM_NEWTFILTER, true, rtnl_held);
                if (err < 0)
                        NL_SET_ERR_MSG(extack, "Failed to send filter notify message");
        }

        tfilter_put(tp, fh);
errout:
        if (chain) {
                if (tp && !IS_ERR(tp))
                        tcf_proto_put(tp, rtnl_held, NULL);
                tcf_chain_put(chain);
        }
        tcf_block_release(q, block, rtnl_held);

        if (rtnl_held)
                rtnl_unlock();

        return err;
}

struct tcf_dump_args {
        struct tcf_walker w;
        struct sk_buff *skb;
        struct netlink_callback *cb;
        struct tcf_block *block;
        struct Qdisc *q;
        u32 parent;
        bool terse_dump;
};

static int tcf_node_dump(struct tcf_proto *tp, void *n, struct tcf_walker *arg)
{
        struct tcf_dump_args *a = (void *)arg;
        struct net *net = sock_net(a->skb->sk);

        return tcf_fill_node(net, a->skb, tp, a->block, a->q, a->parent,
                             n, NETLINK_CB(a->cb->skb).portid,
                             a->cb->nlh->nlmsg_seq, NLM_F_MULTI,
                             RTM_NEWTFILTER, a->terse_dump, true);
}

static bool tcf_chain_dump(struct tcf_chain *chain, struct Qdisc *q, u32 parent,
                           struct sk_buff *skb, struct netlink_callback *cb,
                           long index_start, long *p_index, bool terse)
{
        struct net *net = sock_net(skb->sk);
        struct tcf_block *block = chain->block;
        struct tcmsg *tcm = nlmsg_data(cb->nlh);
        struct tcf_proto *tp, *tp_prev;
        struct tcf_dump_args arg;

        for (tp = __tcf_get_next_proto(chain, NULL);
             tp;
             tp_prev = tp,
                     tp = __tcf_get_next_proto(chain, tp),
                     tcf_proto_put(tp_prev, true, NULL),
                     (*p_index)++) {
                if (*p_index < index_start)
                        continue;
                if (TC_H_MAJ(tcm->tcm_info) &&
                    TC_H_MAJ(tcm->tcm_info) != tp->prio)
                        continue;
                if (TC_H_MIN(tcm->tcm_info) &&
                    TC_H_MIN(tcm->tcm_info) != tp->protocol)
                        continue;
                if (*p_index > index_start)
                        memset(&cb->args[1], 0,
                               sizeof(cb->args) - sizeof(cb->args[0]));
                if (cb->args[1] == 0) {
                        if (tcf_fill_node(net, skb, tp, block, q, parent, NULL,
                                          NETLINK_CB(cb->skb).portid,
                                          cb->nlh->nlmsg_seq, NLM_F_MULTI,
                                          RTM_NEWTFILTER, false, true) <= 0)
                                goto errout;
                        cb->args[1] = 1;
                }
                if (!tp->ops->walk)
                        continue;
                arg.w.fn = tcf_node_dump;
                arg.skb = skb;
                arg.cb = cb;
                arg.block = block;
                arg.q = q;
                arg.parent = parent;
                arg.w.stop = 0;
                arg.w.skip = cb->args[1] - 1;
                arg.w.count = 0;
                arg.w.cookie = cb->args[2];
                arg.terse_dump = terse;
                tp->ops->walk(tp, &arg.w, true);
                cb->args[2] = arg.w.cookie;
                cb->args[1] = arg.w.count + 1;
                if (arg.w.stop)
                        goto errout;
        }
        return true;

errout:
        tcf_proto_put(tp, true, NULL);
        return false;
}

static const struct nla_policy tcf_tfilter_dump_policy[TCA_MAX + 1] = {
        [TCA_DUMP_FLAGS] = NLA_POLICY_BITFIELD32(TCA_DUMP_FLAGS_TERSE),
};

/* called with RTNL */
static int tc_dump_tfilter(struct sk_buff *skb, struct netlink_callback *cb)
{
        struct tcf_chain *chain, *chain_prev;
        struct net *net = sock_net(skb->sk);
        struct nlattr *tca[TCA_MAX + 1];
        struct Qdisc *q = NULL;
        struct tcf_block *block;
        struct tcmsg *tcm = nlmsg_data(cb->nlh);
        bool terse_dump = false;
        long index_start;
        long index;
        u32 parent;
        int err;

        if (nlmsg_len(cb->nlh) < sizeof(*tcm))
                return skb->len;

        err = nlmsg_parse_deprecated(cb->nlh, sizeof(*tcm), tca, TCA_MAX,
                                     tcf_tfilter_dump_policy, cb->extack);
        if (err)
                return err;

        if (tca[TCA_DUMP_FLAGS]) {
                struct nla_bitfield32 flags =
                        nla_get_bitfield32(tca[TCA_DUMP_FLAGS]);

                terse_dump = flags.value & TCA_DUMP_FLAGS_TERSE;
        }

        if (tcm->tcm_ifindex == TCM_IFINDEX_MAGIC_BLOCK) {
                block = tcf_block_refcnt_get(net, tcm->tcm_block_index);
                if (!block)
                        goto out;
                /* If we work with block index, q is NULL and parent value
                 * will never be used in the following code. The check
                 * in tcf_fill_node prevents it. However, compiler does not
                 * see that far, so set parent to zero to silence the warning
                 * about parent being uninitialized.
                 */
                parent = 0;
        } else {
                const struct Qdisc_class_ops *cops;
                struct net_device *dev;
                unsigned long cl = 0;

                dev = __dev_get_by_index(net, tcm->tcm_ifindex);
                if (!dev)
                        return skb->len;

                parent = tcm->tcm_parent;
                if (!parent)
                        q = rtnl_dereference(dev->qdisc);
                else
                        q = qdisc_lookup(dev, TC_H_MAJ(tcm->tcm_parent));
                if (!q)
                        goto out;
                cops = q->ops->cl_ops;
                if (!cops)
                        goto out;
                if (!cops->tcf_block)
                        goto out;
                if (TC_H_MIN(tcm->tcm_parent)) {
                        cl = cops->find(q, tcm->tcm_parent);
                        if (cl == 0)
                                goto out;
                }
                block = cops->tcf_block(q, cl, NULL);
                if (!block)
                        goto out;
                parent = block->classid;
                if (tcf_block_shared(block))
                        q = NULL;
        }

        index_start = cb->args[0];
        index = 0;

        for (chain = __tcf_get_next_chain(block, NULL);
             chain;
             chain_prev = chain,
                     chain = __tcf_get_next_chain(block, chain),
                     tcf_chain_put(chain_prev)) {
                if (tca[TCA_CHAIN] &&
                    nla_get_u32(tca[TCA_CHAIN]) != chain->index)
                        continue;
                if (!tcf_chain_dump(chain, q, parent, skb, cb,
                                    index_start, &index, terse_dump)) {
                        tcf_chain_put(chain);
                        err = -EMSGSIZE;
                        break;
                }
        }

        if (tcm->tcm_ifindex == TCM_IFINDEX_MAGIC_BLOCK)
                tcf_block_refcnt_put(block, true);
        cb->args[0] = index;

out:
        /* If we did no progress, the error (EMSGSIZE) is real */
        if (skb->len == 0 && err)
                return err;
        return skb->len;
}

static int tc_chain_fill_node(const struct tcf_proto_ops *tmplt_ops,
                              void *tmplt_priv, u32 chain_index,
                              struct net *net, struct sk_buff *skb,
                              struct tcf_block *block,
                              u32 portid, u32 seq, u16 flags, int event)
{
        unsigned char *b = skb_tail_pointer(skb);
        const struct tcf_proto_ops *ops;
        struct nlmsghdr *nlh;
        struct tcmsg *tcm;
        void *priv;

        ops = tmplt_ops;
        priv = tmplt_priv;

        nlh = nlmsg_put(skb, portid, seq, event, sizeof(*tcm), flags);
        if (!nlh)
                goto out_nlmsg_trim;
        tcm = nlmsg_data(nlh);
        tcm->tcm_family = AF_UNSPEC;
        tcm->tcm__pad1 = 0;
        tcm->tcm__pad2 = 0;
        tcm->tcm_handle = 0;
        if (block->q) {
                tcm->tcm_ifindex = qdisc_dev(block->q)->ifindex;
                tcm->tcm_parent = block->q->handle;
        } else {
                tcm->tcm_ifindex = TCM_IFINDEX_MAGIC_BLOCK;
                tcm->tcm_block_index = block->index;
        }

        if (nla_put_u32(skb, TCA_CHAIN, chain_index))
                goto nla_put_failure;

        if (ops) {
                if (nla_put_string(skb, TCA_KIND, ops->kind))
                        goto nla_put_failure;
                if (ops->tmplt_dump(skb, net, priv) < 0)
                        goto nla_put_failure;
        }

        nlh->nlmsg_len = skb_tail_pointer(skb) - b;
        return skb->len;

out_nlmsg_trim:
nla_put_failure:
        nlmsg_trim(skb, b);
        return -EMSGSIZE;
}

static int tc_chain_notify(struct tcf_chain *chain, struct sk_buff *oskb,
                           u32 seq, u16 flags, int event, bool unicast)
{
        u32 portid = oskb ? NETLINK_CB(oskb).portid : 0;
        struct tcf_block *block = chain->block;
        struct net *net = block->net;
        struct sk_buff *skb;
        int err = 0;

        skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
        if (!skb)
                return -ENOBUFS;

        if (tc_chain_fill_node(chain->tmplt_ops, chain->tmplt_priv,
                               chain->index, net, skb, block, portid,
                               seq, flags, event) <= 0) {
                kfree_skb(skb);
                return -EINVAL;
        }

        if (unicast)
                err = rtnl_unicast(skb, net, portid);
        else
                err = rtnetlink_send(skb, net, portid, RTNLGRP_TC,
                                     flags & NLM_F_ECHO);

        return err;
}

static int tc_chain_notify_delete(const struct tcf_proto_ops *tmplt_ops,
                                  void *tmplt_priv, u32 chain_index,
                                  struct tcf_block *block, struct sk_buff *oskb,
                                  u32 seq, u16 flags, bool unicast)
{
        u32 portid = oskb ? NETLINK_CB(oskb).portid : 0;
        struct net *net = block->net;
        struct sk_buff *skb;

        skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
        if (!skb)
                return -ENOBUFS;

        if (tc_chain_fill_node(tmplt_ops, tmplt_priv, chain_index, net, skb,
                               block, portid, seq, flags, RTM_DELCHAIN) <= 0) {
                kfree_skb(skb);
                return -EINVAL;
        }

        if (unicast)
                return rtnl_unicast(skb, net, portid);

        return rtnetlink_send(skb, net, portid, RTNLGRP_TC, flags & NLM_F_ECHO);
}

static int tc_chain_tmplt_add(struct tcf_chain *chain, struct net *net,
                              struct nlattr **tca,
                              struct netlink_ext_ack *extack)
{
        const struct tcf_proto_ops *ops;
        char name[IFNAMSIZ];
        void *tmplt_priv;

        /* If kind is not set, user did not specify template. */
        if (!tca[TCA_KIND])
                return 0;

        if (tcf_proto_check_kind(tca[TCA_KIND], name)) {
                NL_SET_ERR_MSG(extack, "Specified TC chain template name too long");
                return -EINVAL;
        }

        ops = tcf_proto_lookup_ops(name, true, extack);
        if (IS_ERR(ops))
                return PTR_ERR(ops);
        if (!ops->tmplt_create || !ops->tmplt_destroy || !ops->tmplt_dump) {
                NL_SET_ERR_MSG(extack, "Chain templates are not supported with specified classifier");
                module_put(ops->owner);
                return -EOPNOTSUPP;
        }

        tmplt_priv = ops->tmplt_create(net, chain, tca, extack);
        if (IS_ERR(tmplt_priv)) {
                module_put(ops->owner);
                return PTR_ERR(tmplt_priv);
        }
        chain->tmplt_ops = ops;
        chain->tmplt_priv = tmplt_priv;
        return 0;
}

static void tc_chain_tmplt_del(const struct tcf_proto_ops *tmplt_ops,
                               void *tmplt_priv)
{
        /* If template ops are set, no work to do for us. */
        if (!tmplt_ops)
                return;

        tmplt_ops->tmplt_destroy(tmplt_priv);
        module_put(tmplt_ops->owner);
}

/* Add/delete/get a chain */

static int tc_ctl_chain(struct sk_buff *skb, struct nlmsghdr *n,
                        struct netlink_ext_ack *extack)
{
        struct net *net = sock_net(skb->sk);
        struct nlattr *tca[TCA_MAX + 1];
        struct tcmsg *t;
        u32 parent;
        u32 chain_index;
        struct Qdisc *q;
        struct tcf_chain *chain;
        struct tcf_block *block;
        unsigned long cl;
        int err;

        if (n->nlmsg_type != RTM_GETCHAIN &&
            !netlink_ns_capable(skb, net->user_ns, CAP_NET_ADMIN))
                return -EPERM;

replay:
        q = NULL;
        err = nlmsg_parse_deprecated(n, sizeof(*t), tca, TCA_MAX,
                                     rtm_tca_policy, extack);
        if (err < 0)
                return err;

        t = nlmsg_data(n);
        parent = t->tcm_parent;
        cl = 0;

        block = tcf_block_find(net, &q, &parent, &cl,
                               t->tcm_ifindex, t->tcm_block_index, extack);
        if (IS_ERR(block))
                return PTR_ERR(block);

        chain_index = tca[TCA_CHAIN] ? nla_get_u32(tca[TCA_CHAIN]) : 0;
        if (chain_index > TC_ACT_EXT_VAL_MASK) {
                NL_SET_ERR_MSG(extack, "Specified chain index exceeds upper limit");
                err = -EINVAL;
                goto errout_block;
        }

        mutex_lock(&block->lock);
        chain = tcf_chain_lookup(block, chain_index);
        if (n->nlmsg_type == RTM_NEWCHAIN) {
                if (chain) {
                        if (tcf_chain_held_by_acts_only(chain)) {
                                /* The chain exists only because there is
                                 * some action referencing it.
                                 */
                                tcf_chain_hold(chain);
                        } else {
                                NL_SET_ERR_MSG(extack, "Filter chain already exists");
                                err = -EEXIST;
                                goto errout_block_locked;
                        }
                } else {
                        if (!(n->nlmsg_flags & NLM_F_CREATE)) {
                                NL_SET_ERR_MSG(extack, "Need both RTM_NEWCHAIN and NLM_F_CREATE to create a new chain");
                                err = -ENOENT;
                                goto errout_block_locked;
                        }
                        chain = tcf_chain_create(block, chain_index);
                        if (!chain) {
                                NL_SET_ERR_MSG(extack, "Failed to create filter chain");
                                err = -ENOMEM;
                                goto errout_block_locked;
                        }
                }
        } else {
                if (!chain || tcf_chain_held_by_acts_only(chain)) {
                        NL_SET_ERR_MSG(extack, "Cannot find specified filter chain");
                        err = -EINVAL;
                        goto errout_block_locked;
                }
                tcf_chain_hold(chain);
        }

        if (n->nlmsg_type == RTM_NEWCHAIN) {
                /* Modifying chain requires holding parent block lock. In case
                 * the chain was successfully added, take a reference to the
                 * chain. This ensures that an empty chain does not disappear at
                 * the end of this function.
                 */
                tcf_chain_hold(chain);
                chain->explicitly_created = true;
        }
        mutex_unlock(&block->lock);

        switch (n->nlmsg_type) {
        case RTM_NEWCHAIN:
                err = tc_chain_tmplt_add(chain, net, tca, extack);
                if (err) {
                        tcf_chain_put_explicitly_created(chain);
                        goto errout;
                }

                tc_chain_notify(chain, NULL, 0, NLM_F_CREATE | NLM_F_EXCL,
                                RTM_NEWCHAIN, false);
                break;
        case RTM_DELCHAIN:
                tfilter_notify_chain(net, skb, block, q, parent, n,
                                     chain, RTM_DELTFILTER);
                /* Flush the chain first as the user requested chain removal. */
                tcf_chain_flush(chain, true);
                /* In case the chain was successfully deleted, put a reference
                 * to the chain previously taken during addition.
                 */
                tcf_chain_put_explicitly_created(chain);
                break;
        case RTM_GETCHAIN:
                err = tc_chain_notify(chain, skb, n->nlmsg_seq,
                                      n->nlmsg_flags, n->nlmsg_type, true);
                if (err < 0)
                        NL_SET_ERR_MSG(extack, "Failed to send chain notify message");
                break;
        default:
                err = -EOPNOTSUPP;
                NL_SET_ERR_MSG(extack, "Unsupported message type");
                goto errout;
        }

errout:
        tcf_chain_put(chain);
errout_block:
        tcf_block_release(q, block, true);
        if (err == -EAGAIN)
                /* Replay the request. */
                goto replay;
        return err;

errout_block_locked:
        mutex_unlock(&block->lock);
        goto errout_block;
}

/* called with RTNL */
static int tc_dump_chain(struct sk_buff *skb, struct netlink_callback *cb)
{
        struct net *net = sock_net(skb->sk);
        struct nlattr *tca[TCA_MAX + 1];
        struct Qdisc *q = NULL;
        struct tcf_block *block;
        struct tcmsg *tcm = nlmsg_data(cb->nlh);
        struct tcf_chain *chain;
        long index_start;
        long index;
        int err;

        if (nlmsg_len(cb->nlh) < sizeof(*tcm))
                return skb->len;

        err = nlmsg_parse_deprecated(cb->nlh, sizeof(*tcm), tca, TCA_MAX,
                                     rtm_tca_policy, cb->extack);
        if (err)
                return err;

        if (tcm->tcm_ifindex == TCM_IFINDEX_MAGIC_BLOCK) {
                block = tcf_block_refcnt_get(net, tcm->tcm_block_index);
                if (!block)
                        goto out;
        } else {
                const struct Qdisc_class_ops *cops;
                struct net_device *dev;
                unsigned long cl = 0;

                dev = __dev_get_by_index(net, tcm->tcm_ifindex);
                if (!dev)
                        return skb->len;

                if (!tcm->tcm_parent)
                        q = rtnl_dereference(dev->qdisc);
                else
                        q = qdisc_lookup(dev, TC_H_MAJ(tcm->tcm_parent));

                if (!q)
                        goto out;
                cops = q->ops->cl_ops;
                if (!cops)
                        goto out;
                if (!cops->tcf_block)
                        goto out;
                if (TC_H_MIN(tcm->tcm_parent)) {
                        cl = cops->find(q, tcm->tcm_parent);
                        if (cl == 0)
                                goto out;
                }
                block = cops->tcf_block(q, cl, NULL);
                if (!block)
                        goto out;
                if (tcf_block_shared(block))
                        q = NULL;
        }

        index_start = cb->args[0];
        index = 0;

        mutex_lock(&block->lock);
        list_for_each_entry(chain, &block->chain_list, list) {
                if ((tca[TCA_CHAIN] &&
                     nla_get_u32(tca[TCA_CHAIN]) != chain->index))
                        continue;
                if (index < index_start) {
                        index++;
                        continue;
                }
                if (tcf_chain_held_by_acts_only(chain))
                        continue;
                err = tc_chain_fill_node(chain->tmplt_ops, chain->tmplt_priv,
                                         chain->index, net, skb, block,
                                         NETLINK_CB(cb->skb).portid,
                                         cb->nlh->nlmsg_seq, NLM_F_MULTI,
                                         RTM_NEWCHAIN);
                if (err <= 0)
                        break;
                index++;
        }
        mutex_unlock(&block->lock);

        if (tcm->tcm_ifindex == TCM_IFINDEX_MAGIC_BLOCK)
                tcf_block_refcnt_put(block, true);
        cb->args[0] = index;

out:
        /* If we did no progress, the error (EMSGSIZE) is real */
        if (skb->len == 0 && err)
                return err;
        return skb->len;
}

void tcf_exts_destroy(struct tcf_exts *exts)
{
#ifdef CONFIG_NET_CLS_ACT
        if (exts->actions) {
                tcf_action_destroy(exts->actions, TCA_ACT_UNBIND);
                kfree(exts->actions);
        }
        exts->nr_actions = 0;
#endif
}
EXPORT_SYMBOL(tcf_exts_destroy);

int tcf_exts_validate(struct net *net, struct tcf_proto *tp, struct nlattr **tb,
                      struct nlattr *rate_tlv, struct tcf_exts *exts,
                      u32 flags, struct netlink_ext_ack *extack)
{
#ifdef CONFIG_NET_CLS_ACT
        {
                int init_res[TCA_ACT_MAX_PRIO] = {};
                struct tc_action *act;
                size_t attr_size = 0;

                if (exts->police && tb[exts->police]) {
                        struct tc_action_ops *a_o;

                        a_o = tc_action_load_ops(tb[exts->police], true,
                                                 !(flags & TCA_ACT_FLAGS_NO_RTNL),
                                                 extack);
                        if (IS_ERR(a_o))
                                return PTR_ERR(a_o);
                        flags |= TCA_ACT_FLAGS_POLICE | TCA_ACT_FLAGS_BIND;
                        act = tcf_action_init_1(net, tp, tb[exts->police],
                                                rate_tlv, a_o, init_res, flags,
                                                extack);
                        module_put(a_o->owner);
                        if (IS_ERR(act))
                                return PTR_ERR(act);

                        act->type = exts->type = TCA_OLD_COMPAT;
                        exts->actions[0] = act;
                        exts->nr_actions = 1;
                        tcf_idr_insert_many(exts->actions);
                } else if (exts->action && tb[exts->action]) {
                        int err;

                        flags |= TCA_ACT_FLAGS_BIND;
                        err = tcf_action_init(net, tp, tb[exts->action],
                                              rate_tlv, exts->actions, init_res,
                                              &attr_size, flags, extack);
                        if (err < 0)
                                return err;
                        exts->nr_actions = err;
                }
        }
#else
        if ((exts->action && tb[exts->action]) ||
            (exts->police && tb[exts->police])) {
                NL_SET_ERR_MSG(extack, "Classifier actions are not supported per compile options (CONFIG_NET_CLS_ACT)");
                return -EOPNOTSUPP;
        }
#endif

        return 0;
}
EXPORT_SYMBOL(tcf_exts_validate);

void tcf_exts_change(struct tcf_exts *dst, struct tcf_exts *src)
{
#ifdef CONFIG_NET_CLS_ACT
        struct tcf_exts old = *dst;

        *dst = *src;
        tcf_exts_destroy(&old);
#endif
}
EXPORT_SYMBOL(tcf_exts_change);

#ifdef CONFIG_NET_CLS_ACT
static struct tc_action *tcf_exts_first_act(struct tcf_exts *exts)
{
        if (exts->nr_actions == 0)
                return NULL;
        else
                return exts->actions[0];
}
#endif

int tcf_exts_dump(struct sk_buff *skb, struct tcf_exts *exts)
{
#ifdef CONFIG_NET_CLS_ACT
        struct nlattr *nest;

        if (exts->action && tcf_exts_has_actions(exts)) {
                /*
                 * again for backward compatible mode - we want
                 * to work with both old and new modes of entering
                 * tc data even if iproute2  was newer - jhs
                 */
                if (exts->type != TCA_OLD_COMPAT) {
                        nest = nla_nest_start_noflag(skb, exts->action);
                        if (nest == NULL)
                                goto nla_put_failure;

                        if (tcf_action_dump(skb, exts->actions, 0, 0, false)
                            < 0)
                                goto nla_put_failure;
                        nla_nest_end(skb, nest);
                } else if (exts->police) {
                        struct tc_action *act = tcf_exts_first_act(exts);
                        nest = nla_nest_start_noflag(skb, exts->police);
                        if (nest == NULL || !act)
                                goto nla_put_failure;
                        if (tcf_action_dump_old(skb, act, 0, 0) < 0)
                                goto nla_put_failure;
                        nla_nest_end(skb, nest);
                }
        }
        return 0;

nla_put_failure:
        nla_nest_cancel(skb, nest);
        return -1;
#else
        return 0;
#endif
}
EXPORT_SYMBOL(tcf_exts_dump);

int tcf_exts_terse_dump(struct sk_buff *skb, struct tcf_exts *exts)
{
#ifdef CONFIG_NET_CLS_ACT
        struct nlattr *nest;

        if (!exts->action || !tcf_exts_has_actions(exts))
                return 0;

        nest = nla_nest_start_noflag(skb, exts->action);
        if (!nest)
                goto nla_put_failure;

        if (tcf_action_dump(skb, exts->actions, 0, 0, true) < 0)
                goto nla_put_failure;
        nla_nest_end(skb, nest);
        return 0;

nla_put_failure:
        nla_nest_cancel(skb, nest);
        return -1;
#else
        return 0;
#endif
}
EXPORT_SYMBOL(tcf_exts_terse_dump);

int tcf_exts_dump_stats(struct sk_buff *skb, struct tcf_exts *exts)
{
#ifdef CONFIG_NET_CLS_ACT
        struct tc_action *a = tcf_exts_first_act(exts);
        if (a != NULL && tcf_action_copy_stats(skb, a, 1) < 0)
                return -1;
#endif
        return 0;
}
EXPORT_SYMBOL(tcf_exts_dump_stats);

static void tcf_block_offload_inc(struct tcf_block *block, u32 *flags)
{
        if (*flags & TCA_CLS_FLAGS_IN_HW)
                return;
        *flags |= TCA_CLS_FLAGS_IN_HW;
        atomic_inc(&block->offloadcnt);
}

static void tcf_block_offload_dec(struct tcf_block *block, u32 *flags)
{
        if (!(*flags & TCA_CLS_FLAGS_IN_HW))
                return;
        *flags &= ~TCA_CLS_FLAGS_IN_HW;
        atomic_dec(&block->offloadcnt);
}

static void tc_cls_offload_cnt_update(struct tcf_block *block,
                                      struct tcf_proto *tp, u32 *cnt,
                                      u32 *flags, u32 diff, bool add)
{
        lockdep_assert_held(&block->cb_lock);

        spin_lock(&tp->lock);
        if (add) {
                if (!*cnt)
                        tcf_block_offload_inc(block, flags);
                *cnt += diff;
        } else {
                *cnt -= diff;
                if (!*cnt)
                        tcf_block_offload_dec(block, flags);
        }
        spin_unlock(&tp->lock);
}

static void
tc_cls_offload_cnt_reset(struct tcf_block *block, struct tcf_proto *tp,
                         u32 *cnt, u32 *flags)
{
        lockdep_assert_held(&block->cb_lock);

        spin_lock(&tp->lock);
        tcf_block_offload_dec(block, flags);
        *cnt = 0;
        spin_unlock(&tp->lock);
}

static int
__tc_setup_cb_call(struct tcf_block *block, enum tc_setup_type type,
                   void *type_data, bool err_stop)
{
        struct flow_block_cb *block_cb;
        int ok_count = 0;
        int err;

        list_for_each_entry(block_cb, &block->flow_block.cb_list, list) {
                err = block_cb->cb(type, type_data, block_cb->cb_priv);
                if (err) {
                        if (err_stop)
                                return err;
                } else {
                        ok_count++;
                }
        }
        return ok_count;
}

int tc_setup_cb_call(struct tcf_block *block, enum tc_setup_type type,
                     void *type_data, bool err_stop, bool rtnl_held)
{
        bool take_rtnl = READ_ONCE(block->lockeddevcnt) && !rtnl_held;
        int ok_count;

retry:
        if (take_rtnl)
                rtnl_lock();
        down_read(&block->cb_lock);
        /* Need to obtain rtnl lock if block is bound to devs that require it.
         * In block bind code cb_lock is obtained while holding rtnl, so we must
         * obtain the locks in same order here.
         */
        if (!rtnl_held && !take_rtnl && block->lockeddevcnt) {
                up_read(&block->cb_lock);
                take_rtnl = true;
                goto retry;
        }

        ok_count = __tc_setup_cb_call(block, type, type_data, err_stop);

        up_read(&block->cb_lock);
        if (take_rtnl)
                rtnl_unlock();
        return ok_count;
}
EXPORT_SYMBOL(tc_setup_cb_call);

/* Non-destructive filter add. If filter that wasn't already in hardware is
 * successfully offloaded, increment block offloads counter. On failure,
 * previously offloaded filter is considered to be intact and offloads counter
 * is not decremented.
 */

int tc_setup_cb_add(struct tcf_block *block, struct tcf_proto *tp,
                    enum tc_setup_type type, void *type_data, bool err_stop,
                    u32 *flags, unsigned int *in_hw_count, bool rtnl_held)
{
        bool take_rtnl = READ_ONCE(block->lockeddevcnt) && !rtnl_held;
        int ok_count;

retry:
        if (take_rtnl)
                rtnl_lock();
        down_read(&block->cb_lock);
        /* Need to obtain rtnl lock if block is bound to devs that require it.
         * In block bind code cb_lock is obtained while holding rtnl, so we must
         * obtain the locks in same order here.
         */
        if (!rtnl_held && !take_rtnl && block->lockeddevcnt) {
                up_read(&block->cb_lock);
                take_rtnl = true;
                goto retry;
        }

        /* Make sure all netdevs sharing this block are offload-capable. */
        if (block->nooffloaddevcnt && err_stop) {
                ok_count = -EOPNOTSUPP;
                goto err_unlock;
        }

        ok_count = __tc_setup_cb_call(block, type, type_data, err_stop);
        if (ok_count < 0)
                goto err_unlock;

        if (tp->ops->hw_add)
                tp->ops->hw_add(tp, type_data);
        if (ok_count > 0)
                tc_cls_offload_cnt_update(block, tp, in_hw_count, flags,
                                          ok_count, true);
err_unlock:
        up_read(&block->cb_lock);
        if (take_rtnl)
                rtnl_unlock();
        return ok_count < 0 ? ok_count : 0;
}
EXPORT_SYMBOL(tc_setup_cb_add);

/* Destructive filter replace. If filter that wasn't already in hardware is
 * successfully offloaded, increment block offload counter. On failure,
 * previously offloaded filter is considered to be destroyed and offload counter
 * is decremented.
 */

int tc_setup_cb_replace(struct tcf_block *block, struct tcf_proto *tp,
                        enum tc_setup_type type, void *type_data, bool err_stop,
                        u32 *old_flags, unsigned int *old_in_hw_count,
                        u32 *new_flags, unsigned int *new_in_hw_count,
                        bool rtnl_held)
{
        bool take_rtnl = READ_ONCE(block->lockeddevcnt) && !rtnl_held;
        int ok_count;

retry:
        if (take_rtnl)
                rtnl_lock();
        down_read(&block->cb_lock);
        /* Need to obtain rtnl lock if block is bound to devs that require it.
         * In block bind code cb_lock is obtained while holding rtnl, so we must
         * obtain the locks in same order here.
         */
        if (!rtnl_held && !take_rtnl && block->lockeddevcnt) {
                up_read(&block->cb_lock);
                take_rtnl = true;
                goto retry;
        }

        /* Make sure all netdevs sharing this block are offload-capable. */
        if (block->nooffloaddevcnt && err_stop) {
                ok_count = -EOPNOTSUPP;
                goto err_unlock;
        }

        tc_cls_offload_cnt_reset(block, tp, old_in_hw_count, old_flags);
        if (tp->ops->hw_del)
                tp->ops->hw_del(tp, type_data);

        ok_count = __tc_setup_cb_call(block, type, type_data, err_stop);
        if (ok_count < 0)
                goto err_unlock;

        if (tp->ops->hw_add)
                tp->ops->hw_add(tp, type_data);
        if (ok_count > 0)
                tc_cls_offload_cnt_update(block, tp, new_in_hw_count,
                                          new_flags, ok_count, true);
err_unlock:
        up_read(&block->cb_lock);
        if (take_rtnl)
                rtnl_unlock();
        return ok_count < 0 ? ok_count : 0;
}
EXPORT_SYMBOL(tc_setup_cb_replace);

/* Destroy filter and decrement block offload counter, if filter was previously
 * offloaded.
 */

int tc_setup_cb_destroy(struct tcf_block *block, struct tcf_proto *tp,
                        enum tc_setup_type type, void *type_data, bool err_stop,
                        u32 *flags, unsigned int *in_hw_count, bool rtnl_held)
{
        bool take_rtnl = READ_ONCE(block->lockeddevcnt) && !rtnl_held;
        int ok_count;

retry:
        if (take_rtnl)
                rtnl_lock();
        down_read(&block->cb_lock);
        /* Need to obtain rtnl lock if block is bound to devs that require it.
         * In block bind code cb_lock is obtained while holding rtnl, so we must
         * obtain the locks in same order here.
         */
        if (!rtnl_held && !take_rtnl && block->lockeddevcnt) {
                up_read(&block->cb_lock);
                take_rtnl = true;
                goto retry;
        }

        ok_count = __tc_setup_cb_call(block, type, type_data, err_stop);

        tc_cls_offload_cnt_reset(block, tp, in_hw_count, flags);
        if (tp->ops->hw_del)
                tp->ops->hw_del(tp, type_data);

        up_read(&block->cb_lock);
        if (take_rtnl)
                rtnl_unlock();
        return ok_count < 0 ? ok_count : 0;
}
EXPORT_SYMBOL(tc_setup_cb_destroy);

int tc_setup_cb_reoffload(struct tcf_block *block, struct tcf_proto *tp,
                          bool add, flow_setup_cb_t *cb,
                          enum tc_setup_type type, void *type_data,
                          void *cb_priv, u32 *flags, unsigned int *in_hw_count)
{
        int err = cb(type, type_data, cb_priv);

        if (err) {
                if (add && tc_skip_sw(*flags))
                        return err;
        } else {
                tc_cls_offload_cnt_update(block, tp, in_hw_count, flags, 1,
                                          add);
        }

        return 0;
}
EXPORT_SYMBOL(tc_setup_cb_reoffload);

static int tcf_act_get_cookie(struct flow_action_entry *entry,
                              const struct tc_action *act)
{
        struct tc_cookie *cookie;
        int err = 0;

        rcu_read_lock();
        cookie = rcu_dereference(act->act_cookie);
        if (cookie) {
                entry->cookie = flow_action_cookie_create(cookie->data,
                                                          cookie->len,
                                                          GFP_ATOMIC);
                if (!entry->cookie)
                        err = -ENOMEM;
        }
        rcu_read_unlock();
        return err;
}

static void tcf_act_put_cookie(struct flow_action_entry *entry)
{
        flow_action_cookie_destroy(entry->cookie);
}

void tc_cleanup_flow_action(struct flow_action *flow_action)
{
        struct flow_action_entry *entry;
        int i;

        flow_action_for_each(i, entry, flow_action) {
                tcf_act_put_cookie(entry);
                if (entry->destructor)
                        entry->destructor(entry->destructor_priv);
        }
}
EXPORT_SYMBOL(tc_cleanup_flow_action);

static void tcf_mirred_get_dev(struct flow_action_entry *entry,
                               const struct tc_action *act)
{
#ifdef CONFIG_NET_CLS_ACT
        entry->dev = act->ops->get_dev(act, &entry->destructor);
        if (!entry->dev)
                return;
        entry->destructor_priv = entry->dev;
#endif
}

static void tcf_tunnel_encap_put_tunnel(void *priv)
{
        struct ip_tunnel_info *tunnel = priv;

        kfree(tunnel);
}

static int tcf_tunnel_encap_get_tunnel(struct flow_action_entry *entry,
                                       const struct tc_action *act)
{
        entry->tunnel = tcf_tunnel_info_copy(act);
        if (!entry->tunnel)
                return -ENOMEM;
        entry->destructor = tcf_tunnel_encap_put_tunnel;
        entry->destructor_priv = entry->tunnel;
        return 0;
}

static void tcf_sample_get_group(struct flow_action_entry *entry,
                                 const struct tc_action *act)
{
#ifdef CONFIG_NET_CLS_ACT
        entry->sample.psample_group =
                act->ops->get_psample_group(act, &entry->destructor);
        entry->destructor_priv = entry->sample.psample_group;
#endif
}

static void tcf_gate_entry_destructor(void *priv)
{
        struct action_gate_entry *oe = priv;

        kfree(oe);
}

static int tcf_gate_get_entries(struct flow_action_entry *entry,
                                const struct tc_action *act)
{
        entry->gate.entries = tcf_gate_get_list(act);

        if (!entry->gate.entries)
                return -EINVAL;

        entry->destructor = tcf_gate_entry_destructor;
        entry->destructor_priv = entry->gate.entries;

        return 0;
}

static enum flow_action_hw_stats tc_act_hw_stats(u8 hw_stats)
{
        if (WARN_ON_ONCE(hw_stats > TCA_ACT_HW_STATS_ANY))
                return FLOW_ACTION_HW_STATS_DONT_CARE;
        else if (!hw_stats)
                return FLOW_ACTION_HW_STATS_DISABLED;

        return hw_stats;
}

int tc_setup_flow_action(struct flow_action *flow_action,
                         const struct tcf_exts *exts)
{
        struct tc_action *act;
        int i, j, k, err = 0;

        BUILD_BUG_ON(TCA_ACT_HW_STATS_ANY != FLOW_ACTION_HW_STATS_ANY);
        BUILD_BUG_ON(TCA_ACT_HW_STATS_IMMEDIATE != FLOW_ACTION_HW_STATS_IMMEDIATE);
        BUILD_BUG_ON(TCA_ACT_HW_STATS_DELAYED != FLOW_ACTION_HW_STATS_DELAYED);

        if (!exts)
                return 0;

        j = 0;
        tcf_exts_for_each_action(i, act, exts) {
                struct flow_action_entry *entry;

                entry = &flow_action->entries[j];
                spin_lock_bh(&act->tcfa_lock);
                err = tcf_act_get_cookie(entry, act);
                if (err)
                        goto err_out_locked;

                entry->hw_stats = tc_act_hw_stats(act->hw_stats);

                if (is_tcf_gact_ok(act)) {
                        entry->id = FLOW_ACTION_ACCEPT;
                } else if (is_tcf_gact_shot(act)) {
                        entry->id = FLOW_ACTION_DROP;
                } else if (is_tcf_gact_trap(act)) {
                        entry->id = FLOW_ACTION_TRAP;
                } else if (is_tcf_gact_goto_chain(act)) {
                        entry->id = FLOW_ACTION_GOTO;
                        entry->chain_index = tcf_gact_goto_chain_index(act);
                } else if (is_tcf_mirred_egress_redirect(act)) {
                        entry->id = FLOW_ACTION_REDIRECT;
                        tcf_mirred_get_dev(entry, act);
                } else if (is_tcf_mirred_egress_mirror(act)) {
                        entry->id = FLOW_ACTION_MIRRED;
                        tcf_mirred_get_dev(entry, act);
                } else if (is_tcf_mirred_ingress_redirect(act)) {
                        entry->id = FLOW_ACTION_REDIRECT_INGRESS;
                        tcf_mirred_get_dev(entry, act);
                } else if (is_tcf_mirred_ingress_mirror(act)) {
                        entry->id = FLOW_ACTION_MIRRED_INGRESS;
                        tcf_mirred_get_dev(entry, act);
                } else if (is_tcf_vlan(act)) {
                        switch (tcf_vlan_action(act)) {
                        case TCA_VLAN_ACT_PUSH:
                                entry->id = FLOW_ACTION_VLAN_PUSH;
                                entry->vlan.vid = tcf_vlan_push_vid(act);
                                entry->vlan.proto = tcf_vlan_push_proto(act);
                                entry->vlan.prio = tcf_vlan_push_prio(act);
                                break;
                        case TCA_VLAN_ACT_POP:
                                entry->id = FLOW_ACTION_VLAN_POP;
                                break;
                        case TCA_VLAN_ACT_MODIFY:
                                entry->id = FLOW_ACTION_VLAN_MANGLE;
                                entry->vlan.vid = tcf_vlan_push_vid(act);
                                entry->vlan.proto = tcf_vlan_push_proto(act);
                                entry->vlan.prio = tcf_vlan_push_prio(act);
                                break;
                        default:
                                err = -EOPNOTSUPP;
                                goto err_out_locked;
                        }
                } else if (is_tcf_tunnel_set(act)) {
                        entry->id = FLOW_ACTION_TUNNEL_ENCAP;
                        err = tcf_tunnel_encap_get_tunnel(entry, act);
                        if (err)
                                goto err_out_locked;
                } else if (is_tcf_tunnel_release(act)) {
                        entry->id = FLOW_ACTION_TUNNEL_DECAP;
                } else if (is_tcf_pedit(act)) {
                        for (k = 0; k < tcf_pedit_nkeys(act); k++) {
                                switch (tcf_pedit_cmd(act, k)) {
                                case TCA_PEDIT_KEY_EX_CMD_SET:
                                        entry->id = FLOW_ACTION_MANGLE;
                                        break;
                                case TCA_PEDIT_KEY_EX_CMD_ADD:
                                        entry->id = FLOW_ACTION_ADD;
                                        break;
                                default:
                                        err = -EOPNOTSUPP;
                                        goto err_out_locked;
                                }
                                entry->mangle.htype = tcf_pedit_htype(act, k);
                                entry->mangle.mask = tcf_pedit_mask(act, k);
                                entry->mangle.val = tcf_pedit_val(act, k);
                                entry->mangle.offset = tcf_pedit_offset(act, k);
                                entry->hw_stats = tc_act_hw_stats(act->hw_stats);
                                entry = &flow_action->entries[++j];
                        }
                } else if (is_tcf_csum(act)) {
                        entry->id = FLOW_ACTION_CSUM;
                        entry->csum_flags = tcf_csum_update_flags(act);
                } else if (is_tcf_skbedit_mark(act)) {
                        entry->id = FLOW_ACTION_MARK;
                        entry->mark = tcf_skbedit_mark(act);
                } else if (is_tcf_sample(act)) {
                        entry->id = FLOW_ACTION_SAMPLE;
                        entry->sample.trunc_size = tcf_sample_trunc_size(act);
                        entry->sample.truncate = tcf_sample_truncate(act);
                        entry->sample.rate = tcf_sample_rate(act);
                        tcf_sample_get_group(entry, act);
                } else if (is_tcf_police(act)) {
                        entry->id = FLOW_ACTION_POLICE;
                        entry->police.burst = tcf_police_burst(act);
                        entry->police.rate_bytes_ps =
                                tcf_police_rate_bytes_ps(act);
                        entry->police.burst_pkt = tcf_police_burst_pkt(act);
                        entry->police.rate_pkt_ps =
                                tcf_police_rate_pkt_ps(act);
                        entry->police.mtu = tcf_police_tcfp_mtu(act);
                        entry->police.index = act->tcfa_index;
                } else if (is_tcf_ct(act)) {
                        entry->id = FLOW_ACTION_CT;
                        entry->ct.action = tcf_ct_action(act);
                        entry->ct.zone = tcf_ct_zone(act);
                        entry->ct.flow_table = tcf_ct_ft(act);
                } else if (is_tcf_mpls(act)) {
                        switch (tcf_mpls_action(act)) {
                        case TCA_MPLS_ACT_PUSH:
                                entry->id = FLOW_ACTION_MPLS_PUSH;
                                entry->mpls_push.proto = tcf_mpls_proto(act);
                                entry->mpls_push.label = tcf_mpls_label(act);
                                entry->mpls_push.tc = tcf_mpls_tc(act);
                                entry->mpls_push.bos = tcf_mpls_bos(act);
                                entry->mpls_push.ttl = tcf_mpls_ttl(act);
                                break;
                        case TCA_MPLS_ACT_POP:
                                entry->id = FLOW_ACTION_MPLS_POP;
                                entry->mpls_pop.proto = tcf_mpls_proto(act);
                                break;
                        case TCA_MPLS_ACT_MODIFY:
                                entry->id = FLOW_ACTION_MPLS_MANGLE;
                                entry->mpls_mangle.label = tcf_mpls_label(act);
                                entry->mpls_mangle.tc = tcf_mpls_tc(act);
                                entry->mpls_mangle.bos = tcf_mpls_bos(act);
                                entry->mpls_mangle.ttl = tcf_mpls_ttl(act);
                                break;
                        default:
                                err = -EOPNOTSUPP;
                                goto err_out_locked;
                        }
                } else if (is_tcf_skbedit_ptype(act)) {
                        entry->id = FLOW_ACTION_PTYPE;
                        entry->ptype = tcf_skbedit_ptype(act);
                } else if (is_tcf_skbedit_priority(act)) {
                        entry->id = FLOW_ACTION_PRIORITY;
                        entry->priority = tcf_skbedit_priority(act);
                } else if (is_tcf_gate(act)) {
                        entry->id = FLOW_ACTION_GATE;
                        entry->gate.index = tcf_gate_index(act);
                        entry->gate.prio = tcf_gate_prio(act);
                        entry->gate.basetime = tcf_gate_basetime(act);
                        entry->gate.cycletime = tcf_gate_cycletime(act);
                        entry->gate.cycletimeext = tcf_gate_cycletimeext(act);
                        entry->gate.num_entries = tcf_gate_num_entries(act);
                        err = tcf_gate_get_entries(entry, act);
                        if (err)
                                goto err_out_locked;
                } else {
                        err = -EOPNOTSUPP;
                        goto err_out_locked;
                }
                spin_unlock_bh(&act->tcfa_lock);

                if (!is_tcf_pedit(act))
                        j++;
        }

err_out:
        if (err)
                tc_cleanup_flow_action(flow_action);

        return err;
err_out_locked:
        spin_unlock_bh(&act->tcfa_lock);
        goto err_out;
}
EXPORT_SYMBOL(tc_setup_flow_action);

unsigned int tcf_exts_num_actions(struct tcf_exts *exts)
{
        unsigned int num_acts = 0;
        struct tc_action *act;
        int i;

        tcf_exts_for_each_action(i, act, exts) {
                if (is_tcf_pedit(act))
                        num_acts += tcf_pedit_nkeys(act);
                else
                        num_acts++;
        }
        return num_acts;
}
EXPORT_SYMBOL(tcf_exts_num_actions);

#ifdef CONFIG_NET_CLS_ACT
static int tcf_qevent_parse_block_index(struct nlattr *block_index_attr,
                                        u32 *p_block_index,
                                        struct netlink_ext_ack *extack)
{
        *p_block_index = nla_get_u32(block_index_attr);
        if (!*p_block_index) {
                NL_SET_ERR_MSG(extack, "Block number may not be zero");
                return -EINVAL;
        }

        return 0;
}

int tcf_qevent_init(struct tcf_qevent *qe, struct Qdisc *sch,
                    enum flow_block_binder_type binder_type,
                    struct nlattr *block_index_attr,
                    struct netlink_ext_ack *extack)
{
        u32 block_index;
        int err;

        if (!block_index_attr)
                return 0;

        err = tcf_qevent_parse_block_index(block_index_attr, &block_index, extack);
        if (err)
                return err;

        if (!block_index)
                return 0;

        qe->info.binder_type = binder_type;
        qe->info.chain_head_change = tcf_chain_head_change_dflt;
        qe->info.chain_head_change_priv = &qe->filter_chain;
        qe->info.block_index = block_index;

        return tcf_block_get_ext(&qe->block, sch, &qe->info, extack);
}
EXPORT_SYMBOL(tcf_qevent_init);

void tcf_qevent_destroy(struct tcf_qevent *qe, struct Qdisc *sch)
{
        if (qe->info.block_index)
                tcf_block_put_ext(qe->block, sch, &qe->info);
}
EXPORT_SYMBOL(tcf_qevent_destroy);

int tcf_qevent_validate_change(struct tcf_qevent *qe, struct nlattr *block_index_attr,
                               struct netlink_ext_ack *extack)
{
        u32 block_index;
        int err;

        if (!block_index_attr)
                return 0;

        err = tcf_qevent_parse_block_index(block_index_attr, &block_index, extack);
        if (err)
                return err;

        /* Bounce newly-configured block or change in block. */
        if (block_index != qe->info.block_index) {
                NL_SET_ERR_MSG(extack, "Change of blocks is not supported");
                return -EINVAL;
        }

        return 0;
}
EXPORT_SYMBOL(tcf_qevent_validate_change);

struct sk_buff *tcf_qevent_handle(struct tcf_qevent *qe, struct Qdisc *sch, struct sk_buff *skb,
                                  struct sk_buff **to_free, int *ret)
{
        struct tcf_result cl_res;
        struct tcf_proto *fl;

        if (!qe->info.block_index)
                return skb;

        fl = rcu_dereference_bh(qe->filter_chain);

        switch (tcf_classify(skb, NULL, fl, &cl_res, false)) {
        case TC_ACT_SHOT:
                qdisc_qstats_drop(sch);
                __qdisc_drop(skb, to_free);
                *ret = __NET_XMIT_BYPASS;
                return NULL;
        case TC_ACT_STOLEN:
        case TC_ACT_QUEUED:
        case TC_ACT_TRAP:
                __qdisc_drop(skb, to_free);
                *ret = __NET_XMIT_STOLEN;
                return NULL;
        case TC_ACT_REDIRECT:
                skb_do_redirect(skb);
                *ret = __NET_XMIT_STOLEN;
                return NULL;
        }

        return skb;
}
EXPORT_SYMBOL(tcf_qevent_handle);

int tcf_qevent_dump(struct sk_buff *skb, int attr_name, struct tcf_qevent *qe)
{
        if (!qe->info.block_index)
                return 0;
        return nla_put_u32(skb, attr_name, qe->info.block_index);
}
EXPORT_SYMBOL(tcf_qevent_dump);
#endif

static __net_init int tcf_net_init(struct net *net)
{
        struct tcf_net *tn = net_generic(net, tcf_net_id);

        spin_lock_init(&tn->idr_lock);
        idr_init(&tn->idr);
        return 0;
}

static void __net_exit tcf_net_exit(struct net *net)
{
        struct tcf_net *tn = net_generic(net, tcf_net_id);

        idr_destroy(&tn->idr);
}

static struct pernet_operations tcf_net_ops = {
        .init = tcf_net_init,
        .exit = tcf_net_exit,
        .id   = &tcf_net_id,
        .size = sizeof(struct tcf_net),
};

static int __init tc_filter_init(void)
{
        int err;

        tc_filter_wq = alloc_ordered_workqueue("tc_filter_workqueue", 0);
        if (!tc_filter_wq)
                return -ENOMEM;

        err = register_pernet_subsys(&tcf_net_ops);
        if (err)
                goto err_register_pernet_subsys;

        rtnl_register(PF_UNSPEC, RTM_NEWTFILTER, tc_new_tfilter, NULL,
                      RTNL_FLAG_DOIT_UNLOCKED);
        rtnl_register(PF_UNSPEC, RTM_DELTFILTER, tc_del_tfilter, NULL,
                      RTNL_FLAG_DOIT_UNLOCKED);
        rtnl_register(PF_UNSPEC, RTM_GETTFILTER, tc_get_tfilter,
                      tc_dump_tfilter, RTNL_FLAG_DOIT_UNLOCKED);
        rtnl_register(PF_UNSPEC, RTM_NEWCHAIN, tc_ctl_chain, NULL, 0);
        rtnl_register(PF_UNSPEC, RTM_DELCHAIN, tc_ctl_chain, NULL, 0);
        rtnl_register(PF_UNSPEC, RTM_GETCHAIN, tc_ctl_chain,
                      tc_dump_chain, 0);

        return 0;

err_register_pernet_subsys:
        destroy_workqueue(tc_filter_wq);
        return err;
}

subsys_initcall(tc_filter_init);























































































































































    8 














    8 























1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
// SPDX-License-Identifier: GPL-2.0-only
#include <linux/fs.h>
#include <linux/fs_struct.h>
#include <linux/kernel_read_file.h>
#include <linux/security.h>
#include <linux/vmalloc.h>

/**
 * kernel_read_file() - read file contents into a kernel buffer
 *
 * @file        file to read from
 * @offset        where to start reading from (see below).
 * @buf                pointer to a "void *" buffer for reading into (if
 *                *@buf is NULL, a buffer will be allocated, and
 *                @buf_size will be ignored)
 * @buf_size        size of buf, if already allocated. If @buf not
 *                allocated, this is the largest size to allocate.
 * @file_size        if non-NULL, the full size of @file will be
 *                written here.
 * @id                the kernel_read_file_id identifying the type of
 *                file contents being read (for LSMs to examine)
 *
 * @offset must be 0 unless both @buf and @file_size are non-NULL
 * (i.e. the caller must be expecting to read partial file contents
 * via an already-allocated @buf, in at most @buf_size chunks, and
 * will be able to determine when the entire file was read by
 * checking @file_size). This isn't a recommended way to read a
 * file, though, since it is possible that the contents might
 * change between calls to kernel_read_file().
 *
 * Returns number of bytes read (no single read will be bigger
 * than INT_MAX), or negative on error.
 *
 */
int kernel_read_file(struct file *file, loff_t offset, void **buf,
                     size_t buf_size, size_t *file_size,
                     enum kernel_read_file_id id)
{
        loff_t i_size, pos;
        size_t copied;
        void *allocated = NULL;
        bool whole_file;
        int ret;

        if (offset != 0 && (!*buf || !file_size))
                return -EINVAL;

        if (!S_ISREG(file_inode(file)->i_mode))
                return -EINVAL;

        ret = deny_write_access(file);
        if (ret)
                return ret;

        i_size = i_size_read(file_inode(file));
        if (i_size <= 0) {
                ret = -EINVAL;
                goto out;
        }
        /* The file is too big for sane activities. */
        if (i_size > INT_MAX) {
                ret = -EFBIG;
                goto out;
        }
        /* The entire file cannot be read in one buffer. */
        if (!file_size && offset == 0 && i_size > buf_size) {
                ret = -EFBIG;
                goto out;
        }

        whole_file = (offset == 0 && i_size <= buf_size);
        ret = security_kernel_read_file(file, id, whole_file);
        if (ret)
                goto out;

        if (file_size)
                *file_size = i_size;

        if (!*buf)
                *buf = allocated = vmalloc(i_size);
        if (!*buf) {
                ret = -ENOMEM;
                goto out;
        }

        pos = offset;
        copied = 0;
        while (copied < buf_size) {
                ssize_t bytes;
                size_t wanted = min_t(size_t, buf_size - copied,
                                              i_size - pos);

                bytes = kernel_read(file, *buf + copied, wanted, &pos);
                if (bytes < 0) {
                        ret = bytes;
                        goto out_free;
                }

                if (bytes == 0)
                        break;
                copied += bytes;
        }

        if (whole_file) {
                if (pos != i_size) {
                        ret = -EIO;
                        goto out_free;
                }

                ret = security_kernel_post_read_file(file, *buf, i_size, id);
        }

out_free:
        if (ret < 0) {
                if (allocated) {
                        vfree(*buf);
                        *buf = NULL;
                }
        }

out:
        allow_write_access(file);
        return ret == 0 ? copied : ret;
}
EXPORT_SYMBOL_GPL(kernel_read_file);

int kernel_read_file_from_path(const char *path, loff_t offset, void **buf,
                               size_t buf_size, size_t *file_size,
                               enum kernel_read_file_id id)
{
        struct file *file;
        int ret;

        if (!path || !*path)
                return -EINVAL;

        file = filp_open(path, O_RDONLY, 0);
        if (IS_ERR(file))
                return PTR_ERR(file);

        ret = kernel_read_file(file, offset, buf, buf_size, file_size, id);
        fput(file);
        return ret;
}
EXPORT_SYMBOL_GPL(kernel_read_file_from_path);

int kernel_read_file_from_path_initns(const char *path, loff_t offset,
                                      void **buf, size_t buf_size,
                                      size_t *file_size,
                                      enum kernel_read_file_id id)
{
        struct file *file;
        struct path root;
        int ret;

        if (!path || !*path)
                return -EINVAL;

        task_lock(&init_task);
        get_fs_root(init_task.fs, &root);
        task_unlock(&init_task);

        file = file_open_root(&root, path, O_RDONLY, 0);
        path_put(&root);
        if (IS_ERR(file))
                return PTR_ERR(file);

        ret = kernel_read_file(file, offset, buf, buf_size, file_size, id);
        fput(file);
        return ret;
}
EXPORT_SYMBOL_GPL(kernel_read_file_from_path_initns);

int kernel_read_file_from_fd(int fd, loff_t offset, void **buf,
                             size_t buf_size, size_t *file_size,
                             enum kernel_read_file_id id)
{
        struct fd f = fdget(fd);
        int ret = -EBADF;

        if (!f.file || !(f.file->f_mode & FMODE_READ))
                goto out;

        ret = kernel_read_file(f.file, offset, buf, buf_size, file_size, id);
out:
        fdput(f);
        return ret;
}
EXPORT_SYMBOL_GPL(kernel_read_file_from_fd);




























































































    9 



    9 
    9 




    6 

















    4 
    4 
    4 
    4 



    9 


    8 






    8 
    2 






    9 





    9 






   42 




   42 




   42 






   42 
   42 

    4 
   40 







   32 
   10 















   46 



   48 

   16 


   42 




   42 







   12 













   32 




   32 

   32 
    1 


   32 
































   23 


    1 





    5 

    3 








    9 








   28 













    7 


   10 

    6 

    5 




    3 








    8 


    9 
    2 

















    2 















    2 











    9 


















    7 



    9 
















    3 










    1 


    2 












    9 












    3 



    2 


    1 



    2 



    3 









    2 



    3 














    2 










    2 



    2 






1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
// SPDX-License-Identifier: GPL-2.0
/*
 * Some IBSS support code for cfg80211.
 *
 * Copyright 2009        Johannes Berg <johannes@sipsolutions.net>
 * Copyright (C) 2020-2021 Intel Corporation
 */

#include <linux/etherdevice.h>
#include <linux/if_arp.h>
#include <linux/slab.h>
#include <linux/export.h>
#include <net/cfg80211.h>
#include "wext-compat.h"
#include "nl80211.h"
#include "rdev-ops.h"


void __cfg80211_ibss_joined(struct net_device *dev, const u8 *bssid,
                            struct ieee80211_channel *channel)
{
        struct wireless_dev *wdev = dev->ieee80211_ptr;
        struct cfg80211_bss *bss;
#ifdef CONFIG_CFG80211_WEXT
        union iwreq_data wrqu;
#endif

        if (WARN_ON(wdev->iftype != NL80211_IFTYPE_ADHOC))
                return;

        if (!wdev->ssid_len)
                return;

        bss = cfg80211_get_bss(wdev->wiphy, channel, bssid, NULL, 0,
                               IEEE80211_BSS_TYPE_IBSS, IEEE80211_PRIVACY_ANY);

        if (WARN_ON(!bss))
                return;

        if (wdev->current_bss) {
                cfg80211_unhold_bss(wdev->current_bss);
                cfg80211_put_bss(wdev->wiphy, &wdev->current_bss->pub);
        }

        cfg80211_hold_bss(bss_from_pub(bss));
        wdev->current_bss = bss_from_pub(bss);

        if (!(wdev->wiphy->flags & WIPHY_FLAG_HAS_STATIC_WEP))
                cfg80211_upload_connect_keys(wdev);

        nl80211_send_ibss_bssid(wiphy_to_rdev(wdev->wiphy), dev, bssid,
                                GFP_KERNEL);
#ifdef CONFIG_CFG80211_WEXT
        memset(&wrqu, 0, sizeof(wrqu));
        memcpy(wrqu.ap_addr.sa_data, bssid, ETH_ALEN);
        wireless_send_event(dev, SIOCGIWAP, &wrqu, NULL);
#endif
}

void cfg80211_ibss_joined(struct net_device *dev, const u8 *bssid,
                          struct ieee80211_channel *channel, gfp_t gfp)
{
        struct wireless_dev *wdev = dev->ieee80211_ptr;
        struct cfg80211_registered_device *rdev = wiphy_to_rdev(wdev->wiphy);
        struct cfg80211_event *ev;
        unsigned long flags;

        trace_cfg80211_ibss_joined(dev, bssid, channel);

        if (WARN_ON(!channel))
                return;

        ev = kzalloc(sizeof(*ev), gfp);
        if (!ev)
                return;

        ev->type = EVENT_IBSS_JOINED;
        memcpy(ev->ij.bssid, bssid, ETH_ALEN);
        ev->ij.channel = channel;

        spin_lock_irqsave(&wdev->event_lock, flags);
        list_add_tail(&ev->list, &wdev->event_list);
        spin_unlock_irqrestore(&wdev->event_lock, flags);
        queue_work(cfg80211_wq, &rdev->event_work);
}
EXPORT_SYMBOL(cfg80211_ibss_joined);

int __cfg80211_join_ibss(struct cfg80211_registered_device *rdev,
                         struct net_device *dev,
                         struct cfg80211_ibss_params *params,
                         struct cfg80211_cached_keys *connkeys)
{
        struct wireless_dev *wdev = dev->ieee80211_ptr;
        int err;

        lockdep_assert_held(&rdev->wiphy.mtx);
        ASSERT_WDEV_LOCK(wdev);

        if (wdev->ssid_len)
                return -EALREADY;

        if (!params->basic_rates) {
                /*
                * If no rates were explicitly configured,
                * use the mandatory rate set for 11b or
                * 11a for maximum compatibility.
                */
                struct ieee80211_supported_band *sband;
                enum nl80211_band band;
                u32 flag;
                int j;

                band = params->chandef.chan->band;
                if (band == NL80211_BAND_5GHZ ||
                    band == NL80211_BAND_6GHZ)
                        flag = IEEE80211_RATE_MANDATORY_A;
                else
                        flag = IEEE80211_RATE_MANDATORY_B;

                sband = rdev->wiphy.bands[band];
                for (j = 0; j < sband->n_bitrates; j++) {
                        if (sband->bitrates[j].flags & flag)
                                params->basic_rates |= BIT(j);
                }
        }

        if (WARN_ON(connkeys && connkeys->def < 0))
                return -EINVAL;

        if (WARN_ON(wdev->connect_keys))
                kfree_sensitive(wdev->connect_keys);
        wdev->connect_keys = connkeys;

        wdev->ibss_fixed = params->channel_fixed;
        wdev->ibss_dfs_possible = params->userspace_handles_dfs;
        wdev->chandef = params->chandef;
        if (connkeys) {
                params->wep_keys = connkeys->params;
                params->wep_tx_key = connkeys->def;
        }

#ifdef CONFIG_CFG80211_WEXT
        wdev->wext.ibss.chandef = params->chandef;
#endif
        err = rdev_join_ibss(rdev, dev, params);
        if (err) {
                wdev->connect_keys = NULL;
                return err;
        }

        memcpy(wdev->ssid, params->ssid, params->ssid_len);
        wdev->ssid_len = params->ssid_len;

        return 0;
}

static void __cfg80211_clear_ibss(struct net_device *dev, bool nowext)
{
        struct wireless_dev *wdev = dev->ieee80211_ptr;
        struct cfg80211_registered_device *rdev = wiphy_to_rdev(wdev->wiphy);
        int i;

        ASSERT_WDEV_LOCK(wdev);

        kfree_sensitive(wdev->connect_keys);
        wdev->connect_keys = NULL;

        rdev_set_qos_map(rdev, dev, NULL);

        /*
         * Delete all the keys ... pairwise keys can't really
         * exist any more anyway, but default keys might.
         */
        if (rdev->ops->del_key)
                for (i = 0; i < 6; i++)
                        rdev_del_key(rdev, dev, i, false, NULL);

        if (wdev->current_bss) {
                cfg80211_unhold_bss(wdev->current_bss);
                cfg80211_put_bss(wdev->wiphy, &wdev->current_bss->pub);
        }

        wdev->current_bss = NULL;
        wdev->ssid_len = 0;
        memset(&wdev->chandef, 0, sizeof(wdev->chandef));
#ifdef CONFIG_CFG80211_WEXT
        if (!nowext)
                wdev->wext.ibss.ssid_len = 0;
#endif
        cfg80211_sched_dfs_chan_update(rdev);
}

void cfg80211_clear_ibss(struct net_device *dev, bool nowext)
{
        struct wireless_dev *wdev = dev->ieee80211_ptr;

        wdev_lock(wdev);
        __cfg80211_clear_ibss(dev, nowext);
        wdev_unlock(wdev);
}

int __cfg80211_leave_ibss(struct cfg80211_registered_device *rdev,
                          struct net_device *dev, bool nowext)
{
        struct wireless_dev *wdev = dev->ieee80211_ptr;
        int err;

        ASSERT_WDEV_LOCK(wdev);

        if (!wdev->ssid_len)
                return -ENOLINK;

        err = rdev_leave_ibss(rdev, dev);

        if (err)
                return err;

        wdev->conn_owner_nlportid = 0;
        __cfg80211_clear_ibss(dev, nowext);

        return 0;
}

int cfg80211_leave_ibss(struct cfg80211_registered_device *rdev,
                        struct net_device *dev, bool nowext)
{
        struct wireless_dev *wdev = dev->ieee80211_ptr;
        int err;

        wdev_lock(wdev);
        err = __cfg80211_leave_ibss(rdev, dev, nowext);
        wdev_unlock(wdev);

        return err;
}

#ifdef CONFIG_CFG80211_WEXT
int cfg80211_ibss_wext_join(struct cfg80211_registered_device *rdev,
                            struct wireless_dev *wdev)
{
        struct cfg80211_cached_keys *ck = NULL;
        enum nl80211_band band;
        int i, err;

        ASSERT_WDEV_LOCK(wdev);

        if (!wdev->wext.ibss.beacon_interval)
                wdev->wext.ibss.beacon_interval = 100;

        /* try to find an IBSS channel if none requested ... */
        if (!wdev->wext.ibss.chandef.chan) {
                struct ieee80211_channel *new_chan = NULL;

                for (band = 0; band < NUM_NL80211_BANDS; band++) {
                        struct ieee80211_supported_band *sband;
                        struct ieee80211_channel *chan;

                        sband = rdev->wiphy.bands[band];
                        if (!sband)
                                continue;

                        for (i = 0; i < sband->n_channels; i++) {
                                chan = &sband->channels[i];
                                if (chan->flags & IEEE80211_CHAN_NO_IR)
                                        continue;
                                if (chan->flags & IEEE80211_CHAN_DISABLED)
                                        continue;
                                new_chan = chan;
                                break;
                        }

                        if (new_chan)
                                break;
                }

                if (!new_chan)
                        return -EINVAL;

                cfg80211_chandef_create(&wdev->wext.ibss.chandef, new_chan,
                                        NL80211_CHAN_NO_HT);
        }

        /* don't join -- SSID is not there */
        if (!wdev->wext.ibss.ssid_len)
                return 0;

        if (!netif_running(wdev->netdev))
                return 0;

        if (wdev->wext.keys)
                wdev->wext.keys->def = wdev->wext.default_key;

        wdev->wext.ibss.privacy = wdev->wext.default_key != -1;

        if (wdev->wext.keys && wdev->wext.keys->def != -1) {
                ck = kmemdup(wdev->wext.keys, sizeof(*ck), GFP_KERNEL);
                if (!ck)
                        return -ENOMEM;
                for (i = 0; i < CFG80211_MAX_WEP_KEYS; i++)
                        ck->params[i].key = ck->data[i];
        }
        err = __cfg80211_join_ibss(rdev, wdev->netdev,
                                   &wdev->wext.ibss, ck);
        if (err)
                kfree(ck);

        return err;
}

int cfg80211_ibss_wext_siwfreq(struct net_device *dev,
                               struct iw_request_info *info,
                               struct iw_freq *wextfreq, char *extra)
{
        struct wireless_dev *wdev = dev->ieee80211_ptr;
        struct cfg80211_registered_device *rdev = wiphy_to_rdev(wdev->wiphy);
        struct ieee80211_channel *chan = NULL;
        int err, freq;

        /* call only for ibss! */
        if (WARN_ON(wdev->iftype != NL80211_IFTYPE_ADHOC))
                return -EINVAL;

        if (!rdev->ops->join_ibss)
                return -EOPNOTSUPP;

        freq = cfg80211_wext_freq(wextfreq);
        if (freq < 0)
                return freq;

        if (freq) {
                chan = ieee80211_get_channel(wdev->wiphy, freq);
                if (!chan)
                        return -EINVAL;
                if (chan->flags & IEEE80211_CHAN_NO_IR ||
                    chan->flags & IEEE80211_CHAN_DISABLED)
                        return -EINVAL;
        }

        if (wdev->wext.ibss.chandef.chan == chan)
                return 0;

        wdev_lock(wdev);
        err = 0;
        if (wdev->ssid_len)
                err = __cfg80211_leave_ibss(rdev, dev, true);
        wdev_unlock(wdev);

        if (err)
                return err;

        if (chan) {
                cfg80211_chandef_create(&wdev->wext.ibss.chandef, chan,
                                        NL80211_CHAN_NO_HT);
                wdev->wext.ibss.channel_fixed = true;
        } else {
                /* cfg80211_ibss_wext_join will pick one if needed */
                wdev->wext.ibss.channel_fixed = false;
        }

        wdev_lock(wdev);
        err = cfg80211_ibss_wext_join(rdev, wdev);
        wdev_unlock(wdev);

        return err;
}

int cfg80211_ibss_wext_giwfreq(struct net_device *dev,
                               struct iw_request_info *info,
                               struct iw_freq *freq, char *extra)
{
        struct wireless_dev *wdev = dev->ieee80211_ptr;
        struct ieee80211_channel *chan = NULL;

        /* call only for ibss! */
        if (WARN_ON(wdev->iftype != NL80211_IFTYPE_ADHOC))
                return -EINVAL;

        wdev_lock(wdev);
        if (wdev->current_bss)
                chan = wdev->current_bss->pub.channel;
        else if (wdev->wext.ibss.chandef.chan)
                chan = wdev->wext.ibss.chandef.chan;
        wdev_unlock(wdev);

        if (chan) {
                freq->m = chan->center_freq;
                freq->e = 6;
                return 0;
        }

        /* no channel if not joining */
        return -EINVAL;
}

int cfg80211_ibss_wext_siwessid(struct net_device *dev,
                                struct iw_request_info *info,
                                struct iw_point *data, char *ssid)
{
        struct wireless_dev *wdev = dev->ieee80211_ptr;
        struct cfg80211_registered_device *rdev = wiphy_to_rdev(wdev->wiphy);
        size_t len = data->length;
        int err;

        /* call only for ibss! */
        if (WARN_ON(wdev->iftype != NL80211_IFTYPE_ADHOC))
                return -EINVAL;

        if (!rdev->ops->join_ibss)
                return -EOPNOTSUPP;

        wdev_lock(wdev);
        err = 0;
        if (wdev->ssid_len)
                err = __cfg80211_leave_ibss(rdev, dev, true);
        wdev_unlock(wdev);

        if (err)
                return err;

        /* iwconfig uses nul termination in SSID.. */
        if (len > 0 && ssid[len - 1] == '\0')
                len--;

        memcpy(wdev->ssid, ssid, len);
        wdev->wext.ibss.ssid = wdev->ssid;
        wdev->wext.ibss.ssid_len = len;

        wdev_lock(wdev);
        err = cfg80211_ibss_wext_join(rdev, wdev);
        wdev_unlock(wdev);

        return err;
}

int cfg80211_ibss_wext_giwessid(struct net_device *dev,
                                struct iw_request_info *info,
                                struct iw_point *data, char *ssid)
{
        struct wireless_dev *wdev = dev->ieee80211_ptr;

        /* call only for ibss! */
        if (WARN_ON(wdev->iftype != NL80211_IFTYPE_ADHOC))
                return -EINVAL;

        data->flags = 0;

        wdev_lock(wdev);
        if (wdev->ssid_len) {
                data->flags = 1;
                data->length = wdev->ssid_len;
                memcpy(ssid, wdev->ssid, data->length);
        } else if (wdev->wext.ibss.ssid && wdev->wext.ibss.ssid_len) {
                data->flags = 1;
                data->length = wdev->wext.ibss.ssid_len;
                memcpy(ssid, wdev->wext.ibss.ssid, data->length);
        }
        wdev_unlock(wdev);

        return 0;
}

int cfg80211_ibss_wext_siwap(struct net_device *dev,
                             struct iw_request_info *info,
                             struct sockaddr *ap_addr, char *extra)
{
        struct wireless_dev *wdev = dev->ieee80211_ptr;
        struct cfg80211_registered_device *rdev = wiphy_to_rdev(wdev->wiphy);
        u8 *bssid = ap_addr->sa_data;
        int err;

        /* call only for ibss! */
        if (WARN_ON(wdev->iftype != NL80211_IFTYPE_ADHOC))
                return -EINVAL;

        if (!rdev->ops->join_ibss)
                return -EOPNOTSUPP;

        if (ap_addr->sa_family != ARPHRD_ETHER)
                return -EINVAL;

        /* automatic mode */
        if (is_zero_ether_addr(bssid) || is_broadcast_ether_addr(bssid))
                bssid = NULL;

        if (bssid && !is_valid_ether_addr(bssid))
                return -EINVAL;

        /* both automatic */
        if (!bssid && !wdev->wext.ibss.bssid)
                return 0;

        /* fixed already - and no change */
        if (wdev->wext.ibss.bssid && bssid &&
            ether_addr_equal(bssid, wdev->wext.ibss.bssid))
                return 0;

        wdev_lock(wdev);
        err = 0;
        if (wdev->ssid_len)
                err = __cfg80211_leave_ibss(rdev, dev, true);
        wdev_unlock(wdev);

        if (err)
                return err;

        if (bssid) {
                memcpy(wdev->wext.bssid, bssid, ETH_ALEN);
                wdev->wext.ibss.bssid = wdev->wext.bssid;
        } else
                wdev->wext.ibss.bssid = NULL;

        wdev_lock(wdev);
        err = cfg80211_ibss_wext_join(rdev, wdev);
        wdev_unlock(wdev);

        return err;
}

int cfg80211_ibss_wext_giwap(struct net_device *dev,
                             struct iw_request_info *info,
                             struct sockaddr *ap_addr, char *extra)
{
        struct wireless_dev *wdev = dev->ieee80211_ptr;

        /* call only for ibss! */
        if (WARN_ON(wdev->iftype != NL80211_IFTYPE_ADHOC))
                return -EINVAL;

        ap_addr->sa_family = ARPHRD_ETHER;

        wdev_lock(wdev);
        if (wdev->current_bss)
                memcpy(ap_addr->sa_data, wdev->current_bss->pub.bssid, ETH_ALEN);
        else if (wdev->wext.ibss.bssid)
                memcpy(ap_addr->sa_data, wdev->wext.ibss.bssid, ETH_ALEN);
        else
                eth_zero_addr(ap_addr->sa_data);

        wdev_unlock(wdev);

        return 0;
}
#endif
































  151 

  151 






  151 
  151 



  151 
  151 
  151 





  151 





















  151 
  151 


  151 
  151 



























  151 











  151 



















































































































































































































































































































































































































































































































  151 











  150 



























































































































































































































































































































































































  151 










  150 

  151 



1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright (C) 2002,2003 by Andreas Gruenbacher <a.gruenbacher@computer.org>
 *
 * Fixes from William Schumacher incorporated on 15 March 2001.
 *    (Reported by Charles Bertsch, <CBertsch@microtest.com>).
 */

/*
 *  This file contains generic functions for manipulating
 *  POSIX 1003.1e draft standard 17 ACLs.
 */

#include <linux/kernel.h>
#include <linux/slab.h>
#include <linux/atomic.h>
#include <linux/fs.h>
#include <linux/sched.h>
#include <linux/cred.h>
#include <linux/posix_acl.h>
#include <linux/posix_acl_xattr.h>
#include <linux/xattr.h>
#include <linux/export.h>
#include <linux/user_namespace.h>
#include <linux/namei.h>
#include <linux/mnt_idmapping.h>

static struct posix_acl **acl_by_type(struct inode *inode, int type)
{
        switch (type) {
        case ACL_TYPE_ACCESS:
                return &inode->i_acl;
        case ACL_TYPE_DEFAULT:
                return &inode->i_default_acl;
        default:
                BUG();
        }
}

struct posix_acl *get_cached_acl(struct inode *inode, int type)
{
        struct posix_acl **p = acl_by_type(inode, type);
        struct posix_acl *acl;

        for (;;) {
                rcu_read_lock();
                acl = rcu_dereference(*p);
                if (!acl || is_uncached_acl(acl) ||
                    refcount_inc_not_zero(&acl->a_refcount))
                        break;
                rcu_read_unlock();
                cpu_relax();
        }
        rcu_read_unlock();
        return acl;
}
EXPORT_SYMBOL(get_cached_acl);

struct posix_acl *get_cached_acl_rcu(struct inode *inode, int type)
{
        struct posix_acl *acl = rcu_dereference(*acl_by_type(inode, type));

        if (acl == ACL_DONT_CACHE) {
                struct posix_acl *ret;

                ret = inode->i_op->get_acl(inode, type, LOOKUP_RCU);
                if (!IS_ERR(ret))
                        acl = ret;
        }

        return acl;
}
EXPORT_SYMBOL(get_cached_acl_rcu);

void set_cached_acl(struct inode *inode, int type, struct posix_acl *acl)
{
        struct posix_acl **p = acl_by_type(inode, type);
        struct posix_acl *old;

        old = xchg(p, posix_acl_dup(acl));
        if (!is_uncached_acl(old))
                posix_acl_release(old);
}
EXPORT_SYMBOL(set_cached_acl);

static void __forget_cached_acl(struct posix_acl **p)
{
        struct posix_acl *old;

        old = xchg(p, ACL_NOT_CACHED);
        if (!is_uncached_acl(old))
                posix_acl_release(old);
}

void forget_cached_acl(struct inode *inode, int type)
{
        __forget_cached_acl(acl_by_type(inode, type));
}
EXPORT_SYMBOL(forget_cached_acl);

void forget_all_cached_acls(struct inode *inode)
{
        __forget_cached_acl(&inode->i_acl);
        __forget_cached_acl(&inode->i_default_acl);
}
EXPORT_SYMBOL(forget_all_cached_acls);

struct posix_acl *get_acl(struct inode *inode, int type)
{
        void *sentinel;
        struct posix_acl **p;
        struct posix_acl *acl;

        /*
         * The sentinel is used to detect when another operation like
         * set_cached_acl() or forget_cached_acl() races with get_acl().
         * It is guaranteed that is_uncached_acl(sentinel) is true.
         */

        acl = get_cached_acl(inode, type);
        if (!is_uncached_acl(acl))
                return acl;

        if (!IS_POSIXACL(inode))
                return NULL;

        sentinel = uncached_acl_sentinel(current);
        p = acl_by_type(inode, type);

        /*
         * If the ACL isn't being read yet, set our sentinel.  Otherwise, the
         * current value of the ACL will not be ACL_NOT_CACHED and so our own
         * sentinel will not be set; another task will update the cache.  We
         * could wait for that other task to complete its job, but it's easier
         * to just call ->get_acl to fetch the ACL ourself.  (This is going to
         * be an unlikely race.)
         */
        if (cmpxchg(p, ACL_NOT_CACHED, sentinel) != ACL_NOT_CACHED)
                /* fall through */ ;

        /*
         * Normally, the ACL returned by ->get_acl will be cached.
         * A filesystem can prevent that by calling
         * forget_cached_acl(inode, type) in ->get_acl.
         *
         * If the filesystem doesn't have a get_acl() function at all, we'll
         * just create the negative cache entry.
         */
        if (!inode->i_op->get_acl) {
                set_cached_acl(inode, type, NULL);
                return NULL;
        }
        acl = inode->i_op->get_acl(inode, type, false);

        if (IS_ERR(acl)) {
                /*
                 * Remove our sentinel so that we don't block future attempts
                 * to cache the ACL.
                 */
                cmpxchg(p, sentinel, ACL_NOT_CACHED);
                return acl;
        }

        /*
         * Cache the result, but only if our sentinel is still in place.
         */
        posix_acl_dup(acl);
        if (unlikely(cmpxchg(p, sentinel, acl) != sentinel))
                posix_acl_release(acl);
        return acl;
}
EXPORT_SYMBOL(get_acl);

/*
 * Init a fresh posix_acl
 */
void
posix_acl_init(struct posix_acl *acl, int count)
{
        refcount_set(&acl->a_refcount, 1);
        acl->a_count = count;
}
EXPORT_SYMBOL(posix_acl_init);

/*
 * Allocate a new ACL with the specified number of entries.
 */
struct posix_acl *
posix_acl_alloc(int count, gfp_t flags)
{
        const size_t size = sizeof(struct posix_acl) +
                            count * sizeof(struct posix_acl_entry);
        struct posix_acl *acl = kmalloc(size, flags);
        if (acl)
                posix_acl_init(acl, count);
        return acl;
}
EXPORT_SYMBOL(posix_acl_alloc);

/*
 * Clone an ACL.
 */
static struct posix_acl *
posix_acl_clone(const struct posix_acl *acl, gfp_t flags)
{
        struct posix_acl *clone = NULL;

        if (acl) {
                int size = sizeof(struct posix_acl) + acl->a_count *
                           sizeof(struct posix_acl_entry);
                clone = kmemdup(acl, size, flags);
                if (clone)
                        refcount_set(&clone->a_refcount, 1);
        }
        return clone;
}

/*
 * Check if an acl is valid. Returns 0 if it is, or -E... otherwise.
 */
int
posix_acl_valid(struct user_namespace *user_ns, const struct posix_acl *acl)
{
        const struct posix_acl_entry *pa, *pe;
        int state = ACL_USER_OBJ;
        int needs_mask = 0;

        FOREACH_ACL_ENTRY(pa, acl, pe) {
                if (pa->e_perm & ~(ACL_READ|ACL_WRITE|ACL_EXECUTE))
                        return -EINVAL;
                switch (pa->e_tag) {
                        case ACL_USER_OBJ:
                                if (state == ACL_USER_OBJ) {
                                        state = ACL_USER;
                                        break;
                                }
                                return -EINVAL;

                        case ACL_USER:
                                if (state != ACL_USER)
                                        return -EINVAL;
                                if (!kuid_has_mapping(user_ns, pa->e_uid))
                                        return -EINVAL;
                                needs_mask = 1;
                                break;

                        case ACL_GROUP_OBJ:
                                if (state == ACL_USER) {
                                        state = ACL_GROUP;
                                        break;
                                }
                                return -EINVAL;

                        case ACL_GROUP:
                                if (state != ACL_GROUP)
                                        return -EINVAL;
                                if (!kgid_has_mapping(user_ns, pa->e_gid))
                                        return -EINVAL;
                                needs_mask = 1;
                                break;

                        case ACL_MASK:
                                if (state != ACL_GROUP)
                                        return -EINVAL;
                                state = ACL_OTHER;
                                break;

                        case ACL_OTHER:
                                if (state == ACL_OTHER ||
                                    (state == ACL_GROUP && !needs_mask)) {
                                        state = 0;
                                        break;
                                }
                                return -EINVAL;

                        default:
                                return -EINVAL;
                }
        }
        if (state == 0)
                return 0;
        return -EINVAL;
}
EXPORT_SYMBOL(posix_acl_valid);

/*
 * Returns 0 if the acl can be exactly represented in the traditional
 * file mode permission bits, or else 1. Returns -E... on error.
 */
int
posix_acl_equiv_mode(const struct posix_acl *acl, umode_t *mode_p)
{
        const struct posix_acl_entry *pa, *pe;
        umode_t mode = 0;
        int not_equiv = 0;

        /*
         * A null ACL can always be presented as mode bits.
         */
        if (!acl)
                return 0;

        FOREACH_ACL_ENTRY(pa, acl, pe) {
                switch (pa->e_tag) {
                        case ACL_USER_OBJ:
                                mode |= (pa->e_perm & S_IRWXO) << 6;
                                break;
                        case ACL_GROUP_OBJ:
                                mode |= (pa->e_perm & S_IRWXO) << 3;
                                break;
                        case ACL_OTHER:
                                mode |= pa->e_perm & S_IRWXO;
                                break;
                        case ACL_MASK:
                                mode = (mode & ~S_IRWXG) |
                                       ((pa->e_perm & S_IRWXO) << 3);
                                not_equiv = 1;
                                break;
                        case ACL_USER:
                        case ACL_GROUP:
                                not_equiv = 1;
                                break;
                        default:
                                return -EINVAL;
                }
        }
        if (mode_p)
                *mode_p = (*mode_p & ~S_IRWXUGO) | mode;
        return not_equiv;
}
EXPORT_SYMBOL(posix_acl_equiv_mode);

/*
 * Create an ACL representing the file mode permission bits of an inode.
 */
struct posix_acl *
posix_acl_from_mode(umode_t mode, gfp_t flags)
{
        struct posix_acl *acl = posix_acl_alloc(3, flags);
        if (!acl)
                return ERR_PTR(-ENOMEM);

        acl->a_entries[0].e_tag  = ACL_USER_OBJ;
        acl->a_entries[0].e_perm = (mode & S_IRWXU) >> 6;

        acl->a_entries[1].e_tag  = ACL_GROUP_OBJ;
        acl->a_entries[1].e_perm = (mode & S_IRWXG) >> 3;

        acl->a_entries[2].e_tag  = ACL_OTHER;
        acl->a_entries[2].e_perm = (mode & S_IRWXO);
        return acl;
}
EXPORT_SYMBOL(posix_acl_from_mode);

/*
 * Return 0 if current is granted want access to the inode
 * by the acl. Returns -E... otherwise.
 */
int
posix_acl_permission(struct user_namespace *mnt_userns, struct inode *inode,
                     const struct posix_acl *acl, int want)
{
        const struct posix_acl_entry *pa, *pe, *mask_obj;
        int found = 0;
        kuid_t uid;
        kgid_t gid;

        want &= MAY_READ | MAY_WRITE | MAY_EXEC;

        FOREACH_ACL_ENTRY(pa, acl, pe) {
                switch(pa->e_tag) {
                        case ACL_USER_OBJ:
                                /* (May have been checked already) */
                                uid = i_uid_into_mnt(mnt_userns, inode);
                                if (uid_eq(uid, current_fsuid()))
                                        goto check_perm;
                                break;
                        case ACL_USER:
                                uid = mapped_kuid_fs(mnt_userns,
                                                     i_user_ns(inode),
                                                     pa->e_uid);
                                if (uid_eq(uid, current_fsuid()))
                                        goto mask;
                                break;
                        case ACL_GROUP_OBJ:
                                gid = i_gid_into_mnt(mnt_userns, inode);
                                if (in_group_p(gid)) {
                                        found = 1;
                                        if ((pa->e_perm & want) == want)
                                                goto mask;
                                }
                                break;
                        case ACL_GROUP:
                                gid = mapped_kgid_fs(mnt_userns,
                                                     i_user_ns(inode),
                                                     pa->e_gid);
                                if (in_group_p(gid)) {
                                        found = 1;
                                        if ((pa->e_perm & want) == want)
                                                goto mask;
                                }
                                break;
                        case ACL_MASK:
                                break;
                        case ACL_OTHER:
                                if (found)
                                        return -EACCES;
                                else
                                        goto check_perm;
                        default:
                                return -EIO;
                }
        }
        return -EIO;

mask:
        for (mask_obj = pa+1; mask_obj != pe; mask_obj++) {
                if (mask_obj->e_tag == ACL_MASK) {
                        if ((pa->e_perm & mask_obj->e_perm & want) == want)
                                return 0;
                        return -EACCES;
                }
        }

check_perm:
        if ((pa->e_perm & want) == want)
                return 0;
        return -EACCES;
}

/*
 * Modify acl when creating a new inode. The caller must ensure the acl is
 * only referenced once.
 *
 * mode_p initially must contain the mode parameter to the open() / creat()
 * system calls. All permissions that are not granted by the acl are removed.
 * The permissions in the acl are changed to reflect the mode_p parameter.
 */
static int posix_acl_create_masq(struct posix_acl *acl, umode_t *mode_p)
{
        struct posix_acl_entry *pa, *pe;
        struct posix_acl_entry *group_obj = NULL, *mask_obj = NULL;
        umode_t mode = *mode_p;
        int not_equiv = 0;

        /* assert(atomic_read(acl->a_refcount) == 1); */

        FOREACH_ACL_ENTRY(pa, acl, pe) {
                switch(pa->e_tag) {
                        case ACL_USER_OBJ:
                                pa->e_perm &= (mode >> 6) | ~S_IRWXO;
                                mode &= (pa->e_perm << 6) | ~S_IRWXU;
                                break;

                        case ACL_USER:
                        case ACL_GROUP:
                                not_equiv = 1;
                                break;

                        case ACL_GROUP_OBJ:
                                group_obj = pa;
                                break;

                        case ACL_OTHER:
                                pa->e_perm &= mode | ~S_IRWXO;
                                mode &= pa->e_perm | ~S_IRWXO;
                                break;

                        case ACL_MASK:
                                mask_obj = pa;
                                not_equiv = 1;
                                break;

                        default:
                                return -EIO;
                }
        }

        if (mask_obj) {
                mask_obj->e_perm &= (mode >> 3) | ~S_IRWXO;
                mode &= (mask_obj->e_perm << 3) | ~S_IRWXG;
        } else {
                if (!group_obj)
                        return -EIO;
                group_obj->e_perm &= (mode >> 3) | ~S_IRWXO;
                mode &= (group_obj->e_perm << 3) | ~S_IRWXG;
        }

        *mode_p = (*mode_p & ~S_IRWXUGO) | mode;
        return not_equiv;
}

/*
 * Modify the ACL for the chmod syscall.
 */
static int __posix_acl_chmod_masq(struct posix_acl *acl, umode_t mode)
{
        struct posix_acl_entry *group_obj = NULL, *mask_obj = NULL;
        struct posix_acl_entry *pa, *pe;

        /* assert(atomic_read(acl->a_refcount) == 1); */

        FOREACH_ACL_ENTRY(pa, acl, pe) {
                switch(pa->e_tag) {
                        case ACL_USER_OBJ:
                                pa->e_perm = (mode & S_IRWXU) >> 6;
                                break;

                        case ACL_USER:
                        case ACL_GROUP:
                                break;

                        case ACL_GROUP_OBJ:
                                group_obj = pa;
                                break;

                        case ACL_MASK:
                                mask_obj = pa;
                                break;

                        case ACL_OTHER:
                                pa->e_perm = (mode & S_IRWXO);
                                break;

                        default:
                                return -EIO;
                }
        }

        if (mask_obj) {
                mask_obj->e_perm = (mode & S_IRWXG) >> 3;
        } else {
                if (!group_obj)
                        return -EIO;
                group_obj->e_perm = (mode & S_IRWXG) >> 3;
        }

        return 0;
}

int
__posix_acl_create(struct posix_acl **acl, gfp_t gfp, umode_t *mode_p)
{
        struct posix_acl *clone = posix_acl_clone(*acl, gfp);
        int err = -ENOMEM;
        if (clone) {
                err = posix_acl_create_masq(clone, mode_p);
                if (err < 0) {
                        posix_acl_release(clone);
                        clone = NULL;
                }
        }
        posix_acl_release(*acl);
        *acl = clone;
        return err;
}
EXPORT_SYMBOL(__posix_acl_create);

int
__posix_acl_chmod(struct posix_acl **acl, gfp_t gfp, umode_t mode)
{
        struct posix_acl *clone = posix_acl_clone(*acl, gfp);
        int err = -ENOMEM;
        if (clone) {
                err = __posix_acl_chmod_masq(clone, mode);
                if (err) {
                        posix_acl_release(clone);
                        clone = NULL;
                }
        }
        posix_acl_release(*acl);
        *acl = clone;
        return err;
}
EXPORT_SYMBOL(__posix_acl_chmod);

/**
 * posix_acl_chmod - chmod a posix acl
 *
 * @mnt_userns:        user namespace of the mount @inode was found from
 * @inode:        inode to check permissions on
 * @mode:        the new mode of @inode
 *
 * If the inode has been found through an idmapped mount the user namespace of
 * the vfsmount must be passed through @mnt_userns. This function will then
 * take care to map the inode according to @mnt_userns before checking
 * permissions. On non-idmapped mounts or if permission checking is to be
 * performed on the raw inode simply passs init_user_ns.
 */
int
 posix_acl_chmod(struct user_namespace *mnt_userns, struct inode *inode,
                    umode_t mode)
{
        struct posix_acl *acl;
        int ret = 0;

        if (!IS_POSIXACL(inode))
                return 0;
        if (!inode->i_op->set_acl)
                return -EOPNOTSUPP;

        acl = get_acl(inode, ACL_TYPE_ACCESS);
        if (IS_ERR_OR_NULL(acl)) {
                if (acl == ERR_PTR(-EOPNOTSUPP))
                        return 0;
                return PTR_ERR(acl);
        }

        ret = __posix_acl_chmod(&acl, GFP_KERNEL, mode);
        if (ret)
                return ret;
        ret = inode->i_op->set_acl(mnt_userns, inode, acl, ACL_TYPE_ACCESS);
        posix_acl_release(acl);
        return ret;
}
EXPORT_SYMBOL(posix_acl_chmod);

int
posix_acl_create(struct inode *dir, umode_t *mode,
                struct posix_acl **default_acl, struct posix_acl **acl)
{
        struct posix_acl *p;
        struct posix_acl *clone;
        int ret;

        *acl = NULL;
        *default_acl = NULL;

        if (S_ISLNK(*mode) || !IS_POSIXACL(dir))
                return 0;

        p = get_acl(dir, ACL_TYPE_DEFAULT);
        if (!p || p == ERR_PTR(-EOPNOTSUPP)) {
                *mode &= ~current_umask();
                return 0;
        }
        if (IS_ERR(p))
                return PTR_ERR(p);

        ret = -ENOMEM;
        clone = posix_acl_clone(p, GFP_NOFS);
        if (!clone)
                goto err_release;

        ret = posix_acl_create_masq(clone, mode);
        if (ret < 0)
                goto err_release_clone;

        if (ret == 0)
                posix_acl_release(clone);
        else
                *acl = clone;

        if (!S_ISDIR(*mode))
                posix_acl_release(p);
        else
                *default_acl = p;

        return 0;

err_release_clone:
        posix_acl_release(clone);
err_release:
        posix_acl_release(p);
        return ret;
}
EXPORT_SYMBOL_GPL(posix_acl_create);

/**
 * posix_acl_update_mode  -  update mode in set_acl
 * @mnt_userns:        user namespace of the mount @inode was found from
 * @inode:        target inode
 * @mode_p:        mode (pointer) for update
 * @acl:        acl pointer
 *
 * Update the file mode when setting an ACL: compute the new file permission
 * bits based on the ACL.  In addition, if the ACL is equivalent to the new
 * file mode, set *@acl to NULL to indicate that no ACL should be set.
 *
 * As with chmod, clear the setgid bit if the caller is not in the owning group
 * or capable of CAP_FSETID (see inode_change_ok).
 *
 * If the inode has been found through an idmapped mount the user namespace of
 * the vfsmount must be passed through @mnt_userns. This function will then
 * take care to map the inode according to @mnt_userns before checking
 * permissions. On non-idmapped mounts or if permission checking is to be
 * performed on the raw inode simply passs init_user_ns.
 *
 * Called from set_acl inode operations.
 */
int posix_acl_update_mode(struct user_namespace *mnt_userns,
                          struct inode *inode, umode_t *mode_p,
                          struct posix_acl **acl)
{
        umode_t mode = inode->i_mode;
        int error;

        error = posix_acl_equiv_mode(*acl, &mode);
        if (error < 0)
                return error;
        if (error == 0)
                *acl = NULL;
        if (!in_group_p(i_gid_into_mnt(mnt_userns, inode)) &&
            !capable_wrt_inode_uidgid(mnt_userns, inode, CAP_FSETID))
                mode &= ~S_ISGID;
        *mode_p = mode;
        return 0;
}
EXPORT_SYMBOL(posix_acl_update_mode);

/*
 * Fix up the uids and gids in posix acl extended attributes in place.
 */
static void posix_acl_fix_xattr_userns(
        struct user_namespace *to, struct user_namespace *from,
        struct user_namespace *mnt_userns,
        void *value, size_t size, bool from_user)
{
        struct posix_acl_xattr_header *header = value;
        struct posix_acl_xattr_entry *entry = (void *)(header + 1), *end;
        int count;
        kuid_t uid;
        kgid_t gid;

        if (!value)
                return;
        if (size < sizeof(struct posix_acl_xattr_header))
                return;
        if (header->a_version != cpu_to_le32(POSIX_ACL_XATTR_VERSION))
                return;

        count = posix_acl_xattr_count(size);
        if (count < 0)
                return;
        if (count == 0)
                return;

        for (end = entry + count; entry != end; entry++) {
                switch(le16_to_cpu(entry->e_tag)) {
                case ACL_USER:
                        uid = make_kuid(from, le32_to_cpu(entry->e_id));
                        if (from_user)
                                uid = mapped_kuid_user(mnt_userns, &init_user_ns, uid);
                        else
                                uid = mapped_kuid_fs(mnt_userns, &init_user_ns, uid);
                        entry->e_id = cpu_to_le32(from_kuid(to, uid));
                        break;
                case ACL_GROUP:
                        gid = make_kgid(from, le32_to_cpu(entry->e_id));
                        if (from_user)
                                gid = mapped_kgid_user(mnt_userns, &init_user_ns, gid);
                        else
                                gid = mapped_kgid_fs(mnt_userns, &init_user_ns, gid);
                        entry->e_id = cpu_to_le32(from_kgid(to, gid));
                        break;
                default:
                        break;
                }
        }
}

void posix_acl_fix_xattr_from_user(struct user_namespace *mnt_userns,
                                   struct inode *inode,
                                   void *value, size_t size)
{
        struct user_namespace *user_ns = current_user_ns();

        /* Leave ids untouched on non-idmapped mounts. */
        if (no_idmapping(mnt_userns, i_user_ns(inode)))
                mnt_userns = &init_user_ns;
        if ((user_ns == &init_user_ns) && (mnt_userns == &init_user_ns))
                return;
        posix_acl_fix_xattr_userns(&init_user_ns, user_ns, mnt_userns, value,
                                   size, true);
}

void posix_acl_fix_xattr_to_user(struct user_namespace *mnt_userns,
                                 struct inode *inode,
                                 void *value, size_t size)
{
        struct user_namespace *user_ns = current_user_ns();

        /* Leave ids untouched on non-idmapped mounts. */
        if (no_idmapping(mnt_userns, i_user_ns(inode)))
                mnt_userns = &init_user_ns;
        if ((user_ns == &init_user_ns) && (mnt_userns == &init_user_ns))
                return;
        posix_acl_fix_xattr_userns(user_ns, &init_user_ns, mnt_userns, value,
                                   size, false);
}

/*
 * Convert from extended attribute to in-memory representation.
 */
struct posix_acl *
posix_acl_from_xattr(struct user_namespace *user_ns,
                     const void *value, size_t size)
{
        const struct posix_acl_xattr_header *header = value;
        const struct posix_acl_xattr_entry *entry = (const void *)(header + 1), *end;
        int count;
        struct posix_acl *acl;
        struct posix_acl_entry *acl_e;

        if (!value)
                return NULL;
        if (size < sizeof(struct posix_acl_xattr_header))
                 return ERR_PTR(-EINVAL);
        if (header->a_version != cpu_to_le32(POSIX_ACL_XATTR_VERSION))
                return ERR_PTR(-EOPNOTSUPP);

        count = posix_acl_xattr_count(size);
        if (count < 0)
                return ERR_PTR(-EINVAL);
        if (count == 0)
                return NULL;
        
        acl = posix_acl_alloc(count, GFP_NOFS);
        if (!acl)
                return ERR_PTR(-ENOMEM);
        acl_e = acl->a_entries;
        
        for (end = entry + count; entry != end; acl_e++, entry++) {
                acl_e->e_tag  = le16_to_cpu(entry->e_tag);
                acl_e->e_perm = le16_to_cpu(entry->e_perm);

                switch(acl_e->e_tag) {
                        case ACL_USER_OBJ:
                        case ACL_GROUP_OBJ:
                        case ACL_MASK:
                        case ACL_OTHER:
                                break;

                        case ACL_USER:
                                acl_e->e_uid =
                                        make_kuid(user_ns,
                                                  le32_to_cpu(entry->e_id));
                                if (!uid_valid(acl_e->e_uid))
                                        goto fail;
                                break;
                        case ACL_GROUP:
                                acl_e->e_gid =
                                        make_kgid(user_ns,
                                                  le32_to_cpu(entry->e_id));
                                if (!gid_valid(acl_e->e_gid))
                                        goto fail;
                                break;

                        default:
                                goto fail;
                }
        }
        return acl;

fail:
        posix_acl_release(acl);
        return ERR_PTR(-EINVAL);
}
EXPORT_SYMBOL (posix_acl_from_xattr);

/*
 * Convert from in-memory to extended attribute representation.
 */
int
posix_acl_to_xattr(struct user_namespace *user_ns, const struct posix_acl *acl,
                   void *buffer, size_t size)
{
        struct posix_acl_xattr_header *ext_acl = buffer;
        struct posix_acl_xattr_entry *ext_entry;
        int real_size, n;

        real_size = posix_acl_xattr_size(acl->a_count);
        if (!buffer)
                return real_size;
        if (real_size > size)
                return -ERANGE;

        ext_entry = (void *)(ext_acl + 1);
        ext_acl->a_version = cpu_to_le32(POSIX_ACL_XATTR_VERSION);

        for (n=0; n < acl->a_count; n++, ext_entry++) {
                const struct posix_acl_entry *acl_e = &acl->a_entries[n];
                ext_entry->e_tag  = cpu_to_le16(acl_e->e_tag);
                ext_entry->e_perm = cpu_to_le16(acl_e->e_perm);
                switch(acl_e->e_tag) {
                case ACL_USER:
                        ext_entry->e_id =
                                cpu_to_le32(from_kuid(user_ns, acl_e->e_uid));
                        break;
                case ACL_GROUP:
                        ext_entry->e_id =
                                cpu_to_le32(from_kgid(user_ns, acl_e->e_gid));
                        break;
                default:
                        ext_entry->e_id = cpu_to_le32(ACL_UNDEFINED_ID);
                        break;
                }
        }
        return real_size;
}
EXPORT_SYMBOL (posix_acl_to_xattr);

static int
posix_acl_xattr_get(const struct xattr_handler *handler,
                    struct dentry *unused, struct inode *inode,
                    const char *name, void *value, size_t size)
{
        struct posix_acl *acl;
        int error;

        if (!IS_POSIXACL(inode))
                return -EOPNOTSUPP;
        if (S_ISLNK(inode->i_mode))
                return -EOPNOTSUPP;

        acl = get_acl(inode, handler->flags);
        if (IS_ERR(acl))
                return PTR_ERR(acl);
        if (acl == NULL)
                return -ENODATA;

        error = posix_acl_to_xattr(&init_user_ns, acl, value, size);
        posix_acl_release(acl);

        return error;
}

int
set_posix_acl(struct user_namespace *mnt_userns, struct inode *inode,
              int type, struct posix_acl *acl)
{
        if (!IS_POSIXACL(inode))
                return -EOPNOTSUPP;
        if (!inode->i_op->set_acl)
                return -EOPNOTSUPP;

        if (type == ACL_TYPE_DEFAULT && !S_ISDIR(inode->i_mode))
                return acl ? -EACCES : 0;
        if (!inode_owner_or_capable(mnt_userns, inode))
                return -EPERM;

        if (acl) {
                int ret = posix_acl_valid(inode->i_sb->s_user_ns, acl);
                if (ret)
                        return ret;
        }
        return inode->i_op->set_acl(mnt_userns, inode, acl, type);
}
EXPORT_SYMBOL(set_posix_acl);

static int
posix_acl_xattr_set(const struct xattr_handler *handler,
                           struct user_namespace *mnt_userns,
                           struct dentry *unused, struct inode *inode,
                           const char *name, const void *value, size_t size,
                           int flags)
{
        struct posix_acl *acl = NULL;
        int ret;

        if (value) {
                acl = posix_acl_from_xattr(&init_user_ns, value, size);
                if (IS_ERR(acl))
                        return PTR_ERR(acl);
        }
        ret = set_posix_acl(mnt_userns, inode, handler->flags, acl);
        posix_acl_release(acl);
        return ret;
}

static bool
posix_acl_xattr_list(struct dentry *dentry)
{
        return IS_POSIXACL(d_backing_inode(dentry));
}

const struct xattr_handler posix_acl_access_xattr_handler = {
        .name = XATTR_NAME_POSIX_ACL_ACCESS,
        .flags = ACL_TYPE_ACCESS,
        .list = posix_acl_xattr_list,
        .get = posix_acl_xattr_get,
        .set = posix_acl_xattr_set,
};
EXPORT_SYMBOL_GPL(posix_acl_access_xattr_handler);

const struct xattr_handler posix_acl_default_xattr_handler = {
        .name = XATTR_NAME_POSIX_ACL_DEFAULT,
        .flags = ACL_TYPE_DEFAULT,
        .list = posix_acl_xattr_list,
        .get = posix_acl_xattr_get,
        .set = posix_acl_xattr_set,
};
EXPORT_SYMBOL_GPL(posix_acl_default_xattr_handler);

int simple_set_acl(struct user_namespace *mnt_userns, struct inode *inode,
                   struct posix_acl *acl, int type)
{
        int error;

        if (type == ACL_TYPE_ACCESS) {
                error = posix_acl_update_mode(mnt_userns, inode,
                                &inode->i_mode, &acl);
                if (error)
                        return error;
        }

        inode->i_ctime = current_time(inode);
        set_cached_acl(inode, type, acl);
        return 0;
}

int simple_acl_create(struct inode *dir, struct inode *inode)
{
        struct posix_acl *default_acl, *acl;
        int error;

        error = posix_acl_create(dir, &inode->i_mode, &default_acl, &acl);
        if (error)
                return error;

        set_cached_acl(inode, ACL_TYPE_DEFAULT, default_acl);
        set_cached_acl(inode, ACL_TYPE_ACCESS, acl);

        if (default_acl)
                posix_acl_release(default_acl);
        if (acl)
                posix_acl_release(acl);
        return 0;
}






























































































   15 
   15 







   15 




   15 




   15 





    8 



   23 


















   15 



    8 











    4 








    7 













































































































































































































































































































   15 

















   15 









































































   15 



   15 






    6 
    9 









    6 







    6 
























































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
// SPDX-License-Identifier: GPL-2.0
/*
 * xfrm_input.c
 *
 * Changes:
 *         YOSHIFUJI Hideaki @USAGI
 *                 Split up af-specific portion
 *
 */

#include <linux/bottom_half.h>
#include <linux/cache.h>
#include <linux/interrupt.h>
#include <linux/slab.h>
#include <linux/module.h>
#include <linux/netdevice.h>
#include <linux/percpu.h>
#include <net/dst.h>
#include <net/ip.h>
#include <net/xfrm.h>
#include <net/ip_tunnels.h>
#include <net/ip6_tunnel.h>

#include "xfrm_inout.h"

struct xfrm_trans_tasklet {
        struct work_struct work;
        spinlock_t queue_lock;
        struct sk_buff_head queue;
};

struct xfrm_trans_cb {
        union {
                struct inet_skb_parm        h4;
#if IS_ENABLED(CONFIG_IPV6)
                struct inet6_skb_parm        h6;
#endif
        } header;
        int (*finish)(struct net *net, struct sock *sk, struct sk_buff *skb);
        struct net *net;
};

#define XFRM_TRANS_SKB_CB(__skb) ((struct xfrm_trans_cb *)&((__skb)->cb[0]))

static DEFINE_SPINLOCK(xfrm_input_afinfo_lock);
static struct xfrm_input_afinfo const __rcu *xfrm_input_afinfo[2][AF_INET6 + 1];

static struct gro_cells gro_cells;
static struct net_device xfrm_napi_dev;

static DEFINE_PER_CPU(struct xfrm_trans_tasklet, xfrm_trans_tasklet);

int xfrm_input_register_afinfo(const struct xfrm_input_afinfo *afinfo)
{
        int err = 0;

        if (WARN_ON(afinfo->family > AF_INET6))
                return -EAFNOSUPPORT;

        spin_lock_bh(&xfrm_input_afinfo_lock);
        if (unlikely(xfrm_input_afinfo[afinfo->is_ipip][afinfo->family]))
                err = -EEXIST;
        else
                rcu_assign_pointer(xfrm_input_afinfo[afinfo->is_ipip][afinfo->family], afinfo);
        spin_unlock_bh(&xfrm_input_afinfo_lock);
        return err;
}
EXPORT_SYMBOL(xfrm_input_register_afinfo);

int xfrm_input_unregister_afinfo(const struct xfrm_input_afinfo *afinfo)
{
        int err = 0;

        spin_lock_bh(&xfrm_input_afinfo_lock);
        if (likely(xfrm_input_afinfo[afinfo->is_ipip][afinfo->family])) {
                if (unlikely(xfrm_input_afinfo[afinfo->is_ipip][afinfo->family] != afinfo))
                        err = -EINVAL;
                else
                        RCU_INIT_POINTER(xfrm_input_afinfo[afinfo->is_ipip][afinfo->family], NULL);
        }
        spin_unlock_bh(&xfrm_input_afinfo_lock);
        synchronize_rcu();
        return err;
}
EXPORT_SYMBOL(xfrm_input_unregister_afinfo);

static const struct xfrm_input_afinfo *xfrm_input_get_afinfo(u8 family, bool is_ipip)
{
        const struct xfrm_input_afinfo *afinfo;

        if (WARN_ON_ONCE(family > AF_INET6))
                return NULL;

        rcu_read_lock();
        afinfo = rcu_dereference(xfrm_input_afinfo[is_ipip][family]);
        if (unlikely(!afinfo))
                rcu_read_unlock();
        return afinfo;
}

static int xfrm_rcv_cb(struct sk_buff *skb, unsigned int family, u8 protocol,
                       int err)
{
        bool is_ipip = (protocol == IPPROTO_IPIP || protocol == IPPROTO_IPV6);
        const struct xfrm_input_afinfo *afinfo;
        int ret;

        afinfo = xfrm_input_get_afinfo(family, is_ipip);
        if (!afinfo)
                return -EAFNOSUPPORT;

        ret = afinfo->callback(skb, protocol, err);
        rcu_read_unlock();

        return ret;
}

struct sec_path *secpath_set(struct sk_buff *skb)
{
        struct sec_path *sp, *tmp = skb_ext_find(skb, SKB_EXT_SEC_PATH);

        sp = skb_ext_add(skb, SKB_EXT_SEC_PATH);
        if (!sp)
                return NULL;

        if (tmp) /* reused existing one (was COW'd if needed) */
                return sp;

        /* allocated new secpath */
        memset(sp->ovec, 0, sizeof(sp->ovec));
        sp->olen = 0;
        sp->len = 0;
        sp->verified_cnt = 0;

        return sp;
}
EXPORT_SYMBOL(secpath_set);

/* Fetch spi and seq from ipsec header */

int xfrm_parse_spi(struct sk_buff *skb, u8 nexthdr, __be32 *spi, __be32 *seq)
{
        int offset, offset_seq;
        int hlen;

        switch (nexthdr) {
        case IPPROTO_AH:
                hlen = sizeof(struct ip_auth_hdr);
                offset = offsetof(struct ip_auth_hdr, spi);
                offset_seq = offsetof(struct ip_auth_hdr, seq_no);
                break;
        case IPPROTO_ESP:
                hlen = sizeof(struct ip_esp_hdr);
                offset = offsetof(struct ip_esp_hdr, spi);
                offset_seq = offsetof(struct ip_esp_hdr, seq_no);
                break;
        case IPPROTO_COMP:
                if (!pskb_may_pull(skb, sizeof(struct ip_comp_hdr)))
                        return -EINVAL;
                *spi = htonl(ntohs(*(__be16 *)(skb_transport_header(skb) + 2)));
                *seq = 0;
                return 0;
        default:
                return 1;
        }

        if (!pskb_may_pull(skb, hlen))
                return -EINVAL;

        *spi = *(__be32 *)(skb_transport_header(skb) + offset);
        *seq = *(__be32 *)(skb_transport_header(skb) + offset_seq);
        return 0;
}
EXPORT_SYMBOL(xfrm_parse_spi);

static int xfrm4_remove_beet_encap(struct xfrm_state *x, struct sk_buff *skb)
{
        struct iphdr *iph;
        int optlen = 0;
        int err = -EINVAL;

        if (unlikely(XFRM_MODE_SKB_CB(skb)->protocol == IPPROTO_BEETPH)) {
                struct ip_beet_phdr *ph;
                int phlen;

                if (!pskb_may_pull(skb, sizeof(*ph)))
                        goto out;

                ph = (struct ip_beet_phdr *)skb->data;

                phlen = sizeof(*ph) + ph->padlen;
                optlen = ph->hdrlen * 8 + (IPV4_BEET_PHMAXLEN - phlen);
                if (optlen < 0 || optlen & 3 || optlen > 250)
                        goto out;

                XFRM_MODE_SKB_CB(skb)->protocol = ph->nexthdr;

                if (!pskb_may_pull(skb, phlen))
                        goto out;
                __skb_pull(skb, phlen);
        }

        skb_push(skb, sizeof(*iph));
        skb_reset_network_header(skb);
        skb_mac_header_rebuild(skb);

        xfrm4_beet_make_header(skb);

        iph = ip_hdr(skb);

        iph->ihl += optlen / 4;
        iph->tot_len = htons(skb->len);
        iph->daddr = x->sel.daddr.a4;
        iph->saddr = x->sel.saddr.a4;
        iph->check = 0;
        iph->check = ip_fast_csum(skb_network_header(skb), iph->ihl);
        err = 0;
out:
        return err;
}

static void ipip_ecn_decapsulate(struct sk_buff *skb)
{
        struct iphdr *inner_iph = ipip_hdr(skb);

        if (INET_ECN_is_ce(XFRM_MODE_SKB_CB(skb)->tos))
                IP_ECN_set_ce(inner_iph);
}

static int xfrm4_remove_tunnel_encap(struct xfrm_state *x, struct sk_buff *skb)
{
        int err = -EINVAL;

        if (XFRM_MODE_SKB_CB(skb)->protocol != IPPROTO_IPIP)
                goto out;

        if (!pskb_may_pull(skb, sizeof(struct iphdr)))
                goto out;

        err = skb_unclone(skb, GFP_ATOMIC);
        if (err)
                goto out;

        if (x->props.flags & XFRM_STATE_DECAP_DSCP)
                ipv4_copy_dscp(XFRM_MODE_SKB_CB(skb)->tos, ipip_hdr(skb));
        if (!(x->props.flags & XFRM_STATE_NOECN))
                ipip_ecn_decapsulate(skb);

        skb_reset_network_header(skb);
        skb_mac_header_rebuild(skb);
        if (skb->mac_len)
                eth_hdr(skb)->h_proto = skb->protocol;

        err = 0;

out:
        return err;
}

static void ipip6_ecn_decapsulate(struct sk_buff *skb)
{
        struct ipv6hdr *inner_iph = ipipv6_hdr(skb);

        if (INET_ECN_is_ce(XFRM_MODE_SKB_CB(skb)->tos))
                IP6_ECN_set_ce(skb, inner_iph);
}

static int xfrm6_remove_tunnel_encap(struct xfrm_state *x, struct sk_buff *skb)
{
        int err = -EINVAL;

        if (XFRM_MODE_SKB_CB(skb)->protocol != IPPROTO_IPV6)
                goto out;
        if (!pskb_may_pull(skb, sizeof(struct ipv6hdr)))
                goto out;

        err = skb_unclone(skb, GFP_ATOMIC);
        if (err)
                goto out;

        if (x->props.flags & XFRM_STATE_DECAP_DSCP)
                ipv6_copy_dscp(XFRM_MODE_SKB_CB(skb)->tos, ipipv6_hdr(skb));
        if (!(x->props.flags & XFRM_STATE_NOECN))
                ipip6_ecn_decapsulate(skb);

        skb_reset_network_header(skb);
        skb_mac_header_rebuild(skb);
        if (skb->mac_len)
                eth_hdr(skb)->h_proto = skb->protocol;

        err = 0;

out:
        return err;
}

static int xfrm6_remove_beet_encap(struct xfrm_state *x, struct sk_buff *skb)
{
        struct ipv6hdr *ip6h;
        int size = sizeof(struct ipv6hdr);
        int err;

        err = skb_cow_head(skb, size + skb->mac_len);
        if (err)
                goto out;

        __skb_push(skb, size);
        skb_reset_network_header(skb);
        skb_mac_header_rebuild(skb);

        xfrm6_beet_make_header(skb);

        ip6h = ipv6_hdr(skb);
        ip6h->payload_len = htons(skb->len - size);
        ip6h->daddr = x->sel.daddr.in6;
        ip6h->saddr = x->sel.saddr.in6;
        err = 0;
out:
        return err;
}

/* Remove encapsulation header.
 *
 * The IP header will be moved over the top of the encapsulation
 * header.
 *
 * On entry, the transport header shall point to where the IP header
 * should be and the network header shall be set to where the IP
 * header currently is.  skb->data shall point to the start of the
 * payload.
 */
static int
xfrm_inner_mode_encap_remove(struct xfrm_state *x,
                             const struct xfrm_mode *inner_mode,
                             struct sk_buff *skb)
{
        switch (inner_mode->encap) {
        case XFRM_MODE_BEET:
                if (inner_mode->family == AF_INET)
                        return xfrm4_remove_beet_encap(x, skb);
                if (inner_mode->family == AF_INET6)
                        return xfrm6_remove_beet_encap(x, skb);
                break;
        case XFRM_MODE_TUNNEL:
                if (inner_mode->family == AF_INET)
                        return xfrm4_remove_tunnel_encap(x, skb);
                if (inner_mode->family == AF_INET6)
                        return xfrm6_remove_tunnel_encap(x, skb);
                break;
        }

        WARN_ON_ONCE(1);
        return -EOPNOTSUPP;
}

static int xfrm_prepare_input(struct xfrm_state *x, struct sk_buff *skb)
{
        const struct xfrm_mode *inner_mode = &x->inner_mode;

        switch (x->outer_mode.family) {
        case AF_INET:
                xfrm4_extract_header(skb);
                break;
        case AF_INET6:
                xfrm6_extract_header(skb);
                break;
        default:
                WARN_ON_ONCE(1);
                return -EAFNOSUPPORT;
        }

        if (x->sel.family == AF_UNSPEC) {
                inner_mode = xfrm_ip2inner_mode(x, XFRM_MODE_SKB_CB(skb)->protocol);
                if (!inner_mode)
                        return -EAFNOSUPPORT;
        }

        switch (inner_mode->family) {
        case AF_INET:
                skb->protocol = htons(ETH_P_IP);
                break;
        case AF_INET6:
                skb->protocol = htons(ETH_P_IPV6);
                break;
        default:
                WARN_ON_ONCE(1);
                break;
        }

        return xfrm_inner_mode_encap_remove(x, inner_mode, skb);
}

/* Remove encapsulation header.
 *
 * The IP header will be moved over the top of the encapsulation header.
 *
 * On entry, skb_transport_header() shall point to where the IP header
 * should be and skb_network_header() shall be set to where the IP header
 * currently is.  skb->data shall point to the start of the payload.
 */
static int xfrm4_transport_input(struct xfrm_state *x, struct sk_buff *skb)
{
        struct xfrm_offload *xo = xfrm_offload(skb);
        int ihl = skb->data - skb_transport_header(skb);

        if (skb->transport_header != skb->network_header) {
                memmove(skb_transport_header(skb),
                        skb_network_header(skb), ihl);
                if (xo)
                        xo->orig_mac_len =
                                skb_mac_header_was_set(skb) ? skb_mac_header_len(skb) : 0;
                skb->network_header = skb->transport_header;
        }
        ip_hdr(skb)->tot_len = htons(skb->len + ihl);
        skb_reset_transport_header(skb);
        return 0;
}

static int xfrm6_transport_input(struct xfrm_state *x, struct sk_buff *skb)
{
#if IS_ENABLED(CONFIG_IPV6)
        struct xfrm_offload *xo = xfrm_offload(skb);
        int ihl = skb->data - skb_transport_header(skb);

        if (skb->transport_header != skb->network_header) {
                memmove(skb_transport_header(skb),
                        skb_network_header(skb), ihl);
                if (xo)
                        xo->orig_mac_len =
                                skb_mac_header_was_set(skb) ? skb_mac_header_len(skb) : 0;
                skb->network_header = skb->transport_header;
        }
        ipv6_hdr(skb)->payload_len = htons(skb->len + ihl -
                                           sizeof(struct ipv6hdr));
        skb_reset_transport_header(skb);
        return 0;
#else
        WARN_ON_ONCE(1);
        return -EAFNOSUPPORT;
#endif
}

static int xfrm_inner_mode_input(struct xfrm_state *x,
                                 const struct xfrm_mode *inner_mode,
                                 struct sk_buff *skb)
{
        switch (inner_mode->encap) {
        case XFRM_MODE_BEET:
        case XFRM_MODE_TUNNEL:
                return xfrm_prepare_input(x, skb);
        case XFRM_MODE_TRANSPORT:
                if (inner_mode->family == AF_INET)
                        return xfrm4_transport_input(x, skb);
                if (inner_mode->family == AF_INET6)
                        return xfrm6_transport_input(x, skb);
                break;
        case XFRM_MODE_ROUTEOPTIMIZATION:
                WARN_ON_ONCE(1);
                break;
        default:
                WARN_ON_ONCE(1);
                break;
        }

        return -EOPNOTSUPP;
}

int xfrm_input(struct sk_buff *skb, int nexthdr, __be32 spi, int encap_type)
{
        const struct xfrm_state_afinfo *afinfo;
        struct net *net = dev_net(skb->dev);
        const struct xfrm_mode *inner_mode;
        int err;
        __be32 seq;
        __be32 seq_hi;
        struct xfrm_state *x = NULL;
        xfrm_address_t *daddr;
        u32 mark = skb->mark;
        unsigned int family = AF_UNSPEC;
        int decaps = 0;
        int async = 0;
        bool xfrm_gro = false;
        bool crypto_done = false;
        struct xfrm_offload *xo = xfrm_offload(skb);
        struct sec_path *sp;

        if (encap_type < 0) {
                x = xfrm_input_state(skb);

                if (unlikely(x->km.state != XFRM_STATE_VALID)) {
                        if (x->km.state == XFRM_STATE_ACQ)
                                XFRM_INC_STATS(net, LINUX_MIB_XFRMACQUIREERROR);
                        else
                                XFRM_INC_STATS(net,
                                               LINUX_MIB_XFRMINSTATEINVALID);

                        if (encap_type == -1)
                                dev_put(skb->dev);
                        goto drop;
                }

                family = x->outer_mode.family;

                /* An encap_type of -1 indicates async resumption. */
                if (encap_type == -1) {
                        async = 1;
                        seq = XFRM_SKB_CB(skb)->seq.input.low;
                        goto resume;
                }

                /* encap_type < -1 indicates a GRO call. */
                encap_type = 0;
                seq = XFRM_SPI_SKB_CB(skb)->seq;

                if (xo && (xo->flags & CRYPTO_DONE)) {
                        crypto_done = true;
                        family = XFRM_SPI_SKB_CB(skb)->family;

                        if (!(xo->status & CRYPTO_SUCCESS)) {
                                if (xo->status &
                                    (CRYPTO_TRANSPORT_AH_AUTH_FAILED |
                                     CRYPTO_TRANSPORT_ESP_AUTH_FAILED |
                                     CRYPTO_TUNNEL_AH_AUTH_FAILED |
                                     CRYPTO_TUNNEL_ESP_AUTH_FAILED)) {

                                        xfrm_audit_state_icvfail(x, skb,
                                                                 x->type->proto);
                                        x->stats.integrity_failed++;
                                        XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEPROTOERROR);
                                        goto drop;
                                }

                                if (xo->status & CRYPTO_INVALID_PROTOCOL) {
                                        XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEPROTOERROR);
                                        goto drop;
                                }

                                XFRM_INC_STATS(net, LINUX_MIB_XFRMINBUFFERERROR);
                                goto drop;
                        }

                        if ((err = xfrm_parse_spi(skb, nexthdr, &spi, &seq)) != 0) {
                                XFRM_INC_STATS(net, LINUX_MIB_XFRMINHDRERROR);
                                goto drop;
                        }
                }

                goto lock;
        }

        family = XFRM_SPI_SKB_CB(skb)->family;

        /* if tunnel is present override skb->mark value with tunnel i_key */
        switch (family) {
        case AF_INET:
                if (XFRM_TUNNEL_SKB_CB(skb)->tunnel.ip4)
                        mark = be32_to_cpu(XFRM_TUNNEL_SKB_CB(skb)->tunnel.ip4->parms.i_key);
                break;
        case AF_INET6:
                if (XFRM_TUNNEL_SKB_CB(skb)->tunnel.ip6)
                        mark = be32_to_cpu(XFRM_TUNNEL_SKB_CB(skb)->tunnel.ip6->parms.i_key);
                break;
        }

        sp = secpath_set(skb);
        if (!sp) {
                XFRM_INC_STATS(net, LINUX_MIB_XFRMINERROR);
                goto drop;
        }

        seq = 0;
        if (!spi && (err = xfrm_parse_spi(skb, nexthdr, &spi, &seq)) != 0) {
                secpath_reset(skb);
                XFRM_INC_STATS(net, LINUX_MIB_XFRMINHDRERROR);
                goto drop;
        }

        daddr = (xfrm_address_t *)(skb_network_header(skb) +
                                   XFRM_SPI_SKB_CB(skb)->daddroff);
        do {
                sp = skb_sec_path(skb);

                if (sp->len == XFRM_MAX_DEPTH) {
                        secpath_reset(skb);
                        XFRM_INC_STATS(net, LINUX_MIB_XFRMINBUFFERERROR);
                        goto drop;
                }

                x = xfrm_state_lookup(net, mark, daddr, spi, nexthdr, family);
                if (x == NULL) {
                        secpath_reset(skb);
                        XFRM_INC_STATS(net, LINUX_MIB_XFRMINNOSTATES);
                        xfrm_audit_state_notfound(skb, family, spi, seq);
                        goto drop;
                }

                skb->mark = xfrm_smark_get(skb->mark, x);

                sp->xvec[sp->len++] = x;

                skb_dst_force(skb);
                if (!skb_dst(skb)) {
                        XFRM_INC_STATS(net, LINUX_MIB_XFRMINERROR);
                        goto drop;
                }

lock:
                spin_lock(&x->lock);

                if (unlikely(x->km.state != XFRM_STATE_VALID)) {
                        if (x->km.state == XFRM_STATE_ACQ)
                                XFRM_INC_STATS(net, LINUX_MIB_XFRMACQUIREERROR);
                        else
                                XFRM_INC_STATS(net,
                                               LINUX_MIB_XFRMINSTATEINVALID);
                        goto drop_unlock;
                }

                if ((x->encap ? x->encap->encap_type : 0) != encap_type) {
                        XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEMISMATCH);
                        goto drop_unlock;
                }

                if (xfrm_replay_check(x, skb, seq)) {
                        XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATESEQERROR);
                        goto drop_unlock;
                }

                if (xfrm_state_check_expire(x)) {
                        XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEEXPIRED);
                        goto drop_unlock;
                }

                spin_unlock(&x->lock);

                if (xfrm_tunnel_check(skb, x, family)) {
                        XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEMODEERROR);
                        goto drop;
                }

                seq_hi = htonl(xfrm_replay_seqhi(x, seq));

                XFRM_SKB_CB(skb)->seq.input.low = seq;
                XFRM_SKB_CB(skb)->seq.input.hi = seq_hi;

                dev_hold(skb->dev);

                if (crypto_done)
                        nexthdr = x->type_offload->input_tail(x, skb);
                else
                        nexthdr = x->type->input(x, skb);

                if (nexthdr == -EINPROGRESS)
                        return 0;
resume:
                dev_put(skb->dev);

                spin_lock(&x->lock);
                if (nexthdr < 0) {
                        if (nexthdr == -EBADMSG) {
                                xfrm_audit_state_icvfail(x, skb,
                                                         x->type->proto);
                                x->stats.integrity_failed++;
                        }
                        XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEPROTOERROR);
                        goto drop_unlock;
                }

                /* only the first xfrm gets the encap type */
                encap_type = 0;

                if (xfrm_replay_recheck(x, skb, seq)) {
                        XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATESEQERROR);
                        goto drop_unlock;
                }

                xfrm_replay_advance(x, seq);

                x->curlft.bytes += skb->len;
                x->curlft.packets++;

                spin_unlock(&x->lock);

                XFRM_MODE_SKB_CB(skb)->protocol = nexthdr;

                inner_mode = &x->inner_mode;

                if (x->sel.family == AF_UNSPEC) {
                        inner_mode = xfrm_ip2inner_mode(x, XFRM_MODE_SKB_CB(skb)->protocol);
                        if (inner_mode == NULL) {
                                XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEMODEERROR);
                                goto drop;
                        }
                }

                if (xfrm_inner_mode_input(x, inner_mode, skb)) {
                        XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEMODEERROR);
                        goto drop;
                }

                if (x->outer_mode.flags & XFRM_MODE_FLAG_TUNNEL) {
                        decaps = 1;
                        break;
                }

                /*
                 * We need the inner address.  However, we only get here for
                 * transport mode so the outer address is identical.
                 */
                daddr = &x->id.daddr;
                family = x->outer_mode.family;

                err = xfrm_parse_spi(skb, nexthdr, &spi, &seq);
                if (err < 0) {
                        XFRM_INC_STATS(net, LINUX_MIB_XFRMINHDRERROR);
                        goto drop;
                }
                crypto_done = false;
        } while (!err);

        err = xfrm_rcv_cb(skb, family, x->type->proto, 0);
        if (err)
                goto drop;

        nf_reset_ct(skb);

        if (decaps) {
                sp = skb_sec_path(skb);
                if (sp)
                        sp->olen = 0;
                skb_dst_drop(skb);
                gro_cells_receive(&gro_cells, skb);
                return 0;
        } else {
                xo = xfrm_offload(skb);
                if (xo)
                        xfrm_gro = xo->flags & XFRM_GRO;

                err = -EAFNOSUPPORT;
                rcu_read_lock();
                afinfo = xfrm_state_afinfo_get_rcu(x->inner_mode.family);
                if (likely(afinfo))
                        err = afinfo->transport_finish(skb, xfrm_gro || async);
                rcu_read_unlock();
                if (xfrm_gro) {
                        sp = skb_sec_path(skb);
                        if (sp)
                                sp->olen = 0;
                        skb_dst_drop(skb);
                        gro_cells_receive(&gro_cells, skb);
                        return err;
                }

                return err;
        }

drop_unlock:
        spin_unlock(&x->lock);
drop:
        xfrm_rcv_cb(skb, family, x && x->type ? x->type->proto : nexthdr, -1);
        kfree_skb(skb);
        return 0;
}
EXPORT_SYMBOL(xfrm_input);

int xfrm_input_resume(struct sk_buff *skb, int nexthdr)
{
        return xfrm_input(skb, nexthdr, 0, -1);
}
EXPORT_SYMBOL(xfrm_input_resume);

static void xfrm_trans_reinject(struct work_struct *work)
{
        struct xfrm_trans_tasklet *trans = container_of(work, struct xfrm_trans_tasklet, work);
        struct sk_buff_head queue;
        struct sk_buff *skb;

        __skb_queue_head_init(&queue);
        spin_lock_bh(&trans->queue_lock);
        skb_queue_splice_init(&trans->queue, &queue);
        spin_unlock_bh(&trans->queue_lock);

        local_bh_disable();
        while ((skb = __skb_dequeue(&queue)))
                XFRM_TRANS_SKB_CB(skb)->finish(XFRM_TRANS_SKB_CB(skb)->net,
                                               NULL, skb);
        local_bh_enable();
}

int xfrm_trans_queue_net(struct net *net, struct sk_buff *skb,
                         int (*finish)(struct net *, struct sock *,
                                       struct sk_buff *))
{
        struct xfrm_trans_tasklet *trans;

        trans = this_cpu_ptr(&xfrm_trans_tasklet);

        if (skb_queue_len(&trans->queue) >= READ_ONCE(netdev_max_backlog))
                return -ENOBUFS;

        BUILD_BUG_ON(sizeof(struct xfrm_trans_cb) > sizeof(skb->cb));

        XFRM_TRANS_SKB_CB(skb)->finish = finish;
        XFRM_TRANS_SKB_CB(skb)->net = net;
        spin_lock_bh(&trans->queue_lock);
        __skb_queue_tail(&trans->queue, skb);
        spin_unlock_bh(&trans->queue_lock);
        schedule_work(&trans->work);
        return 0;
}
EXPORT_SYMBOL(xfrm_trans_queue_net);

int xfrm_trans_queue(struct sk_buff *skb,
                     int (*finish)(struct net *, struct sock *,
                                   struct sk_buff *))
{
        return xfrm_trans_queue_net(dev_net(skb->dev), skb, finish);
}
EXPORT_SYMBOL(xfrm_trans_queue);

void __init xfrm_input_init(void)
{
        int err;
        int i;

        init_dummy_netdev(&xfrm_napi_dev);
        err = gro_cells_init(&gro_cells, &xfrm_napi_dev);
        if (err)
                gro_cells.cells = NULL;

        for_each_possible_cpu(i) {
                struct xfrm_trans_tasklet *trans;

                trans = &per_cpu(xfrm_trans_tasklet, i);
                spin_lock_init(&trans->queue_lock);
                __skb_queue_head_init(&trans->queue);
                INIT_WORK(&trans->work, xfrm_trans_reinject);
        }
}

























  855 


  381 

  848 
  494 





  852 





  855 





    5 


    5 




















1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
// SPDX-License-Identifier: GPL-2.0 OR MIT
/*
 * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
 *
 * This is an implementation of the BLAKE2s hash and PRF functions.
 *
 * Information: https://blake2.net/
 *
 */

#include <crypto/internal/blake2s.h>
#include <linux/types.h>
#include <linux/string.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/init.h>
#include <linux/bug.h>

static inline void blake2s_set_lastblock(struct blake2s_state *state)
{
        state->f[0] = -1;
}

void blake2s_update(struct blake2s_state *state, const u8 *in, size_t inlen)
{
        const size_t fill = BLAKE2S_BLOCK_SIZE - state->buflen;

        if (unlikely(!inlen))
                return;
        if (inlen > fill) {
                memcpy(state->buf + state->buflen, in, fill);
                blake2s_compress(state, state->buf, 1, BLAKE2S_BLOCK_SIZE);
                state->buflen = 0;
                in += fill;
                inlen -= fill;
        }
        if (inlen > BLAKE2S_BLOCK_SIZE) {
                const size_t nblocks = DIV_ROUND_UP(inlen, BLAKE2S_BLOCK_SIZE);
                blake2s_compress(state, in, nblocks - 1, BLAKE2S_BLOCK_SIZE);
                in += BLAKE2S_BLOCK_SIZE * (nblocks - 1);
                inlen -= BLAKE2S_BLOCK_SIZE * (nblocks - 1);
        }
        memcpy(state->buf + state->buflen, in, inlen);
        state->buflen += inlen;
}
EXPORT_SYMBOL(blake2s_update);

void blake2s_final(struct blake2s_state *state, u8 *out)
{
        WARN_ON(IS_ENABLED(DEBUG) && !out);
        blake2s_set_lastblock(state);
        memset(state->buf + state->buflen, 0,
               BLAKE2S_BLOCK_SIZE - state->buflen); /* Padding */
        blake2s_compress(state, state->buf, 1, state->buflen);
        cpu_to_le32_array(state->h, ARRAY_SIZE(state->h));
        memcpy(out, state->h, state->outlen);
        memzero_explicit(state, sizeof(*state));
}
EXPORT_SYMBOL(blake2s_final);

static int __init blake2s_mod_init(void)
{
        if (!IS_ENABLED(CONFIG_CRYPTO_MANAGER_DISABLE_TESTS) &&
            WARN_ON(!blake2s_selftest()))
                return -ENODEV;
        return 0;
}

module_init(blake2s_mod_init);
MODULE_LICENSE("GPL v2");
MODULE_DESCRIPTION("BLAKE2s hash function");
MODULE_AUTHOR("Jason A. Donenfeld <Jason@zx2c4.com>");





























































































    3 


































   33 













   32 
   29 
    1 


    3 


   32 





   33 
















































































    7 











   33 
   12 


   33 













































    2 




    2 





























































































































































   27 




















   28 

   10 


















   12 
    6 



    6 







   27 








































   36 





   37 
   37 



   37 



































































   29 















   28 








   29 




    4 











   21 




















































   22 









   14 

    8 

   21 




   22 












   29 




































   29 








   29 




   28 
   29 

   29 









































































































































































































































































































































































    4 












    4 






































































   13 






    1 









   11 






   13 























































































    4 













    4 




    4 

    4 




































    4 

    4 






































































































































































































































































































































































































































































































































    1 














    1 










































































































































































































































































































































































    4 




    4 

















































   11 





   11 







   11 


















































































































































































































    4 










































































































































































































































































































































































































































































































































































































































































































































































    1 









    1 












































































    4 




















































































































































    4 








    4 
    4 
    4 
    4 




































































































































































































































































































































































   15 









   15 

















































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965
2966
2967
2968
2969
2970
2971
2972
2973
2974
2975
2976
2977
2978
2979
2980
2981
2982
2983
2984
2985
2986
2987
2988
2989
2990
2991
2992
2993
2994
2995
2996
2997
2998
2999
3000
3001
3002
3003
3004
3005
3006
3007
3008
3009
3010
3011
3012
3013
3014
3015
3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029
3030
3031
3032
3033
3034
3035
3036
3037
3038
3039
3040
3041
3042
3043
3044
3045
3046
3047
3048
3049
3050
3051
3052
3053
3054
3055
3056
3057
3058
3059
3060
3061
3062
3063
3064
3065
3066
3067
3068
3069
3070
3071
3072
3073
3074
3075
3076
3077
3078
3079
3080
3081
3082
3083
3084
3085
3086
3087
3088
3089
3090
3091
3092
3093
3094
3095
3096
3097
3098
3099
3100
3101
3102
3103
3104
3105
3106
3107
3108
3109
3110
3111
3112
3113
3114
3115
3116
3117
3118
3119
3120
3121
3122
3123
3124
3125
3126
3127
3128
3129
3130
3131
3132
3133
3134
3135
3136
3137
3138
3139
3140
3141
3142
3143
3144
3145
3146
3147
3148
3149
3150
3151
3152
3153
3154
3155
3156
3157
3158
3159
3160
3161
3162
3163
3164
3165
3166
3167
3168
3169
3170
3171
3172
3173
3174
3175
3176
3177
3178
3179
3180
3181
3182
3183
3184
3185
3186
3187
3188
3189
3190
3191
3192
3193
3194
3195
3196
3197
3198
3199
3200
3201
3202
3203
3204
3205
3206
3207
3208
3209
3210
3211
3212
3213
3214
3215
3216
3217
3218
3219
3220
3221
3222
3223
3224
3225
3226
3227
3228
3229
3230
3231
3232
3233
3234
3235
3236
3237
3238
3239
3240
3241
3242
3243
3244
3245
3246
3247
3248
3249
3250
3251
3252
3253
3254
3255
3256
3257
3258
3259
3260
3261
3262
3263
3264
3265
3266
3267
3268
3269
3270
3271
3272
3273
3274
3275
3276
3277
3278
3279
3280
3281
3282
3283
3284
3285
3286
3287
3288
3289
3290
3291
3292
3293
3294
3295
3296
3297
3298
3299
3300
3301
3302
3303
3304
3305
3306
3307
3308
3309
3310
3311
3312
3313
3314
3315
3316
3317
3318
3319
3320
3321
3322
3323
3324
3325
3326
3327
3328
3329
3330
3331
3332
3333
3334
3335
3336
3337
3338
3339
3340
3341
3342
3343
3344
3345
3346
3347
3348
3349
3350
3351
3352
3353
3354
3355
3356
3357
3358
3359
3360
3361
3362
3363
3364
3365
3366
3367
3368
3369
3370
3371
3372
3373
3374
3375
3376
3377
3378
3379
3380
3381
3382
3383
3384
3385
3386
3387
3388
3389
3390
3391
3392
3393
3394
3395
3396
3397
3398
3399
3400
3401
3402
3403
3404
3405
3406
3407
3408
3409
3410
3411
3412
3413
3414
3415
3416
3417
3418
3419
3420
3421
3422
3423
3424
3425
3426
3427
3428
3429
3430
3431
3432
3433
3434
3435
3436
3437
3438
3439
3440
3441
3442
3443
3444
3445
3446
3447
3448
3449
3450
3451
3452
3453
3454
3455
3456
3457
3458
3459
3460
3461
3462
3463
3464
3465
3466
3467
3468
3469
3470
3471
3472
3473
3474
3475
3476
3477
3478
3479
3480
3481
3482
3483
3484
3485
3486
3487
3488
3489
3490
3491
3492
3493
3494
3495
3496
3497
3498
3499
3500
3501
3502
3503
3504
3505
3506
3507
3508
3509
3510
3511
3512
3513
3514
3515
3516
3517
3518
3519
3520
3521
3522
3523
3524
3525
3526
3527
3528
3529
3530
3531
3532
3533
3534
3535
3536
3537
3538
3539
3540
3541
3542
3543
3544
3545
3546
3547
3548
3549
3550
3551
3552
3553
3554
3555
3556
3557
3558
3559
3560
3561
3562
3563
3564
3565
3566
3567
3568
3569
3570
3571
3572
3573
3574
3575
3576
3577
3578
3579
3580
3581
3582
3583
3584
3585
3586
3587
3588
3589
3590
3591
3592
3593
3594
3595
3596
3597
3598
3599
3600
3601
3602
3603
3604
3605
3606
3607
3608
3609
3610
3611
3612
3613
3614
3615
3616
3617
3618
3619
3620
3621
3622
3623
3624
3625
3626
3627
3628
3629
3630
3631
3632
3633
3634
3635
3636
3637
3638
3639
3640
3641
3642
3643
3644
3645
3646
3647
3648
3649
3650
3651
3652
3653
3654
3655
3656
3657
3658
3659
3660
3661
3662
3663
3664
3665
3666
3667
3668
3669
3670
3671
3672
3673
3674
3675
3676
3677
3678
3679
3680
3681
3682
3683
3684
3685
3686
3687
3688
3689
3690
3691
3692
3693
3694
3695
3696
3697
3698
3699
3700
3701
3702
3703
3704
3705
3706
3707
3708
3709
3710
3711
3712
3713
3714
3715
3716
3717
3718
3719
3720
3721
3722
3723
3724
3725
3726
3727
3728
3729
3730
3731
3732
3733
3734
3735
3736
3737
3738
3739
3740
3741
3742
3743
3744
3745
3746
3747
3748
3749
3750
3751
3752
3753
3754
3755
3756
3757
3758
3759
3760
3761
3762
3763
3764
3765
3766
3767
3768
3769
3770
3771
3772
3773
3774
3775
3776
3777
3778
3779
3780
3781
3782
3783
3784
3785
3786
3787
3788
3789
3790
3791
3792
3793
3794
3795
3796
3797
3798
3799
3800
3801
3802
3803
3804
3805
3806
3807
3808
3809
3810
3811
3812
3813
3814
3815
3816
3817
3818
3819
3820
3821
3822
3823
3824
3825
3826
3827
3828
3829
3830
3831
3832
3833
3834
3835
3836
3837
3838
3839
3840
3841
3842
3843
3844
3845
3846
3847
3848
3849
3850
3851
3852
3853
3854
3855
3856
3857
3858
3859
3860
3861
3862
3863
3864
3865
3866
3867
3868
3869
3870
3871
3872
3873
3874
3875
3876
3877
3878
3879
3880
3881
3882
3883
3884
3885
3886
3887
3888
3889
3890
3891
3892
3893
3894
3895
3896
3897
3898
3899
3900
3901
3902
3903
3904
3905
3906
3907
3908
3909
3910
3911
3912
3913
3914
3915
3916
3917
3918
3919
3920
3921
3922
3923
3924
3925
3926
3927
3928
3929
3930
3931
3932
3933
3934
3935
3936
3937
3938
3939
3940
3941
3942
3943
3944
3945
3946
3947
3948
3949
3950
3951
3952
3953
3954
3955
3956
3957
3958
3959
3960
3961
3962
3963
3964
3965
3966
3967
3968
3969
3970
3971
3972
3973
3974
3975
3976
3977
3978
3979
3980
3981
3982
3983
3984
3985
3986
3987
3988
3989
3990
3991
3992
3993
3994
3995
3996
3997
3998
3999
4000
4001
4002
4003
4004
4005
4006
4007
4008
4009
4010
4011
4012
4013
4014
4015
4016
4017
4018
4019
4020
4021
4022
4023
4024
4025
4026
4027
4028
4029
4030
4031
4032
4033
4034
4035
4036
4037
4038
4039
4040
4041
4042
4043
4044
4045
4046
4047
4048
4049
4050
4051
4052
4053
4054
4055
4056
4057
4058
4059
4060
4061
4062
4063
4064
4065
4066
4067
4068
4069
4070
4071
4072
4073
4074
4075
4076
4077
4078
4079
4080
4081
4082
4083
4084
4085
4086
4087
4088
4089
4090
4091
4092
4093
4094
4095
4096
4097
4098
4099
4100
4101
4102
4103
4104
4105
4106
4107
4108
4109
4110
4111
4112
4113
4114
4115
4116
4117
4118
4119
4120
4121
4122
4123
4124
4125
4126
4127
4128
4129
4130
4131
4132
4133
4134
4135
4136
4137
4138
4139
4140
4141
4142
4143
4144
4145
4146
4147
4148
4149
4150
4151
4152
4153
4154
4155
4156
4157
4158
4159
4160
4161
4162
4163
4164
4165
4166
4167
4168
4169
4170
4171
4172
4173
4174
4175
4176
4177
4178
4179
4180
4181
4182
4183
4184
4185
4186
4187
4188
4189
4190
4191
4192
4193
4194
4195
4196
4197
4198
4199
4200
4201
4202
4203
4204
4205
4206
4207
4208
4209
4210
4211
4212
4213
4214
4215
4216
4217
4218
4219
4220
4221
4222
4223
4224
4225
4226
4227
4228
4229
4230
4231
4232
4233
4234
4235
4236
4237
4238
4239
4240
4241
4242
4243
4244
4245
4246
4247
4248
4249
4250
4251
4252
4253
4254
4255
4256
4257
4258
4259
4260
4261
4262
4263
4264
4265
4266
4267
4268
4269
4270
4271
4272
4273
4274
4275
4276
4277
4278
4279
4280
4281
4282
4283
4284
4285
4286
4287
4288
4289
4290
4291
4292
4293
4294
4295
4296
4297
4298
4299
4300
4301
4302
4303
4304
4305
4306
4307
4308
4309
4310
4311
4312
// SPDX-License-Identifier: GPL-2.0
/* Copyright (C) B.A.T.M.A.N. contributors:
 *
 * Marek Lindner, Simon Wunderlich, Antonio Quartulli
 */

#include "translation-table.h"
#include "main.h"

#include <linux/atomic.h>
#include <linux/bitops.h>
#include <linux/build_bug.h>
#include <linux/byteorder/generic.h>
#include <linux/cache.h>
#include <linux/compiler.h>
#include <linux/crc32c.h>
#include <linux/errno.h>
#include <linux/etherdevice.h>
#include <linux/gfp.h>
#include <linux/if_ether.h>
#include <linux/init.h>
#include <linux/jhash.h>
#include <linux/jiffies.h>
#include <linux/kernel.h>
#include <linux/kref.h>
#include <linux/list.h>
#include <linux/lockdep.h>
#include <linux/net.h>
#include <linux/netdevice.h>
#include <linux/netlink.h>
#include <linux/rculist.h>
#include <linux/rcupdate.h>
#include <linux/skbuff.h>
#include <linux/slab.h>
#include <linux/spinlock.h>
#include <linux/stddef.h>
#include <linux/string.h>
#include <linux/workqueue.h>
#include <net/genetlink.h>
#include <net/netlink.h>
#include <net/sock.h>
#include <uapi/linux/batadv_packet.h>
#include <uapi/linux/batman_adv.h>

#include "bridge_loop_avoidance.h"
#include "hard-interface.h"
#include "hash.h"
#include "log.h"
#include "netlink.h"
#include "originator.h"
#include "soft-interface.h"
#include "tvlv.h"

static struct kmem_cache *batadv_tl_cache __read_mostly;
static struct kmem_cache *batadv_tg_cache __read_mostly;
static struct kmem_cache *batadv_tt_orig_cache __read_mostly;
static struct kmem_cache *batadv_tt_change_cache __read_mostly;
static struct kmem_cache *batadv_tt_req_cache __read_mostly;
static struct kmem_cache *batadv_tt_roam_cache __read_mostly;

/* hash class keys */
static struct lock_class_key batadv_tt_local_hash_lock_class_key;
static struct lock_class_key batadv_tt_global_hash_lock_class_key;

static void batadv_send_roam_adv(struct batadv_priv *bat_priv, u8 *client,
                                 unsigned short vid,
                                 struct batadv_orig_node *orig_node);
static void batadv_tt_purge(struct work_struct *work);
static void
batadv_tt_global_del_orig_list(struct batadv_tt_global_entry *tt_global_entry);
static void batadv_tt_global_del(struct batadv_priv *bat_priv,
                                 struct batadv_orig_node *orig_node,
                                 const unsigned char *addr,
                                 unsigned short vid, const char *message,
                                 bool roaming);

/**
 * batadv_compare_tt() - check if two TT entries are the same
 * @node: the list element pointer of the first TT entry
 * @data2: pointer to the tt_common_entry of the second TT entry
 *
 * Compare the MAC address and the VLAN ID of the two TT entries and check if
 * they are the same TT client.
 * Return: true if the two TT clients are the same, false otherwise
 */
static bool batadv_compare_tt(const struct hlist_node *node, const void *data2)
{
        const void *data1 = container_of(node, struct batadv_tt_common_entry,
                                         hash_entry);
        const struct batadv_tt_common_entry *tt1 = data1;
        const struct batadv_tt_common_entry *tt2 = data2;

        return (tt1->vid == tt2->vid) && batadv_compare_eth(data1, data2);
}

/**
 * batadv_choose_tt() - return the index of the tt entry in the hash table
 * @data: pointer to the tt_common_entry object to map
 * @size: the size of the hash table
 *
 * Return: the hash index where the object represented by 'data' should be
 * stored at.
 */
static inline u32 batadv_choose_tt(const void *data, u32 size)
{
        struct batadv_tt_common_entry *tt;
        u32 hash = 0;

        tt = (struct batadv_tt_common_entry *)data;
        hash = jhash(&tt->addr, ETH_ALEN, hash);
        hash = jhash(&tt->vid, sizeof(tt->vid), hash);

        return hash % size;
}

/**
 * batadv_tt_hash_find() - look for a client in the given hash table
 * @hash: the hash table to search
 * @addr: the mac address of the client to look for
 * @vid: VLAN identifier
 *
 * Return: a pointer to the tt_common struct belonging to the searched client if
 * found, NULL otherwise.
 */
static struct batadv_tt_common_entry *
batadv_tt_hash_find(struct batadv_hashtable *hash, const u8 *addr,
                    unsigned short vid)
{
        struct hlist_head *head;
        struct batadv_tt_common_entry to_search, *tt, *tt_tmp = NULL;
        u32 index;

        if (!hash)
                return NULL;

        ether_addr_copy(to_search.addr, addr);
        to_search.vid = vid;

        index = batadv_choose_tt(&to_search, hash->size);
        head = &hash->table[index];

        rcu_read_lock();
        hlist_for_each_entry_rcu(tt, head, hash_entry) {
                if (!batadv_compare_eth(tt, addr))
                        continue;

                if (tt->vid != vid)
                        continue;

                if (!kref_get_unless_zero(&tt->refcount))
                        continue;

                tt_tmp = tt;
                break;
        }
        rcu_read_unlock();

        return tt_tmp;
}

/**
 * batadv_tt_local_hash_find() - search the local table for a given client
 * @bat_priv: the bat priv with all the soft interface information
 * @addr: the mac address of the client to look for
 * @vid: VLAN identifier
 *
 * Return: a pointer to the corresponding tt_local_entry struct if the client is
 * found, NULL otherwise.
 */
static struct batadv_tt_local_entry *
batadv_tt_local_hash_find(struct batadv_priv *bat_priv, const u8 *addr,
                          unsigned short vid)
{
        struct batadv_tt_common_entry *tt_common_entry;
        struct batadv_tt_local_entry *tt_local_entry = NULL;

        tt_common_entry = batadv_tt_hash_find(bat_priv->tt.local_hash, addr,
                                              vid);
        if (tt_common_entry)
                tt_local_entry = container_of(tt_common_entry,
                                              struct batadv_tt_local_entry,
                                              common);
        return tt_local_entry;
}

/**
 * batadv_tt_global_hash_find() - search the global table for a given client
 * @bat_priv: the bat priv with all the soft interface information
 * @addr: the mac address of the client to look for
 * @vid: VLAN identifier
 *
 * Return: a pointer to the corresponding tt_global_entry struct if the client
 * is found, NULL otherwise.
 */
struct batadv_tt_global_entry *
batadv_tt_global_hash_find(struct batadv_priv *bat_priv, const u8 *addr,
                           unsigned short vid)
{
        struct batadv_tt_common_entry *tt_common_entry;
        struct batadv_tt_global_entry *tt_global_entry = NULL;

        tt_common_entry = batadv_tt_hash_find(bat_priv->tt.global_hash, addr,
                                              vid);
        if (tt_common_entry)
                tt_global_entry = container_of(tt_common_entry,
                                               struct batadv_tt_global_entry,
                                               common);
        return tt_global_entry;
}

/**
 * batadv_tt_local_entry_free_rcu() - free the tt_local_entry
 * @rcu: rcu pointer of the tt_local_entry
 */
static void batadv_tt_local_entry_free_rcu(struct rcu_head *rcu)
{
        struct batadv_tt_local_entry *tt_local_entry;

        tt_local_entry = container_of(rcu, struct batadv_tt_local_entry,
                                      common.rcu);

        kmem_cache_free(batadv_tl_cache, tt_local_entry);
}

/**
 * batadv_tt_local_entry_release() - release tt_local_entry from lists and queue
 *  for free after rcu grace period
 * @ref: kref pointer of the nc_node
 */
static void batadv_tt_local_entry_release(struct kref *ref)
{
        struct batadv_tt_local_entry *tt_local_entry;

        tt_local_entry = container_of(ref, struct batadv_tt_local_entry,
                                      common.refcount);

        batadv_softif_vlan_put(tt_local_entry->vlan);

        call_rcu(&tt_local_entry->common.rcu, batadv_tt_local_entry_free_rcu);
}

/**
 * batadv_tt_local_entry_put() - decrement the tt_local_entry refcounter and
 *  possibly release it
 * @tt_local_entry: tt_local_entry to be free'd
 */
static void
batadv_tt_local_entry_put(struct batadv_tt_local_entry *tt_local_entry)
{
        if (!tt_local_entry)
                return;

        kref_put(&tt_local_entry->common.refcount,
                 batadv_tt_local_entry_release);
}

/**
 * batadv_tt_global_entry_free_rcu() - free the tt_global_entry
 * @rcu: rcu pointer of the tt_global_entry
 */
static void batadv_tt_global_entry_free_rcu(struct rcu_head *rcu)
{
        struct batadv_tt_global_entry *tt_global_entry;

        tt_global_entry = container_of(rcu, struct batadv_tt_global_entry,
                                       common.rcu);

        kmem_cache_free(batadv_tg_cache, tt_global_entry);
}

/**
 * batadv_tt_global_entry_release() - release tt_global_entry from lists and
 *  queue for free after rcu grace period
 * @ref: kref pointer of the nc_node
 */
void batadv_tt_global_entry_release(struct kref *ref)
{
        struct batadv_tt_global_entry *tt_global_entry;

        tt_global_entry = container_of(ref, struct batadv_tt_global_entry,
                                       common.refcount);

        batadv_tt_global_del_orig_list(tt_global_entry);

        call_rcu(&tt_global_entry->common.rcu, batadv_tt_global_entry_free_rcu);
}

/**
 * batadv_tt_global_hash_count() - count the number of orig entries
 * @bat_priv: the bat priv with all the soft interface information
 * @addr: the mac address of the client to count entries for
 * @vid: VLAN identifier
 *
 * Return: the number of originators advertising the given address/data
 * (excluding our self).
 */
int batadv_tt_global_hash_count(struct batadv_priv *bat_priv,
                                const u8 *addr, unsigned short vid)
{
        struct batadv_tt_global_entry *tt_global_entry;
        int count;

        tt_global_entry = batadv_tt_global_hash_find(bat_priv, addr, vid);
        if (!tt_global_entry)
                return 0;

        count = atomic_read(&tt_global_entry->orig_list_count);
        batadv_tt_global_entry_put(tt_global_entry);

        return count;
}

/**
 * batadv_tt_local_size_mod() - change the size by v of the local table
 *  identified by vid
 * @bat_priv: the bat priv with all the soft interface information
 * @vid: the VLAN identifier of the sub-table to change
 * @v: the amount to sum to the local table size
 */
static void batadv_tt_local_size_mod(struct batadv_priv *bat_priv,
                                     unsigned short vid, int v)
{
        struct batadv_softif_vlan *vlan;

        vlan = batadv_softif_vlan_get(bat_priv, vid);
        if (!vlan)
                return;

        atomic_add(v, &vlan->tt.num_entries);

        batadv_softif_vlan_put(vlan);
}

/**
 * batadv_tt_local_size_inc() - increase by one the local table size for the
 *  given vid
 * @bat_priv: the bat priv with all the soft interface information
 * @vid: the VLAN identifier
 */
static void batadv_tt_local_size_inc(struct batadv_priv *bat_priv,
                                     unsigned short vid)
{
        batadv_tt_local_size_mod(bat_priv, vid, 1);
}

/**
 * batadv_tt_local_size_dec() - decrease by one the local table size for the
 *  given vid
 * @bat_priv: the bat priv with all the soft interface information
 * @vid: the VLAN identifier
 */
static void batadv_tt_local_size_dec(struct batadv_priv *bat_priv,
                                     unsigned short vid)
{
        batadv_tt_local_size_mod(bat_priv, vid, -1);
}

/**
 * batadv_tt_global_size_mod() - change the size by v of the global table
 *  for orig_node identified by vid
 * @orig_node: the originator for which the table has to be modified
 * @vid: the VLAN identifier
 * @v: the amount to sum to the global table size
 */
static void batadv_tt_global_size_mod(struct batadv_orig_node *orig_node,
                                      unsigned short vid, int v)
{
        struct batadv_orig_node_vlan *vlan;

        vlan = batadv_orig_node_vlan_new(orig_node, vid);
        if (!vlan)
                return;

        if (atomic_add_return(v, &vlan->tt.num_entries) == 0) {
                spin_lock_bh(&orig_node->vlan_list_lock);
                if (!hlist_unhashed(&vlan->list)) {
                        hlist_del_init_rcu(&vlan->list);
                        batadv_orig_node_vlan_put(vlan);
                }
                spin_unlock_bh(&orig_node->vlan_list_lock);
        }

        batadv_orig_node_vlan_put(vlan);
}

/**
 * batadv_tt_global_size_inc() - increase by one the global table size for the
 *  given vid
 * @orig_node: the originator which global table size has to be decreased
 * @vid: the vlan identifier
 */
static void batadv_tt_global_size_inc(struct batadv_orig_node *orig_node,
                                      unsigned short vid)
{
        batadv_tt_global_size_mod(orig_node, vid, 1);
}

/**
 * batadv_tt_global_size_dec() - decrease by one the global table size for the
 *  given vid
 * @orig_node: the originator which global table size has to be decreased
 * @vid: the vlan identifier
 */
static void batadv_tt_global_size_dec(struct batadv_orig_node *orig_node,
                                      unsigned short vid)
{
        batadv_tt_global_size_mod(orig_node, vid, -1);
}

/**
 * batadv_tt_orig_list_entry_free_rcu() - free the orig_entry
 * @rcu: rcu pointer of the orig_entry
 */
static void batadv_tt_orig_list_entry_free_rcu(struct rcu_head *rcu)
{
        struct batadv_tt_orig_list_entry *orig_entry;

        orig_entry = container_of(rcu, struct batadv_tt_orig_list_entry, rcu);

        kmem_cache_free(batadv_tt_orig_cache, orig_entry);
}

/**
 * batadv_tt_orig_list_entry_release() - release tt orig entry from lists and
 *  queue for free after rcu grace period
 * @ref: kref pointer of the tt orig entry
 */
static void batadv_tt_orig_list_entry_release(struct kref *ref)
{
        struct batadv_tt_orig_list_entry *orig_entry;

        orig_entry = container_of(ref, struct batadv_tt_orig_list_entry,
                                  refcount);

        batadv_orig_node_put(orig_entry->orig_node);
        call_rcu(&orig_entry->rcu, batadv_tt_orig_list_entry_free_rcu);
}

/**
 * batadv_tt_orig_list_entry_put() - decrement the tt orig entry refcounter and
 *  possibly release it
 * @orig_entry: tt orig entry to be free'd
 */
static void
batadv_tt_orig_list_entry_put(struct batadv_tt_orig_list_entry *orig_entry)
{
        if (!orig_entry)
                return;

        kref_put(&orig_entry->refcount, batadv_tt_orig_list_entry_release);
}

/**
 * batadv_tt_local_event() - store a local TT event (ADD/DEL)
 * @bat_priv: the bat priv with all the soft interface information
 * @tt_local_entry: the TT entry involved in the event
 * @event_flags: flags to store in the event structure
 */
static void batadv_tt_local_event(struct batadv_priv *bat_priv,
                                  struct batadv_tt_local_entry *tt_local_entry,
                                  u8 event_flags)
{
        struct batadv_tt_change_node *tt_change_node, *entry, *safe;
        struct batadv_tt_common_entry *common = &tt_local_entry->common;
        u8 flags = common->flags | event_flags;
        bool event_removed = false;
        bool del_op_requested, del_op_entry;

        tt_change_node = kmem_cache_alloc(batadv_tt_change_cache, GFP_ATOMIC);
        if (!tt_change_node)
                return;

        tt_change_node->change.flags = flags;
        memset(tt_change_node->change.reserved, 0,
               sizeof(tt_change_node->change.reserved));
        ether_addr_copy(tt_change_node->change.addr, common->addr);
        tt_change_node->change.vid = htons(common->vid);

        del_op_requested = flags & BATADV_TT_CLIENT_DEL;

        /* check for ADD+DEL or DEL+ADD events */
        spin_lock_bh(&bat_priv->tt.changes_list_lock);
        list_for_each_entry_safe(entry, safe, &bat_priv->tt.changes_list,
                                 list) {
                if (!batadv_compare_eth(entry->change.addr, common->addr))
                        continue;

                /* DEL+ADD in the same orig interval have no effect and can be
                 * removed to avoid silly behaviour on the receiver side. The
                 * other way around (ADD+DEL) can happen in case of roaming of
                 * a client still in the NEW state. Roaming of NEW clients is
                 * now possible due to automatically recognition of "temporary"
                 * clients
                 */
                del_op_entry = entry->change.flags & BATADV_TT_CLIENT_DEL;
                if (!del_op_requested && del_op_entry)
                        goto del;
                if (del_op_requested && !del_op_entry)
                        goto del;

                /* this is a second add in the same originator interval. It
                 * means that flags have been changed: update them!
                 */
                if (!del_op_requested && !del_op_entry)
                        entry->change.flags = flags;

                continue;
del:
                list_del(&entry->list);
                kmem_cache_free(batadv_tt_change_cache, entry);
                kmem_cache_free(batadv_tt_change_cache, tt_change_node);
                event_removed = true;
                goto unlock;
        }

        /* track the change in the OGMinterval list */
        list_add_tail(&tt_change_node->list, &bat_priv->tt.changes_list);

unlock:
        spin_unlock_bh(&bat_priv->tt.changes_list_lock);

        if (event_removed)
                atomic_dec(&bat_priv->tt.local_changes);
        else
                atomic_inc(&bat_priv->tt.local_changes);
}

/**
 * batadv_tt_len() - compute length in bytes of given number of tt changes
 * @changes_num: number of tt changes
 *
 * Return: computed length in bytes.
 */
static int batadv_tt_len(int changes_num)
{
        return changes_num * sizeof(struct batadv_tvlv_tt_change);
}

/**
 * batadv_tt_entries() - compute the number of entries fitting in tt_len bytes
 * @tt_len: available space
 *
 * Return: the number of entries.
 */
static u16 batadv_tt_entries(u16 tt_len)
{
        return tt_len / batadv_tt_len(1);
}

/**
 * batadv_tt_local_table_transmit_size() - calculates the local translation
 *  table size when transmitted over the air
 * @bat_priv: the bat priv with all the soft interface information
 *
 * Return: local translation table size in bytes.
 */
static int batadv_tt_local_table_transmit_size(struct batadv_priv *bat_priv)
{
        u16 num_vlan = 0;
        u16 tt_local_entries = 0;
        struct batadv_softif_vlan *vlan;
        int hdr_size;

        rcu_read_lock();
        hlist_for_each_entry_rcu(vlan, &bat_priv->softif_vlan_list, list) {
                num_vlan++;
                tt_local_entries += atomic_read(&vlan->tt.num_entries);
        }
        rcu_read_unlock();

        /* header size of tvlv encapsulated tt response payload */
        hdr_size = sizeof(struct batadv_unicast_tvlv_packet);
        hdr_size += sizeof(struct batadv_tvlv_hdr);
        hdr_size += sizeof(struct batadv_tvlv_tt_data);
        hdr_size += num_vlan * sizeof(struct batadv_tvlv_tt_vlan_data);

        return hdr_size + batadv_tt_len(tt_local_entries);
}

static int batadv_tt_local_init(struct batadv_priv *bat_priv)
{
        if (bat_priv->tt.local_hash)
                return 0;

        bat_priv->tt.local_hash = batadv_hash_new(1024);

        if (!bat_priv->tt.local_hash)
                return -ENOMEM;

        batadv_hash_set_lock_class(bat_priv->tt.local_hash,
                                   &batadv_tt_local_hash_lock_class_key);

        return 0;
}

static void batadv_tt_global_free(struct batadv_priv *bat_priv,
                                  struct batadv_tt_global_entry *tt_global,
                                  const char *message)
{
        struct batadv_tt_global_entry *tt_removed_entry;
        struct hlist_node *tt_removed_node;

        batadv_dbg(BATADV_DBG_TT, bat_priv,
                   "Deleting global tt entry %pM (vid: %d): %s\n",
                   tt_global->common.addr,
                   batadv_print_vid(tt_global->common.vid), message);

        tt_removed_node = batadv_hash_remove(bat_priv->tt.global_hash,
                                             batadv_compare_tt,
                                             batadv_choose_tt,
                                             &tt_global->common);
        if (!tt_removed_node)
                return;

        /* drop reference of remove hash entry */
        tt_removed_entry = hlist_entry(tt_removed_node,
                                       struct batadv_tt_global_entry,
                                       common.hash_entry);
        batadv_tt_global_entry_put(tt_removed_entry);
}

/**
 * batadv_tt_local_add() - add a new client to the local table or update an
 *  existing client
 * @soft_iface: netdev struct of the mesh interface
 * @addr: the mac address of the client to add
 * @vid: VLAN identifier
 * @ifindex: index of the interface where the client is connected to (useful to
 *  identify wireless clients)
 * @mark: the value contained in the skb->mark field of the received packet (if
 *  any)
 *
 * Return: true if the client was successfully added, false otherwise.
 */
bool batadv_tt_local_add(struct net_device *soft_iface, const u8 *addr,
                         unsigned short vid, int ifindex, u32 mark)
{
        struct batadv_priv *bat_priv = netdev_priv(soft_iface);
        struct batadv_tt_local_entry *tt_local;
        struct batadv_tt_global_entry *tt_global = NULL;
        struct net *net = dev_net(soft_iface);
        struct batadv_softif_vlan *vlan;
        struct net_device *in_dev = NULL;
        struct batadv_hard_iface *in_hardif = NULL;
        struct hlist_head *head;
        struct batadv_tt_orig_list_entry *orig_entry;
        int hash_added, table_size, packet_size_max;
        bool ret = false;
        bool roamed_back = false;
        u8 remote_flags;
        u32 match_mark;

        if (ifindex != BATADV_NULL_IFINDEX)
                in_dev = dev_get_by_index(net, ifindex);

        if (in_dev)
                in_hardif = batadv_hardif_get_by_netdev(in_dev);

        tt_local = batadv_tt_local_hash_find(bat_priv, addr, vid);

        if (!is_multicast_ether_addr(addr))
                tt_global = batadv_tt_global_hash_find(bat_priv, addr, vid);

        if (tt_local) {
                tt_local->last_seen = jiffies;
                if (tt_local->common.flags & BATADV_TT_CLIENT_PENDING) {
                        batadv_dbg(BATADV_DBG_TT, bat_priv,
                                   "Re-adding pending client %pM (vid: %d)\n",
                                   addr, batadv_print_vid(vid));
                        /* whatever the reason why the PENDING flag was set,
                         * this is a client which was enqueued to be removed in
                         * this orig_interval. Since it popped up again, the
                         * flag can be reset like it was never enqueued
                         */
                        tt_local->common.flags &= ~BATADV_TT_CLIENT_PENDING;
                        goto add_event;
                }

                if (tt_local->common.flags & BATADV_TT_CLIENT_ROAM) {
                        batadv_dbg(BATADV_DBG_TT, bat_priv,
                                   "Roaming client %pM (vid: %d) came back to its original location\n",
                                   addr, batadv_print_vid(vid));
                        /* the ROAM flag is set because this client roamed away
                         * and the node got a roaming_advertisement message. Now
                         * that the client popped up again at its original
                         * location such flag can be unset
                         */
                        tt_local->common.flags &= ~BATADV_TT_CLIENT_ROAM;
                        roamed_back = true;
                }
                goto check_roaming;
        }

        /* Ignore the client if we cannot send it in a full table response. */
        table_size = batadv_tt_local_table_transmit_size(bat_priv);
        table_size += batadv_tt_len(1);
        packet_size_max = atomic_read(&bat_priv->packet_size_max);
        if (table_size > packet_size_max) {
                net_ratelimited_function(batadv_info, soft_iface,
                                         "Local translation table size (%i) exceeds maximum packet size (%i); Ignoring new local tt entry: %pM\n",
                                         table_size, packet_size_max, addr);
                goto out;
        }

        tt_local = kmem_cache_alloc(batadv_tl_cache, GFP_ATOMIC);
        if (!tt_local)
                goto out;

        /* increase the refcounter of the related vlan */
        vlan = batadv_softif_vlan_get(bat_priv, vid);
        if (!vlan) {
                net_ratelimited_function(batadv_info, soft_iface,
                                         "adding TT local entry %pM to non-existent VLAN %d\n",
                                         addr, batadv_print_vid(vid));
                kmem_cache_free(batadv_tl_cache, tt_local);
                tt_local = NULL;
                goto out;
        }

        batadv_dbg(BATADV_DBG_TT, bat_priv,
                   "Creating new local tt entry: %pM (vid: %d, ttvn: %d)\n",
                   addr, batadv_print_vid(vid),
                   (u8)atomic_read(&bat_priv->tt.vn));

        ether_addr_copy(tt_local->common.addr, addr);
        /* The local entry has to be marked as NEW to avoid to send it in
         * a full table response going out before the next ttvn increment
         * (consistency check)
         */
        tt_local->common.flags = BATADV_TT_CLIENT_NEW;
        tt_local->common.vid = vid;
        if (batadv_is_wifi_hardif(in_hardif))
                tt_local->common.flags |= BATADV_TT_CLIENT_WIFI;
        kref_init(&tt_local->common.refcount);
        tt_local->last_seen = jiffies;
        tt_local->common.added_at = tt_local->last_seen;
        tt_local->vlan = vlan;

        /* the batman interface mac and multicast addresses should never be
         * purged
         */
        if (batadv_compare_eth(addr, soft_iface->dev_addr) ||
            is_multicast_ether_addr(addr))
                tt_local->common.flags |= BATADV_TT_CLIENT_NOPURGE;

        kref_get(&tt_local->common.refcount);
        hash_added = batadv_hash_add(bat_priv->tt.local_hash, batadv_compare_tt,
                                     batadv_choose_tt, &tt_local->common,
                                     &tt_local->common.hash_entry);

        if (unlikely(hash_added != 0)) {
                /* remove the reference for the hash */
                batadv_tt_local_entry_put(tt_local);
                goto out;
        }

add_event:
        batadv_tt_local_event(bat_priv, tt_local, BATADV_NO_FLAGS);

check_roaming:
        /* Check whether it is a roaming, but don't do anything if the roaming
         * process has already been handled
         */
        if (tt_global && !(tt_global->common.flags & BATADV_TT_CLIENT_ROAM)) {
                /* These node are probably going to update their tt table */
                head = &tt_global->orig_list;
                rcu_read_lock();
                hlist_for_each_entry_rcu(orig_entry, head, list) {
                        batadv_send_roam_adv(bat_priv, tt_global->common.addr,
                                             tt_global->common.vid,
                                             orig_entry->orig_node);
                }
                rcu_read_unlock();
                if (roamed_back) {
                        batadv_tt_global_free(bat_priv, tt_global,
                                              "Roaming canceled");
                } else {
                        /* The global entry has to be marked as ROAMING and
                         * has to be kept for consistency purpose
                         */
                        tt_global->common.flags |= BATADV_TT_CLIENT_ROAM;
                        tt_global->roam_at = jiffies;
                }
        }

        /* store the current remote flags before altering them. This helps
         * understanding is flags are changing or not
         */
        remote_flags = tt_local->common.flags & BATADV_TT_REMOTE_MASK;

        if (batadv_is_wifi_hardif(in_hardif))
                tt_local->common.flags |= BATADV_TT_CLIENT_WIFI;
        else
                tt_local->common.flags &= ~BATADV_TT_CLIENT_WIFI;

        /* check the mark in the skb: if it's equal to the configured
         * isolation_mark, it means the packet is coming from an isolated
         * non-mesh client
         */
        match_mark = (mark & bat_priv->isolation_mark_mask);
        if (bat_priv->isolation_mark_mask &&
            match_mark == bat_priv->isolation_mark)
                tt_local->common.flags |= BATADV_TT_CLIENT_ISOLA;
        else
                tt_local->common.flags &= ~BATADV_TT_CLIENT_ISOLA;

        /* if any "dynamic" flag has been modified, resend an ADD event for this
         * entry so that all the nodes can get the new flags
         */
        if (remote_flags ^ (tt_local->common.flags & BATADV_TT_REMOTE_MASK))
                batadv_tt_local_event(bat_priv, tt_local, BATADV_NO_FLAGS);

        ret = true;
out:
        batadv_hardif_put(in_hardif);
        dev_put(in_dev);
        batadv_tt_local_entry_put(tt_local);
        batadv_tt_global_entry_put(tt_global);
        return ret;
}

/**
 * batadv_tt_prepare_tvlv_global_data() - prepare the TVLV TT header to send
 *  within a TT Response directed to another node
 * @orig_node: originator for which the TT data has to be prepared
 * @tt_data: uninitialised pointer to the address of the TVLV buffer
 * @tt_change: uninitialised pointer to the address of the area where the TT
 *  changed can be stored
 * @tt_len: pointer to the length to reserve to the tt_change. if -1 this
 *  function reserves the amount of space needed to send the entire global TT
 *  table. In case of success the value is updated with the real amount of
 *  reserved bytes
 * Allocate the needed amount of memory for the entire TT TVLV and write its
 * header made up of one tvlv_tt_data object and a series of tvlv_tt_vlan_data
 * objects, one per active VLAN served by the originator node.
 *
 * Return: the size of the allocated buffer or 0 in case of failure.
 */
static u16
batadv_tt_prepare_tvlv_global_data(struct batadv_orig_node *orig_node,
                                   struct batadv_tvlv_tt_data **tt_data,
                                   struct batadv_tvlv_tt_change **tt_change,
                                   s32 *tt_len)
{
        u16 num_vlan = 0;
        u16 num_entries = 0;
        u16 change_offset;
        u16 tvlv_len;
        struct batadv_tvlv_tt_vlan_data *tt_vlan;
        struct batadv_orig_node_vlan *vlan;
        u8 *tt_change_ptr;

        spin_lock_bh(&orig_node->vlan_list_lock);
        hlist_for_each_entry(vlan, &orig_node->vlan_list, list) {
                num_vlan++;
                num_entries += atomic_read(&vlan->tt.num_entries);
        }

        change_offset = sizeof(**tt_data);
        change_offset += num_vlan * sizeof(*tt_vlan);

        /* if tt_len is negative, allocate the space needed by the full table */
        if (*tt_len < 0)
                *tt_len = batadv_tt_len(num_entries);

        tvlv_len = *tt_len;
        tvlv_len += change_offset;

        *tt_data = kmalloc(tvlv_len, GFP_ATOMIC);
        if (!*tt_data) {
                *tt_len = 0;
                goto out;
        }

        (*tt_data)->flags = BATADV_NO_FLAGS;
        (*tt_data)->ttvn = atomic_read(&orig_node->last_ttvn);
        (*tt_data)->num_vlan = htons(num_vlan);

        tt_vlan = (struct batadv_tvlv_tt_vlan_data *)(*tt_data + 1);
        hlist_for_each_entry(vlan, &orig_node->vlan_list, list) {
                tt_vlan->vid = htons(vlan->vid);
                tt_vlan->crc = htonl(vlan->tt.crc);
                tt_vlan->reserved = 0;

                tt_vlan++;
        }

        tt_change_ptr = (u8 *)*tt_data + change_offset;
        *tt_change = (struct batadv_tvlv_tt_change *)tt_change_ptr;

out:
        spin_unlock_bh(&orig_node->vlan_list_lock);
        return tvlv_len;
}

/**
 * batadv_tt_prepare_tvlv_local_data() - allocate and prepare the TT TVLV for
 *  this node
 * @bat_priv: the bat priv with all the soft interface information
 * @tt_data: uninitialised pointer to the address of the TVLV buffer
 * @tt_change: uninitialised pointer to the address of the area where the TT
 *  changes can be stored
 * @tt_len: pointer to the length to reserve to the tt_change. if -1 this
 *  function reserves the amount of space needed to send the entire local TT
 *  table. In case of success the value is updated with the real amount of
 *  reserved bytes
 *
 * Allocate the needed amount of memory for the entire TT TVLV and write its
 * header made up by one tvlv_tt_data object and a series of tvlv_tt_vlan_data
 * objects, one per active VLAN.
 *
 * Return: the size of the allocated buffer or 0 in case of failure.
 */
static u16
batadv_tt_prepare_tvlv_local_data(struct batadv_priv *bat_priv,
                                  struct batadv_tvlv_tt_data **tt_data,
                                  struct batadv_tvlv_tt_change **tt_change,
                                  s32 *tt_len)
{
        struct batadv_tvlv_tt_vlan_data *tt_vlan;
        struct batadv_softif_vlan *vlan;
        u16 num_vlan = 0;
        u16 vlan_entries = 0;
        u16 total_entries = 0;
        u16 tvlv_len;
        u8 *tt_change_ptr;
        int change_offset;

        spin_lock_bh(&bat_priv->softif_vlan_list_lock);
        hlist_for_each_entry(vlan, &bat_priv->softif_vlan_list, list) {
                vlan_entries = atomic_read(&vlan->tt.num_entries);
                if (vlan_entries < 1)
                        continue;

                num_vlan++;
                total_entries += vlan_entries;
        }

        change_offset = sizeof(**tt_data);
        change_offset += num_vlan * sizeof(*tt_vlan);

        /* if tt_len is negative, allocate the space needed by the full table */
        if (*tt_len < 0)
                *tt_len = batadv_tt_len(total_entries);

        tvlv_len = *tt_len;
        tvlv_len += change_offset;

        *tt_data = kmalloc(tvlv_len, GFP_ATOMIC);
        if (!*tt_data) {
                tvlv_len = 0;
                goto out;
        }

        (*tt_data)->flags = BATADV_NO_FLAGS;
        (*tt_data)->ttvn = atomic_read(&bat_priv->tt.vn);
        (*tt_data)->num_vlan = htons(num_vlan);

        tt_vlan = (struct batadv_tvlv_tt_vlan_data *)(*tt_data + 1);
        hlist_for_each_entry(vlan, &bat_priv->softif_vlan_list, list) {
                vlan_entries = atomic_read(&vlan->tt.num_entries);
                if (vlan_entries < 1)
                        continue;

                tt_vlan->vid = htons(vlan->vid);
                tt_vlan->crc = htonl(vlan->tt.crc);
                tt_vlan->reserved = 0;

                tt_vlan++;
        }

        tt_change_ptr = (u8 *)*tt_data + change_offset;
        *tt_change = (struct batadv_tvlv_tt_change *)tt_change_ptr;

out:
        spin_unlock_bh(&bat_priv->softif_vlan_list_lock);
        return tvlv_len;
}

/**
 * batadv_tt_tvlv_container_update() - update the translation table tvlv
 *  container after local tt changes have been committed
 * @bat_priv: the bat priv with all the soft interface information
 */
static void batadv_tt_tvlv_container_update(struct batadv_priv *bat_priv)
{
        struct batadv_tt_change_node *entry, *safe;
        struct batadv_tvlv_tt_data *tt_data;
        struct batadv_tvlv_tt_change *tt_change;
        int tt_diff_len, tt_change_len = 0;
        int tt_diff_entries_num = 0;
        int tt_diff_entries_count = 0;
        bool drop_changes = false;
        size_t tt_extra_len = 0;
        u16 tvlv_len;

        tt_diff_entries_num = atomic_read(&bat_priv->tt.local_changes);
        tt_diff_len = batadv_tt_len(tt_diff_entries_num);

        /* if we have too many changes for one packet don't send any
         * and wait for the tt table request so we can reply with the full
         * (fragmented) table.
         *
         * The local change history should still be cleaned up so the next
         * TT round can start again with a clean state.
         */
        if (tt_diff_len > bat_priv->soft_iface->mtu) {
                tt_diff_len = 0;
                tt_diff_entries_num = 0;
                drop_changes = true;
        }

        tvlv_len = batadv_tt_prepare_tvlv_local_data(bat_priv, &tt_data,
                                                     &tt_change, &tt_diff_len);
        if (!tvlv_len)
                return;

        tt_data->flags = BATADV_TT_OGM_DIFF;

        if (!drop_changes && tt_diff_len == 0)
                goto container_register;

        spin_lock_bh(&bat_priv->tt.changes_list_lock);
        atomic_set(&bat_priv->tt.local_changes, 0);

        list_for_each_entry_safe(entry, safe, &bat_priv->tt.changes_list,
                                 list) {
                if (tt_diff_entries_count < tt_diff_entries_num) {
                        memcpy(tt_change + tt_diff_entries_count,
                               &entry->change,
                               sizeof(struct batadv_tvlv_tt_change));
                        tt_diff_entries_count++;
                }
                list_del(&entry->list);
                kmem_cache_free(batadv_tt_change_cache, entry);
        }
        spin_unlock_bh(&bat_priv->tt.changes_list_lock);

        tt_extra_len = batadv_tt_len(tt_diff_entries_num -
                                     tt_diff_entries_count);

        /* Keep the buffer for possible tt_request */
        spin_lock_bh(&bat_priv->tt.last_changeset_lock);
        kfree(bat_priv->tt.last_changeset);
        bat_priv->tt.last_changeset_len = 0;
        bat_priv->tt.last_changeset = NULL;
        tt_change_len = batadv_tt_len(tt_diff_entries_count);
        /* check whether this new OGM has no changes due to size problems */
        if (tt_diff_entries_count > 0) {
                tt_diff_len -= tt_extra_len;
                /* if kmalloc() fails we will reply with the full table
                 * instead of providing the diff
                 */
                bat_priv->tt.last_changeset = kzalloc(tt_diff_len, GFP_ATOMIC);
                if (bat_priv->tt.last_changeset) {
                        memcpy(bat_priv->tt.last_changeset,
                               tt_change, tt_change_len);
                        bat_priv->tt.last_changeset_len = tt_diff_len;
                }
        }
        spin_unlock_bh(&bat_priv->tt.last_changeset_lock);

        /* Remove extra packet space for OGM */
        tvlv_len -= tt_extra_len;
container_register:
        batadv_tvlv_container_register(bat_priv, BATADV_TVLV_TT, 1, tt_data,
                                       tvlv_len);
        kfree(tt_data);
}

/**
 * batadv_tt_local_dump_entry() - Dump one TT local entry into a message
 * @msg :Netlink message to dump into
 * @portid: Port making netlink request
 * @cb: Control block containing additional options
 * @bat_priv: The bat priv with all the soft interface information
 * @common: tt local & tt global common data
 *
 * Return: Error code, or 0 on success
 */
static int
batadv_tt_local_dump_entry(struct sk_buff *msg, u32 portid,
                           struct netlink_callback *cb,
                           struct batadv_priv *bat_priv,
                           struct batadv_tt_common_entry *common)
{
        void *hdr;
        struct batadv_softif_vlan *vlan;
        struct batadv_tt_local_entry *local;
        unsigned int last_seen_msecs;
        u32 crc;

        local = container_of(common, struct batadv_tt_local_entry, common);
        last_seen_msecs = jiffies_to_msecs(jiffies - local->last_seen);

        vlan = batadv_softif_vlan_get(bat_priv, common->vid);
        if (!vlan)
                return 0;

        crc = vlan->tt.crc;

        batadv_softif_vlan_put(vlan);

        hdr = genlmsg_put(msg, portid, cb->nlh->nlmsg_seq,
                          &batadv_netlink_family,  NLM_F_MULTI,
                          BATADV_CMD_GET_TRANSTABLE_LOCAL);
        if (!hdr)
                return -ENOBUFS;

        genl_dump_check_consistent(cb, hdr);

        if (nla_put(msg, BATADV_ATTR_TT_ADDRESS, ETH_ALEN, common->addr) ||
            nla_put_u32(msg, BATADV_ATTR_TT_CRC32, crc) ||
            nla_put_u16(msg, BATADV_ATTR_TT_VID, common->vid) ||
            nla_put_u32(msg, BATADV_ATTR_TT_FLAGS, common->flags))
                goto nla_put_failure;

        if (!(common->flags & BATADV_TT_CLIENT_NOPURGE) &&
            nla_put_u32(msg, BATADV_ATTR_LAST_SEEN_MSECS, last_seen_msecs))
                goto nla_put_failure;

        genlmsg_end(msg, hdr);
        return 0;

 nla_put_failure:
        genlmsg_cancel(msg, hdr);
        return -EMSGSIZE;
}

/**
 * batadv_tt_local_dump_bucket() - Dump one TT local bucket into a message
 * @msg: Netlink message to dump into
 * @portid: Port making netlink request
 * @cb: Control block containing additional options
 * @bat_priv: The bat priv with all the soft interface information
 * @hash: hash to dump
 * @bucket: bucket index to dump
 * @idx_s: Number of entries to skip
 *
 * Return: Error code, or 0 on success
 */
static int
batadv_tt_local_dump_bucket(struct sk_buff *msg, u32 portid,
                            struct netlink_callback *cb,
                            struct batadv_priv *bat_priv,
                            struct batadv_hashtable *hash, unsigned int bucket,
                            int *idx_s)
{
        struct batadv_tt_common_entry *common;
        int idx = 0;

        spin_lock_bh(&hash->list_locks[bucket]);
        cb->seq = atomic_read(&hash->generation) << 1 | 1;

        hlist_for_each_entry(common, &hash->table[bucket], hash_entry) {
                if (idx++ < *idx_s)
                        continue;

                if (batadv_tt_local_dump_entry(msg, portid, cb, bat_priv,
                                               common)) {
                        spin_unlock_bh(&hash->list_locks[bucket]);
                        *idx_s = idx - 1;
                        return -EMSGSIZE;
                }
        }
        spin_unlock_bh(&hash->list_locks[bucket]);

        *idx_s = 0;
        return 0;
}

/**
 * batadv_tt_local_dump() - Dump TT local entries into a message
 * @msg: Netlink message to dump into
 * @cb: Parameters from query
 *
 * Return: Error code, or 0 on success
 */
int batadv_tt_local_dump(struct sk_buff *msg, struct netlink_callback *cb)
{
        struct net *net = sock_net(cb->skb->sk);
        struct net_device *soft_iface;
        struct batadv_priv *bat_priv;
        struct batadv_hard_iface *primary_if = NULL;
        struct batadv_hashtable *hash;
        int ret;
        int ifindex;
        int bucket = cb->args[0];
        int idx = cb->args[1];
        int portid = NETLINK_CB(cb->skb).portid;

        ifindex = batadv_netlink_get_ifindex(cb->nlh, BATADV_ATTR_MESH_IFINDEX);
        if (!ifindex)
                return -EINVAL;

        soft_iface = dev_get_by_index(net, ifindex);
        if (!soft_iface || !batadv_softif_is_valid(soft_iface)) {
                ret = -ENODEV;
                goto out;
        }

        bat_priv = netdev_priv(soft_iface);

        primary_if = batadv_primary_if_get_selected(bat_priv);
        if (!primary_if || primary_if->if_status != BATADV_IF_ACTIVE) {
                ret = -ENOENT;
                goto out;
        }

        hash = bat_priv->tt.local_hash;

        while (bucket < hash->size) {
                if (batadv_tt_local_dump_bucket(msg, portid, cb, bat_priv,
                                                hash, bucket, &idx))
                        break;

                bucket++;
        }

        ret = msg->len;

 out:
        batadv_hardif_put(primary_if);
        dev_put(soft_iface);

        cb->args[0] = bucket;
        cb->args[1] = idx;

        return ret;
}

static void
batadv_tt_local_set_pending(struct batadv_priv *bat_priv,
                            struct batadv_tt_local_entry *tt_local_entry,
                            u16 flags, const char *message)
{
        batadv_tt_local_event(bat_priv, tt_local_entry, flags);

        /* The local client has to be marked as "pending to be removed" but has
         * to be kept in the table in order to send it in a full table
         * response issued before the net ttvn increment (consistency check)
         */
        tt_local_entry->common.flags |= BATADV_TT_CLIENT_PENDING;

        batadv_dbg(BATADV_DBG_TT, bat_priv,
                   "Local tt entry (%pM, vid: %d) pending to be removed: %s\n",
                   tt_local_entry->common.addr,
                   batadv_print_vid(tt_local_entry->common.vid), message);
}

/**
 * batadv_tt_local_remove() - logically remove an entry from the local table
 * @bat_priv: the bat priv with all the soft interface information
 * @addr: the MAC address of the client to remove
 * @vid: VLAN identifier
 * @message: message to append to the log on deletion
 * @roaming: true if the deletion is due to a roaming event
 *
 * Return: the flags assigned to the local entry before being deleted
 */
u16 batadv_tt_local_remove(struct batadv_priv *bat_priv, const u8 *addr,
                           unsigned short vid, const char *message,
                           bool roaming)
{
        struct batadv_tt_local_entry *tt_removed_entry;
        struct batadv_tt_local_entry *tt_local_entry;
        u16 flags, curr_flags = BATADV_NO_FLAGS;
        struct hlist_node *tt_removed_node;

        tt_local_entry = batadv_tt_local_hash_find(bat_priv, addr, vid);
        if (!tt_local_entry)
                goto out;

        curr_flags = tt_local_entry->common.flags;

        flags = BATADV_TT_CLIENT_DEL;
        /* if this global entry addition is due to a roaming, the node has to
         * mark the local entry as "roamed" in order to correctly reroute
         * packets later
         */
        if (roaming) {
                flags |= BATADV_TT_CLIENT_ROAM;
                /* mark the local client as ROAMed */
                tt_local_entry->common.flags |= BATADV_TT_CLIENT_ROAM;
        }

        if (!(tt_local_entry->common.flags & BATADV_TT_CLIENT_NEW)) {
                batadv_tt_local_set_pending(bat_priv, tt_local_entry, flags,
                                            message);
                goto out;
        }
        /* if this client has been added right now, it is possible to
         * immediately purge it
         */
        batadv_tt_local_event(bat_priv, tt_local_entry, BATADV_TT_CLIENT_DEL);

        tt_removed_node = batadv_hash_remove(bat_priv->tt.local_hash,
                                             batadv_compare_tt,
                                             batadv_choose_tt,
                                             &tt_local_entry->common);
        if (!tt_removed_node)
                goto out;

        /* drop reference of remove hash entry */
        tt_removed_entry = hlist_entry(tt_removed_node,
                                       struct batadv_tt_local_entry,
                                       common.hash_entry);
        batadv_tt_local_entry_put(tt_removed_entry);

out:
        batadv_tt_local_entry_put(tt_local_entry);

        return curr_flags;
}

/**
 * batadv_tt_local_purge_list() - purge inactive tt local entries
 * @bat_priv: the bat priv with all the soft interface information
 * @head: pointer to the list containing the local tt entries
 * @timeout: parameter deciding whether a given tt local entry is considered
 *  inactive or not
 */
static void batadv_tt_local_purge_list(struct batadv_priv *bat_priv,
                                       struct hlist_head *head,
                                       int timeout)
{
        struct batadv_tt_local_entry *tt_local_entry;
        struct batadv_tt_common_entry *tt_common_entry;
        struct hlist_node *node_tmp;

        hlist_for_each_entry_safe(tt_common_entry, node_tmp, head,
                                  hash_entry) {
                tt_local_entry = container_of(tt_common_entry,
                                              struct batadv_tt_local_entry,
                                              common);
                if (tt_local_entry->common.flags & BATADV_TT_CLIENT_NOPURGE)
                        continue;

                /* entry already marked for deletion */
                if (tt_local_entry->common.flags & BATADV_TT_CLIENT_PENDING)
                        continue;

                if (!batadv_has_timed_out(tt_local_entry->last_seen, timeout))
                        continue;

                batadv_tt_local_set_pending(bat_priv, tt_local_entry,
                                            BATADV_TT_CLIENT_DEL, "timed out");
        }
}

/**
 * batadv_tt_local_purge() - purge inactive tt local entries
 * @bat_priv: the bat priv with all the soft interface information
 * @timeout: parameter deciding whether a given tt local entry is considered
 *  inactive or not
 */
static void batadv_tt_local_purge(struct batadv_priv *bat_priv,
                                  int timeout)
{
        struct batadv_hashtable *hash = bat_priv->tt.local_hash;
        struct hlist_head *head;
        spinlock_t *list_lock; /* protects write access to the hash lists */
        u32 i;

        for (i = 0; i < hash->size; i++) {
                head = &hash->table[i];
                list_lock = &hash->list_locks[i];

                spin_lock_bh(list_lock);
                batadv_tt_local_purge_list(bat_priv, head, timeout);
                spin_unlock_bh(list_lock);
        }
}

static void batadv_tt_local_table_free(struct batadv_priv *bat_priv)
{
        struct batadv_hashtable *hash;
        spinlock_t *list_lock; /* protects write access to the hash lists */
        struct batadv_tt_common_entry *tt_common_entry;
        struct batadv_tt_local_entry *tt_local;
        struct hlist_node *node_tmp;
        struct hlist_head *head;
        u32 i;

        if (!bat_priv->tt.local_hash)
                return;

        hash = bat_priv->tt.local_hash;

        for (i = 0; i < hash->size; i++) {
                head = &hash->table[i];
                list_lock = &hash->list_locks[i];

                spin_lock_bh(list_lock);
                hlist_for_each_entry_safe(tt_common_entry, node_tmp,
                                          head, hash_entry) {
                        hlist_del_rcu(&tt_common_entry->hash_entry);
                        tt_local = container_of(tt_common_entry,
                                                struct batadv_tt_local_entry,
                                                common);

                        batadv_tt_local_entry_put(tt_local);
                }
                spin_unlock_bh(list_lock);
        }

        batadv_hash_destroy(hash);

        bat_priv->tt.local_hash = NULL;
}

static int batadv_tt_global_init(struct batadv_priv *bat_priv)
{
        if (bat_priv->tt.global_hash)
                return 0;

        bat_priv->tt.global_hash = batadv_hash_new(1024);

        if (!bat_priv->tt.global_hash)
                return -ENOMEM;

        batadv_hash_set_lock_class(bat_priv->tt.global_hash,
                                   &batadv_tt_global_hash_lock_class_key);

        return 0;
}

static void batadv_tt_changes_list_free(struct batadv_priv *bat_priv)
{
        struct batadv_tt_change_node *entry, *safe;

        spin_lock_bh(&bat_priv->tt.changes_list_lock);

        list_for_each_entry_safe(entry, safe, &bat_priv->tt.changes_list,
                                 list) {
                list_del(&entry->list);
                kmem_cache_free(batadv_tt_change_cache, entry);
        }

        atomic_set(&bat_priv->tt.local_changes, 0);
        spin_unlock_bh(&bat_priv->tt.changes_list_lock);
}

/**
 * batadv_tt_global_orig_entry_find() - find a TT orig_list_entry
 * @entry: the TT global entry where the orig_list_entry has to be
 *  extracted from
 * @orig_node: the originator for which the orig_list_entry has to be found
 *
 * retrieve the orig_tt_list_entry belonging to orig_node from the
 * batadv_tt_global_entry list
 *
 * Return: it with an increased refcounter, NULL if not found
 */
static struct batadv_tt_orig_list_entry *
batadv_tt_global_orig_entry_find(const struct batadv_tt_global_entry *entry,
                                 const struct batadv_orig_node *orig_node)
{
        struct batadv_tt_orig_list_entry *tmp_orig_entry, *orig_entry = NULL;
        const struct hlist_head *head;

        rcu_read_lock();
        head = &entry->orig_list;
        hlist_for_each_entry_rcu(tmp_orig_entry, head, list) {
                if (tmp_orig_entry->orig_node != orig_node)
                        continue;
                if (!kref_get_unless_zero(&tmp_orig_entry->refcount))
                        continue;

                orig_entry = tmp_orig_entry;
                break;
        }
        rcu_read_unlock();

        return orig_entry;
}

/**
 * batadv_tt_global_entry_has_orig() - check if a TT global entry is also
 *  handled by a given originator
 * @entry: the TT global entry to check
 * @orig_node: the originator to search in the list
 * @flags: a pointer to store TT flags for the given @entry received
 *  from @orig_node
 *
 * find out if an orig_node is already in the list of a tt_global_entry.
 *
 * Return: true if found, false otherwise
 */
static bool
batadv_tt_global_entry_has_orig(const struct batadv_tt_global_entry *entry,
                                const struct batadv_orig_node *orig_node,
                                u8 *flags)
{
        struct batadv_tt_orig_list_entry *orig_entry;
        bool found = false;

        orig_entry = batadv_tt_global_orig_entry_find(entry, orig_node);
        if (orig_entry) {
                found = true;

                if (flags)
                        *flags = orig_entry->flags;

                batadv_tt_orig_list_entry_put(orig_entry);
        }

        return found;
}

/**
 * batadv_tt_global_sync_flags() - update TT sync flags
 * @tt_global: the TT global entry to update sync flags in
 *
 * Updates the sync flag bits in the tt_global flag attribute with a logical
 * OR of all sync flags from any of its TT orig entries.
 */
static void
batadv_tt_global_sync_flags(struct batadv_tt_global_entry *tt_global)
{
        struct batadv_tt_orig_list_entry *orig_entry;
        const struct hlist_head *head;
        u16 flags = BATADV_NO_FLAGS;

        rcu_read_lock();
        head = &tt_global->orig_list;
        hlist_for_each_entry_rcu(orig_entry, head, list)
                flags |= orig_entry->flags;
        rcu_read_unlock();

        flags |= tt_global->common.flags & (~BATADV_TT_SYNC_MASK);
        tt_global->common.flags = flags;
}

/**
 * batadv_tt_global_orig_entry_add() - add or update a TT orig entry
 * @tt_global: the TT global entry to add an orig entry in
 * @orig_node: the originator to add an orig entry for
 * @ttvn: translation table version number of this changeset
 * @flags: TT sync flags
 */
static void
batadv_tt_global_orig_entry_add(struct batadv_tt_global_entry *tt_global,
                                struct batadv_orig_node *orig_node, int ttvn,
                                u8 flags)
{
        struct batadv_tt_orig_list_entry *orig_entry;

        spin_lock_bh(&tt_global->list_lock);

        orig_entry = batadv_tt_global_orig_entry_find(tt_global, orig_node);
        if (orig_entry) {
                /* refresh the ttvn: the current value could be a bogus one that
                 * was added during a "temporary client detection"
                 */
                orig_entry->ttvn = ttvn;
                orig_entry->flags = flags;
                goto sync_flags;
        }

        orig_entry = kmem_cache_zalloc(batadv_tt_orig_cache, GFP_ATOMIC);
        if (!orig_entry)
                goto out;

        INIT_HLIST_NODE(&orig_entry->list);
        kref_get(&orig_node->refcount);
        batadv_tt_global_size_inc(orig_node, tt_global->common.vid);
        orig_entry->orig_node = orig_node;
        orig_entry->ttvn = ttvn;
        orig_entry->flags = flags;
        kref_init(&orig_entry->refcount);

        kref_get(&orig_entry->refcount);
        hlist_add_head_rcu(&orig_entry->list,
                           &tt_global->orig_list);
        atomic_inc(&tt_global->orig_list_count);

sync_flags:
        batadv_tt_global_sync_flags(tt_global);
out:
        batadv_tt_orig_list_entry_put(orig_entry);

        spin_unlock_bh(&tt_global->list_lock);
}

/**
 * batadv_tt_global_add() - add a new TT global entry or update an existing one
 * @bat_priv: the bat priv with all the soft interface information
 * @orig_node: the originator announcing the client
 * @tt_addr: the mac address of the non-mesh client
 * @vid: VLAN identifier
 * @flags: TT flags that have to be set for this non-mesh client
 * @ttvn: the tt version number ever announcing this non-mesh client
 *
 * Add a new TT global entry for the given originator. If the entry already
 * exists add a new reference to the given originator (a global entry can have
 * references to multiple originators) and adjust the flags attribute to reflect
 * the function argument.
 * If a TT local entry exists for this non-mesh client remove it.
 *
 * The caller must hold the orig_node refcount.
 *
 * Return: true if the new entry has been added, false otherwise
 */
static bool batadv_tt_global_add(struct batadv_priv *bat_priv,
                                 struct batadv_orig_node *orig_node,
                                 const unsigned char *tt_addr,
                                 unsigned short vid, u16 flags, u8 ttvn)
{
        struct batadv_tt_global_entry *tt_global_entry;
        struct batadv_tt_local_entry *tt_local_entry;
        bool ret = false;
        int hash_added;
        struct batadv_tt_common_entry *common;
        u16 local_flags;

        /* ignore global entries from backbone nodes */
        if (batadv_bla_is_backbone_gw_orig(bat_priv, orig_node->orig, vid))
                return true;

        tt_global_entry = batadv_tt_global_hash_find(bat_priv, tt_addr, vid);
        tt_local_entry = batadv_tt_local_hash_find(bat_priv, tt_addr, vid);

        /* if the node already has a local client for this entry, it has to wait
         * for a roaming advertisement instead of manually messing up the global
         * table
         */
        if ((flags & BATADV_TT_CLIENT_TEMP) && tt_local_entry &&
            !(tt_local_entry->common.flags & BATADV_TT_CLIENT_NEW))
                goto out;

        if (!tt_global_entry) {
                tt_global_entry = kmem_cache_zalloc(batadv_tg_cache,
                                                    GFP_ATOMIC);
                if (!tt_global_entry)
                        goto out;

                common = &tt_global_entry->common;
                ether_addr_copy(common->addr, tt_addr);
                common->vid = vid;

                if (!is_multicast_ether_addr(common->addr))
                        common->flags = flags & (~BATADV_TT_SYNC_MASK);

                tt_global_entry->roam_at = 0;
                /* node must store current time in case of roaming. This is
                 * needed to purge this entry out on timeout (if nobody claims
                 * it)
                 */
                if (flags & BATADV_TT_CLIENT_ROAM)
                        tt_global_entry->roam_at = jiffies;
                kref_init(&common->refcount);
                common->added_at = jiffies;

                INIT_HLIST_HEAD(&tt_global_entry->orig_list);
                atomic_set(&tt_global_entry->orig_list_count, 0);
                spin_lock_init(&tt_global_entry->list_lock);

                kref_get(&common->refcount);
                hash_added = batadv_hash_add(bat_priv->tt.global_hash,
                                             batadv_compare_tt,
                                             batadv_choose_tt, common,
                                             &common->hash_entry);

                if (unlikely(hash_added != 0)) {
                        /* remove the reference for the hash */
                        batadv_tt_global_entry_put(tt_global_entry);
                        goto out_remove;
                }
        } else {
                common = &tt_global_entry->common;
                /* If there is already a global entry, we can use this one for
                 * our processing.
                 * But if we are trying to add a temporary client then here are
                 * two options at this point:
                 * 1) the global client is not a temporary client: the global
                 *    client has to be left as it is, temporary information
                 *    should never override any already known client state
                 * 2) the global client is a temporary client: purge the
                 *    originator list and add the new one orig_entry
                 */
                if (flags & BATADV_TT_CLIENT_TEMP) {
                        if (!(common->flags & BATADV_TT_CLIENT_TEMP))
                                goto out;
                        if (batadv_tt_global_entry_has_orig(tt_global_entry,
                                                            orig_node, NULL))
                                goto out_remove;
                        batadv_tt_global_del_orig_list(tt_global_entry);
                        goto add_orig_entry;
                }

                /* if the client was temporary added before receiving the first
                 * OGM announcing it, we have to clear the TEMP flag. Also,
                 * remove the previous temporary orig node and re-add it
                 * if required. If the orig entry changed, the new one which
                 * is a non-temporary entry is preferred.
                 */
                if (common->flags & BATADV_TT_CLIENT_TEMP) {
                        batadv_tt_global_del_orig_list(tt_global_entry);
                        common->flags &= ~BATADV_TT_CLIENT_TEMP;
                }

                /* the change can carry possible "attribute" flags like the
                 * TT_CLIENT_TEMP, therefore they have to be copied in the
                 * client entry
                 */
                if (!is_multicast_ether_addr(common->addr))
                        common->flags |= flags & (~BATADV_TT_SYNC_MASK);

                /* If there is the BATADV_TT_CLIENT_ROAM flag set, there is only
                 * one originator left in the list and we previously received a
                 * delete + roaming change for this originator.
                 *
                 * We should first delete the old originator before adding the
                 * new one.
                 */
                if (common->flags & BATADV_TT_CLIENT_ROAM) {
                        batadv_tt_global_del_orig_list(tt_global_entry);
                        common->flags &= ~BATADV_TT_CLIENT_ROAM;
                        tt_global_entry->roam_at = 0;
                }
        }
add_orig_entry:
        /* add the new orig_entry (if needed) or update it */
        batadv_tt_global_orig_entry_add(tt_global_entry, orig_node, ttvn,
                                        flags & BATADV_TT_SYNC_MASK);

        batadv_dbg(BATADV_DBG_TT, bat_priv,
                   "Creating new global tt entry: %pM (vid: %d, via %pM)\n",
                   common->addr, batadv_print_vid(common->vid),
                   orig_node->orig);
        ret = true;

out_remove:
        /* Do not remove multicast addresses from the local hash on
         * global additions
         */
        if (is_multicast_ether_addr(tt_addr))
                goto out;

        /* remove address from local hash if present */
        local_flags = batadv_tt_local_remove(bat_priv, tt_addr, vid,
                                             "global tt received",
                                             flags & BATADV_TT_CLIENT_ROAM);
        tt_global_entry->common.flags |= local_flags & BATADV_TT_CLIENT_WIFI;

        if (!(flags & BATADV_TT_CLIENT_ROAM))
                /* this is a normal global add. Therefore the client is not in a
                 * roaming state anymore.
                 */
                tt_global_entry->common.flags &= ~BATADV_TT_CLIENT_ROAM;

out:
        batadv_tt_global_entry_put(tt_global_entry);
        batadv_tt_local_entry_put(tt_local_entry);
        return ret;
}

/**
 * batadv_transtable_best_orig() - Get best originator list entry from tt entry
 * @bat_priv: the bat priv with all the soft interface information
 * @tt_global_entry: global translation table entry to be analyzed
 *
 * This function assumes the caller holds rcu_read_lock().
 * Return: best originator list entry or NULL on errors.
 */
static struct batadv_tt_orig_list_entry *
batadv_transtable_best_orig(struct batadv_priv *bat_priv,
                            struct batadv_tt_global_entry *tt_global_entry)
{
        struct batadv_neigh_node *router, *best_router = NULL;
        struct batadv_algo_ops *bao = bat_priv->algo_ops;
        struct hlist_head *head;
        struct batadv_tt_orig_list_entry *orig_entry, *best_entry = NULL;

        head = &tt_global_entry->orig_list;
        hlist_for_each_entry_rcu(orig_entry, head, list) {
                router = batadv_orig_router_get(orig_entry->orig_node,
                                                BATADV_IF_DEFAULT);
                if (!router)
                        continue;

                if (best_router &&
                    bao->neigh.cmp(router, BATADV_IF_DEFAULT, best_router,
                                   BATADV_IF_DEFAULT) <= 0) {
                        batadv_neigh_node_put(router);
                        continue;
                }

                /* release the refcount for the "old" best */
                batadv_neigh_node_put(best_router);

                best_entry = orig_entry;
                best_router = router;
        }

        batadv_neigh_node_put(best_router);

        return best_entry;
}

/**
 * batadv_tt_global_dump_subentry() - Dump all TT local entries into a message
 * @msg: Netlink message to dump into
 * @portid: Port making netlink request
 * @seq: Sequence number of netlink message
 * @common: tt local & tt global common data
 * @orig: Originator node announcing a non-mesh client
 * @best: Is the best originator for the TT entry
 *
 * Return: Error code, or 0 on success
 */
static int
batadv_tt_global_dump_subentry(struct sk_buff *msg, u32 portid, u32 seq,
                               struct batadv_tt_common_entry *common,
                               struct batadv_tt_orig_list_entry *orig,
                               bool best)
{
        u16 flags = (common->flags & (~BATADV_TT_SYNC_MASK)) | orig->flags;
        void *hdr;
        struct batadv_orig_node_vlan *vlan;
        u8 last_ttvn;
        u32 crc;

        vlan = batadv_orig_node_vlan_get(orig->orig_node,
                                         common->vid);
        if (!vlan)
                return 0;

        crc = vlan->tt.crc;

        batadv_orig_node_vlan_put(vlan);

        hdr = genlmsg_put(msg, portid, seq, &batadv_netlink_family,
                          NLM_F_MULTI,
                          BATADV_CMD_GET_TRANSTABLE_GLOBAL);
        if (!hdr)
                return -ENOBUFS;

        last_ttvn = atomic_read(&orig->orig_node->last_ttvn);

        if (nla_put(msg, BATADV_ATTR_TT_ADDRESS, ETH_ALEN, common->addr) ||
            nla_put(msg, BATADV_ATTR_ORIG_ADDRESS, ETH_ALEN,
                    orig->orig_node->orig) ||
            nla_put_u8(msg, BATADV_ATTR_TT_TTVN, orig->ttvn) ||
            nla_put_u8(msg, BATADV_ATTR_TT_LAST_TTVN, last_ttvn) ||
            nla_put_u32(msg, BATADV_ATTR_TT_CRC32, crc) ||
            nla_put_u16(msg, BATADV_ATTR_TT_VID, common->vid) ||
            nla_put_u32(msg, BATADV_ATTR_TT_FLAGS, flags))
                goto nla_put_failure;

        if (best && nla_put_flag(msg, BATADV_ATTR_FLAG_BEST))
                goto nla_put_failure;

        genlmsg_end(msg, hdr);
        return 0;

 nla_put_failure:
        genlmsg_cancel(msg, hdr);
        return -EMSGSIZE;
}

/**
 * batadv_tt_global_dump_entry() - Dump one TT global entry into a message
 * @msg: Netlink message to dump into
 * @portid: Port making netlink request
 * @seq: Sequence number of netlink message
 * @bat_priv: The bat priv with all the soft interface information
 * @common: tt local & tt global common data
 * @sub_s: Number of entries to skip
 *
 * This function assumes the caller holds rcu_read_lock().
 *
 * Return: Error code, or 0 on success
 */
static int
batadv_tt_global_dump_entry(struct sk_buff *msg, u32 portid, u32 seq,
                            struct batadv_priv *bat_priv,
                            struct batadv_tt_common_entry *common, int *sub_s)
{
        struct batadv_tt_orig_list_entry *orig_entry, *best_entry;
        struct batadv_tt_global_entry *global;
        struct hlist_head *head;
        int sub = 0;
        bool best;

        global = container_of(common, struct batadv_tt_global_entry, common);
        best_entry = batadv_transtable_best_orig(bat_priv, global);
        head = &global->orig_list;

        hlist_for_each_entry_rcu(orig_entry, head, list) {
                if (sub++ < *sub_s)
                        continue;

                best = (orig_entry == best_entry);

                if (batadv_tt_global_dump_subentry(msg, portid, seq, common,
                                                   orig_entry, best)) {
                        *sub_s = sub - 1;
                        return -EMSGSIZE;
                }
        }

        *sub_s = 0;
        return 0;
}

/**
 * batadv_tt_global_dump_bucket() - Dump one TT local bucket into a message
 * @msg: Netlink message to dump into
 * @portid: Port making netlink request
 * @seq: Sequence number of netlink message
 * @bat_priv: The bat priv with all the soft interface information
 * @head: Pointer to the list containing the global tt entries
 * @idx_s: Number of entries to skip
 * @sub: Number of entries to skip
 *
 * Return: Error code, or 0 on success
 */
static int
batadv_tt_global_dump_bucket(struct sk_buff *msg, u32 portid, u32 seq,
                             struct batadv_priv *bat_priv,
                             struct hlist_head *head, int *idx_s, int *sub)
{
        struct batadv_tt_common_entry *common;
        int idx = 0;

        rcu_read_lock();
        hlist_for_each_entry_rcu(common, head, hash_entry) {
                if (idx++ < *idx_s)
                        continue;

                if (batadv_tt_global_dump_entry(msg, portid, seq, bat_priv,
                                                common, sub)) {
                        rcu_read_unlock();
                        *idx_s = idx - 1;
                        return -EMSGSIZE;
                }
        }
        rcu_read_unlock();

        *idx_s = 0;
        *sub = 0;
        return 0;
}

/**
 * batadv_tt_global_dump() -  Dump TT global entries into a message
 * @msg: Netlink message to dump into
 * @cb: Parameters from query
 *
 * Return: Error code, or length of message on success
 */
int batadv_tt_global_dump(struct sk_buff *msg, struct netlink_callback *cb)
{
        struct net *net = sock_net(cb->skb->sk);
        struct net_device *soft_iface;
        struct batadv_priv *bat_priv;
        struct batadv_hard_iface *primary_if = NULL;
        struct batadv_hashtable *hash;
        struct hlist_head *head;
        int ret;
        int ifindex;
        int bucket = cb->args[0];
        int idx = cb->args[1];
        int sub = cb->args[2];
        int portid = NETLINK_CB(cb->skb).portid;

        ifindex = batadv_netlink_get_ifindex(cb->nlh, BATADV_ATTR_MESH_IFINDEX);
        if (!ifindex)
                return -EINVAL;

        soft_iface = dev_get_by_index(net, ifindex);
        if (!soft_iface || !batadv_softif_is_valid(soft_iface)) {
                ret = -ENODEV;
                goto out;
        }

        bat_priv = netdev_priv(soft_iface);

        primary_if = batadv_primary_if_get_selected(bat_priv);
        if (!primary_if || primary_if->if_status != BATADV_IF_ACTIVE) {
                ret = -ENOENT;
                goto out;
        }

        hash = bat_priv->tt.global_hash;

        while (bucket < hash->size) {
                head = &hash->table[bucket];

                if (batadv_tt_global_dump_bucket(msg, portid,
                                                 cb->nlh->nlmsg_seq, bat_priv,
                                                 head, &idx, &sub))
                        break;

                bucket++;
        }

        ret = msg->len;

 out:
        batadv_hardif_put(primary_if);
        dev_put(soft_iface);

        cb->args[0] = bucket;
        cb->args[1] = idx;
        cb->args[2] = sub;

        return ret;
}

/**
 * _batadv_tt_global_del_orig_entry() - remove and free an orig_entry
 * @tt_global_entry: the global entry to remove the orig_entry from
 * @orig_entry: the orig entry to remove and free
 *
 * Remove an orig_entry from its list in the given tt_global_entry and
 * free this orig_entry afterwards.
 *
 * Caller must hold tt_global_entry->list_lock and ensure orig_entry->list is
 * part of a list.
 */
static void
_batadv_tt_global_del_orig_entry(struct batadv_tt_global_entry *tt_global_entry,
                                 struct batadv_tt_orig_list_entry *orig_entry)
{
        lockdep_assert_held(&tt_global_entry->list_lock);

        batadv_tt_global_size_dec(orig_entry->orig_node,
                                  tt_global_entry->common.vid);
        atomic_dec(&tt_global_entry->orig_list_count);
        /* requires holding tt_global_entry->list_lock and orig_entry->list
         * being part of a list
         */
        hlist_del_rcu(&orig_entry->list);
        batadv_tt_orig_list_entry_put(orig_entry);
}

/* deletes the orig list of a tt_global_entry */
static void
batadv_tt_global_del_orig_list(struct batadv_tt_global_entry *tt_global_entry)
{
        struct hlist_head *head;
        struct hlist_node *safe;
        struct batadv_tt_orig_list_entry *orig_entry;

        spin_lock_bh(&tt_global_entry->list_lock);
        head = &tt_global_entry->orig_list;
        hlist_for_each_entry_safe(orig_entry, safe, head, list)
                _batadv_tt_global_del_orig_entry(tt_global_entry, orig_entry);
        spin_unlock_bh(&tt_global_entry->list_lock);
}

/**
 * batadv_tt_global_del_orig_node() - remove orig_node from a global tt entry
 * @bat_priv: the bat priv with all the soft interface information
 * @tt_global_entry: the global entry to remove the orig_node from
 * @orig_node: the originator announcing the client
 * @message: message to append to the log on deletion
 *
 * Remove the given orig_node and its according orig_entry from the given
 * global tt entry.
 */
static void
batadv_tt_global_del_orig_node(struct batadv_priv *bat_priv,
                               struct batadv_tt_global_entry *tt_global_entry,
                               struct batadv_orig_node *orig_node,
                               const char *message)
{
        struct hlist_head *head;
        struct hlist_node *safe;
        struct batadv_tt_orig_list_entry *orig_entry;
        unsigned short vid;

        spin_lock_bh(&tt_global_entry->list_lock);
        head = &tt_global_entry->orig_list;
        hlist_for_each_entry_safe(orig_entry, safe, head, list) {
                if (orig_entry->orig_node == orig_node) {
                        vid = tt_global_entry->common.vid;
                        batadv_dbg(BATADV_DBG_TT, bat_priv,
                                   "Deleting %pM from global tt entry %pM (vid: %d): %s\n",
                                   orig_node->orig,
                                   tt_global_entry->common.addr,
                                   batadv_print_vid(vid), message);
                        _batadv_tt_global_del_orig_entry(tt_global_entry,
                                                         orig_entry);
                }
        }
        spin_unlock_bh(&tt_global_entry->list_lock);
}

/* If the client is to be deleted, we check if it is the last origantor entry
 * within tt_global entry. If yes, we set the BATADV_TT_CLIENT_ROAM flag and the
 * timer, otherwise we simply remove the originator scheduled for deletion.
 */
static void
batadv_tt_global_del_roaming(struct batadv_priv *bat_priv,
                             struct batadv_tt_global_entry *tt_global_entry,
                             struct batadv_orig_node *orig_node,
                             const char *message)
{
        bool last_entry = true;
        struct hlist_head *head;
        struct batadv_tt_orig_list_entry *orig_entry;

        /* no local entry exists, case 1:
         * Check if this is the last one or if other entries exist.
         */

        rcu_read_lock();
        head = &tt_global_entry->orig_list;
        hlist_for_each_entry_rcu(orig_entry, head, list) {
                if (orig_entry->orig_node != orig_node) {
                        last_entry = false;
                        break;
                }
        }
        rcu_read_unlock();

        if (last_entry) {
                /* its the last one, mark for roaming. */
                tt_global_entry->common.flags |= BATADV_TT_CLIENT_ROAM;
                tt_global_entry->roam_at = jiffies;
        } else {
                /* there is another entry, we can simply delete this
                 * one and can still use the other one.
                 */
                batadv_tt_global_del_orig_node(bat_priv, tt_global_entry,
                                               orig_node, message);
        }
}

/**
 * batadv_tt_global_del() - remove a client from the global table
 * @bat_priv: the bat priv with all the soft interface information
 * @orig_node: an originator serving this client
 * @addr: the mac address of the client
 * @vid: VLAN identifier
 * @message: a message explaining the reason for deleting the client to print
 *  for debugging purpose
 * @roaming: true if the deletion has been triggered by a roaming event
 */
static void batadv_tt_global_del(struct batadv_priv *bat_priv,
                                 struct batadv_orig_node *orig_node,
                                 const unsigned char *addr, unsigned short vid,
                                 const char *message, bool roaming)
{
        struct batadv_tt_global_entry *tt_global_entry;
        struct batadv_tt_local_entry *local_entry = NULL;

        tt_global_entry = batadv_tt_global_hash_find(bat_priv, addr, vid);
        if (!tt_global_entry)
                goto out;

        if (!roaming) {
                batadv_tt_global_del_orig_node(bat_priv, tt_global_entry,
                                               orig_node, message);

                if (hlist_empty(&tt_global_entry->orig_list))
                        batadv_tt_global_free(bat_priv, tt_global_entry,
                                              message);

                goto out;
        }

        /* if we are deleting a global entry due to a roam
         * event, there are two possibilities:
         * 1) the client roamed from node A to node B => if there
         *    is only one originator left for this client, we mark
         *    it with BATADV_TT_CLIENT_ROAM, we start a timer and we
         *    wait for node B to claim it. In case of timeout
         *    the entry is purged.
         *
         *    If there are other originators left, we directly delete
         *    the originator.
         * 2) the client roamed to us => we can directly delete
         *    the global entry, since it is useless now.
         */
        local_entry = batadv_tt_local_hash_find(bat_priv,
                                                tt_global_entry->common.addr,
                                                vid);
        if (local_entry) {
                /* local entry exists, case 2: client roamed to us. */
                batadv_tt_global_del_orig_list(tt_global_entry);
                batadv_tt_global_free(bat_priv, tt_global_entry, message);
        } else {
                /* no local entry exists, case 1: check for roaming */
                batadv_tt_global_del_roaming(bat_priv, tt_global_entry,
                                             orig_node, message);
        }

out:
        batadv_tt_global_entry_put(tt_global_entry);
        batadv_tt_local_entry_put(local_entry);
}

/**
 * batadv_tt_global_del_orig() - remove all the TT global entries belonging to
 *  the given originator matching the provided vid
 * @bat_priv: the bat priv with all the soft interface information
 * @orig_node: the originator owning the entries to remove
 * @match_vid: the VLAN identifier to match. If negative all the entries will be
 *  removed
 * @message: debug message to print as "reason"
 */
void batadv_tt_global_del_orig(struct batadv_priv *bat_priv,
                               struct batadv_orig_node *orig_node,
                               s32 match_vid,
                               const char *message)
{
        struct batadv_tt_global_entry *tt_global;
        struct batadv_tt_common_entry *tt_common_entry;
        u32 i;
        struct batadv_hashtable *hash = bat_priv->tt.global_hash;
        struct hlist_node *safe;
        struct hlist_head *head;
        spinlock_t *list_lock; /* protects write access to the hash lists */
        unsigned short vid;

        if (!hash)
                return;

        for (i = 0; i < hash->size; i++) {
                head = &hash->table[i];
                list_lock = &hash->list_locks[i];

                spin_lock_bh(list_lock);
                hlist_for_each_entry_safe(tt_common_entry, safe,
                                          head, hash_entry) {
                        /* remove only matching entries */
                        if (match_vid >= 0 && tt_common_entry->vid != match_vid)
                                continue;

                        tt_global = container_of(tt_common_entry,
                                                 struct batadv_tt_global_entry,
                                                 common);

                        batadv_tt_global_del_orig_node(bat_priv, tt_global,
                                                       orig_node, message);

                        if (hlist_empty(&tt_global->orig_list)) {
                                vid = tt_global->common.vid;
                                batadv_dbg(BATADV_DBG_TT, bat_priv,
                                           "Deleting global tt entry %pM (vid: %d): %s\n",
                                           tt_global->common.addr,
                                           batadv_print_vid(vid), message);
                                hlist_del_rcu(&tt_common_entry->hash_entry);
                                batadv_tt_global_entry_put(tt_global);
                        }
                }
                spin_unlock_bh(list_lock);
        }
        clear_bit(BATADV_ORIG_CAPA_HAS_TT, &orig_node->capa_initialized);
}

static bool batadv_tt_global_to_purge(struct batadv_tt_global_entry *tt_global,
                                      char **msg)
{
        bool purge = false;
        unsigned long roam_timeout = BATADV_TT_CLIENT_ROAM_TIMEOUT;
        unsigned long temp_timeout = BATADV_TT_CLIENT_TEMP_TIMEOUT;

        if ((tt_global->common.flags & BATADV_TT_CLIENT_ROAM) &&
            batadv_has_timed_out(tt_global->roam_at, roam_timeout)) {
                purge = true;
                *msg = "Roaming timeout\n";
        }

        if ((tt_global->common.flags & BATADV_TT_CLIENT_TEMP) &&
            batadv_has_timed_out(tt_global->common.added_at, temp_timeout)) {
                purge = true;
                *msg = "Temporary client timeout\n";
        }

        return purge;
}

static void batadv_tt_global_purge(struct batadv_priv *bat_priv)
{
        struct batadv_hashtable *hash = bat_priv->tt.global_hash;
        struct hlist_head *head;
        struct hlist_node *node_tmp;
        spinlock_t *list_lock; /* protects write access to the hash lists */
        u32 i;
        char *msg = NULL;
        struct batadv_tt_common_entry *tt_common;
        struct batadv_tt_global_entry *tt_global;

        for (i = 0; i < hash->size; i++) {
                head = &hash->table[i];
                list_lock = &hash->list_locks[i];

                spin_lock_bh(list_lock);
                hlist_for_each_entry_safe(tt_common, node_tmp, head,
                                          hash_entry) {
                        tt_global = container_of(tt_common,
                                                 struct batadv_tt_global_entry,
                                                 common);

                        if (!batadv_tt_global_to_purge(tt_global, &msg))
                                continue;

                        batadv_dbg(BATADV_DBG_TT, bat_priv,
                                   "Deleting global tt entry %pM (vid: %d): %s\n",
                                   tt_global->common.addr,
                                   batadv_print_vid(tt_global->common.vid),
                                   msg);

                        hlist_del_rcu(&tt_common->hash_entry);

                        batadv_tt_global_entry_put(tt_global);
                }
                spin_unlock_bh(list_lock);
        }
}

static void batadv_tt_global_table_free(struct batadv_priv *bat_priv)
{
        struct batadv_hashtable *hash;
        spinlock_t *list_lock; /* protects write access to the hash lists */
        struct batadv_tt_common_entry *tt_common_entry;
        struct batadv_tt_global_entry *tt_global;
        struct hlist_node *node_tmp;
        struct hlist_head *head;
        u32 i;

        if (!bat_priv->tt.global_hash)
                return;

        hash = bat_priv->tt.global_hash;

        for (i = 0; i < hash->size; i++) {
                head = &hash->table[i];
                list_lock = &hash->list_locks[i];

                spin_lock_bh(list_lock);
                hlist_for_each_entry_safe(tt_common_entry, node_tmp,
                                          head, hash_entry) {
                        hlist_del_rcu(&tt_common_entry->hash_entry);
                        tt_global = container_of(tt_common_entry,
                                                 struct batadv_tt_global_entry,
                                                 common);
                        batadv_tt_global_entry_put(tt_global);
                }
                spin_unlock_bh(list_lock);
        }

        batadv_hash_destroy(hash);

        bat_priv->tt.global_hash = NULL;
}

static bool
_batadv_is_ap_isolated(struct batadv_tt_local_entry *tt_local_entry,
                       struct batadv_tt_global_entry *tt_global_entry)
{
        if (tt_local_entry->common.flags & BATADV_TT_CLIENT_WIFI &&
            tt_global_entry->common.flags & BATADV_TT_CLIENT_WIFI)
                return true;

        /* check if the two clients are marked as isolated */
        if (tt_local_entry->common.flags & BATADV_TT_CLIENT_ISOLA &&
            tt_global_entry->common.flags & BATADV_TT_CLIENT_ISOLA)
                return true;

        return false;
}

/**
 * batadv_transtable_search() - get the mesh destination for a given client
 * @bat_priv: the bat priv with all the soft interface information
 * @src: mac address of the source client
 * @addr: mac address of the destination client
 * @vid: VLAN identifier
 *
 * Return: a pointer to the originator that was selected as destination in the
 * mesh for contacting the client 'addr', NULL otherwise.
 * In case of multiple originators serving the same client, the function returns
 * the best one (best in terms of metric towards the destination node).
 *
 * If the two clients are AP isolated the function returns NULL.
 */
struct batadv_orig_node *batadv_transtable_search(struct batadv_priv *bat_priv,
                                                  const u8 *src,
                                                  const u8 *addr,
                                                  unsigned short vid)
{
        struct batadv_tt_local_entry *tt_local_entry = NULL;
        struct batadv_tt_global_entry *tt_global_entry = NULL;
        struct batadv_orig_node *orig_node = NULL;
        struct batadv_tt_orig_list_entry *best_entry;

        if (src && batadv_vlan_ap_isola_get(bat_priv, vid)) {
                tt_local_entry = batadv_tt_local_hash_find(bat_priv, src, vid);
                if (!tt_local_entry ||
                    (tt_local_entry->common.flags & BATADV_TT_CLIENT_PENDING))
                        goto out;
        }

        tt_global_entry = batadv_tt_global_hash_find(bat_priv, addr, vid);
        if (!tt_global_entry)
                goto out;

        /* check whether the clients should not communicate due to AP
         * isolation
         */
        if (tt_local_entry &&
            _batadv_is_ap_isolated(tt_local_entry, tt_global_entry))
                goto out;

        rcu_read_lock();
        best_entry = batadv_transtable_best_orig(bat_priv, tt_global_entry);
        /* found anything? */
        if (best_entry)
                orig_node = best_entry->orig_node;
        if (orig_node && !kref_get_unless_zero(&orig_node->refcount))
                orig_node = NULL;
        rcu_read_unlock();

out:
        batadv_tt_global_entry_put(tt_global_entry);
        batadv_tt_local_entry_put(tt_local_entry);

        return orig_node;
}

/**
 * batadv_tt_global_crc() - calculates the checksum of the local table belonging
 *  to the given orig_node
 * @bat_priv: the bat priv with all the soft interface information
 * @orig_node: originator for which the CRC should be computed
 * @vid: VLAN identifier for which the CRC32 has to be computed
 *
 * This function computes the checksum for the global table corresponding to a
 * specific originator. In particular, the checksum is computed as follows: For
 * each client connected to the originator the CRC32C of the MAC address and the
 * VID is computed and then all the CRC32Cs of the various clients are xor'ed
 * together.
 *
 * The idea behind is that CRC32C should be used as much as possible in order to
 * produce a unique hash of the table, but since the order which is used to feed
 * the CRC32C function affects the result and since every node in the network
 * probably sorts the clients differently, the hash function cannot be directly
 * computed over the entire table. Hence the CRC32C is used only on
 * the single client entry, while all the results are then xor'ed together
 * because the XOR operation can combine them all while trying to reduce the
 * noise as much as possible.
 *
 * Return: the checksum of the global table of a given originator.
 */
static u32 batadv_tt_global_crc(struct batadv_priv *bat_priv,
                                struct batadv_orig_node *orig_node,
                                unsigned short vid)
{
        struct batadv_hashtable *hash = bat_priv->tt.global_hash;
        struct batadv_tt_orig_list_entry *tt_orig;
        struct batadv_tt_common_entry *tt_common;
        struct batadv_tt_global_entry *tt_global;
        struct hlist_head *head;
        u32 i, crc_tmp, crc = 0;
        u8 flags;
        __be16 tmp_vid;

        for (i = 0; i < hash->size; i++) {
                head = &hash->table[i];

                rcu_read_lock();
                hlist_for_each_entry_rcu(tt_common, head, hash_entry) {
                        tt_global = container_of(tt_common,
                                                 struct batadv_tt_global_entry,
                                                 common);
                        /* compute the CRC only for entries belonging to the
                         * VLAN identified by the vid passed as parameter
                         */
                        if (tt_common->vid != vid)
                                continue;

                        /* Roaming clients are in the global table for
                         * consistency only. They don't have to be
                         * taken into account while computing the
                         * global crc
                         */
                        if (tt_common->flags & BATADV_TT_CLIENT_ROAM)
                                continue;
                        /* Temporary clients have not been announced yet, so
                         * they have to be skipped while computing the global
                         * crc
                         */
                        if (tt_common->flags & BATADV_TT_CLIENT_TEMP)
                                continue;

                        /* find out if this global entry is announced by this
                         * originator
                         */
                        tt_orig = batadv_tt_global_orig_entry_find(tt_global,
                                                                   orig_node);
                        if (!tt_orig)
                                continue;

                        /* use network order to read the VID: this ensures that
                         * every node reads the bytes in the same order.
                         */
                        tmp_vid = htons(tt_common->vid);
                        crc_tmp = crc32c(0, &tmp_vid, sizeof(tmp_vid));

                        /* compute the CRC on flags that have to be kept in sync
                         * among nodes
                         */
                        flags = tt_orig->flags;
                        crc_tmp = crc32c(crc_tmp, &flags, sizeof(flags));

                        crc ^= crc32c(crc_tmp, tt_common->addr, ETH_ALEN);

                        batadv_tt_orig_list_entry_put(tt_orig);
                }
                rcu_read_unlock();
        }

        return crc;
}

/**
 * batadv_tt_local_crc() - calculates the checksum of the local table
 * @bat_priv: the bat priv with all the soft interface information
 * @vid: VLAN identifier for which the CRC32 has to be computed
 *
 * For details about the computation, please refer to the documentation for
 * batadv_tt_global_crc().
 *
 * Return: the checksum of the local table
 */
static u32 batadv_tt_local_crc(struct batadv_priv *bat_priv,
                               unsigned short vid)
{
        struct batadv_hashtable *hash = bat_priv->tt.local_hash;
        struct batadv_tt_common_entry *tt_common;
        struct hlist_head *head;
        u32 i, crc_tmp, crc = 0;
        u8 flags;
        __be16 tmp_vid;

        for (i = 0; i < hash->size; i++) {
                head = &hash->table[i];

                rcu_read_lock();
                hlist_for_each_entry_rcu(tt_common, head, hash_entry) {
                        /* compute the CRC only for entries belonging to the
                         * VLAN identified by vid
                         */
                        if (tt_common->vid != vid)
                                continue;

                        /* not yet committed clients have not to be taken into
                         * account while computing the CRC
                         */
                        if (tt_common->flags & BATADV_TT_CLIENT_NEW)
                                continue;

                        /* use network order to read the VID: this ensures that
                         * every node reads the bytes in the same order.
                         */
                        tmp_vid = htons(tt_common->vid);
                        crc_tmp = crc32c(0, &tmp_vid, sizeof(tmp_vid));

                        /* compute the CRC on flags that have to be kept in sync
                         * among nodes
                         */
                        flags = tt_common->flags & BATADV_TT_SYNC_MASK;
                        crc_tmp = crc32c(crc_tmp, &flags, sizeof(flags));

                        crc ^= crc32c(crc_tmp, tt_common->addr, ETH_ALEN);
                }
                rcu_read_unlock();
        }

        return crc;
}

/**
 * batadv_tt_req_node_release() - free tt_req node entry
 * @ref: kref pointer of the tt req_node entry
 */
static void batadv_tt_req_node_release(struct kref *ref)
{
        struct batadv_tt_req_node *tt_req_node;

        tt_req_node = container_of(ref, struct batadv_tt_req_node, refcount);

        kmem_cache_free(batadv_tt_req_cache, tt_req_node);
}

/**
 * batadv_tt_req_node_put() - decrement the tt_req_node refcounter and
 *  possibly release it
 * @tt_req_node: tt_req_node to be free'd
 */
static void batadv_tt_req_node_put(struct batadv_tt_req_node *tt_req_node)
{
        if (!tt_req_node)
                return;

        kref_put(&tt_req_node->refcount, batadv_tt_req_node_release);
}

static void batadv_tt_req_list_free(struct batadv_priv *bat_priv)
{
        struct batadv_tt_req_node *node;
        struct hlist_node *safe;

        spin_lock_bh(&bat_priv->tt.req_list_lock);

        hlist_for_each_entry_safe(node, safe, &bat_priv->tt.req_list, list) {
                hlist_del_init(&node->list);
                batadv_tt_req_node_put(node);
        }

        spin_unlock_bh(&bat_priv->tt.req_list_lock);
}

static void batadv_tt_save_orig_buffer(struct batadv_priv *bat_priv,
                                       struct batadv_orig_node *orig_node,
                                       const void *tt_buff,
                                       u16 tt_buff_len)
{
        /* Replace the old buffer only if I received something in the
         * last OGM (the OGM could carry no changes)
         */
        spin_lock_bh(&orig_node->tt_buff_lock);
        if (tt_buff_len > 0) {
                kfree(orig_node->tt_buff);
                orig_node->tt_buff_len = 0;
                orig_node->tt_buff = kmalloc(tt_buff_len, GFP_ATOMIC);
                if (orig_node->tt_buff) {
                        memcpy(orig_node->tt_buff, tt_buff, tt_buff_len);
                        orig_node->tt_buff_len = tt_buff_len;
                }
        }
        spin_unlock_bh(&orig_node->tt_buff_lock);
}

static void batadv_tt_req_purge(struct batadv_priv *bat_priv)
{
        struct batadv_tt_req_node *node;
        struct hlist_node *safe;

        spin_lock_bh(&bat_priv->tt.req_list_lock);
        hlist_for_each_entry_safe(node, safe, &bat_priv->tt.req_list, list) {
                if (batadv_has_timed_out(node->issued_at,
                                         BATADV_TT_REQUEST_TIMEOUT)) {
                        hlist_del_init(&node->list);
                        batadv_tt_req_node_put(node);
                }
        }
        spin_unlock_bh(&bat_priv->tt.req_list_lock);
}

/**
 * batadv_tt_req_node_new() - search and possibly create a tt_req_node object
 * @bat_priv: the bat priv with all the soft interface information
 * @orig_node: orig node this request is being issued for
 *
 * Return: the pointer to the new tt_req_node struct if no request
 * has already been issued for this orig_node, NULL otherwise.
 */
static struct batadv_tt_req_node *
batadv_tt_req_node_new(struct batadv_priv *bat_priv,
                       struct batadv_orig_node *orig_node)
{
        struct batadv_tt_req_node *tt_req_node_tmp, *tt_req_node = NULL;

        spin_lock_bh(&bat_priv->tt.req_list_lock);
        hlist_for_each_entry(tt_req_node_tmp, &bat_priv->tt.req_list, list) {
                if (batadv_compare_eth(tt_req_node_tmp, orig_node) &&
                    !batadv_has_timed_out(tt_req_node_tmp->issued_at,
                                          BATADV_TT_REQUEST_TIMEOUT))
                        goto unlock;
        }

        tt_req_node = kmem_cache_alloc(batadv_tt_req_cache, GFP_ATOMIC);
        if (!tt_req_node)
                goto unlock;

        kref_init(&tt_req_node->refcount);
        ether_addr_copy(tt_req_node->addr, orig_node->orig);
        tt_req_node->issued_at = jiffies;

        kref_get(&tt_req_node->refcount);
        hlist_add_head(&tt_req_node->list, &bat_priv->tt.req_list);
unlock:
        spin_unlock_bh(&bat_priv->tt.req_list_lock);
        return tt_req_node;
}

/**
 * batadv_tt_local_valid() - verify local tt entry and get flags
 * @entry_ptr: to be checked local tt entry
 * @data_ptr: not used but definition required to satisfy the callback prototype
 * @flags: a pointer to store TT flags for this client to
 *
 * Checks the validity of the given local TT entry. If it is, then the provided
 * flags pointer is updated.
 *
 * Return: true if the entry is a valid, false otherwise.
 */
static bool batadv_tt_local_valid(const void *entry_ptr,
                                  const void *data_ptr,
                                  u8 *flags)
{
        const struct batadv_tt_common_entry *tt_common_entry = entry_ptr;

        if (tt_common_entry->flags & BATADV_TT_CLIENT_NEW)
                return false;

        if (flags)
                *flags = tt_common_entry->flags;

        return true;
}

/**
 * batadv_tt_global_valid() - verify global tt entry and get flags
 * @entry_ptr: to be checked global tt entry
 * @data_ptr: an orig_node object (may be NULL)
 * @flags: a pointer to store TT flags for this client to
 *
 * Checks the validity of the given global TT entry. If it is, then the provided
 * flags pointer is updated either with the common (summed) TT flags if data_ptr
 * is NULL or the specific, per originator TT flags otherwise.
 *
 * Return: true if the entry is a valid, false otherwise.
 */
static bool batadv_tt_global_valid(const void *entry_ptr,
                                   const void *data_ptr,
                                   u8 *flags)
{
        const struct batadv_tt_common_entry *tt_common_entry = entry_ptr;
        const struct batadv_tt_global_entry *tt_global_entry;
        const struct batadv_orig_node *orig_node = data_ptr;

        if (tt_common_entry->flags & BATADV_TT_CLIENT_ROAM ||
            tt_common_entry->flags & BATADV_TT_CLIENT_TEMP)
                return false;

        tt_global_entry = container_of(tt_common_entry,
                                       struct batadv_tt_global_entry,
                                       common);

        return batadv_tt_global_entry_has_orig(tt_global_entry, orig_node,
                                               flags);
}

/**
 * batadv_tt_tvlv_generate() - fill the tvlv buff with the tt entries from the
 *  specified tt hash
 * @bat_priv: the bat priv with all the soft interface information
 * @hash: hash table containing the tt entries
 * @tt_len: expected tvlv tt data buffer length in number of bytes
 * @tvlv_buff: pointer to the buffer to fill with the TT data
 * @valid_cb: function to filter tt change entries and to return TT flags
 * @cb_data: data passed to the filter function as argument
 *
 * Fills the tvlv buff with the tt entries from the specified hash. If valid_cb
 * is not provided then this becomes a no-op.
 *
 * Return: Remaining unused length in tvlv_buff.
 */
static u16 batadv_tt_tvlv_generate(struct batadv_priv *bat_priv,
                                   struct batadv_hashtable *hash,
                                   void *tvlv_buff, u16 tt_len,
                                   bool (*valid_cb)(const void *,
                                                    const void *,
                                                    u8 *flags),
                                   void *cb_data)
{
        struct batadv_tt_common_entry *tt_common_entry;
        struct batadv_tvlv_tt_change *tt_change;
        struct hlist_head *head;
        u16 tt_tot, tt_num_entries = 0;
        u8 flags;
        bool ret;
        u32 i;

        tt_tot = batadv_tt_entries(tt_len);
        tt_change = (struct batadv_tvlv_tt_change *)tvlv_buff;

        if (!valid_cb)
                return tt_len;

        rcu_read_lock();
        for (i = 0; i < hash->size; i++) {
                head = &hash->table[i];

                hlist_for_each_entry_rcu(tt_common_entry,
                                         head, hash_entry) {
                        if (tt_tot == tt_num_entries)
                                break;

                        ret = valid_cb(tt_common_entry, cb_data, &flags);
                        if (!ret)
                                continue;

                        ether_addr_copy(tt_change->addr, tt_common_entry->addr);
                        tt_change->flags = flags;
                        tt_change->vid = htons(tt_common_entry->vid);
                        memset(tt_change->reserved, 0,
                               sizeof(tt_change->reserved));

                        tt_num_entries++;
                        tt_change++;
                }
        }
        rcu_read_unlock();

        return batadv_tt_len(tt_tot - tt_num_entries);
}

/**
 * batadv_tt_global_check_crc() - check if all the CRCs are correct
 * @orig_node: originator for which the CRCs have to be checked
 * @tt_vlan: pointer to the first tvlv VLAN entry
 * @num_vlan: number of tvlv VLAN entries
 *
 * Return: true if all the received CRCs match the locally stored ones, false
 * otherwise
 */
static bool batadv_tt_global_check_crc(struct batadv_orig_node *orig_node,
                                       struct batadv_tvlv_tt_vlan_data *tt_vlan,
                                       u16 num_vlan)
{
        struct batadv_tvlv_tt_vlan_data *tt_vlan_tmp;
        struct batadv_orig_node_vlan *vlan;
        int i, orig_num_vlan;
        u32 crc;

        /* check if each received CRC matches the locally stored one */
        for (i = 0; i < num_vlan; i++) {
                tt_vlan_tmp = tt_vlan + i;

                /* if orig_node is a backbone node for this VLAN, don't check
                 * the CRC as we ignore all the global entries over it
                 */
                if (batadv_bla_is_backbone_gw_orig(orig_node->bat_priv,
                                                   orig_node->orig,
                                                   ntohs(tt_vlan_tmp->vid)))
                        continue;

                vlan = batadv_orig_node_vlan_get(orig_node,
                                                 ntohs(tt_vlan_tmp->vid));
                if (!vlan)
                        return false;

                crc = vlan->tt.crc;
                batadv_orig_node_vlan_put(vlan);

                if (crc != ntohl(tt_vlan_tmp->crc))
                        return false;
        }

        /* check if any excess VLANs exist locally for the originator
         * which are not mentioned in the TVLV from the originator.
         */
        rcu_read_lock();
        orig_num_vlan = 0;
        hlist_for_each_entry_rcu(vlan, &orig_node->vlan_list, list)
                orig_num_vlan++;
        rcu_read_unlock();

        if (orig_num_vlan > num_vlan)
                return false;

        return true;
}

/**
 * batadv_tt_local_update_crc() - update all the local CRCs
 * @bat_priv: the bat priv with all the soft interface information
 */
static void batadv_tt_local_update_crc(struct batadv_priv *bat_priv)
{
        struct batadv_softif_vlan *vlan;

        /* recompute the global CRC for each VLAN */
        rcu_read_lock();
        hlist_for_each_entry_rcu(vlan, &bat_priv->softif_vlan_list, list) {
                vlan->tt.crc = batadv_tt_local_crc(bat_priv, vlan->vid);
        }
        rcu_read_unlock();
}

/**
 * batadv_tt_global_update_crc() - update all the global CRCs for this orig_node
 * @bat_priv: the bat priv with all the soft interface information
 * @orig_node: the orig_node for which the CRCs have to be updated
 */
static void batadv_tt_global_update_crc(struct batadv_priv *bat_priv,
                                        struct batadv_orig_node *orig_node)
{
        struct batadv_orig_node_vlan *vlan;
        u32 crc;

        /* recompute the global CRC for each VLAN */
        rcu_read_lock();
        hlist_for_each_entry_rcu(vlan, &orig_node->vlan_list, list) {
                /* if orig_node is a backbone node for this VLAN, don't compute
                 * the CRC as we ignore all the global entries over it
                 */
                if (batadv_bla_is_backbone_gw_orig(bat_priv, orig_node->orig,
                                                   vlan->vid))
                        continue;

                crc = batadv_tt_global_crc(bat_priv, orig_node, vlan->vid);
                vlan->tt.crc = crc;
        }
        rcu_read_unlock();
}

/**
 * batadv_send_tt_request() - send a TT Request message to a given node
 * @bat_priv: the bat priv with all the soft interface information
 * @dst_orig_node: the destination of the message
 * @ttvn: the version number that the source of the message is looking for
 * @tt_vlan: pointer to the first tvlv VLAN object to request
 * @num_vlan: number of tvlv VLAN entries
 * @full_table: ask for the entire translation table if true, while only for the
 *  last TT diff otherwise
 *
 * Return: true if the TT Request was sent, false otherwise
 */
static bool batadv_send_tt_request(struct batadv_priv *bat_priv,
                                   struct batadv_orig_node *dst_orig_node,
                                   u8 ttvn,
                                   struct batadv_tvlv_tt_vlan_data *tt_vlan,
                                   u16 num_vlan, bool full_table)
{
        struct batadv_tvlv_tt_data *tvlv_tt_data = NULL;
        struct batadv_tt_req_node *tt_req_node = NULL;
        struct batadv_tvlv_tt_vlan_data *tt_vlan_req;
        struct batadv_hard_iface *primary_if;
        bool ret = false;
        int i, size;

        primary_if = batadv_primary_if_get_selected(bat_priv);
        if (!primary_if)
                goto out;

        /* The new tt_req will be issued only if I'm not waiting for a
         * reply from the same orig_node yet
         */
        tt_req_node = batadv_tt_req_node_new(bat_priv, dst_orig_node);
        if (!tt_req_node)
                goto out;

        size = sizeof(*tvlv_tt_data) + sizeof(*tt_vlan_req) * num_vlan;
        tvlv_tt_data = kzalloc(size, GFP_ATOMIC);
        if (!tvlv_tt_data)
                goto out;

        tvlv_tt_data->flags = BATADV_TT_REQUEST;
        tvlv_tt_data->ttvn = ttvn;
        tvlv_tt_data->num_vlan = htons(num_vlan);

        /* send all the CRCs within the request. This is needed by intermediate
         * nodes to ensure they have the correct table before replying
         */
        tt_vlan_req = (struct batadv_tvlv_tt_vlan_data *)(tvlv_tt_data + 1);
        for (i = 0; i < num_vlan; i++) {
                tt_vlan_req->vid = tt_vlan->vid;
                tt_vlan_req->crc = tt_vlan->crc;

                tt_vlan_req++;
                tt_vlan++;
        }

        if (full_table)
                tvlv_tt_data->flags |= BATADV_TT_FULL_TABLE;

        batadv_dbg(BATADV_DBG_TT, bat_priv, "Sending TT_REQUEST to %pM [%c]\n",
                   dst_orig_node->orig, full_table ? 'F' : '.');

        batadv_inc_counter(bat_priv, BATADV_CNT_TT_REQUEST_TX);
        batadv_tvlv_unicast_send(bat_priv, primary_if->net_dev->dev_addr,
                                 dst_orig_node->orig, BATADV_TVLV_TT, 1,
                                 tvlv_tt_data, size);
        ret = true;

out:
        batadv_hardif_put(primary_if);

        if (ret && tt_req_node) {
                spin_lock_bh(&bat_priv->tt.req_list_lock);
                if (!hlist_unhashed(&tt_req_node->list)) {
                        hlist_del_init(&tt_req_node->list);
                        batadv_tt_req_node_put(tt_req_node);
                }
                spin_unlock_bh(&bat_priv->tt.req_list_lock);
        }

        batadv_tt_req_node_put(tt_req_node);

        kfree(tvlv_tt_data);
        return ret;
}

/**
 * batadv_send_other_tt_response() - send reply to tt request concerning another
 *  node's translation table
 * @bat_priv: the bat priv with all the soft interface information
 * @tt_data: tt data containing the tt request information
 * @req_src: mac address of tt request sender
 * @req_dst: mac address of tt request recipient
 *
 * Return: true if tt request reply was sent, false otherwise.
 */
static bool batadv_send_other_tt_response(struct batadv_priv *bat_priv,
                                          struct batadv_tvlv_tt_data *tt_data,
                                          u8 *req_src, u8 *req_dst)
{
        struct batadv_orig_node *req_dst_orig_node;
        struct batadv_orig_node *res_dst_orig_node = NULL;
        struct batadv_tvlv_tt_change *tt_change;
        struct batadv_tvlv_tt_data *tvlv_tt_data = NULL;
        struct batadv_tvlv_tt_vlan_data *tt_vlan;
        bool ret = false, full_table;
        u8 orig_ttvn, req_ttvn;
        u16 tvlv_len;
        s32 tt_len;

        batadv_dbg(BATADV_DBG_TT, bat_priv,
                   "Received TT_REQUEST from %pM for ttvn: %u (%pM) [%c]\n",
                   req_src, tt_data->ttvn, req_dst,
                   ((tt_data->flags & BATADV_TT_FULL_TABLE) ? 'F' : '.'));

        /* Let's get the orig node of the REAL destination */
        req_dst_orig_node = batadv_orig_hash_find(bat_priv, req_dst);
        if (!req_dst_orig_node)
                goto out;

        res_dst_orig_node = batadv_orig_hash_find(bat_priv, req_src);
        if (!res_dst_orig_node)
                goto out;

        orig_ttvn = (u8)atomic_read(&req_dst_orig_node->last_ttvn);
        req_ttvn = tt_data->ttvn;

        tt_vlan = (struct batadv_tvlv_tt_vlan_data *)(tt_data + 1);
        /* this node doesn't have the requested data */
        if (orig_ttvn != req_ttvn ||
            !batadv_tt_global_check_crc(req_dst_orig_node, tt_vlan,
                                        ntohs(tt_data->num_vlan)))
                goto out;

        /* If the full table has been explicitly requested */
        if (tt_data->flags & BATADV_TT_FULL_TABLE ||
            !req_dst_orig_node->tt_buff)
                full_table = true;
        else
                full_table = false;

        /* TT fragmentation hasn't been implemented yet, so send as many
         * TT entries fit a single packet as possible only
         */
        if (!full_table) {
                spin_lock_bh(&req_dst_orig_node->tt_buff_lock);
                tt_len = req_dst_orig_node->tt_buff_len;

                tvlv_len = batadv_tt_prepare_tvlv_global_data(req_dst_orig_node,
                                                              &tvlv_tt_data,
                                                              &tt_change,
                                                              &tt_len);
                if (!tt_len)
                        goto unlock;

                /* Copy the last orig_node's OGM buffer */
                memcpy(tt_change, req_dst_orig_node->tt_buff,
                       req_dst_orig_node->tt_buff_len);
                spin_unlock_bh(&req_dst_orig_node->tt_buff_lock);
        } else {
                /* allocate the tvlv, put the tt_data and all the tt_vlan_data
                 * in the initial part
                 */
                tt_len = -1;
                tvlv_len = batadv_tt_prepare_tvlv_global_data(req_dst_orig_node,
                                                              &tvlv_tt_data,
                                                              &tt_change,
                                                              &tt_len);
                if (!tt_len)
                        goto out;

                /* fill the rest of the tvlv with the real TT entries */
                tvlv_len -= batadv_tt_tvlv_generate(bat_priv,
                                                    bat_priv->tt.global_hash,
                                                    tt_change, tt_len,
                                                    batadv_tt_global_valid,
                                                    req_dst_orig_node);
        }

        /* Don't send the response, if larger than fragmented packet. */
        tt_len = sizeof(struct batadv_unicast_tvlv_packet) + tvlv_len;
        if (tt_len > atomic_read(&bat_priv->packet_size_max)) {
                net_ratelimited_function(batadv_info, bat_priv->soft_iface,
                                         "Ignoring TT_REQUEST from %pM; Response size exceeds max packet size.\n",
                                         res_dst_orig_node->orig);
                goto out;
        }

        tvlv_tt_data->flags = BATADV_TT_RESPONSE;
        tvlv_tt_data->ttvn = req_ttvn;

        if (full_table)
                tvlv_tt_data->flags |= BATADV_TT_FULL_TABLE;

        batadv_dbg(BATADV_DBG_TT, bat_priv,
                   "Sending TT_RESPONSE %pM for %pM [%c] (ttvn: %u)\n",
                   res_dst_orig_node->orig, req_dst_orig_node->orig,
                   full_table ? 'F' : '.', req_ttvn);

        batadv_inc_counter(bat_priv, BATADV_CNT_TT_RESPONSE_TX);

        batadv_tvlv_unicast_send(bat_priv, req_dst_orig_node->orig,
                                 req_src, BATADV_TVLV_TT, 1, tvlv_tt_data,
                                 tvlv_len);

        ret = true;
        goto out;

unlock:
        spin_unlock_bh(&req_dst_orig_node->tt_buff_lock);

out:
        batadv_orig_node_put(res_dst_orig_node);
        batadv_orig_node_put(req_dst_orig_node);
        kfree(tvlv_tt_data);
        return ret;
}

/**
 * batadv_send_my_tt_response() - send reply to tt request concerning this
 *  node's translation table
 * @bat_priv: the bat priv with all the soft interface information
 * @tt_data: tt data containing the tt request information
 * @req_src: mac address of tt request sender
 *
 * Return: true if tt request reply was sent, false otherwise.
 */
static bool batadv_send_my_tt_response(struct batadv_priv *bat_priv,
                                       struct batadv_tvlv_tt_data *tt_data,
                                       u8 *req_src)
{
        struct batadv_tvlv_tt_data *tvlv_tt_data = NULL;
        struct batadv_hard_iface *primary_if = NULL;
        struct batadv_tvlv_tt_change *tt_change;
        struct batadv_orig_node *orig_node;
        u8 my_ttvn, req_ttvn;
        u16 tvlv_len;
        bool full_table;
        s32 tt_len;

        batadv_dbg(BATADV_DBG_TT, bat_priv,
                   "Received TT_REQUEST from %pM for ttvn: %u (me) [%c]\n",
                   req_src, tt_data->ttvn,
                   ((tt_data->flags & BATADV_TT_FULL_TABLE) ? 'F' : '.'));

        spin_lock_bh(&bat_priv->tt.commit_lock);

        my_ttvn = (u8)atomic_read(&bat_priv->tt.vn);
        req_ttvn = tt_data->ttvn;

        orig_node = batadv_orig_hash_find(bat_priv, req_src);
        if (!orig_node)
                goto out;

        primary_if = batadv_primary_if_get_selected(bat_priv);
        if (!primary_if)
                goto out;

        /* If the full table has been explicitly requested or the gap
         * is too big send the whole local translation table
         */
        if (tt_data->flags & BATADV_TT_FULL_TABLE || my_ttvn != req_ttvn ||
            !bat_priv->tt.last_changeset)
                full_table = true;
        else
                full_table = false;

        /* TT fragmentation hasn't been implemented yet, so send as many
         * TT entries fit a single packet as possible only
         */
        if (!full_table) {
                spin_lock_bh(&bat_priv->tt.last_changeset_lock);

                tt_len = bat_priv->tt.last_changeset_len;
                tvlv_len = batadv_tt_prepare_tvlv_local_data(bat_priv,
                                                             &tvlv_tt_data,
                                                             &tt_change,
                                                             &tt_len);
                if (!tt_len || !tvlv_len)
                        goto unlock;

                /* Copy the last orig_node's OGM buffer */
                memcpy(tt_change, bat_priv->tt.last_changeset,
                       bat_priv->tt.last_changeset_len);
                spin_unlock_bh(&bat_priv->tt.last_changeset_lock);
        } else {
                req_ttvn = (u8)atomic_read(&bat_priv->tt.vn);

                /* allocate the tvlv, put the tt_data and all the tt_vlan_data
                 * in the initial part
                 */
                tt_len = -1;
                tvlv_len = batadv_tt_prepare_tvlv_local_data(bat_priv,
                                                             &tvlv_tt_data,
                                                             &tt_change,
                                                             &tt_len);
                if (!tt_len || !tvlv_len)
                        goto out;

                /* fill the rest of the tvlv with the real TT entries */
                tvlv_len -= batadv_tt_tvlv_generate(bat_priv,
                                                    bat_priv->tt.local_hash,
                                                    tt_change, tt_len,
                                                    batadv_tt_local_valid,
                                                    NULL);
        }

        tvlv_tt_data->flags = BATADV_TT_RESPONSE;
        tvlv_tt_data->ttvn = req_ttvn;

        if (full_table)
                tvlv_tt_data->flags |= BATADV_TT_FULL_TABLE;

        batadv_dbg(BATADV_DBG_TT, bat_priv,
                   "Sending TT_RESPONSE to %pM [%c] (ttvn: %u)\n",
                   orig_node->orig, full_table ? 'F' : '.', req_ttvn);

        batadv_inc_counter(bat_priv, BATADV_CNT_TT_RESPONSE_TX);

        batadv_tvlv_unicast_send(bat_priv, primary_if->net_dev->dev_addr,
                                 req_src, BATADV_TVLV_TT, 1, tvlv_tt_data,
                                 tvlv_len);

        goto out;

unlock:
        spin_unlock_bh(&bat_priv->tt.last_changeset_lock);
out:
        spin_unlock_bh(&bat_priv->tt.commit_lock);
        batadv_orig_node_put(orig_node);
        batadv_hardif_put(primary_if);
        kfree(tvlv_tt_data);
        /* The packet was for this host, so it doesn't need to be re-routed */
        return true;
}

/**
 * batadv_send_tt_response() - send reply to tt request
 * @bat_priv: the bat priv with all the soft interface information
 * @tt_data: tt data containing the tt request information
 * @req_src: mac address of tt request sender
 * @req_dst: mac address of tt request recipient
 *
 * Return: true if tt request reply was sent, false otherwise.
 */
static bool batadv_send_tt_response(struct batadv_priv *bat_priv,
                                    struct batadv_tvlv_tt_data *tt_data,
                                    u8 *req_src, u8 *req_dst)
{
        if (batadv_is_my_mac(bat_priv, req_dst))
                return batadv_send_my_tt_response(bat_priv, tt_data, req_src);
        return batadv_send_other_tt_response(bat_priv, tt_data, req_src,
                                             req_dst);
}

static void _batadv_tt_update_changes(struct batadv_priv *bat_priv,
                                      struct batadv_orig_node *orig_node,
                                      struct batadv_tvlv_tt_change *tt_change,
                                      u16 tt_num_changes, u8 ttvn)
{
        int i;
        int roams;

        for (i = 0; i < tt_num_changes; i++) {
                if ((tt_change + i)->flags & BATADV_TT_CLIENT_DEL) {
                        roams = (tt_change + i)->flags & BATADV_TT_CLIENT_ROAM;
                        batadv_tt_global_del(bat_priv, orig_node,
                                             (tt_change + i)->addr,
                                             ntohs((tt_change + i)->vid),
                                             "tt removed by changes",
                                             roams);
                } else {
                        if (!batadv_tt_global_add(bat_priv, orig_node,
                                                  (tt_change + i)->addr,
                                                  ntohs((tt_change + i)->vid),
                                                  (tt_change + i)->flags, ttvn))
                                /* In case of problem while storing a
                                 * global_entry, we stop the updating
                                 * procedure without committing the
                                 * ttvn change. This will avoid to send
                                 * corrupted data on tt_request
                                 */
                                return;
                }
        }
        set_bit(BATADV_ORIG_CAPA_HAS_TT, &orig_node->capa_initialized);
}

static void batadv_tt_fill_gtable(struct batadv_priv *bat_priv,
                                  struct batadv_tvlv_tt_change *tt_change,
                                  u8 ttvn, u8 *resp_src,
                                  u16 num_entries)
{
        struct batadv_orig_node *orig_node;

        orig_node = batadv_orig_hash_find(bat_priv, resp_src);
        if (!orig_node)
                goto out;

        /* Purge the old table first.. */
        batadv_tt_global_del_orig(bat_priv, orig_node, -1,
                                  "Received full table");

        _batadv_tt_update_changes(bat_priv, orig_node, tt_change, num_entries,
                                  ttvn);

        spin_lock_bh(&orig_node->tt_buff_lock);
        kfree(orig_node->tt_buff);
        orig_node->tt_buff_len = 0;
        orig_node->tt_buff = NULL;
        spin_unlock_bh(&orig_node->tt_buff_lock);

        atomic_set(&orig_node->last_ttvn, ttvn);

out:
        batadv_orig_node_put(orig_node);
}

static void batadv_tt_update_changes(struct batadv_priv *bat_priv,
                                     struct batadv_orig_node *orig_node,
                                     u16 tt_num_changes, u8 ttvn,
                                     struct batadv_tvlv_tt_change *tt_change)
{
        _batadv_tt_update_changes(bat_priv, orig_node, tt_change,
                                  tt_num_changes, ttvn);

        batadv_tt_save_orig_buffer(bat_priv, orig_node, tt_change,
                                   batadv_tt_len(tt_num_changes));
        atomic_set(&orig_node->last_ttvn, ttvn);
}

/**
 * batadv_is_my_client() - check if a client is served by the local node
 * @bat_priv: the bat priv with all the soft interface information
 * @addr: the mac address of the client to check
 * @vid: VLAN identifier
 *
 * Return: true if the client is served by this node, false otherwise.
 */
bool batadv_is_my_client(struct batadv_priv *bat_priv, const u8 *addr,
                         unsigned short vid)
{
        struct batadv_tt_local_entry *tt_local_entry;
        bool ret = false;

        tt_local_entry = batadv_tt_local_hash_find(bat_priv, addr, vid);
        if (!tt_local_entry)
                goto out;
        /* Check if the client has been logically deleted (but is kept for
         * consistency purpose)
         */
        if ((tt_local_entry->common.flags & BATADV_TT_CLIENT_PENDING) ||
            (tt_local_entry->common.flags & BATADV_TT_CLIENT_ROAM))
                goto out;
        ret = true;
out:
        batadv_tt_local_entry_put(tt_local_entry);
        return ret;
}

/**
 * batadv_handle_tt_response() - process incoming tt reply
 * @bat_priv: the bat priv with all the soft interface information
 * @tt_data: tt data containing the tt request information
 * @resp_src: mac address of tt reply sender
 * @num_entries: number of tt change entries appended to the tt data
 */
static void batadv_handle_tt_response(struct batadv_priv *bat_priv,
                                      struct batadv_tvlv_tt_data *tt_data,
                                      u8 *resp_src, u16 num_entries)
{
        struct batadv_tt_req_node *node;
        struct hlist_node *safe;
        struct batadv_orig_node *orig_node = NULL;
        struct batadv_tvlv_tt_change *tt_change;
        u8 *tvlv_ptr = (u8 *)tt_data;
        u16 change_offset;

        batadv_dbg(BATADV_DBG_TT, bat_priv,
                   "Received TT_RESPONSE from %pM for ttvn %d t_size: %d [%c]\n",
                   resp_src, tt_data->ttvn, num_entries,
                   ((tt_data->flags & BATADV_TT_FULL_TABLE) ? 'F' : '.'));

        orig_node = batadv_orig_hash_find(bat_priv, resp_src);
        if (!orig_node)
                goto out;

        spin_lock_bh(&orig_node->tt_lock);

        change_offset = sizeof(struct batadv_tvlv_tt_vlan_data);
        change_offset *= ntohs(tt_data->num_vlan);
        change_offset += sizeof(*tt_data);
        tvlv_ptr += change_offset;

        tt_change = (struct batadv_tvlv_tt_change *)tvlv_ptr;
        if (tt_data->flags & BATADV_TT_FULL_TABLE) {
                batadv_tt_fill_gtable(bat_priv, tt_change, tt_data->ttvn,
                                      resp_src, num_entries);
        } else {
                batadv_tt_update_changes(bat_priv, orig_node, num_entries,
                                         tt_data->ttvn, tt_change);
        }

        /* Recalculate the CRC for this orig_node and store it */
        batadv_tt_global_update_crc(bat_priv, orig_node);

        spin_unlock_bh(&orig_node->tt_lock);

        /* Delete the tt_req_node from pending tt_requests list */
        spin_lock_bh(&bat_priv->tt.req_list_lock);
        hlist_for_each_entry_safe(node, safe, &bat_priv->tt.req_list, list) {
                if (!batadv_compare_eth(node->addr, resp_src))
                        continue;
                hlist_del_init(&node->list);
                batadv_tt_req_node_put(node);
        }

        spin_unlock_bh(&bat_priv->tt.req_list_lock);
out:
        batadv_orig_node_put(orig_node);
}

static void batadv_tt_roam_list_free(struct batadv_priv *bat_priv)
{
        struct batadv_tt_roam_node *node, *safe;

        spin_lock_bh(&bat_priv->tt.roam_list_lock);

        list_for_each_entry_safe(node, safe, &bat_priv->tt.roam_list, list) {
                list_del(&node->list);
                kmem_cache_free(batadv_tt_roam_cache, node);
        }

        spin_unlock_bh(&bat_priv->tt.roam_list_lock);
}

static void batadv_tt_roam_purge(struct batadv_priv *bat_priv)
{
        struct batadv_tt_roam_node *node, *safe;

        spin_lock_bh(&bat_priv->tt.roam_list_lock);
        list_for_each_entry_safe(node, safe, &bat_priv->tt.roam_list, list) {
                if (!batadv_has_timed_out(node->first_time,
                                          BATADV_ROAMING_MAX_TIME))
                        continue;

                list_del(&node->list);
                kmem_cache_free(batadv_tt_roam_cache, node);
        }
        spin_unlock_bh(&bat_priv->tt.roam_list_lock);
}

/**
 * batadv_tt_check_roam_count() - check if a client has roamed too frequently
 * @bat_priv: the bat priv with all the soft interface information
 * @client: mac address of the roaming client
 *
 * This function checks whether the client already reached the
 * maximum number of possible roaming phases. In this case the ROAMING_ADV
 * will not be sent.
 *
 * Return: true if the ROAMING_ADV can be sent, false otherwise
 */
static bool batadv_tt_check_roam_count(struct batadv_priv *bat_priv, u8 *client)
{
        struct batadv_tt_roam_node *tt_roam_node;
        bool ret = false;

        spin_lock_bh(&bat_priv->tt.roam_list_lock);
        /* The new tt_req will be issued only if I'm not waiting for a
         * reply from the same orig_node yet
         */
        list_for_each_entry(tt_roam_node, &bat_priv->tt.roam_list, list) {
                if (!batadv_compare_eth(tt_roam_node->addr, client))
                        continue;

                if (batadv_has_timed_out(tt_roam_node->first_time,
                                         BATADV_ROAMING_MAX_TIME))
                        continue;

                if (!batadv_atomic_dec_not_zero(&tt_roam_node->counter))
                        /* Sorry, you roamed too many times! */
                        goto unlock;
                ret = true;
                break;
        }

        if (!ret) {
                tt_roam_node = kmem_cache_alloc(batadv_tt_roam_cache,
                                                GFP_ATOMIC);
                if (!tt_roam_node)
                        goto unlock;

                tt_roam_node->first_time = jiffies;
                atomic_set(&tt_roam_node->counter,
                           BATADV_ROAMING_MAX_COUNT - 1);
                ether_addr_copy(tt_roam_node->addr, client);

                list_add(&tt_roam_node->list, &bat_priv->tt.roam_list);
                ret = true;
        }

unlock:
        spin_unlock_bh(&bat_priv->tt.roam_list_lock);
        return ret;
}

/**
 * batadv_send_roam_adv() - send a roaming advertisement message
 * @bat_priv: the bat priv with all the soft interface information
 * @client: mac address of the roaming client
 * @vid: VLAN identifier
 * @orig_node: message destination
 *
 * Send a ROAMING_ADV message to the node which was previously serving this
 * client. This is done to inform the node that from now on all traffic destined
 * for this particular roamed client has to be forwarded to the sender of the
 * roaming message.
 */
static void batadv_send_roam_adv(struct batadv_priv *bat_priv, u8 *client,
                                 unsigned short vid,
                                 struct batadv_orig_node *orig_node)
{
        struct batadv_hard_iface *primary_if;
        struct batadv_tvlv_roam_adv tvlv_roam;

        primary_if = batadv_primary_if_get_selected(bat_priv);
        if (!primary_if)
                goto out;

        /* before going on we have to check whether the client has
         * already roamed to us too many times
         */
        if (!batadv_tt_check_roam_count(bat_priv, client))
                goto out;

        batadv_dbg(BATADV_DBG_TT, bat_priv,
                   "Sending ROAMING_ADV to %pM (client %pM, vid: %d)\n",
                   orig_node->orig, client, batadv_print_vid(vid));

        batadv_inc_counter(bat_priv, BATADV_CNT_TT_ROAM_ADV_TX);

        memcpy(tvlv_roam.client, client, sizeof(tvlv_roam.client));
        tvlv_roam.vid = htons(vid);

        batadv_tvlv_unicast_send(bat_priv, primary_if->net_dev->dev_addr,
                                 orig_node->orig, BATADV_TVLV_ROAM, 1,
                                 &tvlv_roam, sizeof(tvlv_roam));

out:
        batadv_hardif_put(primary_if);
}

static void batadv_tt_purge(struct work_struct *work)
{
        struct delayed_work *delayed_work;
        struct batadv_priv_tt *priv_tt;
        struct batadv_priv *bat_priv;

        delayed_work = to_delayed_work(work);
        priv_tt = container_of(delayed_work, struct batadv_priv_tt, work);
        bat_priv = container_of(priv_tt, struct batadv_priv, tt);

        batadv_tt_local_purge(bat_priv, BATADV_TT_LOCAL_TIMEOUT);
        batadv_tt_global_purge(bat_priv);
        batadv_tt_req_purge(bat_priv);
        batadv_tt_roam_purge(bat_priv);

        queue_delayed_work(batadv_event_workqueue, &bat_priv->tt.work,
                           msecs_to_jiffies(BATADV_TT_WORK_PERIOD));
}

/**
 * batadv_tt_free() - Free translation table of soft interface
 * @bat_priv: the bat priv with all the soft interface information
 */
void batadv_tt_free(struct batadv_priv *bat_priv)
{
        batadv_tvlv_handler_unregister(bat_priv, BATADV_TVLV_ROAM, 1);

        batadv_tvlv_container_unregister(bat_priv, BATADV_TVLV_TT, 1);
        batadv_tvlv_handler_unregister(bat_priv, BATADV_TVLV_TT, 1);

        cancel_delayed_work_sync(&bat_priv->tt.work);

        batadv_tt_local_table_free(bat_priv);
        batadv_tt_global_table_free(bat_priv);
        batadv_tt_req_list_free(bat_priv);
        batadv_tt_changes_list_free(bat_priv);
        batadv_tt_roam_list_free(bat_priv);

        kfree(bat_priv->tt.last_changeset);
}

/**
 * batadv_tt_local_set_flags() - set or unset the specified flags on the local
 *  table and possibly count them in the TT size
 * @bat_priv: the bat priv with all the soft interface information
 * @flags: the flag to switch
 * @enable: whether to set or unset the flag
 * @count: whether to increase the TT size by the number of changed entries
 */
static void batadv_tt_local_set_flags(struct batadv_priv *bat_priv, u16 flags,
                                      bool enable, bool count)
{
        struct batadv_hashtable *hash = bat_priv->tt.local_hash;
        struct batadv_tt_common_entry *tt_common_entry;
        struct hlist_head *head;
        u32 i;

        if (!hash)
                return;

        for (i = 0; i < hash->size; i++) {
                head = &hash->table[i];

                rcu_read_lock();
                hlist_for_each_entry_rcu(tt_common_entry,
                                         head, hash_entry) {
                        if (enable) {
                                if ((tt_common_entry->flags & flags) == flags)
                                        continue;
                                tt_common_entry->flags |= flags;
                        } else {
                                if (!(tt_common_entry->flags & flags))
                                        continue;
                                tt_common_entry->flags &= ~flags;
                        }

                        if (!count)
                                continue;

                        batadv_tt_local_size_inc(bat_priv,
                                                 tt_common_entry->vid);
                }
                rcu_read_unlock();
        }
}

/* Purge out all the tt local entries marked with BATADV_TT_CLIENT_PENDING */
static void batadv_tt_local_purge_pending_clients(struct batadv_priv *bat_priv)
{
        struct batadv_hashtable *hash = bat_priv->tt.local_hash;
        struct batadv_tt_common_entry *tt_common;
        struct batadv_tt_local_entry *tt_local;
        struct hlist_node *node_tmp;
        struct hlist_head *head;
        spinlock_t *list_lock; /* protects write access to the hash lists */
        u32 i;

        if (!hash)
                return;

        for (i = 0; i < hash->size; i++) {
                head = &hash->table[i];
                list_lock = &hash->list_locks[i];

                spin_lock_bh(list_lock);
                hlist_for_each_entry_safe(tt_common, node_tmp, head,
                                          hash_entry) {
                        if (!(tt_common->flags & BATADV_TT_CLIENT_PENDING))
                                continue;

                        batadv_dbg(BATADV_DBG_TT, bat_priv,
                                   "Deleting local tt entry (%pM, vid: %d): pending\n",
                                   tt_common->addr,
                                   batadv_print_vid(tt_common->vid));

                        batadv_tt_local_size_dec(bat_priv, tt_common->vid);
                        hlist_del_rcu(&tt_common->hash_entry);
                        tt_local = container_of(tt_common,
                                                struct batadv_tt_local_entry,
                                                common);

                        batadv_tt_local_entry_put(tt_local);
                }
                spin_unlock_bh(list_lock);
        }
}

/**
 * batadv_tt_local_commit_changes_nolock() - commit all pending local tt changes
 *  which have been queued in the time since the last commit
 * @bat_priv: the bat priv with all the soft interface information
 *
 * Caller must hold tt->commit_lock.
 */
static void batadv_tt_local_commit_changes_nolock(struct batadv_priv *bat_priv)
{
        lockdep_assert_held(&bat_priv->tt.commit_lock);

        if (atomic_read(&bat_priv->tt.local_changes) < 1) {
                if (!batadv_atomic_dec_not_zero(&bat_priv->tt.ogm_append_cnt))
                        batadv_tt_tvlv_container_update(bat_priv);
                return;
        }

        batadv_tt_local_set_flags(bat_priv, BATADV_TT_CLIENT_NEW, false, true);

        batadv_tt_local_purge_pending_clients(bat_priv);
        batadv_tt_local_update_crc(bat_priv);

        /* Increment the TTVN only once per OGM interval */
        atomic_inc(&bat_priv->tt.vn);
        batadv_dbg(BATADV_DBG_TT, bat_priv,
                   "Local changes committed, updating to ttvn %u\n",
                   (u8)atomic_read(&bat_priv->tt.vn));

        /* reset the sending counter */
        atomic_set(&bat_priv->tt.ogm_append_cnt, BATADV_TT_OGM_APPEND_MAX);
        batadv_tt_tvlv_container_update(bat_priv);
}

/**
 * batadv_tt_local_commit_changes() - commit all pending local tt changes which
 *  have been queued in the time since the last commit
 * @bat_priv: the bat priv with all the soft interface information
 */
void batadv_tt_local_commit_changes(struct batadv_priv *bat_priv)
{
        spin_lock_bh(&bat_priv->tt.commit_lock);
        batadv_tt_local_commit_changes_nolock(bat_priv);
        spin_unlock_bh(&bat_priv->tt.commit_lock);
}

/**
 * batadv_is_ap_isolated() - Check if packet from upper layer should be dropped
 * @bat_priv: the bat priv with all the soft interface information
 * @src: source mac address of packet
 * @dst: destination mac address of packet
 * @vid: vlan id of packet
 *
 * Return: true when src+dst(+vid) pair should be isolated, false otherwise
 */
bool batadv_is_ap_isolated(struct batadv_priv *bat_priv, u8 *src, u8 *dst,
                           unsigned short vid)
{
        struct batadv_tt_local_entry *tt_local_entry;
        struct batadv_tt_global_entry *tt_global_entry;
        struct batadv_softif_vlan *vlan;
        bool ret = false;

        vlan = batadv_softif_vlan_get(bat_priv, vid);
        if (!vlan)
                return false;

        if (!atomic_read(&vlan->ap_isolation))
                goto vlan_put;

        tt_local_entry = batadv_tt_local_hash_find(bat_priv, dst, vid);
        if (!tt_local_entry)
                goto vlan_put;

        tt_global_entry = batadv_tt_global_hash_find(bat_priv, src, vid);
        if (!tt_global_entry)
                goto local_entry_put;

        if (_batadv_is_ap_isolated(tt_local_entry, tt_global_entry))
                ret = true;

        batadv_tt_global_entry_put(tt_global_entry);
local_entry_put:
        batadv_tt_local_entry_put(tt_local_entry);
vlan_put:
        batadv_softif_vlan_put(vlan);
        return ret;
}

/**
 * batadv_tt_update_orig() - update global translation table with new tt
 *  information received via ogms
 * @bat_priv: the bat priv with all the soft interface information
 * @orig_node: the orig_node of the ogm
 * @tt_buff: pointer to the first tvlv VLAN entry
 * @tt_num_vlan: number of tvlv VLAN entries
 * @tt_change: pointer to the first entry in the TT buffer
 * @tt_num_changes: number of tt changes inside the tt buffer
 * @ttvn: translation table version number of this changeset
 */
static void batadv_tt_update_orig(struct batadv_priv *bat_priv,
                                  struct batadv_orig_node *orig_node,
                                  const void *tt_buff, u16 tt_num_vlan,
                                  struct batadv_tvlv_tt_change *tt_change,
                                  u16 tt_num_changes, u8 ttvn)
{
        u8 orig_ttvn = (u8)atomic_read(&orig_node->last_ttvn);
        struct batadv_tvlv_tt_vlan_data *tt_vlan;
        bool full_table = true;
        bool has_tt_init;

        tt_vlan = (struct batadv_tvlv_tt_vlan_data *)tt_buff;
        has_tt_init = test_bit(BATADV_ORIG_CAPA_HAS_TT,
                               &orig_node->capa_initialized);

        /* orig table not initialised AND first diff is in the OGM OR the ttvn
         * increased by one -> we can apply the attached changes
         */
        if ((!has_tt_init && ttvn == 1) || ttvn - orig_ttvn == 1) {
                /* the OGM could not contain the changes due to their size or
                 * because they have already been sent BATADV_TT_OGM_APPEND_MAX
                 * times.
                 * In this case send a tt request
                 */
                if (!tt_num_changes) {
                        full_table = false;
                        goto request_table;
                }

                spin_lock_bh(&orig_node->tt_lock);

                batadv_tt_update_changes(bat_priv, orig_node, tt_num_changes,
                                         ttvn, tt_change);

                /* Even if we received the precomputed crc with the OGM, we
                 * prefer to recompute it to spot any possible inconsistency
                 * in the global table
                 */
                batadv_tt_global_update_crc(bat_priv, orig_node);

                spin_unlock_bh(&orig_node->tt_lock);

                /* The ttvn alone is not enough to guarantee consistency
                 * because a single value could represent different states
                 * (due to the wrap around). Thus a node has to check whether
                 * the resulting table (after applying the changes) is still
                 * consistent or not. E.g. a node could disconnect while its
                 * ttvn is X and reconnect on ttvn = X + TTVN_MAX: in this case
                 * checking the CRC value is mandatory to detect the
                 * inconsistency
                 */
                if (!batadv_tt_global_check_crc(orig_node, tt_vlan,
                                                tt_num_vlan))
                        goto request_table;
        } else {
                /* if we missed more than one change or our tables are not
                 * in sync anymore -> request fresh tt data
                 */
                if (!has_tt_init || ttvn != orig_ttvn ||
                    !batadv_tt_global_check_crc(orig_node, tt_vlan,
                                                tt_num_vlan)) {
request_table:
                        batadv_dbg(BATADV_DBG_TT, bat_priv,
                                   "TT inconsistency for %pM. Need to retrieve the correct information (ttvn: %u last_ttvn: %u num_changes: %u)\n",
                                   orig_node->orig, ttvn, orig_ttvn,
                                   tt_num_changes);
                        batadv_send_tt_request(bat_priv, orig_node, ttvn,
                                               tt_vlan, tt_num_vlan,
                                               full_table);
                        return;
                }
        }
}

/**
 * batadv_tt_global_client_is_roaming() - check if a client is marked as roaming
 * @bat_priv: the bat priv with all the soft interface information
 * @addr: the mac address of the client to check
 * @vid: VLAN identifier
 *
 * Return: true if we know that the client has moved from its old originator
 * to another one. This entry is still kept for consistency purposes and will be
 * deleted later by a DEL or because of timeout
 */
bool batadv_tt_global_client_is_roaming(struct batadv_priv *bat_priv,
                                        u8 *addr, unsigned short vid)
{
        struct batadv_tt_global_entry *tt_global_entry;
        bool ret = false;

        tt_global_entry = batadv_tt_global_hash_find(bat_priv, addr, vid);
        if (!tt_global_entry)
                goto out;

        ret = tt_global_entry->common.flags & BATADV_TT_CLIENT_ROAM;
        batadv_tt_global_entry_put(tt_global_entry);
out:
        return ret;
}

/**
 * batadv_tt_local_client_is_roaming() - tells whether the client is roaming
 * @bat_priv: the bat priv with all the soft interface information
 * @addr: the mac address of the local client to query
 * @vid: VLAN identifier
 *
 * Return: true if the local client is known to be roaming (it is not served by
 * this node anymore) or not. If yes, the client is still present in the table
 * to keep the latter consistent with the node TTVN
 */
bool batadv_tt_local_client_is_roaming(struct batadv_priv *bat_priv,
                                       u8 *addr, unsigned short vid)
{
        struct batadv_tt_local_entry *tt_local_entry;
        bool ret = false;

        tt_local_entry = batadv_tt_local_hash_find(bat_priv, addr, vid);
        if (!tt_local_entry)
                goto out;

        ret = tt_local_entry->common.flags & BATADV_TT_CLIENT_ROAM;
        batadv_tt_local_entry_put(tt_local_entry);
out:
        return ret;
}

/**
 * batadv_tt_add_temporary_global_entry() - Add temporary entry to global TT
 * @bat_priv: the bat priv with all the soft interface information
 * @orig_node: orig node which the temporary entry should be associated with
 * @addr: mac address of the client
 * @vid: VLAN id of the new temporary global translation table
 *
 * Return: true when temporary tt entry could be added, false otherwise
 */
bool batadv_tt_add_temporary_global_entry(struct batadv_priv *bat_priv,
                                          struct batadv_orig_node *orig_node,
                                          const unsigned char *addr,
                                          unsigned short vid)
{
        /* ignore loop detect macs, they are not supposed to be in the tt local
         * data as well.
         */
        if (batadv_bla_is_loopdetect_mac(addr))
                return false;

        if (!batadv_tt_global_add(bat_priv, orig_node, addr, vid,
                                  BATADV_TT_CLIENT_TEMP,
                                  atomic_read(&orig_node->last_ttvn)))
                return false;

        batadv_dbg(BATADV_DBG_TT, bat_priv,
                   "Added temporary global client (addr: %pM, vid: %d, orig: %pM)\n",
                   addr, batadv_print_vid(vid), orig_node->orig);

        return true;
}

/**
 * batadv_tt_local_resize_to_mtu() - resize the local translation table fit the
 *  maximum packet size that can be transported through the mesh
 * @soft_iface: netdev struct of the mesh interface
 *
 * Remove entries older than 'timeout' and half timeout if more entries need
 * to be removed.
 */
void batadv_tt_local_resize_to_mtu(struct net_device *soft_iface)
{
        struct batadv_priv *bat_priv = netdev_priv(soft_iface);
        int packet_size_max = atomic_read(&bat_priv->packet_size_max);
        int table_size, timeout = BATADV_TT_LOCAL_TIMEOUT / 2;
        bool reduced = false;

        spin_lock_bh(&bat_priv->tt.commit_lock);

        while (timeout) {
                table_size = batadv_tt_local_table_transmit_size(bat_priv);
                if (packet_size_max >= table_size)
                        break;

                batadv_tt_local_purge(bat_priv, timeout);
                batadv_tt_local_purge_pending_clients(bat_priv);

                timeout /= 2;
                reduced = true;
                net_ratelimited_function(batadv_info, soft_iface,
                                         "Forced to purge local tt entries to fit new maximum fragment MTU (%i)\n",
                                         packet_size_max);
        }

        /* commit these changes immediately, to avoid synchronization problem
         * with the TTVN
         */
        if (reduced)
                batadv_tt_local_commit_changes_nolock(bat_priv);

        spin_unlock_bh(&bat_priv->tt.commit_lock);
}

/**
 * batadv_tt_tvlv_ogm_handler_v1() - process incoming tt tvlv container
 * @bat_priv: the bat priv with all the soft interface information
 * @orig: the orig_node of the ogm
 * @flags: flags indicating the tvlv state (see batadv_tvlv_handler_flags)
 * @tvlv_value: tvlv buffer containing the gateway data
 * @tvlv_value_len: tvlv buffer length
 */
static void batadv_tt_tvlv_ogm_handler_v1(struct batadv_priv *bat_priv,
                                          struct batadv_orig_node *orig,
                                          u8 flags, void *tvlv_value,
                                          u16 tvlv_value_len)
{
        struct batadv_tvlv_tt_vlan_data *tt_vlan;
        struct batadv_tvlv_tt_change *tt_change;
        struct batadv_tvlv_tt_data *tt_data;
        u16 num_entries, num_vlan;

        if (tvlv_value_len < sizeof(*tt_data))
                return;

        tt_data = (struct batadv_tvlv_tt_data *)tvlv_value;
        tvlv_value_len -= sizeof(*tt_data);

        num_vlan = ntohs(tt_data->num_vlan);

        if (tvlv_value_len < sizeof(*tt_vlan) * num_vlan)
                return;

        tt_vlan = (struct batadv_tvlv_tt_vlan_data *)(tt_data + 1);
        tt_change = (struct batadv_tvlv_tt_change *)(tt_vlan + num_vlan);
        tvlv_value_len -= sizeof(*tt_vlan) * num_vlan;

        num_entries = batadv_tt_entries(tvlv_value_len);

        batadv_tt_update_orig(bat_priv, orig, tt_vlan, num_vlan, tt_change,
                              num_entries, tt_data->ttvn);
}

/**
 * batadv_tt_tvlv_unicast_handler_v1() - process incoming (unicast) tt tvlv
 *  container
 * @bat_priv: the bat priv with all the soft interface information
 * @src: mac address of tt tvlv sender
 * @dst: mac address of tt tvlv recipient
 * @tvlv_value: tvlv buffer containing the tt data
 * @tvlv_value_len: tvlv buffer length
 *
 * Return: NET_RX_DROP if the tt tvlv is to be re-routed, NET_RX_SUCCESS
 * otherwise.
 */
static int batadv_tt_tvlv_unicast_handler_v1(struct batadv_priv *bat_priv,
                                             u8 *src, u8 *dst,
                                             void *tvlv_value,
                                             u16 tvlv_value_len)
{
        struct batadv_tvlv_tt_data *tt_data;
        u16 tt_vlan_len, tt_num_entries;
        char tt_flag;
        bool ret;

        if (tvlv_value_len < sizeof(*tt_data))
                return NET_RX_SUCCESS;

        tt_data = (struct batadv_tvlv_tt_data *)tvlv_value;
        tvlv_value_len -= sizeof(*tt_data);

        tt_vlan_len = sizeof(struct batadv_tvlv_tt_vlan_data);
        tt_vlan_len *= ntohs(tt_data->num_vlan);

        if (tvlv_value_len < tt_vlan_len)
                return NET_RX_SUCCESS;

        tvlv_value_len -= tt_vlan_len;
        tt_num_entries = batadv_tt_entries(tvlv_value_len);

        switch (tt_data->flags & BATADV_TT_DATA_TYPE_MASK) {
        case BATADV_TT_REQUEST:
                batadv_inc_counter(bat_priv, BATADV_CNT_TT_REQUEST_RX);

                /* If this node cannot provide a TT response the tt_request is
                 * forwarded
                 */
                ret = batadv_send_tt_response(bat_priv, tt_data, src, dst);
                if (!ret) {
                        if (tt_data->flags & BATADV_TT_FULL_TABLE)
                                tt_flag = 'F';
                        else
                                tt_flag = '.';

                        batadv_dbg(BATADV_DBG_TT, bat_priv,
                                   "Routing TT_REQUEST to %pM [%c]\n",
                                   dst, tt_flag);
                        /* tvlv API will re-route the packet */
                        return NET_RX_DROP;
                }
                break;
        case BATADV_TT_RESPONSE:
                batadv_inc_counter(bat_priv, BATADV_CNT_TT_RESPONSE_RX);

                if (batadv_is_my_mac(bat_priv, dst)) {
                        batadv_handle_tt_response(bat_priv, tt_data,
                                                  src, tt_num_entries);
                        return NET_RX_SUCCESS;
                }

                if (tt_data->flags & BATADV_TT_FULL_TABLE)
                        tt_flag =  'F';
                else
                        tt_flag = '.';

                batadv_dbg(BATADV_DBG_TT, bat_priv,
                           "Routing TT_RESPONSE to %pM [%c]\n", dst, tt_flag);

                /* tvlv API will re-route the packet */
                return NET_RX_DROP;
        }

        return NET_RX_SUCCESS;
}

/**
 * batadv_roam_tvlv_unicast_handler_v1() - process incoming tt roam tvlv
 *  container
 * @bat_priv: the bat priv with all the soft interface information
 * @src: mac address of tt tvlv sender
 * @dst: mac address of tt tvlv recipient
 * @tvlv_value: tvlv buffer containing the tt data
 * @tvlv_value_len: tvlv buffer length
 *
 * Return: NET_RX_DROP if the tt roam tvlv is to be re-routed, NET_RX_SUCCESS
 * otherwise.
 */
static int batadv_roam_tvlv_unicast_handler_v1(struct batadv_priv *bat_priv,
                                               u8 *src, u8 *dst,
                                               void *tvlv_value,
                                               u16 tvlv_value_len)
{
        struct batadv_tvlv_roam_adv *roaming_adv;
        struct batadv_orig_node *orig_node = NULL;

        /* If this node is not the intended recipient of the
         * roaming advertisement the packet is forwarded
         * (the tvlv API will re-route the packet).
         */
        if (!batadv_is_my_mac(bat_priv, dst))
                return NET_RX_DROP;

        if (tvlv_value_len < sizeof(*roaming_adv))
                goto out;

        orig_node = batadv_orig_hash_find(bat_priv, src);
        if (!orig_node)
                goto out;

        batadv_inc_counter(bat_priv, BATADV_CNT_TT_ROAM_ADV_RX);
        roaming_adv = (struct batadv_tvlv_roam_adv *)tvlv_value;

        batadv_dbg(BATADV_DBG_TT, bat_priv,
                   "Received ROAMING_ADV from %pM (client %pM)\n",
                   src, roaming_adv->client);

        batadv_tt_global_add(bat_priv, orig_node, roaming_adv->client,
                             ntohs(roaming_adv->vid), BATADV_TT_CLIENT_ROAM,
                             atomic_read(&orig_node->last_ttvn) + 1);

out:
        batadv_orig_node_put(orig_node);
        return NET_RX_SUCCESS;
}

/**
 * batadv_tt_init() - initialise the translation table internals
 * @bat_priv: the bat priv with all the soft interface information
 *
 * Return: 0 on success or negative error number in case of failure.
 */
int batadv_tt_init(struct batadv_priv *bat_priv)
{
        int ret;

        /* synchronized flags must be remote */
        BUILD_BUG_ON(!(BATADV_TT_SYNC_MASK & BATADV_TT_REMOTE_MASK));

        ret = batadv_tt_local_init(bat_priv);
        if (ret < 0)
                return ret;

        ret = batadv_tt_global_init(bat_priv);
        if (ret < 0) {
                batadv_tt_local_table_free(bat_priv);
                return ret;
        }

        batadv_tvlv_handler_register(bat_priv, batadv_tt_tvlv_ogm_handler_v1,
                                     batadv_tt_tvlv_unicast_handler_v1,
                                     BATADV_TVLV_TT, 1, BATADV_NO_FLAGS);

        batadv_tvlv_handler_register(bat_priv, NULL,
                                     batadv_roam_tvlv_unicast_handler_v1,
                                     BATADV_TVLV_ROAM, 1, BATADV_NO_FLAGS);

        INIT_DELAYED_WORK(&bat_priv->tt.work, batadv_tt_purge);
        queue_delayed_work(batadv_event_workqueue, &bat_priv->tt.work,
                           msecs_to_jiffies(BATADV_TT_WORK_PERIOD));

        return 1;
}

/**
 * batadv_tt_global_is_isolated() - check if a client is marked as isolated
 * @bat_priv: the bat priv with all the soft interface information
 * @addr: the mac address of the client
 * @vid: the identifier of the VLAN where this client is connected
 *
 * Return: true if the client is marked with the TT_CLIENT_ISOLA flag, false
 * otherwise
 */
bool batadv_tt_global_is_isolated(struct batadv_priv *bat_priv,
                                  const u8 *addr, unsigned short vid)
{
        struct batadv_tt_global_entry *tt;
        bool ret;

        tt = batadv_tt_global_hash_find(bat_priv, addr, vid);
        if (!tt)
                return false;

        ret = tt->common.flags & BATADV_TT_CLIENT_ISOLA;

        batadv_tt_global_entry_put(tt);

        return ret;
}

/**
 * batadv_tt_cache_init() - Initialize tt memory object cache
 *
 * Return: 0 on success or negative error number in case of failure.
 */
int __init batadv_tt_cache_init(void)
{
        size_t tl_size = sizeof(struct batadv_tt_local_entry);
        size_t tg_size = sizeof(struct batadv_tt_global_entry);
        size_t tt_orig_size = sizeof(struct batadv_tt_orig_list_entry);
        size_t tt_change_size = sizeof(struct batadv_tt_change_node);
        size_t tt_req_size = sizeof(struct batadv_tt_req_node);
        size_t tt_roam_size = sizeof(struct batadv_tt_roam_node);

        batadv_tl_cache = kmem_cache_create("batadv_tl_cache", tl_size, 0,
                                            SLAB_HWCACHE_ALIGN, NULL);
        if (!batadv_tl_cache)
                return -ENOMEM;

        batadv_tg_cache = kmem_cache_create("batadv_tg_cache", tg_size, 0,
                                            SLAB_HWCACHE_ALIGN, NULL);
        if (!batadv_tg_cache)
                goto err_tt_tl_destroy;

        batadv_tt_orig_cache = kmem_cache_create("batadv_tt_orig_cache",
                                                 tt_orig_size, 0,
                                                 SLAB_HWCACHE_ALIGN, NULL);
        if (!batadv_tt_orig_cache)
                goto err_tt_tg_destroy;

        batadv_tt_change_cache = kmem_cache_create("batadv_tt_change_cache",
                                                   tt_change_size, 0,
                                                   SLAB_HWCACHE_ALIGN, NULL);
        if (!batadv_tt_change_cache)
                goto err_tt_orig_destroy;

        batadv_tt_req_cache = kmem_cache_create("batadv_tt_req_cache",
                                                tt_req_size, 0,
                                                SLAB_HWCACHE_ALIGN, NULL);
        if (!batadv_tt_req_cache)
                goto err_tt_change_destroy;

        batadv_tt_roam_cache = kmem_cache_create("batadv_tt_roam_cache",
                                                 tt_roam_size, 0,
                                                 SLAB_HWCACHE_ALIGN, NULL);
        if (!batadv_tt_roam_cache)
                goto err_tt_req_destroy;

        return 0;

err_tt_req_destroy:
        kmem_cache_destroy(batadv_tt_req_cache);
        batadv_tt_req_cache = NULL;
err_tt_change_destroy:
        kmem_cache_destroy(batadv_tt_change_cache);
        batadv_tt_change_cache = NULL;
err_tt_orig_destroy:
        kmem_cache_destroy(batadv_tt_orig_cache);
        batadv_tt_orig_cache = NULL;
err_tt_tg_destroy:
        kmem_cache_destroy(batadv_tg_cache);
        batadv_tg_cache = NULL;
err_tt_tl_destroy:
        kmem_cache_destroy(batadv_tl_cache);
        batadv_tl_cache = NULL;

        return -ENOMEM;
}

/**
 * batadv_tt_cache_destroy() - Destroy tt memory object cache
 */
void batadv_tt_cache_destroy(void)
{
        kmem_cache_destroy(batadv_tl_cache);
        kmem_cache_destroy(batadv_tg_cache);
        kmem_cache_destroy(batadv_tt_orig_cache);
        kmem_cache_destroy(batadv_tt_change_cache);
        kmem_cache_destroy(batadv_tt_req_cache);
        kmem_cache_destroy(batadv_tt_roam_cache);
}






























































































































































































































































































































































































































































  980 



  980 












































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 *        "LAPB via ethernet" driver release 001
 *
 *        This code REQUIRES 2.1.15 or higher/ NET3.038
 *
 *        This is a "pseudo" network driver to allow LAPB over Ethernet.
 *
 *        This driver can use any ethernet destination address, and can be
 *        limited to accept frames from one dedicated ethernet card only.
 *
 *        History
 *        LAPBETH 001        Jonathan Naylor                Cloned from bpqether.c
 *        2000-10-29        Henner Eisen        lapb_data_indication() return status.
 *        2000-11-14        Henner Eisen        dev_hold/put, NETDEV_GOING_DOWN support
 */

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#include <linux/errno.h>
#include <linux/types.h>
#include <linux/socket.h>
#include <linux/in.h>
#include <linux/slab.h>
#include <linux/kernel.h>
#include <linux/string.h>
#include <linux/net.h>
#include <linux/inet.h>
#include <linux/netdevice.h>
#include <linux/if_arp.h>
#include <linux/skbuff.h>
#include <net/sock.h>
#include <linux/uaccess.h>
#include <linux/mm.h>
#include <linux/interrupt.h>
#include <linux/notifier.h>
#include <linux/stat.h>
#include <linux/module.h>
#include <linux/lapb.h>
#include <linux/init.h>

#include <net/x25device.h>

static const u8 bcast_addr[6] = { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF };

/* If this number is made larger, check that the temporary string buffer
 * in lapbeth_new_device is large enough to store the probe device name.
 */
#define MAXLAPBDEV 100

struct lapbethdev {
        struct list_head        node;
        struct net_device        *ethdev;        /* link to ethernet device */
        struct net_device        *axdev;                /* lapbeth device (lapb#) */
        bool                        up;
        spinlock_t                up_lock;        /* Protects "up" */
        struct sk_buff_head        rx_queue;
        struct napi_struct        napi;
};

static LIST_HEAD(lapbeth_devices);

static void lapbeth_connected(struct net_device *dev, int reason);
static void lapbeth_disconnected(struct net_device *dev, int reason);

/* ------------------------------------------------------------------------ */

/*        Get the LAPB device for the ethernet device
 */
static struct lapbethdev *lapbeth_get_x25_dev(struct net_device *dev)
{
        struct lapbethdev *lapbeth;

        list_for_each_entry_rcu(lapbeth, &lapbeth_devices, node, lockdep_rtnl_is_held()) {
                if (lapbeth->ethdev == dev)
                        return lapbeth;
        }
        return NULL;
}

static __inline__ int dev_is_ethdev(struct net_device *dev)
{
        return dev->type == ARPHRD_ETHER && strncmp(dev->name, "dummy", 5);
}

/* ------------------------------------------------------------------------ */

static int lapbeth_napi_poll(struct napi_struct *napi, int budget)
{
        struct lapbethdev *lapbeth = container_of(napi, struct lapbethdev,
                                                  napi);
        struct sk_buff *skb;
        int processed = 0;

        for (; processed < budget; ++processed) {
                skb = skb_dequeue(&lapbeth->rx_queue);
                if (!skb)
                        break;
                netif_receive_skb_core(skb);
        }

        if (processed < budget)
                napi_complete(napi);

        return processed;
}

/*        Receive a LAPB frame via an ethernet interface.
 */
static int lapbeth_rcv(struct sk_buff *skb, struct net_device *dev,
                       struct packet_type *ptype, struct net_device *orig_dev)
{
        int len, err;
        struct lapbethdev *lapbeth;

        if (dev_net(dev) != &init_net)
                goto drop;

        skb = skb_share_check(skb, GFP_ATOMIC);
        if (!skb)
                return NET_RX_DROP;

        if (!pskb_may_pull(skb, 2))
                goto drop;

        rcu_read_lock();
        lapbeth = lapbeth_get_x25_dev(dev);
        if (!lapbeth)
                goto drop_unlock_rcu;
        spin_lock_bh(&lapbeth->up_lock);
        if (!lapbeth->up)
                goto drop_unlock;

        len = skb->data[0] + skb->data[1] * 256;
        dev->stats.rx_packets++;
        dev->stats.rx_bytes += len;

        skb_pull(skb, 2);        /* Remove the length bytes */
        skb_trim(skb, len);        /* Set the length of the data */

        err = lapb_data_received(lapbeth->axdev, skb);
        if (err != LAPB_OK) {
                printk(KERN_DEBUG "lapbether: lapb_data_received err - %d\n", err);
                goto drop_unlock;
        }
out:
        spin_unlock_bh(&lapbeth->up_lock);
        rcu_read_unlock();
        return 0;
drop_unlock:
        kfree_skb(skb);
        goto out;
drop_unlock_rcu:
        rcu_read_unlock();
drop:
        kfree_skb(skb);
        return 0;
}

static int lapbeth_data_indication(struct net_device *dev, struct sk_buff *skb)
{
        struct lapbethdev *lapbeth = netdev_priv(dev);
        unsigned char *ptr;

        if (skb_cow(skb, 1)) {
                kfree_skb(skb);
                return NET_RX_DROP;
        }

        skb_push(skb, 1);

        ptr  = skb->data;
        *ptr = X25_IFACE_DATA;

        skb->protocol = x25_type_trans(skb, dev);

        skb_queue_tail(&lapbeth->rx_queue, skb);
        napi_schedule(&lapbeth->napi);
        return NET_RX_SUCCESS;
}

/*        Send a LAPB frame via an ethernet interface
 */
static netdev_tx_t lapbeth_xmit(struct sk_buff *skb,
                                struct net_device *dev)
{
        struct lapbethdev *lapbeth = netdev_priv(dev);
        int err;

        spin_lock_bh(&lapbeth->up_lock);
        if (!lapbeth->up)
                goto drop;

        /* There should be a pseudo header of 1 byte added by upper layers.
         * Check to make sure it is there before reading it.
         */
        if (skb->len < 1)
                goto drop;

        switch (skb->data[0]) {
        case X25_IFACE_DATA:
                break;
        case X25_IFACE_CONNECT:
                err = lapb_connect_request(dev);
                if (err == LAPB_CONNECTED)
                        lapbeth_connected(dev, LAPB_OK);
                else if (err != LAPB_OK)
                        pr_err("lapb_connect_request error: %d\n", err);
                goto drop;
        case X25_IFACE_DISCONNECT:
                err = lapb_disconnect_request(dev);
                if (err == LAPB_NOTCONNECTED)
                        lapbeth_disconnected(dev, LAPB_OK);
                else if (err != LAPB_OK)
                        pr_err("lapb_disconnect_request err: %d\n", err);
                fallthrough;
        default:
                goto drop;
        }

        skb_pull(skb, 1);

        err = lapb_data_request(dev, skb);
        if (err != LAPB_OK) {
                pr_err("lapb_data_request error - %d\n", err);
                goto drop;
        }
out:
        spin_unlock_bh(&lapbeth->up_lock);
        return NETDEV_TX_OK;
drop:
        kfree_skb(skb);
        goto out;
}

static void lapbeth_data_transmit(struct net_device *ndev, struct sk_buff *skb)
{
        struct lapbethdev *lapbeth = netdev_priv(ndev);
        unsigned char *ptr;
        struct net_device *dev;
        int size = skb->len;

        ptr = skb_push(skb, 2);

        *ptr++ = size % 256;
        *ptr++ = size / 256;

        ndev->stats.tx_packets++;
        ndev->stats.tx_bytes += size;

        skb->dev = dev = lapbeth->ethdev;

        skb->protocol = htons(ETH_P_DEC);

        skb_reset_network_header(skb);

        dev_hard_header(skb, dev, ETH_P_DEC, bcast_addr, NULL, 0);

        dev_queue_xmit(skb);
}

static void lapbeth_connected(struct net_device *dev, int reason)
{
        struct lapbethdev *lapbeth = netdev_priv(dev);
        unsigned char *ptr;
        struct sk_buff *skb = __dev_alloc_skb(1, GFP_ATOMIC | __GFP_NOMEMALLOC);

        if (!skb)
                return;

        ptr  = skb_put(skb, 1);
        *ptr = X25_IFACE_CONNECT;

        skb->protocol = x25_type_trans(skb, dev);

        skb_queue_tail(&lapbeth->rx_queue, skb);
        napi_schedule(&lapbeth->napi);
}

static void lapbeth_disconnected(struct net_device *dev, int reason)
{
        struct lapbethdev *lapbeth = netdev_priv(dev);
        unsigned char *ptr;
        struct sk_buff *skb = __dev_alloc_skb(1, GFP_ATOMIC | __GFP_NOMEMALLOC);

        if (!skb)
                return;

        ptr  = skb_put(skb, 1);
        *ptr = X25_IFACE_DISCONNECT;

        skb->protocol = x25_type_trans(skb, dev);

        skb_queue_tail(&lapbeth->rx_queue, skb);
        napi_schedule(&lapbeth->napi);
}

/*        Set AX.25 callsign
 */
static int lapbeth_set_mac_address(struct net_device *dev, void *addr)
{
        struct sockaddr *sa = addr;

        memcpy(dev->dev_addr, sa->sa_data, dev->addr_len);
        return 0;
}

static const struct lapb_register_struct lapbeth_callbacks = {
        .connect_confirmation    = lapbeth_connected,
        .connect_indication      = lapbeth_connected,
        .disconnect_confirmation = lapbeth_disconnected,
        .disconnect_indication   = lapbeth_disconnected,
        .data_indication         = lapbeth_data_indication,
        .data_transmit           = lapbeth_data_transmit,
};

/* open/close a device
 */
static int lapbeth_open(struct net_device *dev)
{
        struct lapbethdev *lapbeth = netdev_priv(dev);
        int err;

        napi_enable(&lapbeth->napi);

        err = lapb_register(dev, &lapbeth_callbacks);
        if (err != LAPB_OK) {
                napi_disable(&lapbeth->napi);
                pr_err("lapb_register error: %d\n", err);
                return -ENODEV;
        }

        spin_lock_bh(&lapbeth->up_lock);
        lapbeth->up = true;
        spin_unlock_bh(&lapbeth->up_lock);

        return 0;
}

static int lapbeth_close(struct net_device *dev)
{
        struct lapbethdev *lapbeth = netdev_priv(dev);
        int err;

        spin_lock_bh(&lapbeth->up_lock);
        lapbeth->up = false;
        spin_unlock_bh(&lapbeth->up_lock);

        err = lapb_unregister(dev);
        if (err != LAPB_OK)
                pr_err("lapb_unregister error: %d\n", err);

        napi_disable(&lapbeth->napi);

        return 0;
}

/* ------------------------------------------------------------------------ */

static const struct net_device_ops lapbeth_netdev_ops = {
        .ndo_open             = lapbeth_open,
        .ndo_stop             = lapbeth_close,
        .ndo_start_xmit             = lapbeth_xmit,
        .ndo_set_mac_address = lapbeth_set_mac_address,
};

static void lapbeth_setup(struct net_device *dev)
{
        dev->netdev_ops             = &lapbeth_netdev_ops;
        dev->needs_free_netdev = true;
        dev->type            = ARPHRD_X25;
        dev->hard_header_len = 0;
        dev->mtu             = 1000;
        dev->addr_len        = 0;
}

/*        Setup a new device.
 */
static int lapbeth_new_device(struct net_device *dev)
{
        struct net_device *ndev;
        struct lapbethdev *lapbeth;
        int rc = -ENOMEM;

        ASSERT_RTNL();

        if (dev->type != ARPHRD_ETHER)
                return -EINVAL;

        ndev = alloc_netdev(sizeof(*lapbeth), "lapb%d", NET_NAME_UNKNOWN,
                            lapbeth_setup);
        if (!ndev)
                goto out;

        /* When transmitting data:
         * first this driver removes a pseudo header of 1 byte,
         * then the lapb module prepends an LAPB header of at most 3 bytes,
         * then this driver prepends a length field of 2 bytes,
         * then the underlying Ethernet device prepends its own header.
         */
        ndev->needed_headroom = -1 + 3 + 2 + dev->hard_header_len
                                           + dev->needed_headroom;
        ndev->needed_tailroom = dev->needed_tailroom;

        lapbeth = netdev_priv(ndev);
        lapbeth->axdev = ndev;

        dev_hold(dev);
        lapbeth->ethdev = dev;

        lapbeth->up = false;
        spin_lock_init(&lapbeth->up_lock);

        skb_queue_head_init(&lapbeth->rx_queue);
        netif_napi_add(ndev, &lapbeth->napi, lapbeth_napi_poll, 16);

        rc = -EIO;
        if (register_netdevice(ndev))
                goto fail;

        list_add_rcu(&lapbeth->node, &lapbeth_devices);
        rc = 0;
out:
        return rc;
fail:
        dev_put(dev);
        free_netdev(ndev);
        goto out;
}

/*        Free a lapb network device.
 */
static void lapbeth_free_device(struct lapbethdev *lapbeth)
{
        dev_put(lapbeth->ethdev);
        list_del_rcu(&lapbeth->node);
        unregister_netdevice(lapbeth->axdev);
}

/*        Handle device status changes.
 *
 * Called from notifier with RTNL held.
 */
static int lapbeth_device_event(struct notifier_block *this,
                                unsigned long event, void *ptr)
{
        struct lapbethdev *lapbeth;
        struct net_device *dev = netdev_notifier_info_to_dev(ptr);

        if (dev_net(dev) != &init_net)
                return NOTIFY_DONE;

        if (!dev_is_ethdev(dev) && !lapbeth_get_x25_dev(dev))
                return NOTIFY_DONE;

        switch (event) {
        case NETDEV_UP:
                /* New ethernet device -> new LAPB interface         */
                if (!lapbeth_get_x25_dev(dev))
                        lapbeth_new_device(dev);
                break;
        case NETDEV_GOING_DOWN:
                /* ethernet device closes -> close LAPB interface */
                lapbeth = lapbeth_get_x25_dev(dev);
                if (lapbeth)
                        dev_close(lapbeth->axdev);
                break;
        case NETDEV_UNREGISTER:
                /* ethernet device disappears -> remove LAPB interface */
                lapbeth = lapbeth_get_x25_dev(dev);
                if (lapbeth)
                        lapbeth_free_device(lapbeth);
                break;
        }

        return NOTIFY_DONE;
}

/* ------------------------------------------------------------------------ */

static struct packet_type lapbeth_packet_type __read_mostly = {
        .type = cpu_to_be16(ETH_P_DEC),
        .func = lapbeth_rcv,
};

static struct notifier_block lapbeth_dev_notifier = {
        .notifier_call = lapbeth_device_event,
};

static const char banner[] __initconst =
        KERN_INFO "LAPB Ethernet driver version 0.02\n";

static int __init lapbeth_init_driver(void)
{
        dev_add_pack(&lapbeth_packet_type);

        register_netdevice_notifier(&lapbeth_dev_notifier);

        printk(banner);

        return 0;
}
module_init(lapbeth_init_driver);

static void __exit lapbeth_cleanup_driver(void)
{
        struct lapbethdev *lapbeth;
        struct list_head *entry, *tmp;

        dev_remove_pack(&lapbeth_packet_type);
        unregister_netdevice_notifier(&lapbeth_dev_notifier);

        rtnl_lock();
        list_for_each_safe(entry, tmp, &lapbeth_devices) {
                lapbeth = list_entry(entry, struct lapbethdev, node);

                dev_put(lapbeth->ethdev);
                unregister_netdevice(lapbeth->axdev);
        }
        rtnl_unlock();
}
module_exit(lapbeth_cleanup_driver);

MODULE_AUTHOR("Jonathan Naylor <g4klx@g4klx.demon.co.uk>");
MODULE_DESCRIPTION("The unofficial LAPB over Ethernet driver");
MODULE_LICENSE("GPL");





































































   54 







   54 

















































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
// SPDX-License-Identifier: GPL-2.0-or-later
/* RxRPC security handling
 *
 * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved.
 * Written by David Howells (dhowells@redhat.com)
 */

#include <linux/module.h>
#include <linux/net.h>
#include <linux/skbuff.h>
#include <linux/udp.h>
#include <linux/crypto.h>
#include <net/sock.h>
#include <net/af_rxrpc.h>
#include <keys/rxrpc-type.h>
#include "ar-internal.h"

static const struct rxrpc_security *rxrpc_security_types[] = {
        [RXRPC_SECURITY_NONE]        = &rxrpc_no_security,
#ifdef CONFIG_RXKAD
        [RXRPC_SECURITY_RXKAD]        = &rxkad,
#endif
};

int __init rxrpc_init_security(void)
{
        int i, ret;

        for (i = 0; i < ARRAY_SIZE(rxrpc_security_types); i++) {
                if (rxrpc_security_types[i]) {
                        ret = rxrpc_security_types[i]->init();
                        if (ret < 0)
                                goto failed;
                }
        }

        return 0;

failed:
        for (i--; i >= 0; i--)
                if (rxrpc_security_types[i])
                        rxrpc_security_types[i]->exit();
        return ret;
}

void rxrpc_exit_security(void)
{
        int i;

        for (i = 0; i < ARRAY_SIZE(rxrpc_security_types); i++)
                if (rxrpc_security_types[i])
                        rxrpc_security_types[i]->exit();
}

/*
 * look up an rxrpc security module
 */
const struct rxrpc_security *rxrpc_security_lookup(u8 security_index)
{
        if (security_index >= ARRAY_SIZE(rxrpc_security_types))
                return NULL;
        return rxrpc_security_types[security_index];
}

/*
 * initialise the security on a client connection
 */
int rxrpc_init_client_conn_security(struct rxrpc_connection *conn)
{
        const struct rxrpc_security *sec;
        struct rxrpc_key_token *token;
        struct key *key = conn->params.key;
        int ret;

        _enter("{%d},{%x}", conn->debug_id, key_serial(key));

        if (!key)
                return 0;

        ret = key_validate(key);
        if (ret < 0)
                return ret;

        for (token = key->payload.data[0]; token; token = token->next) {
                sec = rxrpc_security_lookup(token->security_index);
                if (sec)
                        goto found;
        }
        return -EKEYREJECTED;

found:
        conn->security = sec;

        ret = conn->security->init_connection_security(conn, token);
        if (ret < 0) {
                conn->security = &rxrpc_no_security;
                return ret;
        }

        _leave(" = 0");
        return 0;
}

/*
 * Set the ops a server connection.
 */
const struct rxrpc_security *rxrpc_get_incoming_security(struct rxrpc_sock *rx,
                                                         struct sk_buff *skb)
{
        const struct rxrpc_security *sec;
        struct rxrpc_skb_priv *sp = rxrpc_skb(skb);

        _enter("");

        sec = rxrpc_security_lookup(sp->hdr.securityIndex);
        if (!sec) {
                trace_rxrpc_abort(0, "SVS",
                                  sp->hdr.cid, sp->hdr.callNumber, sp->hdr.seq,
                                  RX_INVALID_OPERATION, EKEYREJECTED);
                skb->mark = RXRPC_SKB_MARK_REJECT_ABORT;
                skb->priority = RX_INVALID_OPERATION;
                return NULL;
        }

        if (sp->hdr.securityIndex != RXRPC_SECURITY_NONE &&
            !rx->securities) {
                trace_rxrpc_abort(0, "SVR",
                                  sp->hdr.cid, sp->hdr.callNumber, sp->hdr.seq,
                                  RX_INVALID_OPERATION, EKEYREJECTED);
                skb->mark = RXRPC_SKB_MARK_REJECT_ABORT;
                skb->priority = sec->no_key_abort;
                return NULL;
        }

        return sec;
}

/*
 * Find the security key for a server connection.
 */
struct key *rxrpc_look_up_server_security(struct rxrpc_connection *conn,
                                          struct sk_buff *skb,
                                          u32 kvno, u32 enctype)
{
        struct rxrpc_skb_priv *sp = rxrpc_skb(skb);
        struct rxrpc_sock *rx;
        struct key *key = ERR_PTR(-EKEYREJECTED);
        key_ref_t kref = NULL;
        char kdesc[5 + 1 + 3 + 1 + 12 + 1 + 12 + 1];
        int ret;

        _enter("");

        if (enctype)
                sprintf(kdesc, "%u:%u:%u:%u",
                        sp->hdr.serviceId, sp->hdr.securityIndex, kvno, enctype);
        else if (kvno)
                sprintf(kdesc, "%u:%u:%u",
                        sp->hdr.serviceId, sp->hdr.securityIndex, kvno);
        else
                sprintf(kdesc, "%u:%u",
                        sp->hdr.serviceId, sp->hdr.securityIndex);

        rcu_read_lock();

        rx = rcu_dereference(conn->params.local->service);
        if (!rx)
                goto out;

        /* look through the service's keyring */
        kref = keyring_search(make_key_ref(rx->securities, 1UL),
                              &key_type_rxrpc_s, kdesc, true);
        if (IS_ERR(kref)) {
                key = ERR_CAST(kref);
                goto out;
        }

        key = key_ref_to_ptr(kref);

        ret = key_validate(key);
        if (ret < 0) {
                key_put(key);
                key = ERR_PTR(ret);
                goto out;
        }

out:
        rcu_read_unlock();
        return key;
}










































































































































































   91 













































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
/*
 * include/linux/topology.h
 *
 * Written by: Matthew Dobson, IBM Corporation
 *
 * Copyright (C) 2002, IBM Corp.
 *
 * All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
 * NON INFRINGEMENT.  See the GNU General Public License for more
 * details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 *
 * Send feedback to <colpatch@us.ibm.com>
 */
#ifndef _LINUX_TOPOLOGY_H
#define _LINUX_TOPOLOGY_H

#include <linux/arch_topology.h>
#include <linux/cpumask.h>
#include <linux/bitops.h>
#include <linux/mmzone.h>
#include <linux/smp.h>
#include <linux/percpu.h>
#include <asm/topology.h>

#ifndef nr_cpus_node
#define nr_cpus_node(node) cpumask_weight(cpumask_of_node(node))
#endif

#define for_each_node_with_cpus(node)                        \
        for_each_online_node(node)                        \
                if (nr_cpus_node(node))

int arch_update_cpu_topology(void);

/* Conform to ACPI 2.0 SLIT distance definitions */
#define LOCAL_DISTANCE                10
#define REMOTE_DISTANCE                20
#define DISTANCE_BITS           8
#ifndef node_distance
#define node_distance(from,to)        ((from) == (to) ? LOCAL_DISTANCE : REMOTE_DISTANCE)
#endif
#ifndef RECLAIM_DISTANCE
/*
 * If the distance between nodes in a system is larger than RECLAIM_DISTANCE
 * (in whatever arch specific measurement units returned by node_distance())
 * and node_reclaim_mode is enabled then the VM will only call node_reclaim()
 * on nodes within this distance.
 */
#define RECLAIM_DISTANCE 30
#endif

/*
 * The following tunable allows platforms to override the default node
 * reclaim distance (RECLAIM_DISTANCE) if remote memory accesses are
 * sufficiently fast that the default value actually hurts
 * performance.
 *
 * AMD EPYC machines use this because even though the 2-hop distance
 * is 32 (3.2x slower than a local memory access) performance actually
 * *improves* if allowed to reclaim memory and load balance tasks
 * between NUMA nodes 2-hops apart.
 */
extern int __read_mostly node_reclaim_distance;

#ifndef PENALTY_FOR_NODE_WITH_CPUS
#define PENALTY_FOR_NODE_WITH_CPUS        (1)
#endif

#ifdef CONFIG_USE_PERCPU_NUMA_NODE_ID
DECLARE_PER_CPU(int, numa_node);

#ifndef numa_node_id
/* Returns the number of the current Node. */
static inline int numa_node_id(void)
{
        return raw_cpu_read(numa_node);
}
#endif

#ifndef cpu_to_node
static inline int cpu_to_node(int cpu)
{
        return per_cpu(numa_node, cpu);
}
#endif

#ifndef set_numa_node
static inline void set_numa_node(int node)
{
        this_cpu_write(numa_node, node);
}
#endif

#ifndef set_cpu_numa_node
static inline void set_cpu_numa_node(int cpu, int node)
{
        per_cpu(numa_node, cpu) = node;
}
#endif

#else        /* !CONFIG_USE_PERCPU_NUMA_NODE_ID */

/* Returns the number of the current Node. */
#ifndef numa_node_id
static inline int numa_node_id(void)
{
        return cpu_to_node(raw_smp_processor_id());
}
#endif

#endif        /* [!]CONFIG_USE_PERCPU_NUMA_NODE_ID */

#ifdef CONFIG_HAVE_MEMORYLESS_NODES

/*
 * N.B., Do NOT reference the '_numa_mem_' per cpu variable directly.
 * It will not be defined when CONFIG_HAVE_MEMORYLESS_NODES is not defined.
 * Use the accessor functions set_numa_mem(), numa_mem_id() and cpu_to_mem().
 */
DECLARE_PER_CPU(int, _numa_mem_);

#ifndef set_numa_mem
static inline void set_numa_mem(int node)
{
        this_cpu_write(_numa_mem_, node);
}
#endif

#ifndef numa_mem_id
/* Returns the number of the nearest Node with memory */
static inline int numa_mem_id(void)
{
        return raw_cpu_read(_numa_mem_);
}
#endif

#ifndef cpu_to_mem
static inline int cpu_to_mem(int cpu)
{
        return per_cpu(_numa_mem_, cpu);
}
#endif

#ifndef set_cpu_numa_mem
static inline void set_cpu_numa_mem(int cpu, int node)
{
        per_cpu(_numa_mem_, cpu) = node;
}
#endif

#else        /* !CONFIG_HAVE_MEMORYLESS_NODES */

#ifndef numa_mem_id
/* Returns the number of the nearest Node with memory */
static inline int numa_mem_id(void)
{
        return numa_node_id();
}
#endif

#ifndef cpu_to_mem
static inline int cpu_to_mem(int cpu)
{
        return cpu_to_node(cpu);
}
#endif

#endif        /* [!]CONFIG_HAVE_MEMORYLESS_NODES */

#ifndef topology_physical_package_id
#define topology_physical_package_id(cpu)        ((void)(cpu), -1)
#endif
#ifndef topology_die_id
#define topology_die_id(cpu)                        ((void)(cpu), -1)
#endif
#ifndef topology_core_id
#define topology_core_id(cpu)                        ((void)(cpu), 0)
#endif
#ifndef topology_sibling_cpumask
#define topology_sibling_cpumask(cpu)                cpumask_of(cpu)
#endif
#ifndef topology_core_cpumask
#define topology_core_cpumask(cpu)                cpumask_of(cpu)
#endif
#ifndef topology_die_cpumask
#define topology_die_cpumask(cpu)                cpumask_of(cpu)
#endif

#if defined(CONFIG_SCHED_SMT) && !defined(cpu_smt_mask)
static inline const struct cpumask *cpu_smt_mask(int cpu)
{
        return topology_sibling_cpumask(cpu);
}
#endif

static inline const struct cpumask *cpu_cpu_mask(int cpu)
{
        return cpumask_of_node(cpu_to_node(cpu));
}


#endif /* _LINUX_TOPOLOGY_H */



































































































  415 































  414 




  416 



  416 


  415 









  133 



  133 



  132 












































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































  521 



   33 









  522 




  522 

  520 
  521 
  519 
  520 
   31 








1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
// SPDX-License-Identifier: GPL-2.0
/*
 * drivers/base/power/main.c - Where the driver meets power management.
 *
 * Copyright (c) 2003 Patrick Mochel
 * Copyright (c) 2003 Open Source Development Lab
 *
 * The driver model core calls device_pm_add() when a device is registered.
 * This will initialize the embedded device_pm_info object in the device
 * and add it to the list of power-controlled devices. sysfs entries for
 * controlling device power management will also be added.
 *
 * A separate list is used for keeping track of power info, because the power
 * domain dependencies may differ from the ancestral dependencies that the
 * subsystem list maintains.
 */

#define pr_fmt(fmt) "PM: " fmt
#define dev_fmt pr_fmt

#include <linux/device.h>
#include <linux/export.h>
#include <linux/mutex.h>
#include <linux/pm.h>
#include <linux/pm_runtime.h>
#include <linux/pm-trace.h>
#include <linux/pm_wakeirq.h>
#include <linux/interrupt.h>
#include <linux/sched.h>
#include <linux/sched/debug.h>
#include <linux/async.h>
#include <linux/suspend.h>
#include <trace/events/power.h>
#include <linux/cpufreq.h>
#include <linux/cpuidle.h>
#include <linux/devfreq.h>
#include <linux/timer.h>

#include "../base.h"
#include "power.h"

typedef int (*pm_callback_t)(struct device *);

#define list_for_each_entry_rcu_locked(pos, head, member) \
        list_for_each_entry_rcu(pos, head, member, \
                        device_links_read_lock_held())

/*
 * The entries in the dpm_list list are in a depth first order, simply
 * because children are guaranteed to be discovered after parents, and
 * are inserted at the back of the list on discovery.
 *
 * Since device_pm_add() may be called with a device lock held,
 * we must never try to acquire a device lock while holding
 * dpm_list_mutex.
 */

LIST_HEAD(dpm_list);
static LIST_HEAD(dpm_prepared_list);
static LIST_HEAD(dpm_suspended_list);
static LIST_HEAD(dpm_late_early_list);
static LIST_HEAD(dpm_noirq_list);

struct suspend_stats suspend_stats;
static DEFINE_MUTEX(dpm_list_mtx);
static pm_message_t pm_transition;

static int async_error;

static const char *pm_verb(int event)
{
        switch (event) {
        case PM_EVENT_SUSPEND:
                return "suspend";
        case PM_EVENT_RESUME:
                return "resume";
        case PM_EVENT_FREEZE:
                return "freeze";
        case PM_EVENT_QUIESCE:
                return "quiesce";
        case PM_EVENT_HIBERNATE:
                return "hibernate";
        case PM_EVENT_THAW:
                return "thaw";
        case PM_EVENT_RESTORE:
                return "restore";
        case PM_EVENT_RECOVER:
                return "recover";
        default:
                return "(unknown PM event)";
        }
}

/**
 * device_pm_sleep_init - Initialize system suspend-related device fields.
 * @dev: Device object being initialized.
 */
void device_pm_sleep_init(struct device *dev)
{
        dev->power.is_prepared = false;
        dev->power.is_suspended = false;
        dev->power.is_noirq_suspended = false;
        dev->power.is_late_suspended = false;
        init_completion(&dev->power.completion);
        complete_all(&dev->power.completion);
        dev->power.wakeup = NULL;
        INIT_LIST_HEAD(&dev->power.entry);
}

/**
 * device_pm_lock - Lock the list of active devices used by the PM core.
 */
void device_pm_lock(void)
{
        mutex_lock(&dpm_list_mtx);
}

/**
 * device_pm_unlock - Unlock the list of active devices used by the PM core.
 */
void device_pm_unlock(void)
{
        mutex_unlock(&dpm_list_mtx);
}

/**
 * device_pm_add - Add a device to the PM core's list of active devices.
 * @dev: Device to add to the list.
 */
void device_pm_add(struct device *dev)
{
        /* Skip PM setup/initialization. */
        if (device_pm_not_required(dev))
                return;

        pr_debug("Adding info for %s:%s\n",
                 dev->bus ? dev->bus->name : "No Bus", dev_name(dev));
        device_pm_check_callbacks(dev);
        mutex_lock(&dpm_list_mtx);
        if (dev->parent && dev->parent->power.is_prepared)
                dev_warn(dev, "parent %s should not be sleeping\n",
                        dev_name(dev->parent));
        list_add_tail(&dev->power.entry, &dpm_list);
        dev->power.in_dpm_list = true;
        mutex_unlock(&dpm_list_mtx);
}

/**
 * device_pm_remove - Remove a device from the PM core's list of active devices.
 * @dev: Device to be removed from the list.
 */
void device_pm_remove(struct device *dev)
{
        if (device_pm_not_required(dev))
                return;

        pr_debug("Removing info for %s:%s\n",
                 dev->bus ? dev->bus->name : "No Bus", dev_name(dev));
        complete_all(&dev->power.completion);
        mutex_lock(&dpm_list_mtx);
        list_del_init(&dev->power.entry);
        dev->power.in_dpm_list = false;
        mutex_unlock(&dpm_list_mtx);
        device_wakeup_disable(dev);
        pm_runtime_remove(dev);
        device_pm_check_callbacks(dev);
}

/**
 * device_pm_move_before - Move device in the PM core's list of active devices.
 * @deva: Device to move in dpm_list.
 * @devb: Device @deva should come before.
 */
void device_pm_move_before(struct device *deva, struct device *devb)
{
        pr_debug("Moving %s:%s before %s:%s\n",
                 deva->bus ? deva->bus->name : "No Bus", dev_name(deva),
                 devb->bus ? devb->bus->name : "No Bus", dev_name(devb));
        /* Delete deva from dpm_list and reinsert before devb. */
        list_move_tail(&deva->power.entry, &devb->power.entry);
}

/**
 * device_pm_move_after - Move device in the PM core's list of active devices.
 * @deva: Device to move in dpm_list.
 * @devb: Device @deva should come after.
 */
void device_pm_move_after(struct device *deva, struct device *devb)
{
        pr_debug("Moving %s:%s after %s:%s\n",
                 deva->bus ? deva->bus->name : "No Bus", dev_name(deva),
                 devb->bus ? devb->bus->name : "No Bus", dev_name(devb));
        /* Delete deva from dpm_list and reinsert after devb. */
        list_move(&deva->power.entry, &devb->power.entry);
}

/**
 * device_pm_move_last - Move device to end of the PM core's list of devices.
 * @dev: Device to move in dpm_list.
 */
void device_pm_move_last(struct device *dev)
{
        pr_debug("Moving %s:%s to end of list\n",
                 dev->bus ? dev->bus->name : "No Bus", dev_name(dev));
        list_move_tail(&dev->power.entry, &dpm_list);
}

static ktime_t initcall_debug_start(struct device *dev, void *cb)
{
        if (!pm_print_times_enabled)
                return 0;

        dev_info(dev, "calling %pS @ %i, parent: %s\n", cb,
                 task_pid_nr(current),
                 dev->parent ? dev_name(dev->parent) : "none");
        return ktime_get();
}

static void initcall_debug_report(struct device *dev, ktime_t calltime,
                                  void *cb, int error)
{
        ktime_t rettime;

        if (!pm_print_times_enabled)
                return;

        rettime = ktime_get();
        dev_info(dev, "%pS returned %d after %Ld usecs\n", cb, error,
                 (unsigned long long)ktime_us_delta(rettime, calltime));
}

/**
 * dpm_wait - Wait for a PM operation to complete.
 * @dev: Device to wait for.
 * @async: If unset, wait only if the device's power.async_suspend flag is set.
 */
static void dpm_wait(struct device *dev, bool async)
{
        if (!dev)
                return;

        if (async || (pm_async_enabled && dev->power.async_suspend))
                wait_for_completion(&dev->power.completion);
}

static int dpm_wait_fn(struct device *dev, void *async_ptr)
{
        dpm_wait(dev, *((bool *)async_ptr));
        return 0;
}

static void dpm_wait_for_children(struct device *dev, bool async)
{
       device_for_each_child(dev, &async, dpm_wait_fn);
}

static void dpm_wait_for_suppliers(struct device *dev, bool async)
{
        struct device_link *link;
        int idx;

        idx = device_links_read_lock();

        /*
         * If the supplier goes away right after we've checked the link to it,
         * we'll wait for its completion to change the state, but that's fine,
         * because the only things that will block as a result are the SRCU
         * callbacks freeing the link objects for the links in the list we're
         * walking.
         */
        list_for_each_entry_rcu_locked(link, &dev->links.suppliers, c_node)
                if (READ_ONCE(link->status) != DL_STATE_DORMANT)
                        dpm_wait(link->supplier, async);

        device_links_read_unlock(idx);
}

static bool dpm_wait_for_superior(struct device *dev, bool async)
{
        struct device *parent;

        /*
         * If the device is resumed asynchronously and the parent's callback
         * deletes both the device and the parent itself, the parent object may
         * be freed while this function is running, so avoid that by reference
         * counting the parent once more unless the device has been deleted
         * already (in which case return right away).
         */
        mutex_lock(&dpm_list_mtx);

        if (!device_pm_initialized(dev)) {
                mutex_unlock(&dpm_list_mtx);
                return false;
        }

        parent = get_device(dev->parent);

        mutex_unlock(&dpm_list_mtx);

        dpm_wait(parent, async);
        put_device(parent);

        dpm_wait_for_suppliers(dev, async);

        /*
         * If the parent's callback has deleted the device, attempting to resume
         * it would be invalid, so avoid doing that then.
         */
        return device_pm_initialized(dev);
}

static void dpm_wait_for_consumers(struct device *dev, bool async)
{
        struct device_link *link;
        int idx;

        idx = device_links_read_lock();

        /*
         * The status of a device link can only be changed from "dormant" by a
         * probe, but that cannot happen during system suspend/resume.  In
         * theory it can change to "dormant" at that time, but then it is
         * reasonable to wait for the target device anyway (eg. if it goes
         * away, it's better to wait for it to go away completely and then
         * continue instead of trying to continue in parallel with its
         * unregistration).
         */
        list_for_each_entry_rcu_locked(link, &dev->links.consumers, s_node)
                if (READ_ONCE(link->status) != DL_STATE_DORMANT)
                        dpm_wait(link->consumer, async);

        device_links_read_unlock(idx);
}

static void dpm_wait_for_subordinate(struct device *dev, bool async)
{
        dpm_wait_for_children(dev, async);
        dpm_wait_for_consumers(dev, async);
}

/**
 * pm_op - Return the PM operation appropriate for given PM event.
 * @ops: PM operations to choose from.
 * @state: PM transition of the system being carried out.
 */
static pm_callback_t pm_op(const struct dev_pm_ops *ops, pm_message_t state)
{
        switch (state.event) {
#ifdef CONFIG_SUSPEND
        case PM_EVENT_SUSPEND:
                return ops->suspend;
        case PM_EVENT_RESUME:
                return ops->resume;
#endif /* CONFIG_SUSPEND */
#ifdef CONFIG_HIBERNATE_CALLBACKS
        case PM_EVENT_FREEZE:
        case PM_EVENT_QUIESCE:
                return ops->freeze;
        case PM_EVENT_HIBERNATE:
                return ops->poweroff;
        case PM_EVENT_THAW:
        case PM_EVENT_RECOVER:
                return ops->thaw;
        case PM_EVENT_RESTORE:
                return ops->restore;
#endif /* CONFIG_HIBERNATE_CALLBACKS */
        }

        return NULL;
}

/**
 * pm_late_early_op - Return the PM operation appropriate for given PM event.
 * @ops: PM operations to choose from.
 * @state: PM transition of the system being carried out.
 *
 * Runtime PM is disabled for @dev while this function is being executed.
 */
static pm_callback_t pm_late_early_op(const struct dev_pm_ops *ops,
                                      pm_message_t state)
{
        switch (state.event) {
#ifdef CONFIG_SUSPEND
        case PM_EVENT_SUSPEND:
                return ops->suspend_late;
        case PM_EVENT_RESUME:
                return ops->resume_early;
#endif /* CONFIG_SUSPEND */
#ifdef CONFIG_HIBERNATE_CALLBACKS
        case PM_EVENT_FREEZE:
        case PM_EVENT_QUIESCE:
                return ops->freeze_late;
        case PM_EVENT_HIBERNATE:
                return ops->poweroff_late;
        case PM_EVENT_THAW:
        case PM_EVENT_RECOVER:
                return ops->thaw_early;
        case PM_EVENT_RESTORE:
                return ops->restore_early;
#endif /* CONFIG_HIBERNATE_CALLBACKS */
        }

        return NULL;
}

/**
 * pm_noirq_op - Return the PM operation appropriate for given PM event.
 * @ops: PM operations to choose from.
 * @state: PM transition of the system being carried out.
 *
 * The driver of @dev will not receive interrupts while this function is being
 * executed.
 */
static pm_callback_t pm_noirq_op(const struct dev_pm_ops *ops, pm_message_t state)
{
        switch (state.event) {
#ifdef CONFIG_SUSPEND
        case PM_EVENT_SUSPEND:
                return ops->suspend_noirq;
        case PM_EVENT_RESUME:
                return ops->resume_noirq;
#endif /* CONFIG_SUSPEND */
#ifdef CONFIG_HIBERNATE_CALLBACKS
        case PM_EVENT_FREEZE:
        case PM_EVENT_QUIESCE:
                return ops->freeze_noirq;
        case PM_EVENT_HIBERNATE:
                return ops->poweroff_noirq;
        case PM_EVENT_THAW:
        case PM_EVENT_RECOVER:
                return ops->thaw_noirq;
        case PM_EVENT_RESTORE:
                return ops->restore_noirq;
#endif /* CONFIG_HIBERNATE_CALLBACKS */
        }

        return NULL;
}

static void pm_dev_dbg(struct device *dev, pm_message_t state, const char *info)
{
        dev_dbg(dev, "%s%s%s driver flags: %x\n", info, pm_verb(state.event),
                ((state.event & PM_EVENT_SLEEP) && device_may_wakeup(dev)) ?
                ", may wakeup" : "", dev->power.driver_flags);
}

static void pm_dev_err(struct device *dev, pm_message_t state, const char *info,
                        int error)
{
        dev_err(dev, "failed to %s%s: error %d\n", pm_verb(state.event), info,
                error);
}

static void dpm_show_time(ktime_t starttime, pm_message_t state, int error,
                          const char *info)
{
        ktime_t calltime;
        u64 usecs64;
        int usecs;

        calltime = ktime_get();
        usecs64 = ktime_to_ns(ktime_sub(calltime, starttime));
        do_div(usecs64, NSEC_PER_USEC);
        usecs = usecs64;
        if (usecs == 0)
                usecs = 1;

        pm_pr_dbg("%s%s%s of devices %s after %ld.%03ld msecs\n",
                  info ?: "", info ? " " : "", pm_verb(state.event),
                  error ? "aborted" : "complete",
                  usecs / USEC_PER_MSEC, usecs % USEC_PER_MSEC);
}

static int dpm_run_callback(pm_callback_t cb, struct device *dev,
                            pm_message_t state, const char *info)
{
        ktime_t calltime;
        int error;

        if (!cb)
                return 0;

        calltime = initcall_debug_start(dev, cb);

        pm_dev_dbg(dev, state, info);
        trace_device_pm_callback_start(dev, info, state.event);
        error = cb(dev);
        trace_device_pm_callback_end(dev, error);
        suspend_report_result(cb, error);

        initcall_debug_report(dev, calltime, cb, error);

        return error;
}

#ifdef CONFIG_DPM_WATCHDOG
struct dpm_watchdog {
        struct device                *dev;
        struct task_struct        *tsk;
        struct timer_list        timer;
};

#define DECLARE_DPM_WATCHDOG_ON_STACK(wd) \
        struct dpm_watchdog wd

/**
 * dpm_watchdog_handler - Driver suspend / resume watchdog handler.
 * @t: The timer that PM watchdog depends on.
 *
 * Called when a driver has timed out suspending or resuming.
 * There's not much we can do here to recover so panic() to
 * capture a crash-dump in pstore.
 */
static void dpm_watchdog_handler(struct timer_list *t)
{
        struct dpm_watchdog *wd = from_timer(wd, t, timer);

        dev_emerg(wd->dev, "**** DPM device timeout ****\n");
        show_stack(wd->tsk, NULL, KERN_EMERG);
        panic("%s %s: unrecoverable failure\n",
                dev_driver_string(wd->dev), dev_name(wd->dev));
}

/**
 * dpm_watchdog_set - Enable pm watchdog for given device.
 * @wd: Watchdog. Must be allocated on the stack.
 * @dev: Device to handle.
 */
static void dpm_watchdog_set(struct dpm_watchdog *wd, struct device *dev)
{
        struct timer_list *timer = &wd->timer;

        wd->dev = dev;
        wd->tsk = current;

        timer_setup_on_stack(timer, dpm_watchdog_handler, 0);
        /* use same timeout value for both suspend and resume */
        timer->expires = jiffies + HZ * CONFIG_DPM_WATCHDOG_TIMEOUT;
        add_timer(timer);
}

/**
 * dpm_watchdog_clear - Disable suspend/resume watchdog.
 * @wd: Watchdog to disable.
 */
static void dpm_watchdog_clear(struct dpm_watchdog *wd)
{
        struct timer_list *timer = &wd->timer;

        del_timer_sync(timer);
        destroy_timer_on_stack(timer);
}
#else
#define DECLARE_DPM_WATCHDOG_ON_STACK(wd)
#define dpm_watchdog_set(x, y)
#define dpm_watchdog_clear(x)
#endif

/*------------------------- Resume routines -------------------------*/

/**
 * dev_pm_skip_resume - System-wide device resume optimization check.
 * @dev: Target device.
 *
 * Return:
 * - %false if the transition under way is RESTORE.
 * - Return value of dev_pm_skip_suspend() if the transition under way is THAW.
 * - The logical negation of %power.must_resume otherwise (that is, when the
 *   transition under way is RESUME).
 */
bool dev_pm_skip_resume(struct device *dev)
{
        if (pm_transition.event == PM_EVENT_RESTORE)
                return false;

        if (pm_transition.event == PM_EVENT_THAW)
                return dev_pm_skip_suspend(dev);

        return !dev->power.must_resume;
}

/**
 * __device_resume_noirq - Execute a "noirq resume" callback for given device.
 * @dev: Device to handle.
 * @state: PM transition of the system being carried out.
 * @async: If true, the device is being resumed asynchronously.
 *
 * The driver of @dev will not receive interrupts while this function is being
 * executed.
 */
static void __device_resume_noirq(struct device *dev, pm_message_t state, bool async)
{
        pm_callback_t callback = NULL;
        const char *info = NULL;
        bool skip_resume;
        int error = 0;

        TRACE_DEVICE(dev);
        TRACE_RESUME(0);

        if (dev->power.syscore || dev->power.direct_complete)
                goto Out;

        if (!dev->power.is_noirq_suspended)
                goto Out;

        if (!dpm_wait_for_superior(dev, async))
                goto Out;

        skip_resume = dev_pm_skip_resume(dev);
        /*
         * If the driver callback is skipped below or by the middle layer
         * callback and device_resume_early() also skips the driver callback for
         * this device later, it needs to appear as "suspended" to PM-runtime,
         * so change its status accordingly.
         *
         * Otherwise, the device is going to be resumed, so set its PM-runtime
         * status to "active", but do that only if DPM_FLAG_SMART_SUSPEND is set
         * to avoid confusing drivers that don't use it.
         */
        if (skip_resume)
                pm_runtime_set_suspended(dev);
        else if (dev_pm_skip_suspend(dev))
                pm_runtime_set_active(dev);

        if (dev->pm_domain) {
                info = "noirq power domain ";
                callback = pm_noirq_op(&dev->pm_domain->ops, state);
        } else if (dev->type && dev->type->pm) {
                info = "noirq type ";
                callback = pm_noirq_op(dev->type->pm, state);
        } else if (dev->class && dev->class->pm) {
                info = "noirq class ";
                callback = pm_noirq_op(dev->class->pm, state);
        } else if (dev->bus && dev->bus->pm) {
                info = "noirq bus ";
                callback = pm_noirq_op(dev->bus->pm, state);
        }
        if (callback)
                goto Run;

        if (skip_resume)
                goto Skip;

        if (dev->driver && dev->driver->pm) {
                info = "noirq driver ";
                callback = pm_noirq_op(dev->driver->pm, state);
        }

Run:
        error = dpm_run_callback(callback, dev, state, info);

Skip:
        dev->power.is_noirq_suspended = false;

Out:
        complete_all(&dev->power.completion);
        TRACE_RESUME(error);

        if (error) {
                suspend_stats.failed_resume_noirq++;
                dpm_save_failed_step(SUSPEND_RESUME_NOIRQ);
                dpm_save_failed_dev(dev_name(dev));
                pm_dev_err(dev, state, async ? " async noirq" : " noirq", error);
        }
}

static bool is_async(struct device *dev)
{
        return dev->power.async_suspend && pm_async_enabled
                && !pm_trace_is_enabled();
}

static bool dpm_async_fn(struct device *dev, async_func_t func)
{
        reinit_completion(&dev->power.completion);

        if (!is_async(dev))
                return false;

        get_device(dev);

        if (async_schedule_dev_nocall(func, dev))
                return true;

        put_device(dev);

        return false;
}

static void async_resume_noirq(void *data, async_cookie_t cookie)
{
        struct device *dev = data;

        __device_resume_noirq(dev, pm_transition, true);
        put_device(dev);
}

static void device_resume_noirq(struct device *dev)
{
        if (dpm_async_fn(dev, async_resume_noirq))
                return;

        __device_resume_noirq(dev, pm_transition, false);
}

static void dpm_noirq_resume_devices(pm_message_t state)
{
        struct device *dev;
        ktime_t starttime = ktime_get();

        trace_suspend_resume(TPS("dpm_resume_noirq"), state.event, true);
        mutex_lock(&dpm_list_mtx);
        pm_transition = state;

        while (!list_empty(&dpm_noirq_list)) {
                dev = to_device(dpm_noirq_list.next);
                get_device(dev);
                list_move_tail(&dev->power.entry, &dpm_late_early_list);

                mutex_unlock(&dpm_list_mtx);

                device_resume_noirq(dev);

                put_device(dev);

                mutex_lock(&dpm_list_mtx);
        }
        mutex_unlock(&dpm_list_mtx);
        async_synchronize_full();
        dpm_show_time(starttime, state, 0, "noirq");
        trace_suspend_resume(TPS("dpm_resume_noirq"), state.event, false);
}

/**
 * dpm_resume_noirq - Execute "noirq resume" callbacks for all devices.
 * @state: PM transition of the system being carried out.
 *
 * Invoke the "noirq" resume callbacks for all devices in dpm_noirq_list and
 * allow device drivers' interrupt handlers to be called.
 */
void dpm_resume_noirq(pm_message_t state)
{
        dpm_noirq_resume_devices(state);

        resume_device_irqs();
        device_wakeup_disarm_wake_irqs();

        cpuidle_resume();
}

/**
 * __device_resume_early - Execute an "early resume" callback for given device.
 * @dev: Device to handle.
 * @state: PM transition of the system being carried out.
 * @async: If true, the device is being resumed asynchronously.
 *
 * Runtime PM is disabled for @dev while this function is being executed.
 */
static void __device_resume_early(struct device *dev, pm_message_t state, bool async)
{
        pm_callback_t callback = NULL;
        const char *info = NULL;
        int error = 0;

        TRACE_DEVICE(dev);
        TRACE_RESUME(0);

        if (dev->power.syscore || dev->power.direct_complete)
                goto Out;

        if (!dev->power.is_late_suspended)
                goto Out;

        if (!dpm_wait_for_superior(dev, async))
                goto Out;

        if (dev->pm_domain) {
                info = "early power domain ";
                callback = pm_late_early_op(&dev->pm_domain->ops, state);
        } else if (dev->type && dev->type->pm) {
                info = "early type ";
                callback = pm_late_early_op(dev->type->pm, state);
        } else if (dev->class && dev->class->pm) {
                info = "early class ";
                callback = pm_late_early_op(dev->class->pm, state);
        } else if (dev->bus && dev->bus->pm) {
                info = "early bus ";
                callback = pm_late_early_op(dev->bus->pm, state);
        }
        if (callback)
                goto Run;

        if (dev_pm_skip_resume(dev))
                goto Skip;

        if (dev->driver && dev->driver->pm) {
                info = "early driver ";
                callback = pm_late_early_op(dev->driver->pm, state);
        }

Run:
        error = dpm_run_callback(callback, dev, state, info);

Skip:
        dev->power.is_late_suspended = false;

Out:
        TRACE_RESUME(error);

        pm_runtime_enable(dev);
        complete_all(&dev->power.completion);

        if (error) {
                suspend_stats.failed_resume_early++;
                dpm_save_failed_step(SUSPEND_RESUME_EARLY);
                dpm_save_failed_dev(dev_name(dev));
                pm_dev_err(dev, state, async ? " async early" : " early", error);
        }
}

static void async_resume_early(void *data, async_cookie_t cookie)
{
        struct device *dev = data;

        __device_resume_early(dev, pm_transition, true);
        put_device(dev);
}

static void device_resume_early(struct device *dev)
{
        if (dpm_async_fn(dev, async_resume_early))
                return;

        __device_resume_early(dev, pm_transition, false);
}

/**
 * dpm_resume_early - Execute "early resume" callbacks for all devices.
 * @state: PM transition of the system being carried out.
 */
void dpm_resume_early(pm_message_t state)
{
        struct device *dev;
        ktime_t starttime = ktime_get();

        trace_suspend_resume(TPS("dpm_resume_early"), state.event, true);
        mutex_lock(&dpm_list_mtx);
        pm_transition = state;

        while (!list_empty(&dpm_late_early_list)) {
                dev = to_device(dpm_late_early_list.next);
                get_device(dev);
                list_move_tail(&dev->power.entry, &dpm_suspended_list);

                mutex_unlock(&dpm_list_mtx);

                device_resume_early(dev);

                put_device(dev);

                mutex_lock(&dpm_list_mtx);
        }
        mutex_unlock(&dpm_list_mtx);
        async_synchronize_full();
        dpm_show_time(starttime, state, 0, "early");
        trace_suspend_resume(TPS("dpm_resume_early"), state.event, false);
}

/**
 * dpm_resume_start - Execute "noirq" and "early" device callbacks.
 * @state: PM transition of the system being carried out.
 */
void dpm_resume_start(pm_message_t state)
{
        dpm_resume_noirq(state);
        dpm_resume_early(state);
}
EXPORT_SYMBOL_GPL(dpm_resume_start);

/**
 * __device_resume - Execute "resume" callbacks for given device.
 * @dev: Device to handle.
 * @state: PM transition of the system being carried out.
 * @async: If true, the device is being resumed asynchronously.
 */
static void __device_resume(struct device *dev, pm_message_t state, bool async)
{
        pm_callback_t callback = NULL;
        const char *info = NULL;
        int error = 0;
        DECLARE_DPM_WATCHDOG_ON_STACK(wd);

        TRACE_DEVICE(dev);
        TRACE_RESUME(0);

        if (dev->power.syscore)
                goto Complete;

        if (dev->power.direct_complete) {
                /* Match the pm_runtime_disable() in __device_suspend(). */
                pm_runtime_enable(dev);
                goto Complete;
        }

        if (!dpm_wait_for_superior(dev, async))
                goto Complete;

        dpm_watchdog_set(&wd, dev);
        device_lock(dev);

        /*
         * This is a fib.  But we'll allow new children to be added below
         * a resumed device, even if the device hasn't been completed yet.
         */
        dev->power.is_prepared = false;

        if (!dev->power.is_suspended)
                goto Unlock;

        if (dev->pm_domain) {
                info = "power domain ";
                callback = pm_op(&dev->pm_domain->ops, state);
                goto Driver;
        }

        if (dev->type && dev->type->pm) {
                info = "type ";
                callback = pm_op(dev->type->pm, state);
                goto Driver;
        }

        if (dev->class && dev->class->pm) {
                info = "class ";
                callback = pm_op(dev->class->pm, state);
                goto Driver;
        }

        if (dev->bus) {
                if (dev->bus->pm) {
                        info = "bus ";
                        callback = pm_op(dev->bus->pm, state);
                } else if (dev->bus->resume) {
                        info = "legacy bus ";
                        callback = dev->bus->resume;
                        goto End;
                }
        }

 Driver:
        if (!callback && dev->driver && dev->driver->pm) {
                info = "driver ";
                callback = pm_op(dev->driver->pm, state);
        }

 End:
        error = dpm_run_callback(callback, dev, state, info);
        dev->power.is_suspended = false;

 Unlock:
        device_unlock(dev);
        dpm_watchdog_clear(&wd);

 Complete:
        complete_all(&dev->power.completion);

        TRACE_RESUME(error);

        if (error) {
                suspend_stats.failed_resume++;
                dpm_save_failed_step(SUSPEND_RESUME);
                dpm_save_failed_dev(dev_name(dev));
                pm_dev_err(dev, state, async ? " async" : "", error);
        }
}

static void async_resume(void *data, async_cookie_t cookie)
{
        struct device *dev = data;

        __device_resume(dev, pm_transition, true);
        put_device(dev);
}

static void device_resume(struct device *dev)
{
        if (dpm_async_fn(dev, async_resume))
                return;

        __device_resume(dev, pm_transition, false);
}

/**
 * dpm_resume - Execute "resume" callbacks for non-sysdev devices.
 * @state: PM transition of the system being carried out.
 *
 * Execute the appropriate "resume" callback for all devices whose status
 * indicates that they are suspended.
 */
void dpm_resume(pm_message_t state)
{
        struct device *dev;
        ktime_t starttime = ktime_get();

        trace_suspend_resume(TPS("dpm_resume"), state.event, true);
        might_sleep();

        mutex_lock(&dpm_list_mtx);
        pm_transition = state;
        async_error = 0;

        while (!list_empty(&dpm_suspended_list)) {
                dev = to_device(dpm_suspended_list.next);

                get_device(dev);

                mutex_unlock(&dpm_list_mtx);

                device_resume(dev);

                mutex_lock(&dpm_list_mtx);

                if (!list_empty(&dev->power.entry))
                        list_move_tail(&dev->power.entry, &dpm_prepared_list);

                mutex_unlock(&dpm_list_mtx);

                put_device(dev);

                mutex_lock(&dpm_list_mtx);
        }
        mutex_unlock(&dpm_list_mtx);
        async_synchronize_full();
        dpm_show_time(starttime, state, 0, NULL);

        cpufreq_resume();
        devfreq_resume();
        trace_suspend_resume(TPS("dpm_resume"), state.event, false);
}

/**
 * device_complete - Complete a PM transition for given device.
 * @dev: Device to handle.
 * @state: PM transition of the system being carried out.
 */
static void device_complete(struct device *dev, pm_message_t state)
{
        void (*callback)(struct device *) = NULL;
        const char *info = NULL;

        if (dev->power.syscore)
                goto out;

        device_lock(dev);

        if (dev->pm_domain) {
                info = "completing power domain ";
                callback = dev->pm_domain->ops.complete;
        } else if (dev->type && dev->type->pm) {
                info = "completing type ";
                callback = dev->type->pm->complete;
        } else if (dev->class && dev->class->pm) {
                info = "completing class ";
                callback = dev->class->pm->complete;
        } else if (dev->bus && dev->bus->pm) {
                info = "completing bus ";
                callback = dev->bus->pm->complete;
        }

        if (!callback && dev->driver && dev->driver->pm) {
                info = "completing driver ";
                callback = dev->driver->pm->complete;
        }

        if (callback) {
                pm_dev_dbg(dev, state, info);
                callback(dev);
        }

        device_unlock(dev);

out:
        pm_runtime_put(dev);
}

/**
 * dpm_complete - Complete a PM transition for all non-sysdev devices.
 * @state: PM transition of the system being carried out.
 *
 * Execute the ->complete() callbacks for all devices whose PM status is not
 * DPM_ON (this allows new devices to be registered).
 */
void dpm_complete(pm_message_t state)
{
        struct list_head list;

        trace_suspend_resume(TPS("dpm_complete"), state.event, true);
        might_sleep();

        INIT_LIST_HEAD(&list);
        mutex_lock(&dpm_list_mtx);
        while (!list_empty(&dpm_prepared_list)) {
                struct device *dev = to_device(dpm_prepared_list.prev);

                get_device(dev);
                dev->power.is_prepared = false;
                list_move(&dev->power.entry, &list);

                mutex_unlock(&dpm_list_mtx);

                trace_device_pm_callback_start(dev, "", state.event);
                device_complete(dev, state);
                trace_device_pm_callback_end(dev, 0);

                put_device(dev);

                mutex_lock(&dpm_list_mtx);
        }
        list_splice(&list, &dpm_list);
        mutex_unlock(&dpm_list_mtx);

        /* Allow device probing and trigger re-probing of deferred devices */
        device_unblock_probing();
        trace_suspend_resume(TPS("dpm_complete"), state.event, false);
}

/**
 * dpm_resume_end - Execute "resume" callbacks and complete system transition.
 * @state: PM transition of the system being carried out.
 *
 * Execute "resume" callbacks for all devices and complete the PM transition of
 * the system.
 */
void dpm_resume_end(pm_message_t state)
{
        dpm_resume(state);
        dpm_complete(state);
}
EXPORT_SYMBOL_GPL(dpm_resume_end);


/*------------------------- Suspend routines -------------------------*/

/**
 * resume_event - Return a "resume" message for given "suspend" sleep state.
 * @sleep_state: PM message representing a sleep state.
 *
 * Return a PM message representing the resume event corresponding to given
 * sleep state.
 */
static pm_message_t resume_event(pm_message_t sleep_state)
{
        switch (sleep_state.event) {
        case PM_EVENT_SUSPEND:
                return PMSG_RESUME;
        case PM_EVENT_FREEZE:
        case PM_EVENT_QUIESCE:
                return PMSG_RECOVER;
        case PM_EVENT_HIBERNATE:
                return PMSG_RESTORE;
        }
        return PMSG_ON;
}

static void dpm_superior_set_must_resume(struct device *dev)
{
        struct device_link *link;
        int idx;

        if (dev->parent)
                dev->parent->power.must_resume = true;

        idx = device_links_read_lock();

        list_for_each_entry_rcu_locked(link, &dev->links.suppliers, c_node)
                link->supplier->power.must_resume = true;

        device_links_read_unlock(idx);
}

/**
 * __device_suspend_noirq - Execute a "noirq suspend" callback for given device.
 * @dev: Device to handle.
 * @state: PM transition of the system being carried out.
 * @async: If true, the device is being suspended asynchronously.
 *
 * The driver of @dev will not receive interrupts while this function is being
 * executed.
 */
static int __device_suspend_noirq(struct device *dev, pm_message_t state, bool async)
{
        pm_callback_t callback = NULL;
        const char *info = NULL;
        int error = 0;

        TRACE_DEVICE(dev);
        TRACE_SUSPEND(0);

        dpm_wait_for_subordinate(dev, async);

        if (async_error)
                goto Complete;

        if (dev->power.syscore || dev->power.direct_complete)
                goto Complete;

        if (dev->pm_domain) {
                info = "noirq power domain ";
                callback = pm_noirq_op(&dev->pm_domain->ops, state);
        } else if (dev->type && dev->type->pm) {
                info = "noirq type ";
                callback = pm_noirq_op(dev->type->pm, state);
        } else if (dev->class && dev->class->pm) {
                info = "noirq class ";
                callback = pm_noirq_op(dev->class->pm, state);
        } else if (dev->bus && dev->bus->pm) {
                info = "noirq bus ";
                callback = pm_noirq_op(dev->bus->pm, state);
        }
        if (callback)
                goto Run;

        if (dev_pm_skip_suspend(dev))
                goto Skip;

        if (dev->driver && dev->driver->pm) {
                info = "noirq driver ";
                callback = pm_noirq_op(dev->driver->pm, state);
        }

Run:
        error = dpm_run_callback(callback, dev, state, info);
        if (error) {
                async_error = error;
                goto Complete;
        }

Skip:
        dev->power.is_noirq_suspended = true;

        /*
         * Skipping the resume of devices that were in use right before the
         * system suspend (as indicated by their PM-runtime usage counters)
         * would be suboptimal.  Also resume them if doing that is not allowed
         * to be skipped.
         */
        if (atomic_read(&dev->power.usage_count) > 1 ||
            !(dev_pm_test_driver_flags(dev, DPM_FLAG_MAY_SKIP_RESUME) &&
              dev->power.may_skip_resume))
                dev->power.must_resume = true;

        if (dev->power.must_resume)
                dpm_superior_set_must_resume(dev);

Complete:
        complete_all(&dev->power.completion);
        TRACE_SUSPEND(error);
        return error;
}

static void async_suspend_noirq(void *data, async_cookie_t cookie)
{
        struct device *dev = data;
        int error;

        error = __device_suspend_noirq(dev, pm_transition, true);
        if (error) {
                dpm_save_failed_dev(dev_name(dev));
                pm_dev_err(dev, pm_transition, " async", error);
        }

        put_device(dev);
}

static int device_suspend_noirq(struct device *dev)
{
        if (dpm_async_fn(dev, async_suspend_noirq))
                return 0;

        return __device_suspend_noirq(dev, pm_transition, false);
}

static int dpm_noirq_suspend_devices(pm_message_t state)
{
        ktime_t starttime = ktime_get();
        int error = 0;

        trace_suspend_resume(TPS("dpm_suspend_noirq"), state.event, true);
        mutex_lock(&dpm_list_mtx);
        pm_transition = state;
        async_error = 0;

        while (!list_empty(&dpm_late_early_list)) {
                struct device *dev = to_device(dpm_late_early_list.prev);

                get_device(dev);
                mutex_unlock(&dpm_list_mtx);

                error = device_suspend_noirq(dev);

                mutex_lock(&dpm_list_mtx);

                if (error) {
                        pm_dev_err(dev, state, " noirq", error);
                        dpm_save_failed_dev(dev_name(dev));
                } else if (!list_empty(&dev->power.entry)) {
                        list_move(&dev->power.entry, &dpm_noirq_list);
                }

                mutex_unlock(&dpm_list_mtx);

                put_device(dev);

                mutex_lock(&dpm_list_mtx);

                if (error || async_error)
                        break;
        }
        mutex_unlock(&dpm_list_mtx);
        async_synchronize_full();
        if (!error)
                error = async_error;

        if (error) {
                suspend_stats.failed_suspend_noirq++;
                dpm_save_failed_step(SUSPEND_SUSPEND_NOIRQ);
        }
        dpm_show_time(starttime, state, error, "noirq");
        trace_suspend_resume(TPS("dpm_suspend_noirq"), state.event, false);
        return error;
}

/**
 * dpm_suspend_noirq - Execute "noirq suspend" callbacks for all devices.
 * @state: PM transition of the system being carried out.
 *
 * Prevent device drivers' interrupt handlers from being called and invoke
 * "noirq" suspend callbacks for all non-sysdev devices.
 */
int dpm_suspend_noirq(pm_message_t state)
{
        int ret;

        cpuidle_pause();

        device_wakeup_arm_wake_irqs();
        suspend_device_irqs();

        ret = dpm_noirq_suspend_devices(state);
        if (ret)
                dpm_resume_noirq(resume_event(state));

        return ret;
}

static void dpm_propagate_wakeup_to_parent(struct device *dev)
{
        struct device *parent = dev->parent;

        if (!parent)
                return;

        spin_lock_irq(&parent->power.lock);

        if (device_wakeup_path(dev) && !parent->power.ignore_children)
                parent->power.wakeup_path = true;

        spin_unlock_irq(&parent->power.lock);
}

/**
 * __device_suspend_late - Execute a "late suspend" callback for given device.
 * @dev: Device to handle.
 * @state: PM transition of the system being carried out.
 * @async: If true, the device is being suspended asynchronously.
 *
 * Runtime PM is disabled for @dev while this function is being executed.
 */
static int __device_suspend_late(struct device *dev, pm_message_t state, bool async)
{
        pm_callback_t callback = NULL;
        const char *info = NULL;
        int error = 0;

        TRACE_DEVICE(dev);
        TRACE_SUSPEND(0);

        __pm_runtime_disable(dev, false);

        dpm_wait_for_subordinate(dev, async);

        if (async_error)
                goto Complete;

        if (pm_wakeup_pending()) {
                async_error = -EBUSY;
                goto Complete;
        }

        if (dev->power.syscore || dev->power.direct_complete)
                goto Complete;

        if (dev->pm_domain) {
                info = "late power domain ";
                callback = pm_late_early_op(&dev->pm_domain->ops, state);
        } else if (dev->type && dev->type->pm) {
                info = "late type ";
                callback = pm_late_early_op(dev->type->pm, state);
        } else if (dev->class && dev->class->pm) {
                info = "late class ";
                callback = pm_late_early_op(dev->class->pm, state);
        } else if (dev->bus && dev->bus->pm) {
                info = "late bus ";
                callback = pm_late_early_op(dev->bus->pm, state);
        }
        if (callback)
                goto Run;

        if (dev_pm_skip_suspend(dev))
                goto Skip;

        if (dev->driver && dev->driver->pm) {
                info = "late driver ";
                callback = pm_late_early_op(dev->driver->pm, state);
        }

Run:
        error = dpm_run_callback(callback, dev, state, info);
        if (error) {
                async_error = error;
                goto Complete;
        }
        dpm_propagate_wakeup_to_parent(dev);

Skip:
        dev->power.is_late_suspended = true;

Complete:
        TRACE_SUSPEND(error);
        complete_all(&dev->power.completion);
        return error;
}

static void async_suspend_late(void *data, async_cookie_t cookie)
{
        struct device *dev = data;
        int error;

        error = __device_suspend_late(dev, pm_transition, true);
        if (error) {
                dpm_save_failed_dev(dev_name(dev));
                pm_dev_err(dev, pm_transition, " async", error);
        }
        put_device(dev);
}

static int device_suspend_late(struct device *dev)
{
        if (dpm_async_fn(dev, async_suspend_late))
                return 0;

        return __device_suspend_late(dev, pm_transition, false);
}

/**
 * dpm_suspend_late - Execute "late suspend" callbacks for all devices.
 * @state: PM transition of the system being carried out.
 */
int dpm_suspend_late(pm_message_t state)
{
        ktime_t starttime = ktime_get();
        int error = 0;

        trace_suspend_resume(TPS("dpm_suspend_late"), state.event, true);
        mutex_lock(&dpm_list_mtx);
        pm_transition = state;
        async_error = 0;

        while (!list_empty(&dpm_suspended_list)) {
                struct device *dev = to_device(dpm_suspended_list.prev);

                get_device(dev);

                mutex_unlock(&dpm_list_mtx);

                error = device_suspend_late(dev);

                mutex_lock(&dpm_list_mtx);

                if (!list_empty(&dev->power.entry))
                        list_move(&dev->power.entry, &dpm_late_early_list);

                if (error) {
                        pm_dev_err(dev, state, " late", error);
                        dpm_save_failed_dev(dev_name(dev));
                }

                mutex_unlock(&dpm_list_mtx);

                put_device(dev);

                mutex_lock(&dpm_list_mtx);

                if (error || async_error)
                        break;
        }
        mutex_unlock(&dpm_list_mtx);
        async_synchronize_full();
        if (!error)
                error = async_error;
        if (error) {
                suspend_stats.failed_suspend_late++;
                dpm_save_failed_step(SUSPEND_SUSPEND_LATE);
                dpm_resume_early(resume_event(state));
        }
        dpm_show_time(starttime, state, error, "late");
        trace_suspend_resume(TPS("dpm_suspend_late"), state.event, false);
        return error;
}

/**
 * dpm_suspend_end - Execute "late" and "noirq" device suspend callbacks.
 * @state: PM transition of the system being carried out.
 */
int dpm_suspend_end(pm_message_t state)
{
        ktime_t starttime = ktime_get();
        int error;

        error = dpm_suspend_late(state);
        if (error)
                goto out;

        error = dpm_suspend_noirq(state);
        if (error)
                dpm_resume_early(resume_event(state));

out:
        dpm_show_time(starttime, state, error, "end");
        return error;
}
EXPORT_SYMBOL_GPL(dpm_suspend_end);

/**
 * legacy_suspend - Execute a legacy (bus or class) suspend callback for device.
 * @dev: Device to suspend.
 * @state: PM transition of the system being carried out.
 * @cb: Suspend callback to execute.
 * @info: string description of caller.
 */
static int legacy_suspend(struct device *dev, pm_message_t state,
                          int (*cb)(struct device *dev, pm_message_t state),
                          const char *info)
{
        int error;
        ktime_t calltime;

        calltime = initcall_debug_start(dev, cb);

        trace_device_pm_callback_start(dev, info, state.event);
        error = cb(dev, state);
        trace_device_pm_callback_end(dev, error);
        suspend_report_result(cb, error);

        initcall_debug_report(dev, calltime, cb, error);

        return error;
}

static void dpm_clear_superiors_direct_complete(struct device *dev)
{
        struct device_link *link;
        int idx;

        if (dev->parent) {
                spin_lock_irq(&dev->parent->power.lock);
                dev->parent->power.direct_complete = false;
                spin_unlock_irq(&dev->parent->power.lock);
        }

        idx = device_links_read_lock();

        list_for_each_entry_rcu_locked(link, &dev->links.suppliers, c_node) {
                spin_lock_irq(&link->supplier->power.lock);
                link->supplier->power.direct_complete = false;
                spin_unlock_irq(&link->supplier->power.lock);
        }

        device_links_read_unlock(idx);
}

/**
 * __device_suspend - Execute "suspend" callbacks for given device.
 * @dev: Device to handle.
 * @state: PM transition of the system being carried out.
 * @async: If true, the device is being suspended asynchronously.
 */
static int __device_suspend(struct device *dev, pm_message_t state, bool async)
{
        pm_callback_t callback = NULL;
        const char *info = NULL;
        int error = 0;
        DECLARE_DPM_WATCHDOG_ON_STACK(wd);

        TRACE_DEVICE(dev);
        TRACE_SUSPEND(0);

        dpm_wait_for_subordinate(dev, async);

        if (async_error) {
                dev->power.direct_complete = false;
                goto Complete;
        }

        /*
         * Wait for possible runtime PM transitions of the device in progress
         * to complete and if there's a runtime resume request pending for it,
         * resume it before proceeding with invoking the system-wide suspend
         * callbacks for it.
         *
         * If the system-wide suspend callbacks below change the configuration
         * of the device, they must disable runtime PM for it or otherwise
         * ensure that its runtime-resume callbacks will not be confused by that
         * change in case they are invoked going forward.
         */
        pm_runtime_barrier(dev);

        if (pm_wakeup_pending()) {
                dev->power.direct_complete = false;
                async_error = -EBUSY;
                goto Complete;
        }

        if (dev->power.syscore)
                goto Complete;

        /* Avoid direct_complete to let wakeup_path propagate. */
        if (device_may_wakeup(dev) || device_wakeup_path(dev))
                dev->power.direct_complete = false;

        if (dev->power.direct_complete) {
                if (pm_runtime_status_suspended(dev)) {
                        pm_runtime_disable(dev);
                        if (pm_runtime_status_suspended(dev)) {
                                pm_dev_dbg(dev, state, "direct-complete ");
                                goto Complete;
                        }

                        pm_runtime_enable(dev);
                }
                dev->power.direct_complete = false;
        }

        dev->power.may_skip_resume = true;
        dev->power.must_resume = !dev_pm_test_driver_flags(dev, DPM_FLAG_MAY_SKIP_RESUME);

        dpm_watchdog_set(&wd, dev);
        device_lock(dev);

        if (dev->pm_domain) {
                info = "power domain ";
                callback = pm_op(&dev->pm_domain->ops, state);
                goto Run;
        }

        if (dev->type && dev->type->pm) {
                info = "type ";
                callback = pm_op(dev->type->pm, state);
                goto Run;
        }

        if (dev->class && dev->class->pm) {
                info = "class ";
                callback = pm_op(dev->class->pm, state);
                goto Run;
        }

        if (dev->bus) {
                if (dev->bus->pm) {
                        info = "bus ";
                        callback = pm_op(dev->bus->pm, state);
                } else if (dev->bus->suspend) {
                        pm_dev_dbg(dev, state, "legacy bus ");
                        error = legacy_suspend(dev, state, dev->bus->suspend,
                                                "legacy bus ");
                        goto End;
                }
        }

 Run:
        if (!callback && dev->driver && dev->driver->pm) {
                info = "driver ";
                callback = pm_op(dev->driver->pm, state);
        }

        error = dpm_run_callback(callback, dev, state, info);

 End:
        if (!error) {
                dev->power.is_suspended = true;
                if (device_may_wakeup(dev))
                        dev->power.wakeup_path = true;

                dpm_propagate_wakeup_to_parent(dev);
                dpm_clear_superiors_direct_complete(dev);
        }

        device_unlock(dev);
        dpm_watchdog_clear(&wd);

 Complete:
        if (error)
                async_error = error;

        complete_all(&dev->power.completion);
        TRACE_SUSPEND(error);
        return error;
}

static void async_suspend(void *data, async_cookie_t cookie)
{
        struct device *dev = data;
        int error;

        error = __device_suspend(dev, pm_transition, true);
        if (error) {
                dpm_save_failed_dev(dev_name(dev));
                pm_dev_err(dev, pm_transition, " async", error);
        }

        put_device(dev);
}

static int device_suspend(struct device *dev)
{
        if (dpm_async_fn(dev, async_suspend))
                return 0;

        return __device_suspend(dev, pm_transition, false);
}

/**
 * dpm_suspend - Execute "suspend" callbacks for all non-sysdev devices.
 * @state: PM transition of the system being carried out.
 */
int dpm_suspend(pm_message_t state)
{
        ktime_t starttime = ktime_get();
        int error = 0;

        trace_suspend_resume(TPS("dpm_suspend"), state.event, true);
        might_sleep();

        devfreq_suspend();
        cpufreq_suspend();

        mutex_lock(&dpm_list_mtx);
        pm_transition = state;
        async_error = 0;
        while (!list_empty(&dpm_prepared_list)) {
                struct device *dev = to_device(dpm_prepared_list.prev);

                get_device(dev);

                mutex_unlock(&dpm_list_mtx);

                error = device_suspend(dev);

                mutex_lock(&dpm_list_mtx);

                if (error) {
                        pm_dev_err(dev, state, "", error);
                        dpm_save_failed_dev(dev_name(dev));
                } else if (!list_empty(&dev->power.entry)) {
                        list_move(&dev->power.entry, &dpm_suspended_list);
                }

                mutex_unlock(&dpm_list_mtx);

                put_device(dev);

                mutex_lock(&dpm_list_mtx);

                if (error || async_error)
                        break;
        }
        mutex_unlock(&dpm_list_mtx);
        async_synchronize_full();
        if (!error)
                error = async_error;
        if (error) {
                suspend_stats.failed_suspend++;
                dpm_save_failed_step(SUSPEND_SUSPEND);
        }
        dpm_show_time(starttime, state, error, NULL);
        trace_suspend_resume(TPS("dpm_suspend"), state.event, false);
        return error;
}

/**
 * device_prepare - Prepare a device for system power transition.
 * @dev: Device to handle.
 * @state: PM transition of the system being carried out.
 *
 * Execute the ->prepare() callback(s) for given device.  No new children of the
 * device may be registered after this function has returned.
 */
static int device_prepare(struct device *dev, pm_message_t state)
{
        int (*callback)(struct device *) = NULL;
        int ret = 0;

        /*
         * If a device's parent goes into runtime suspend at the wrong time,
         * it won't be possible to resume the device.  To prevent this we
         * block runtime suspend here, during the prepare phase, and allow
         * it again during the complete phase.
         */
        pm_runtime_get_noresume(dev);

        if (dev->power.syscore)
                return 0;

        device_lock(dev);

        dev->power.wakeup_path = false;

        if (dev->power.no_pm_callbacks)
                goto unlock;

        if (dev->pm_domain)
                callback = dev->pm_domain->ops.prepare;
        else if (dev->type && dev->type->pm)
                callback = dev->type->pm->prepare;
        else if (dev->class && dev->class->pm)
                callback = dev->class->pm->prepare;
        else if (dev->bus && dev->bus->pm)
                callback = dev->bus->pm->prepare;

        if (!callback && dev->driver && dev->driver->pm)
                callback = dev->driver->pm->prepare;

        if (callback)
                ret = callback(dev);

unlock:
        device_unlock(dev);

        if (ret < 0) {
                suspend_report_result(callback, ret);
                pm_runtime_put(dev);
                return ret;
        }
        /*
         * A positive return value from ->prepare() means "this device appears
         * to be runtime-suspended and its state is fine, so if it really is
         * runtime-suspended, you can leave it in that state provided that you
         * will do the same thing with all of its descendants".  This only
         * applies to suspend transitions, however.
         */
        spin_lock_irq(&dev->power.lock);
        dev->power.direct_complete = state.event == PM_EVENT_SUSPEND &&
                (ret > 0 || dev->power.no_pm_callbacks) &&
                !dev_pm_test_driver_flags(dev, DPM_FLAG_NO_DIRECT_COMPLETE);
        spin_unlock_irq(&dev->power.lock);
        return 0;
}

/**
 * dpm_prepare - Prepare all non-sysdev devices for a system PM transition.
 * @state: PM transition of the system being carried out.
 *
 * Execute the ->prepare() callback(s) for all devices.
 */
int dpm_prepare(pm_message_t state)
{
        int error = 0;

        trace_suspend_resume(TPS("dpm_prepare"), state.event, true);
        might_sleep();

        /*
         * Give a chance for the known devices to complete their probes, before
         * disable probing of devices. This sync point is important at least
         * at boot time + hibernation restore.
         */
        wait_for_device_probe();
        /*
         * It is unsafe if probing of devices will happen during suspend or
         * hibernation and system behavior will be unpredictable in this case.
         * So, let's prohibit device's probing here and defer their probes
         * instead. The normal behavior will be restored in dpm_complete().
         */
        device_block_probing();

        mutex_lock(&dpm_list_mtx);
        while (!list_empty(&dpm_list) && !error) {
                struct device *dev = to_device(dpm_list.next);

                get_device(dev);

                mutex_unlock(&dpm_list_mtx);

                trace_device_pm_callback_start(dev, "", state.event);
                error = device_prepare(dev, state);
                trace_device_pm_callback_end(dev, error);

                mutex_lock(&dpm_list_mtx);

                if (!error) {
                        dev->power.is_prepared = true;
                        if (!list_empty(&dev->power.entry))
                                list_move_tail(&dev->power.entry, &dpm_prepared_list);
                } else if (error == -EAGAIN) {
                        error = 0;
                } else {
                        dev_info(dev, "not prepared for power transition: code %d\n",
                                 error);
                }

                mutex_unlock(&dpm_list_mtx);

                put_device(dev);

                mutex_lock(&dpm_list_mtx);
        }
        mutex_unlock(&dpm_list_mtx);
        trace_suspend_resume(TPS("dpm_prepare"), state.event, false);
        return error;
}

/**
 * dpm_suspend_start - Prepare devices for PM transition and suspend them.
 * @state: PM transition of the system being carried out.
 *
 * Prepare all non-sysdev devices for system PM transition and execute "suspend"
 * callbacks for them.
 */
int dpm_suspend_start(pm_message_t state)
{
        ktime_t starttime = ktime_get();
        int error;

        error = dpm_prepare(state);
        if (error) {
                suspend_stats.failed_prepare++;
                dpm_save_failed_step(SUSPEND_PREPARE);
        } else
                error = dpm_suspend(state);
        dpm_show_time(starttime, state, error, "start");
        return error;
}
EXPORT_SYMBOL_GPL(dpm_suspend_start);

void __suspend_report_result(const char *function, void *fn, int ret)
{
        if (ret)
                pr_err("%s(): %pS returns %d\n", function, fn, ret);
}
EXPORT_SYMBOL_GPL(__suspend_report_result);

/**
 * device_pm_wait_for_dev - Wait for suspend/resume of a device to complete.
 * @subordinate: Device that needs to wait for @dev.
 * @dev: Device to wait for.
 */
int device_pm_wait_for_dev(struct device *subordinate, struct device *dev)
{
        dpm_wait(dev, subordinate->power.async_suspend);
        return async_error;
}
EXPORT_SYMBOL_GPL(device_pm_wait_for_dev);

/**
 * dpm_for_each_dev - device iterator.
 * @data: data for the callback.
 * @fn: function to be called for each device.
 *
 * Iterate over devices in dpm_list, and call @fn for each device,
 * passing it @data.
 */
void dpm_for_each_dev(void *data, void (*fn)(struct device *, void *))
{
        struct device *dev;

        if (!fn)
                return;

        device_pm_lock();
        list_for_each_entry(dev, &dpm_list, power.entry)
                fn(dev, data);
        device_pm_unlock();
}
EXPORT_SYMBOL_GPL(dpm_for_each_dev);

static bool pm_ops_is_empty(const struct dev_pm_ops *ops)
{
        if (!ops)
                return true;

        return !ops->prepare &&
               !ops->suspend &&
               !ops->suspend_late &&
               !ops->suspend_noirq &&
               !ops->resume_noirq &&
               !ops->resume_early &&
               !ops->resume &&
               !ops->complete;
}

void device_pm_check_callbacks(struct device *dev)
{
        unsigned long flags;

        spin_lock_irqsave(&dev->power.lock, flags);
        dev->power.no_pm_callbacks =
                (!dev->bus || (pm_ops_is_empty(dev->bus->pm) &&
                 !dev->bus->suspend && !dev->bus->resume)) &&
                (!dev->class || pm_ops_is_empty(dev->class->pm)) &&
                (!dev->type || pm_ops_is_empty(dev->type->pm)) &&
                (!dev->pm_domain || pm_ops_is_empty(&dev->pm_domain->ops)) &&
                (!dev->driver || (pm_ops_is_empty(dev->driver->pm) &&
                 !dev->driver->suspend && !dev->driver->resume));
        spin_unlock_irqrestore(&dev->power.lock, flags);
}

bool dev_pm_skip_suspend(struct device *dev)
{
        return dev_pm_test_driver_flags(dev, DPM_FLAG_SMART_SUSPEND) &&
                pm_runtime_status_suspended(dev);
}
























































    8 
    8 








1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
// SPDX-License-Identifier: GPL-2.0-only
/*
 *      crc16.c
 */

#include <linux/types.h>
#include <linux/module.h>
#include <linux/crc16.h>

/** CRC table for the CRC-16. The poly is 0x8005 (x^16 + x^15 + x^2 + 1) */
u16 const crc16_table[256] = {
        0x0000, 0xC0C1, 0xC181, 0x0140, 0xC301, 0x03C0, 0x0280, 0xC241,
        0xC601, 0x06C0, 0x0780, 0xC741, 0x0500, 0xC5C1, 0xC481, 0x0440,
        0xCC01, 0x0CC0, 0x0D80, 0xCD41, 0x0F00, 0xCFC1, 0xCE81, 0x0E40,
        0x0A00, 0xCAC1, 0xCB81, 0x0B40, 0xC901, 0x09C0, 0x0880, 0xC841,
        0xD801, 0x18C0, 0x1980, 0xD941, 0x1B00, 0xDBC1, 0xDA81, 0x1A40,
        0x1E00, 0xDEC1, 0xDF81, 0x1F40, 0xDD01, 0x1DC0, 0x1C80, 0xDC41,
        0x1400, 0xD4C1, 0xD581, 0x1540, 0xD701, 0x17C0, 0x1680, 0xD641,
        0xD201, 0x12C0, 0x1380, 0xD341, 0x1100, 0xD1C1, 0xD081, 0x1040,
        0xF001, 0x30C0, 0x3180, 0xF141, 0x3300, 0xF3C1, 0xF281, 0x3240,
        0x3600, 0xF6C1, 0xF781, 0x3740, 0xF501, 0x35C0, 0x3480, 0xF441,
        0x3C00, 0xFCC1, 0xFD81, 0x3D40, 0xFF01, 0x3FC0, 0x3E80, 0xFE41,
        0xFA01, 0x3AC0, 0x3B80, 0xFB41, 0x3900, 0xF9C1, 0xF881, 0x3840,
        0x2800, 0xE8C1, 0xE981, 0x2940, 0xEB01, 0x2BC0, 0x2A80, 0xEA41,
        0xEE01, 0x2EC0, 0x2F80, 0xEF41, 0x2D00, 0xEDC1, 0xEC81, 0x2C40,
        0xE401, 0x24C0, 0x2580, 0xE541, 0x2700, 0xE7C1, 0xE681, 0x2640,
        0x2200, 0xE2C1, 0xE381, 0x2340, 0xE101, 0x21C0, 0x2080, 0xE041,
        0xA001, 0x60C0, 0x6180, 0xA141, 0x6300, 0xA3C1, 0xA281, 0x6240,
        0x6600, 0xA6C1, 0xA781, 0x6740, 0xA501, 0x65C0, 0x6480, 0xA441,
        0x6C00, 0xACC1, 0xAD81, 0x6D40, 0xAF01, 0x6FC0, 0x6E80, 0xAE41,
        0xAA01, 0x6AC0, 0x6B80, 0xAB41, 0x6900, 0xA9C1, 0xA881, 0x6840,
        0x7800, 0xB8C1, 0xB981, 0x7940, 0xBB01, 0x7BC0, 0x7A80, 0xBA41,
        0xBE01, 0x7EC0, 0x7F80, 0xBF41, 0x7D00, 0xBDC1, 0xBC81, 0x7C40,
        0xB401, 0x74C0, 0x7580, 0xB541, 0x7700, 0xB7C1, 0xB681, 0x7640,
        0x7200, 0xB2C1, 0xB381, 0x7340, 0xB101, 0x71C0, 0x7080, 0xB041,
        0x5000, 0x90C1, 0x9181, 0x5140, 0x9301, 0x53C0, 0x5280, 0x9241,
        0x9601, 0x56C0, 0x5780, 0x9741, 0x5500, 0x95C1, 0x9481, 0x5440,
        0x9C01, 0x5CC0, 0x5D80, 0x9D41, 0x5F00, 0x9FC1, 0x9E81, 0x5E40,
        0x5A00, 0x9AC1, 0x9B81, 0x5B40, 0x9901, 0x59C0, 0x5880, 0x9841,
        0x8801, 0x48C0, 0x4980, 0x8941, 0x4B00, 0x8BC1, 0x8A81, 0x4A40,
        0x4E00, 0x8EC1, 0x8F81, 0x4F40, 0x8D01, 0x4DC0, 0x4C80, 0x8C41,
        0x4400, 0x84C1, 0x8581, 0x4540, 0x8701, 0x47C0, 0x4680, 0x8641,
        0x8201, 0x42C0, 0x4380, 0x8341, 0x4100, 0x81C1, 0x8081, 0x4040
};
EXPORT_SYMBOL(crc16_table);

/**
 * crc16 - compute the CRC-16 for the data buffer
 * @crc:        previous CRC value
 * @buffer:        data pointer
 * @len:        number of bytes in the buffer
 *
 * Returns the updated CRC value.
 */
u16 crc16(u16 crc, u8 const *buffer, size_t len)
{
        while (len--)
                crc = crc16_byte(crc, *buffer++);
        return crc;
}
EXPORT_SYMBOL(crc16);

MODULE_DESCRIPTION("CRC16 calculations");
MODULE_LICENSE("GPL");












































































































































  978 







  979 






























































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
// SPDX-License-Identifier: GPL-2.0-only
#include <linux/etherdevice.h>
#include "ipvlan.h"
#include <linux/if_vlan.h>
#include <linux/if_tap.h>
#include <linux/interrupt.h>
#include <linux/nsproxy.h>
#include <linux/compat.h>
#include <linux/if_tun.h>
#include <linux/module.h>
#include <linux/skbuff.h>
#include <linux/cache.h>
#include <linux/sched.h>
#include <linux/types.h>
#include <linux/slab.h>
#include <linux/wait.h>
#include <linux/cdev.h>
#include <linux/idr.h>
#include <linux/fs.h>
#include <linux/uio.h>

#include <net/net_namespace.h>
#include <net/rtnetlink.h>
#include <net/sock.h>
#include <linux/virtio_net.h>

#define TUN_OFFLOADS (NETIF_F_HW_CSUM | NETIF_F_TSO_ECN | NETIF_F_TSO | \
                      NETIF_F_TSO6)

static dev_t ipvtap_major;
static struct cdev ipvtap_cdev;

static const void *ipvtap_net_namespace(struct device *d)
{
        struct net_device *dev = to_net_dev(d->parent);
        return dev_net(dev);
}

static struct class ipvtap_class = {
         .name = "ipvtap",
         .owner = THIS_MODULE,
         .ns_type = &net_ns_type_operations,
         .namespace = ipvtap_net_namespace,
};

struct ipvtap_dev {
        struct ipvl_dev vlan;
        struct tap_dev          tap;
};

static void ipvtap_count_tx_dropped(struct tap_dev *tap)
{
        struct ipvtap_dev *vlantap = container_of(tap, struct ipvtap_dev, tap);
        struct ipvl_dev *vlan = &vlantap->vlan;

        this_cpu_inc(vlan->pcpu_stats->tx_drps);
}

static void ipvtap_count_rx_dropped(struct tap_dev *tap)
{
        struct ipvtap_dev *vlantap = container_of(tap, struct ipvtap_dev, tap);
        struct ipvl_dev *vlan = &vlantap->vlan;

        ipvlan_count_rx(vlan, 0, 0, 0);
}

static void ipvtap_update_features(struct tap_dev *tap,
                                   netdev_features_t features)
{
        struct ipvtap_dev *vlantap = container_of(tap, struct ipvtap_dev, tap);
        struct ipvl_dev *vlan = &vlantap->vlan;

        vlan->sfeatures = features;
        netdev_update_features(vlan->dev);
}

static int ipvtap_newlink(struct net *src_net, struct net_device *dev,
                          struct nlattr *tb[], struct nlattr *data[],
                          struct netlink_ext_ack *extack)
{
        struct ipvtap_dev *vlantap = netdev_priv(dev);
        int err;

        INIT_LIST_HEAD(&vlantap->tap.queue_list);

        /* Since macvlan supports all offloads by default, make
         * tap support all offloads also.
         */
        vlantap->tap.tap_features = TUN_OFFLOADS;
        vlantap->tap.count_tx_dropped = ipvtap_count_tx_dropped;
        vlantap->tap.update_features =        ipvtap_update_features;
        vlantap->tap.count_rx_dropped = ipvtap_count_rx_dropped;

        err = netdev_rx_handler_register(dev, tap_handle_frame, &vlantap->tap);
        if (err)
                return err;

        /* Don't put anything that may fail after macvlan_common_newlink
         * because we can't undo what it does.
         */
        err =  ipvlan_link_new(src_net, dev, tb, data, extack);
        if (err) {
                netdev_rx_handler_unregister(dev);
                return err;
        }

        vlantap->tap.dev = vlantap->vlan.dev;

        return err;
}

static void ipvtap_dellink(struct net_device *dev,
                           struct list_head *head)
{
        struct ipvtap_dev *vlan = netdev_priv(dev);

        netdev_rx_handler_unregister(dev);
        tap_del_queues(&vlan->tap);
        ipvlan_link_delete(dev, head);
}

static void ipvtap_setup(struct net_device *dev)
{
        ipvlan_link_setup(dev);
        dev->tx_queue_len = TUN_READQ_SIZE;
        dev->priv_flags &= ~IFF_NO_QUEUE;
}

static struct rtnl_link_ops ipvtap_link_ops __read_mostly = {
        .kind                = "ipvtap",
        .setup                = ipvtap_setup,
        .newlink        = ipvtap_newlink,
        .dellink        = ipvtap_dellink,
        .priv_size        = sizeof(struct ipvtap_dev),
};

static int ipvtap_device_event(struct notifier_block *unused,
                               unsigned long event, void *ptr)
{
        struct net_device *dev = netdev_notifier_info_to_dev(ptr);
        struct ipvtap_dev *vlantap;
        struct device *classdev;
        dev_t devt;
        int err;
        char tap_name[IFNAMSIZ];

        if (dev->rtnl_link_ops != &ipvtap_link_ops)
                return NOTIFY_DONE;

        snprintf(tap_name, IFNAMSIZ, "tap%d", dev->ifindex);
        vlantap = netdev_priv(dev);

        switch (event) {
        case NETDEV_REGISTER:
                /* Create the device node here after the network device has
                 * been registered but before register_netdevice has
                 * finished running.
                 */
                err = tap_get_minor(ipvtap_major, &vlantap->tap);
                if (err)
                        return notifier_from_errno(err);

                devt = MKDEV(MAJOR(ipvtap_major), vlantap->tap.minor);
                classdev = device_create(&ipvtap_class, &dev->dev, devt,
                                         dev, tap_name);
                if (IS_ERR(classdev)) {
                        tap_free_minor(ipvtap_major, &vlantap->tap);
                        return notifier_from_errno(PTR_ERR(classdev));
                }
                err = sysfs_create_link(&dev->dev.kobj, &classdev->kobj,
                                        tap_name);
                if (err)
                        return notifier_from_errno(err);
                break;
        case NETDEV_UNREGISTER:
                /* vlan->minor == 0 if NETDEV_REGISTER above failed */
                if (vlantap->tap.minor == 0)
                        break;
                sysfs_remove_link(&dev->dev.kobj, tap_name);
                devt = MKDEV(MAJOR(ipvtap_major), vlantap->tap.minor);
                device_destroy(&ipvtap_class, devt);
                tap_free_minor(ipvtap_major, &vlantap->tap);
                break;
        case NETDEV_CHANGE_TX_QUEUE_LEN:
                if (tap_queue_resize(&vlantap->tap))
                        return NOTIFY_BAD;
                break;
        }

        return NOTIFY_DONE;
}

static struct notifier_block ipvtap_notifier_block __read_mostly = {
        .notifier_call        = ipvtap_device_event,
};

static int __init ipvtap_init(void)
{
        int err;

        err = tap_create_cdev(&ipvtap_cdev, &ipvtap_major, "ipvtap",
                              THIS_MODULE);
        if (err)
                goto out1;

        err = class_register(&ipvtap_class);
        if (err)
                goto out2;

        err = register_netdevice_notifier(&ipvtap_notifier_block);
        if (err)
                goto out3;

        err = ipvlan_link_register(&ipvtap_link_ops);
        if (err)
                goto out4;

        return 0;

out4:
        unregister_netdevice_notifier(&ipvtap_notifier_block);
out3:
        class_unregister(&ipvtap_class);
out2:
        tap_destroy_cdev(ipvtap_major, &ipvtap_cdev);
out1:
        return err;
}
module_init(ipvtap_init);

static void __exit ipvtap_exit(void)
{
        rtnl_link_unregister(&ipvtap_link_ops);
        unregister_netdevice_notifier(&ipvtap_notifier_block);
        class_unregister(&ipvtap_class);
        tap_destroy_cdev(ipvtap_major, &ipvtap_cdev);
}
module_exit(ipvtap_exit);
MODULE_ALIAS_RTNL_LINK("ipvtap");
MODULE_AUTHOR("Sainath Grandhi <sainath.grandhi@intel.com>");
MODULE_LICENSE("GPL");

































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































   47 
   47 







































































































































































































































































































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
/* SPDX-License-Identifier: GPL-2.0-or-later */
/* internal AFS stuff
 *
 * Copyright (C) 2002, 2007 Red Hat, Inc. All Rights Reserved.
 * Written by David Howells (dhowells@redhat.com)
 */

#include <linux/compiler.h>
#include <linux/kernel.h>
#include <linux/ktime.h>
#include <linux/fs.h>
#include <linux/pagemap.h>
#include <linux/rxrpc.h>
#include <linux/key.h>
#include <linux/workqueue.h>
#include <linux/sched.h>
#define FSCACHE_USE_NEW_IO_API
#include <linux/fscache.h>
#include <linux/backing-dev.h>
#include <linux/uuid.h>
#include <linux/mm_types.h>
#include <linux/dns_resolver.h>
#include <net/net_namespace.h>
#include <net/netns/generic.h>
#include <net/sock.h>
#include <net/af_rxrpc.h>

#include "afs.h"
#include "afs_vl.h"

#define AFS_CELL_MAX_ADDRS 15

struct pagevec;
struct afs_call;
struct afs_vnode;

/*
 * Partial file-locking emulation mode.  (The problem being that AFS3 only
 * allows whole-file locks and no upgrading/downgrading).
 */
enum afs_flock_mode {
        afs_flock_mode_unset,
        afs_flock_mode_local,        /* Local locking only */
        afs_flock_mode_openafs,        /* Don't get server lock for a partial lock */
        afs_flock_mode_strict,        /* Always get a server lock for a partial lock */
        afs_flock_mode_write,        /* Get an exclusive server lock for a partial lock */
};

struct afs_fs_context {
        bool                        force;                /* T to force cell type */
        bool                        autocell;        /* T if set auto mount operation */
        bool                        dyn_root;        /* T if dynamic root */
        bool                        no_cell;        /* T if the source is "none" (for dynroot) */
        enum afs_flock_mode        flock_mode;        /* Partial file-locking emulation mode */
        afs_voltype_t                type;                /* type of volume requested */
        unsigned int                volnamesz;        /* size of volume name */
        const char                *volname;        /* name of volume to mount */
        struct afs_net                *net;                /* the AFS net namespace stuff */
        struct afs_cell                *cell;                /* cell in which to find volume */
        struct afs_volume        *volume;        /* volume record */
        struct key                *key;                /* key to use for secure mounting */
};

enum afs_call_state {
        AFS_CALL_CL_REQUESTING,                /* Client: Request is being sent */
        AFS_CALL_CL_AWAIT_REPLY,        /* Client: Awaiting reply */
        AFS_CALL_CL_PROC_REPLY,                /* Client: rxrpc call complete; processing reply */
        AFS_CALL_SV_AWAIT_OP_ID,        /* Server: Awaiting op ID */
        AFS_CALL_SV_AWAIT_REQUEST,        /* Server: Awaiting request data */
        AFS_CALL_SV_REPLYING,                /* Server: Replying */
        AFS_CALL_SV_AWAIT_ACK,                /* Server: Awaiting final ACK */
        AFS_CALL_COMPLETE,                /* Completed or failed */
};

/*
 * List of server addresses.
 */
struct afs_addr_list {
        struct rcu_head                rcu;
        refcount_t                usage;
        u32                        version;        /* Version */
        unsigned char                max_addrs;
        unsigned char                nr_addrs;
        unsigned char                preferred;        /* Preferred address */
        unsigned char                nr_ipv4;        /* Number of IPv4 addresses */
        enum dns_record_source        source:8;
        enum dns_lookup_status        status:8;
        unsigned long                failed;                /* Mask of addrs that failed locally/ICMP */
        unsigned long                responded;        /* Mask of addrs that responded */
        struct sockaddr_rxrpc        addrs[];
#define AFS_MAX_ADDRESSES ((unsigned int)(sizeof(unsigned long) * 8))
};

/*
 * a record of an in-progress RxRPC call
 */
struct afs_call {
        const struct afs_call_type *type;        /* type of call */
        struct afs_addr_list        *alist;                /* Address is alist[addr_ix] */
        wait_queue_head_t        waitq;                /* processes awaiting completion */
        struct work_struct        async_work;        /* async I/O processor */
        struct work_struct        work;                /* actual work processor */
        struct rxrpc_call        *rxcall;        /* RxRPC call handle */
        struct key                *key;                /* security for this call */
        struct afs_net                *net;                /* The network namespace */
        struct afs_server        *server;        /* The fileserver record if fs op (pins ref) */
        struct afs_vlserver        *vlserver;        /* The vlserver record if vl op */
        void                        *request;        /* request data (first part) */
        size_t                        iov_len;        /* Size of *iter to be used */
        struct iov_iter                def_iter;        /* Default buffer/data iterator */
        struct iov_iter                *write_iter;        /* Iterator defining write to be made */
        struct iov_iter                *iter;                /* Iterator currently in use */
        union {        /* Convenience for ->def_iter */
                struct kvec        kvec[1];
                struct bio_vec        bvec[1];
        };
        void                        *buffer;        /* reply receive buffer */
        union {
                long                        ret0;        /* Value to reply with instead of 0 */
                struct afs_addr_list        *ret_alist;
                struct afs_vldb_entry        *ret_vldb;
                char                        *ret_str;
        };
        struct afs_operation        *op;
        unsigned int                server_index;
        refcount_t                ref;
        enum afs_call_state        state;
        spinlock_t                state_lock;
        int                        error;                /* error code */
        u32                        abort_code;        /* Remote abort ID or 0 */
        unsigned int                max_lifespan;        /* Maximum lifespan to set if not 0 */
        unsigned                request_size;        /* size of request data */
        unsigned                reply_max;        /* maximum size of reply */
        unsigned                count2;                /* count used in unmarshalling */
        unsigned char                unmarshall;        /* unmarshalling phase */
        unsigned char                addr_ix;        /* Address in ->alist */
        bool                        drop_ref;        /* T if need to drop ref for incoming call */
        bool                        need_attention;        /* T if RxRPC poked us */
        bool                        async;                /* T if asynchronous */
        bool                        upgrade;        /* T to request service upgrade */
        bool                        intr;                /* T if interruptible */
        bool                        unmarshalling_error; /* T if an unmarshalling error occurred */
        u16                        service_id;        /* Actual service ID (after upgrade) */
        unsigned int                debug_id;        /* Trace ID */
        u32                        operation_ID;        /* operation ID for an incoming call */
        u32                        count;                /* count for use in unmarshalling */
        union {                                        /* place to extract temporary data */
                struct {
                        __be32        tmp_u;
                        __be32        tmp;
                } __attribute__((packed));
                __be64                tmp64;
        };
        ktime_t                        issue_time;        /* Time of issue of operation */
};

struct afs_call_type {
        const char *name;
        unsigned int op; /* Really enum afs_fs_operation */

        /* deliver request or reply data to an call
         * - returning an error will cause the call to be aborted
         */
        int (*deliver)(struct afs_call *call);

        /* clean up a call */
        void (*destructor)(struct afs_call *call);

        /* Work function */
        void (*work)(struct work_struct *work);

        /* Call done function (gets called immediately on success or failure) */
        void (*done)(struct afs_call *call);
};

/*
 * Key available for writeback on a file.
 */
struct afs_wb_key {
        refcount_t                usage;
        struct key                *key;
        struct list_head        vnode_link;        /* Link in vnode->wb_keys */
};

/*
 * AFS open file information record.  Pointed to by file->private_data.
 */
struct afs_file {
        struct key                *key;                /* The key this file was opened with */
        struct afs_wb_key        *wb;                /* Writeback key record for this file */
};

static inline struct key *afs_file_key(struct file *file)
{
        struct afs_file *af = file->private_data;

        return af->key;
}

/*
 * Record of an outstanding read operation on a vnode.
 */
struct afs_read {
        loff_t                        pos;                /* Where to start reading */
        loff_t                        len;                /* How much we're asking for */
        loff_t                        actual_len;        /* How much we're actually getting */
        loff_t                        file_size;        /* File size returned by server */
        struct key                *key;                /* The key to use to reissue the read */
        struct afs_vnode        *vnode;                /* The file being read into. */
        struct netfs_read_subrequest *subreq;        /* Fscache helper read request this belongs to */
        afs_dataversion_t        data_version;        /* Version number returned by server */
        refcount_t                usage;
        unsigned int                call_debug_id;
        unsigned int                nr_pages;
        int                        error;
        void (*done)(struct afs_read *);
        void (*cleanup)(struct afs_read *);
        struct iov_iter                *iter;                /* Iterator representing the buffer */
        struct iov_iter                def_iter;        /* Default iterator */
};

/*
 * AFS superblock private data
 * - there's one superblock per volume
 */
struct afs_super_info {
        struct net                *net_ns;        /* Network namespace */
        struct afs_cell                *cell;                /* The cell in which the volume resides */
        struct afs_volume        *volume;        /* volume record */
        enum afs_flock_mode        flock_mode:8;        /* File locking emulation mode */
        bool                        dyn_root;        /* True if dynamic root */
};

static inline struct afs_super_info *AFS_FS_S(struct super_block *sb)
{
        return sb->s_fs_info;
}

extern struct file_system_type afs_fs_type;

/*
 * Set of substitutes for @sys.
 */
struct afs_sysnames {
#define AFS_NR_SYSNAME 16
        char                        *subs[AFS_NR_SYSNAME];
        refcount_t                usage;
        unsigned short                nr;
        char                        blank[1];
};

/*
 * AFS network namespace record.
 */
struct afs_net {
        struct net                *net;                /* Backpointer to the owning net namespace */
        struct afs_uuid                uuid;
        bool                        live;                /* F if this namespace is being removed */

        /* AF_RXRPC I/O stuff */
        struct socket                *socket;
        struct afs_call                *spare_incoming_call;
        struct work_struct        charge_preallocation_work;
        struct mutex                socket_mutex;
        atomic_t                nr_outstanding_calls;
        atomic_t                nr_superblocks;

        /* Cell database */
        struct rb_root                cells;
        struct afs_cell                *ws_cell;
        struct work_struct        cells_manager;
        struct timer_list        cells_timer;
        atomic_t                cells_outstanding;
        struct rw_semaphore        cells_lock;
        struct mutex                cells_alias_lock;

        struct mutex                proc_cells_lock;
        struct hlist_head        proc_cells;

        /* Known servers.  Theoretically each fileserver can only be in one
         * cell, but in practice, people create aliases and subsets and there's
         * no easy way to distinguish them.
         */
        seqlock_t                fs_lock;        /* For fs_servers, fs_probe_*, fs_proc */
        struct rb_root                fs_servers;        /* afs_server (by server UUID or address) */
        struct list_head        fs_probe_fast;        /* List of afs_server to probe at 30s intervals */
        struct list_head        fs_probe_slow;        /* List of afs_server to probe at 5m intervals */
        struct hlist_head        fs_proc;        /* procfs servers list */

        struct hlist_head        fs_addresses4;        /* afs_server (by lowest IPv4 addr) */
        struct hlist_head        fs_addresses6;        /* afs_server (by lowest IPv6 addr) */
        seqlock_t                fs_addr_lock;        /* For fs_addresses[46] */

        struct work_struct        fs_manager;
        struct timer_list        fs_timer;

        struct work_struct        fs_prober;
        struct timer_list        fs_probe_timer;
        atomic_t                servers_outstanding;

        /* File locking renewal management */
        struct mutex                lock_manager_mutex;

        /* Misc */
        struct super_block        *dynroot_sb;        /* Dynamic root mount superblock */
        struct proc_dir_entry        *proc_afs;        /* /proc/net/afs directory */
        struct afs_sysnames        *sysnames;
        rwlock_t                sysnames_lock;

        /* Statistics counters */
        atomic_t                n_lookup;        /* Number of lookups done */
        atomic_t                n_reval;        /* Number of dentries needing revalidation */
        atomic_t                n_inval;        /* Number of invalidations by the server */
        atomic_t                n_relpg;        /* Number of invalidations by releasepage */
        atomic_t                n_read_dir;        /* Number of directory pages read */
        atomic_t                n_dir_cr;        /* Number of directory entry creation edits */
        atomic_t                n_dir_rm;        /* Number of directory entry removal edits */
        atomic_t                n_stores;        /* Number of store ops */
        atomic_long_t                n_store_bytes;        /* Number of bytes stored */
        atomic_long_t                n_fetch_bytes;        /* Number of bytes fetched */
        atomic_t                n_fetches;        /* Number of data fetch ops */
};

extern const char afs_init_sysname[];

enum afs_cell_state {
        AFS_CELL_UNSET,
        AFS_CELL_ACTIVATING,
        AFS_CELL_ACTIVE,
        AFS_CELL_DEACTIVATING,
        AFS_CELL_INACTIVE,
        AFS_CELL_FAILED,
        AFS_CELL_REMOVED,
};

/*
 * AFS cell record.
 *
 * This is a tricky concept to get right as it is possible to create aliases
 * simply by pointing AFSDB/SRV records for two names at the same set of VL
 * servers; it is also possible to do things like setting up two sets of VL
 * servers, one of which provides a superset of the volumes provided by the
 * other (for internal/external division, for example).
 *
 * Cells only exist in the sense that (a) a cell's name maps to a set of VL
 * servers and (b) a cell's name is used by the client to select the key to use
 * for authentication and encryption.  The cell name is not typically used in
 * the protocol.
 *
 * Two cells are determined to be aliases if they have an explicit alias (YFS
 * only), share any VL servers in common or have at least one volume in common.
 * "In common" means that the address list of the VL servers or the fileservers
 * share at least one endpoint.
 */
struct afs_cell {
        union {
                struct rcu_head        rcu;
                struct rb_node        net_node;        /* Node in net->cells */
        };
        struct afs_net                *net;
        struct afs_cell                *alias_of;        /* The cell this is an alias of */
        struct afs_volume        *root_volume;        /* The root.cell volume if there is one */
        struct key                *anonymous_key;        /* anonymous user key for this cell */
        struct work_struct        manager;        /* Manager for init/deinit/dns */
        struct hlist_node        proc_link;        /* /proc cell list link */
#ifdef CONFIG_AFS_FSCACHE
        struct fscache_cookie        *cache;                /* caching cookie */
#endif
        time64_t                dns_expiry;        /* Time AFSDB/SRV record expires */
        time64_t                last_inactive;        /* Time of last drop of usage count */
        refcount_t                ref;                /* Struct refcount */
        atomic_t                active;                /* Active usage counter */
        unsigned long                flags;
#define AFS_CELL_FL_NO_GC        0                /* The cell was added manually, don't auto-gc */
#define AFS_CELL_FL_DO_LOOKUP        1                /* DNS lookup requested */
#define AFS_CELL_FL_CHECK_ALIAS        2                /* Need to check for aliases */
        enum afs_cell_state        state;
        short                        error;
        enum dns_record_source        dns_source:8;        /* Latest source of data from lookup */
        enum dns_lookup_status        dns_status:8;        /* Latest status of data from lookup */
        unsigned int                dns_lookup_count; /* Counter of DNS lookups */
        unsigned int                debug_id;

        /* The volumes belonging to this cell */
        struct rb_root                volumes;        /* Tree of volumes on this server */
        struct hlist_head        proc_volumes;        /* procfs volume list */
        seqlock_t                volume_lock;        /* For volumes */

        /* Active fileserver interaction state. */
        struct rb_root                fs_servers;        /* afs_server (by server UUID) */
        seqlock_t                fs_lock;        /* For fs_servers  */
        struct rw_semaphore        fs_open_mmaps_lock;
        struct list_head        fs_open_mmaps;        /* List of vnodes that are mmapped */
        atomic_t                fs_s_break;        /* Counter of CB.InitCallBackState messages */

        /* VL server list. */
        rwlock_t                vl_servers_lock; /* Lock on vl_servers */
        struct afs_vlserver_list __rcu *vl_servers;

        u8                        name_len;        /* Length of name */
        char                        *name;                /* Cell name, case-flattened and NUL-padded */
};

/*
 * Volume Location server record.
 */
struct afs_vlserver {
        struct rcu_head                rcu;
        struct afs_addr_list        __rcu *addresses; /* List of addresses for this VL server */
        unsigned long                flags;
#define AFS_VLSERVER_FL_PROBED        0                /* The VL server has been probed */
#define AFS_VLSERVER_FL_PROBING        1                /* VL server is being probed */
#define AFS_VLSERVER_FL_IS_YFS        2                /* Server is YFS not AFS */
#define AFS_VLSERVER_FL_RESPONDING 3                /* VL server is responding */
        rwlock_t                lock;                /* Lock on addresses */
        refcount_t                ref;
        unsigned int                rtt;                /* Server's current RTT in uS */

        /* Probe state */
        wait_queue_head_t        probe_wq;
        atomic_t                probe_outstanding;
        spinlock_t                probe_lock;
        struct {
                unsigned int        rtt;                /* RTT in uS */
                u32                abort_code;
                short                error;
                unsigned short        flags;
#define AFS_VLSERVER_PROBE_RESPONDED                0x01 /* At least once response (may be abort) */
#define AFS_VLSERVER_PROBE_IS_YFS                0x02 /* The peer appears to be YFS */
#define AFS_VLSERVER_PROBE_NOT_YFS                0x04 /* The peer appears not to be YFS */
#define AFS_VLSERVER_PROBE_LOCAL_FAILURE        0x08 /* A local failure prevented a probe */
        } probe;

        u16                        port;
        u16                        name_len;        /* Length of name */
        char                        name[];                /* Server name, case-flattened */
};

/*
 * Weighted list of Volume Location servers.
 */
struct afs_vlserver_entry {
        u16                        priority;        /* Preference (as SRV) */
        u16                        weight;                /* Weight (as SRV) */
        enum dns_record_source        source:8;
        enum dns_lookup_status        status:8;
        struct afs_vlserver        *server;
};

struct afs_vlserver_list {
        struct rcu_head                rcu;
        refcount_t                ref;
        u8                        nr_servers;
        u8                        index;                /* Server currently in use */
        u8                        preferred;        /* Preferred server */
        enum dns_record_source        source:8;
        enum dns_lookup_status        status:8;
        rwlock_t                lock;
        struct afs_vlserver_entry servers[];
};

/*
 * Cached VLDB entry.
 *
 * This is pointed to by cell->vldb_entries, indexed by name.
 */
struct afs_vldb_entry {
        afs_volid_t                vid[3];                /* Volume IDs for R/W, R/O and Bak volumes */

        unsigned long                flags;
#define AFS_VLDB_HAS_RW                0                /* - R/W volume exists */
#define AFS_VLDB_HAS_RO                1                /* - R/O volume exists */
#define AFS_VLDB_HAS_BAK        2                /* - Backup volume exists */
#define AFS_VLDB_QUERY_VALID        3                /* - Record is valid */
#define AFS_VLDB_QUERY_ERROR        4                /* - VL server returned error */

        uuid_t                        fs_server[AFS_NMAXNSERVERS];
        u32                        addr_version[AFS_NMAXNSERVERS]; /* Registration change counters */
        u8                        fs_mask[AFS_NMAXNSERVERS];
#define AFS_VOL_VTM_RW        0x01 /* R/W version of the volume is available (on this server) */
#define AFS_VOL_VTM_RO        0x02 /* R/O version of the volume is available (on this server) */
#define AFS_VOL_VTM_BAK        0x04 /* backup version of the volume is available (on this server) */
        short                        error;
        u8                        nr_servers;        /* Number of server records */
        u8                        name_len;
        u8                        name[AFS_MAXVOLNAME + 1]; /* NUL-padded volume name */
};

/*
 * Record of fileserver with which we're actively communicating.
 */
struct afs_server {
        struct rcu_head                rcu;
        union {
                uuid_t                uuid;                /* Server ID */
                struct afs_uuid        _uuid;
        };

        struct afs_addr_list        __rcu *addresses;
        struct afs_cell                *cell;                /* Cell to which belongs (pins ref) */
        struct rb_node                uuid_rb;        /* Link in net->fs_servers */
        struct afs_server __rcu        *uuid_next;        /* Next server with same UUID */
        struct afs_server        *uuid_prev;        /* Previous server with same UUID */
        struct list_head        probe_link;        /* Link in net->fs_probe_list */
        struct hlist_node        addr4_link;        /* Link in net->fs_addresses4 */
        struct hlist_node        addr6_link;        /* Link in net->fs_addresses6 */
        struct hlist_node        proc_link;        /* Link in net->fs_proc */
        struct work_struct        initcb_work;        /* Work for CB.InitCallBackState* */
        struct afs_server        *gc_next;        /* Next server in manager's list */
        time64_t                unuse_time;        /* Time at which last unused */
        unsigned long                flags;
#define AFS_SERVER_FL_RESPONDING 0                /* The server is responding */
#define AFS_SERVER_FL_UPDATING        1
#define AFS_SERVER_FL_NEEDS_UPDATE 2                /* Fileserver address list is out of date */
#define AFS_SERVER_FL_NOT_READY        4                /* The record is not ready for use */
#define AFS_SERVER_FL_NOT_FOUND        5                /* VL server says no such server */
#define AFS_SERVER_FL_VL_FAIL        6                /* Failed to access VL server */
#define AFS_SERVER_FL_MAY_HAVE_CB 8                /* May have callbacks on this fileserver */
#define AFS_SERVER_FL_IS_YFS        16                /* Server is YFS not AFS */
#define AFS_SERVER_FL_NO_IBULK        17                /* Fileserver doesn't support FS.InlineBulkStatus */
#define AFS_SERVER_FL_NO_RM2        18                /* Fileserver doesn't support YFS.RemoveFile2 */
#define AFS_SERVER_FL_HAS_FS64        19                /* Fileserver supports FS.{Fetch,Store}Data64 */
        refcount_t                ref;                /* Object refcount */
        atomic_t                active;                /* Active user count */
        u32                        addr_version;        /* Address list version */
        unsigned int                rtt;                /* Server's current RTT in uS */
        unsigned int                debug_id;        /* Debugging ID for traces */

        /* file service access */
        rwlock_t                fs_lock;        /* access lock */

        /* callback promise management */
        unsigned                cb_s_break;        /* Break-everything counter. */

        /* Probe state */
        unsigned long                probed_at;        /* Time last probe was dispatched (jiffies) */
        wait_queue_head_t        probe_wq;
        atomic_t                probe_outstanding;
        spinlock_t                probe_lock;
        struct {
                unsigned int        rtt;                /* RTT in uS */
                u32                abort_code;
                short                error;
                bool                responded:1;
                bool                is_yfs:1;
                bool                not_yfs:1;
                bool                local_failure:1;
        } probe;
};

/*
 * Replaceable volume server list.
 */
struct afs_server_entry {
        struct afs_server        *server;
};

struct afs_server_list {
        struct rcu_head                rcu;
        afs_volid_t                vids[AFS_MAXTYPES]; /* Volume IDs */
        refcount_t                usage;
        unsigned char                nr_servers;
        unsigned char                preferred;        /* Preferred server */
        unsigned short                vnovol_mask;        /* Servers to be skipped due to VNOVOL */
        unsigned int                seq;                /* Set to ->servers_seq when installed */
        rwlock_t                lock;
        struct afs_server_entry        servers[];
};

/*
 * Live AFS volume management.
 */
struct afs_volume {
        union {
                struct rcu_head        rcu;
                afs_volid_t        vid;                /* volume ID */
        };
        refcount_t                ref;
        time64_t                update_at;        /* Time at which to next update */
        struct afs_cell                *cell;                /* Cell to which belongs (pins ref) */
        struct rb_node                cell_node;        /* Link in cell->volumes */
        struct hlist_node        proc_link;        /* Link in cell->proc_volumes */
        struct super_block __rcu *sb;                /* Superblock on which inodes reside */
        unsigned long                flags;
#define AFS_VOLUME_NEEDS_UPDATE        0        /* - T if an update needs performing */
#define AFS_VOLUME_UPDATING        1        /* - T if an update is in progress */
#define AFS_VOLUME_WAIT                2        /* - T if users must wait for update */
#define AFS_VOLUME_DELETED        3        /* - T if volume appears deleted */
#define AFS_VOLUME_OFFLINE        4        /* - T if volume offline notice given */
#define AFS_VOLUME_BUSY                5        /* - T if volume busy notice given */
#define AFS_VOLUME_MAYBE_NO_IBULK 6        /* - T if some servers don't have InlineBulkStatus */
#define AFS_VOLUME_RM_TREE        7        /* - Set if volume removed from cell->volumes */
#ifdef CONFIG_AFS_FSCACHE
        struct fscache_cookie        *cache;                /* caching cookie */
#endif
        struct afs_server_list __rcu *servers;        /* List of servers on which volume resides */
        rwlock_t                servers_lock;        /* Lock for ->servers */
        unsigned int                servers_seq;        /* Incremented each time ->servers changes */

        unsigned                cb_v_break;        /* Break-everything counter. */
        rwlock_t                cb_v_break_lock;

        afs_voltype_t                type;                /* type of volume */
        char                        type_force;        /* force volume type (suppress R/O -> R/W) */
        u8                        name_len;
        u8                        name[AFS_MAXVOLNAME + 1]; /* NUL-padded volume name */
};

enum afs_lock_state {
        AFS_VNODE_LOCK_NONE,                /* The vnode has no lock on the server */
        AFS_VNODE_LOCK_WAITING_FOR_CB,        /* We're waiting for the server to break the callback */
        AFS_VNODE_LOCK_SETTING,                /* We're asking the server for a lock */
        AFS_VNODE_LOCK_GRANTED,                /* We have a lock on the server */
        AFS_VNODE_LOCK_EXTENDING,        /* We're extending a lock on the server */
        AFS_VNODE_LOCK_NEED_UNLOCK,        /* We need to unlock on the server */
        AFS_VNODE_LOCK_UNLOCKING,        /* We're telling the server to unlock */
        AFS_VNODE_LOCK_DELETED,                /* The vnode has been deleted whilst we have a lock */
};

/*
 * AFS inode private data.
 *
 * Note that afs_alloc_inode() *must* reset anything that could incorrectly
 * leak from one inode to another.
 */
struct afs_vnode {
        struct inode                vfs_inode;        /* the VFS's inode record */

        struct afs_volume        *volume;        /* volume on which vnode resides */
        struct afs_fid                fid;                /* the file identifier for this inode */
        struct afs_file_status        status;                /* AFS status info for this file */
        afs_dataversion_t        invalid_before;        /* Child dentries are invalid before this */
#ifdef CONFIG_AFS_FSCACHE
        struct fscache_cookie        *cache;                /* caching cookie */
#endif
        struct afs_permits __rcu *permit_cache;        /* cache of permits so far obtained */
        struct mutex                io_lock;        /* Lock for serialising I/O on this mutex */
        struct rw_semaphore        validate_lock;        /* lock for validating this vnode */
        struct rw_semaphore        rmdir_lock;        /* Lock for rmdir vs sillyrename */
        struct key                *silly_key;        /* Silly rename key */
        spinlock_t                wb_lock;        /* lock for wb_keys */
        spinlock_t                lock;                /* waitqueue/flags lock */
        unsigned long                flags;
#define AFS_VNODE_CB_PROMISED        0                /* Set if vnode has a callback promise */
#define AFS_VNODE_UNSET                1                /* set if vnode attributes not yet set */
#define AFS_VNODE_DIR_VALID        2                /* Set if dir contents are valid */
#define AFS_VNODE_ZAP_DATA        3                /* set if vnode's data should be invalidated */
#define AFS_VNODE_DELETED        4                /* set if vnode deleted on server */
#define AFS_VNODE_MOUNTPOINT        5                /* set if vnode is a mountpoint symlink */
#define AFS_VNODE_AUTOCELL        6                /* set if Vnode is an auto mount point */
#define AFS_VNODE_PSEUDODIR        7                 /* set if Vnode is a pseudo directory */
#define AFS_VNODE_NEW_CONTENT        8                /* Set if file has new content (create/trunc-0) */
#define AFS_VNODE_SILLY_DELETED        9                /* Set if file has been silly-deleted */
#define AFS_VNODE_MODIFYING        10                /* Set if we're performing a modification op */

        struct list_head        wb_keys;        /* List of keys available for writeback */
        struct list_head        pending_locks;        /* locks waiting to be granted */
        struct list_head        granted_locks;        /* locks granted on this file */
        struct delayed_work        lock_work;        /* work to be done in locking */
        struct key                *lock_key;        /* Key to be used in lock ops */
        ktime_t                        locked_at;        /* Time at which lock obtained */
        enum afs_lock_state        lock_state : 8;
        afs_lock_type_t                lock_type : 8;

        /* outstanding callback notification on this file */
        struct work_struct        cb_work;        /* Work for mmap'd files */
        struct list_head        cb_mmap_link;        /* Link in cell->fs_open_mmaps */
        void                        *cb_server;        /* Server with callback/filelock */
        atomic_t                cb_nr_mmap;        /* Number of mmaps */
        unsigned int                cb_fs_s_break;        /* Mass server break counter (cell->fs_s_break) */
        unsigned int                cb_s_break;        /* Mass break counter on ->server */
        unsigned int                cb_v_break;        /* Mass break counter on ->volume */
        unsigned int                cb_break;        /* Break counter on vnode */
        seqlock_t                cb_lock;        /* Lock for ->cb_server, ->status, ->cb_*break */

        time64_t                cb_expires_at;        /* time at which callback expires */
};

static inline struct fscache_cookie *afs_vnode_cache(struct afs_vnode *vnode)
{
#ifdef CONFIG_AFS_FSCACHE
        return vnode->cache;
#else
        return NULL;
#endif
}

/*
 * cached security record for one user's attempt to access a vnode
 */
struct afs_permit {
        struct key                *key;                /* RxRPC ticket holding a security context */
        afs_access_t                access;                /* CallerAccess value for this key */
};

/*
 * Immutable cache of CallerAccess records from attempts to access vnodes.
 * These may be shared between multiple vnodes.
 */
struct afs_permits {
        struct rcu_head                rcu;
        struct hlist_node        hash_node;        /* Link in hash */
        unsigned long                h;                /* Hash value for this permit list */
        refcount_t                usage;
        unsigned short                nr_permits;        /* Number of records */
        bool                        invalidated;        /* Invalidated due to key change */
        struct afs_permit        permits[];        /* List of permits sorted by key pointer */
};

/*
 * Error prioritisation and accumulation.
 */
struct afs_error {
        short        error;                        /* Accumulated error */
        bool        responded;                /* T if server responded */
};

/*
 * Cursor for iterating over a server's address list.
 */
struct afs_addr_cursor {
        struct afs_addr_list        *alist;                /* Current address list (pins ref) */
        unsigned long                tried;                /* Tried addresses */
        signed char                index;                /* Current address */
        bool                        responded;        /* T if the current address responded */
        unsigned short                nr_iterations;        /* Number of address iterations */
        short                        error;
        u32                        abort_code;
};

/*
 * Cursor for iterating over a set of volume location servers.
 */
struct afs_vl_cursor {
        struct afs_addr_cursor        ac;
        struct afs_cell                *cell;                /* The cell we're querying */
        struct afs_vlserver_list *server_list;        /* Current server list (pins ref) */
        struct afs_vlserver        *server;        /* Server on which this resides */
        struct key                *key;                /* Key for the server */
        unsigned long                untried;        /* Bitmask of untried servers */
        short                        index;                /* Current server */
        short                        error;
        unsigned short                flags;
#define AFS_VL_CURSOR_STOP        0x0001                /* Set to cease iteration */
#define AFS_VL_CURSOR_RETRY        0x0002                /* Set to do a retry */
#define AFS_VL_CURSOR_RETRIED        0x0004                /* Set if started a retry */
        unsigned short                nr_iterations;        /* Number of server iterations */
};

/*
 * Fileserver operation methods.
 */
struct afs_operation_ops {
        void (*issue_afs_rpc)(struct afs_operation *op);
        void (*issue_yfs_rpc)(struct afs_operation *op);
        void (*success)(struct afs_operation *op);
        void (*aborted)(struct afs_operation *op);
        void (*failed)(struct afs_operation *op);
        void (*edit_dir)(struct afs_operation *op);
        void (*put)(struct afs_operation *op);
};

struct afs_vnode_param {
        struct afs_vnode        *vnode;
        struct afs_fid                fid;                /* Fid to access */
        struct afs_status_cb        scb;                /* Returned status and callback promise */
        afs_dataversion_t        dv_before;        /* Data version before the call */
        unsigned int                cb_break_before; /* cb_break + cb_s_break before the call */
        u8                        dv_delta;        /* Expected change in data version */
        bool                        put_vnode:1;        /* T if we have a ref on the vnode */
        bool                        need_io_lock:1;        /* T if we need the I/O lock on this */
        bool                        update_ctime:1;        /* Need to update the ctime */
        bool                        set_size:1;        /* Must update i_size */
        bool                        op_unlinked:1;        /* True if file was unlinked by op */
        bool                        speculative:1;        /* T if speculative status fetch (no vnode lock) */
        bool                        modification:1;        /* Set if the content gets modified */
};

/*
 * Fileserver operation wrapper, handling server and address rotation
 * asynchronously.  May make simultaneous calls to multiple servers.
 */
struct afs_operation {
        struct afs_net                *net;                /* Network namespace */
        struct key                *key;                /* Key for the cell */
        const struct afs_call_type *type;        /* Type of call done */
        const struct afs_operation_ops *ops;

        /* Parameters/results for the operation */
        struct afs_volume        *volume;        /* Volume being accessed */
        struct afs_vnode_param        file[2];
        struct afs_vnode_param        *more_files;
        struct afs_volsync        volsync;
        struct dentry                *dentry;        /* Dentry to be altered */
        struct dentry                *dentry_2;        /* Second dentry to be altered */
        struct timespec64        mtime;                /* Modification time to record */
        struct timespec64        ctime;                /* Change time to set */
        short                        nr_files;        /* Number of entries in file[], more_files */
        short                        error;
        unsigned int                debug_id;

        unsigned int                cb_v_break;        /* Volume break counter before op */
        unsigned int                cb_s_break;        /* Server break counter before op */

        union {
                struct {
                        int        which;                /* Which ->file[] to fetch for */
                } fetch_status;
                struct {
                        int        reason;                /* enum afs_edit_dir_reason */
                        mode_t        mode;
                        const char *symlink;
                } create;
                struct {
                        bool        need_rehash;
                } unlink;
                struct {
                        struct dentry *rehash;
                        struct dentry *tmp;
                        bool        new_negative;
                } rename;
                struct {
                        struct afs_read *req;
                } fetch;
                struct {
                        afs_lock_type_t type;
                } lock;
                struct {
                        struct iov_iter        *write_iter;
                        loff_t        pos;
                        loff_t        size;
                        loff_t        i_size;
                        bool        laundering;        /* Laundering page, PG_writeback not set */
                } store;
                struct {
                        struct iattr        *attr;
                        loff_t                old_i_size;
                } setattr;
                struct afs_acl        *acl;
                struct yfs_acl        *yacl;
                struct {
                        struct afs_volume_status vs;
                        struct kstatfs                *buf;
                } volstatus;
        };

        /* Fileserver iteration state */
        struct afs_addr_cursor        ac;
        struct afs_server_list        *server_list;        /* Current server list (pins ref) */
        struct afs_server        *server;        /* Server we're using (ref pinned by server_list) */
        struct afs_call                *call;
        unsigned long                untried;        /* Bitmask of untried servers */
        short                        index;                /* Current server */
        unsigned short                nr_iterations;        /* Number of server iterations */

        unsigned int                flags;
#define AFS_OPERATION_STOP                0x0001        /* Set to cease iteration */
#define AFS_OPERATION_VBUSY                0x0002        /* Set if seen VBUSY */
#define AFS_OPERATION_VMOVED                0x0004        /* Set if seen VMOVED */
#define AFS_OPERATION_VNOVOL                0x0008        /* Set if seen VNOVOL */
#define AFS_OPERATION_CUR_ONLY                0x0010        /* Set if current server only (file lock held) */
#define AFS_OPERATION_NO_VSLEEP                0x0020        /* Set to prevent sleep on VBUSY, VOFFLINE, ... */
#define AFS_OPERATION_UNINTR                0x0040        /* Set if op is uninterruptible */
#define AFS_OPERATION_DOWNGRADE                0x0080        /* Set to retry with downgraded opcode */
#define AFS_OPERATION_LOCK_0                0x0100        /* Set if have io_lock on file[0] */
#define AFS_OPERATION_LOCK_1                0x0200        /* Set if have io_lock on file[1] */
#define AFS_OPERATION_TRIED_ALL                0x0400        /* Set if we've tried all the fileservers */
#define AFS_OPERATION_RETRY_SERVER        0x0800        /* Set if we should retry the current server */
#define AFS_OPERATION_DIR_CONFLICT        0x1000        /* Set if we detected a 3rd-party dir change */
};

/*
 * Cache auxiliary data.
 */
struct afs_vnode_cache_aux {
        u64                        data_version;
} __packed;

/*
 * We use page->private to hold the amount of the page that we've written to,
 * splitting the field into two parts.  However, we need to represent a range
 * 0...PAGE_SIZE, so we reduce the resolution if the size of the page
 * exceeds what we can encode.
 */
#ifdef CONFIG_64BIT
#define __AFS_PAGE_PRIV_MASK        0x7fffffffUL
#define __AFS_PAGE_PRIV_SHIFT        32
#define __AFS_PAGE_PRIV_MMAPPED        0x80000000UL
#else
#define __AFS_PAGE_PRIV_MASK        0x7fffUL
#define __AFS_PAGE_PRIV_SHIFT        16
#define __AFS_PAGE_PRIV_MMAPPED        0x8000UL
#endif

static inline unsigned int afs_page_dirty_resolution(struct page *page)
{
        int shift = thp_order(page) + PAGE_SHIFT - (__AFS_PAGE_PRIV_SHIFT - 1);
        return (shift > 0) ? shift : 0;
}

static inline size_t afs_page_dirty_from(struct page *page, unsigned long priv)
{
        unsigned long x = priv & __AFS_PAGE_PRIV_MASK;

        /* The lower bound is inclusive */
        return x << afs_page_dirty_resolution(page);
}

static inline size_t afs_page_dirty_to(struct page *page, unsigned long priv)
{
        unsigned long x = (priv >> __AFS_PAGE_PRIV_SHIFT) & __AFS_PAGE_PRIV_MASK;

        /* The upper bound is immediately beyond the region */
        return (x + 1) << afs_page_dirty_resolution(page);
}

static inline unsigned long afs_page_dirty(struct page *page, size_t from, size_t to)
{
        unsigned int res = afs_page_dirty_resolution(page);
        from >>= res;
        to = (to - 1) >> res;
        return (to << __AFS_PAGE_PRIV_SHIFT) | from;
}

static inline unsigned long afs_page_dirty_mmapped(unsigned long priv)
{
        return priv | __AFS_PAGE_PRIV_MMAPPED;
}

static inline bool afs_is_page_dirty_mmapped(unsigned long priv)
{
        return priv & __AFS_PAGE_PRIV_MMAPPED;
}

#include <trace/events/afs.h>

/*****************************************************************************/
/*
 * addr_list.c
 */
static inline struct afs_addr_list *afs_get_addrlist(struct afs_addr_list *alist)
{
        if (alist)
                refcount_inc(&alist->usage);
        return alist;
}
extern struct afs_addr_list *afs_alloc_addrlist(unsigned int,
                                                unsigned short,
                                                unsigned short);
extern void afs_put_addrlist(struct afs_addr_list *);
extern struct afs_vlserver_list *afs_parse_text_addrs(struct afs_net *,
                                                      const char *, size_t, char,
                                                      unsigned short, unsigned short);
extern struct afs_vlserver_list *afs_dns_query(struct afs_cell *, time64_t *);
extern bool afs_iterate_addresses(struct afs_addr_cursor *);
extern int afs_end_cursor(struct afs_addr_cursor *);

extern void afs_merge_fs_addr4(struct afs_addr_list *, __be32, u16);
extern void afs_merge_fs_addr6(struct afs_addr_list *, __be32 *, u16);

/*
 * cache.c
 */
#ifdef CONFIG_AFS_FSCACHE
extern struct fscache_netfs afs_cache_netfs;
extern struct fscache_cookie_def afs_cell_cache_index_def;
extern struct fscache_cookie_def afs_volume_cache_index_def;
extern struct fscache_cookie_def afs_vnode_cache_index_def;
#else
#define afs_cell_cache_index_def        (*(struct fscache_cookie_def *) NULL)
#define afs_volume_cache_index_def        (*(struct fscache_cookie_def *) NULL)
#define afs_vnode_cache_index_def        (*(struct fscache_cookie_def *) NULL)
#endif

/*
 * callback.c
 */
extern void afs_invalidate_mmap_work(struct work_struct *);
extern void afs_server_init_callback_work(struct work_struct *work);
extern void afs_init_callback_state(struct afs_server *);
extern void __afs_break_callback(struct afs_vnode *, enum afs_cb_break_reason);
extern void afs_break_callback(struct afs_vnode *, enum afs_cb_break_reason);
extern void afs_break_callbacks(struct afs_server *, size_t, struct afs_callback_break *);

static inline unsigned int afs_calc_vnode_cb_break(struct afs_vnode *vnode)
{
        return vnode->cb_break + vnode->cb_v_break;
}

static inline bool afs_cb_is_broken(unsigned int cb_break,
                                    const struct afs_vnode *vnode)
{
        return cb_break != (vnode->cb_break + vnode->volume->cb_v_break);
}

/*
 * cell.c
 */
extern int afs_cell_init(struct afs_net *, const char *);
extern struct afs_cell *afs_find_cell(struct afs_net *, const char *, unsigned,
                                      enum afs_cell_trace);
extern struct afs_cell *afs_lookup_cell(struct afs_net *, const char *, unsigned,
                                        const char *, bool);
extern struct afs_cell *afs_use_cell(struct afs_cell *, enum afs_cell_trace);
extern void afs_unuse_cell(struct afs_net *, struct afs_cell *, enum afs_cell_trace);
extern struct afs_cell *afs_get_cell(struct afs_cell *, enum afs_cell_trace);
extern void afs_see_cell(struct afs_cell *, enum afs_cell_trace);
extern void afs_put_cell(struct afs_cell *, enum afs_cell_trace);
extern void afs_queue_cell(struct afs_cell *, enum afs_cell_trace);
extern void afs_manage_cells(struct work_struct *);
extern void afs_cells_timer(struct timer_list *);
extern void __net_exit afs_cell_purge(struct afs_net *);

/*
 * cmservice.c
 */
extern bool afs_cm_incoming_call(struct afs_call *);

/*
 * dir.c
 */
extern const struct file_operations afs_dir_file_operations;
extern const struct inode_operations afs_dir_inode_operations;
extern const struct address_space_operations afs_dir_aops;
extern const struct dentry_operations afs_fs_dentry_operations;

extern void afs_d_release(struct dentry *);
extern void afs_check_for_remote_deletion(struct afs_operation *);

/*
 * dir_edit.c
 */
extern void afs_edit_dir_add(struct afs_vnode *, struct qstr *, struct afs_fid *,
                             enum afs_edit_dir_reason);
extern void afs_edit_dir_remove(struct afs_vnode *, struct qstr *, enum afs_edit_dir_reason);

/*
 * dir_silly.c
 */
extern int afs_sillyrename(struct afs_vnode *, struct afs_vnode *,
                           struct dentry *, struct key *);
extern int afs_silly_iput(struct dentry *, struct inode *);

/*
 * dynroot.c
 */
extern const struct inode_operations afs_dynroot_inode_operations;
extern const struct dentry_operations afs_dynroot_dentry_operations;

extern struct inode *afs_try_auto_mntpt(struct dentry *, struct inode *);
extern int afs_dynroot_mkdir(struct afs_net *, struct afs_cell *);
extern void afs_dynroot_rmdir(struct afs_net *, struct afs_cell *);
extern int afs_dynroot_populate(struct super_block *);
extern void afs_dynroot_depopulate(struct super_block *);

/*
 * file.c
 */
extern const struct address_space_operations afs_fs_aops;
extern const struct inode_operations afs_file_inode_operations;
extern const struct file_operations afs_file_operations;
extern const struct netfs_read_request_ops afs_req_ops;

extern int afs_cache_wb_key(struct afs_vnode *, struct afs_file *);
extern void afs_put_wb_key(struct afs_wb_key *);
extern int afs_open(struct inode *, struct file *);
extern int afs_release(struct inode *, struct file *);
extern int afs_fetch_data(struct afs_vnode *, struct afs_read *);
extern struct afs_read *afs_alloc_read(gfp_t);
extern void afs_put_read(struct afs_read *);

static inline struct afs_read *afs_get_read(struct afs_read *req)
{
        refcount_inc(&req->usage);
        return req;
}

/*
 * flock.c
 */
extern struct workqueue_struct *afs_lock_manager;

extern void afs_lock_op_done(struct afs_call *);
extern void afs_lock_work(struct work_struct *);
extern void afs_lock_may_be_available(struct afs_vnode *);
extern int afs_lock(struct file *, int, struct file_lock *);
extern int afs_flock(struct file *, int, struct file_lock *);

/*
 * fsclient.c
 */
extern void afs_fs_fetch_status(struct afs_operation *);
extern void afs_fs_fetch_data(struct afs_operation *);
extern void afs_fs_create_file(struct afs_operation *);
extern void afs_fs_make_dir(struct afs_operation *);
extern void afs_fs_remove_file(struct afs_operation *);
extern void afs_fs_remove_dir(struct afs_operation *);
extern void afs_fs_link(struct afs_operation *);
extern void afs_fs_symlink(struct afs_operation *);
extern void afs_fs_rename(struct afs_operation *);
extern void afs_fs_store_data(struct afs_operation *);
extern void afs_fs_setattr(struct afs_operation *);
extern void afs_fs_get_volume_status(struct afs_operation *);
extern void afs_fs_set_lock(struct afs_operation *);
extern void afs_fs_extend_lock(struct afs_operation *);
extern void afs_fs_release_lock(struct afs_operation *);
extern int afs_fs_give_up_all_callbacks(struct afs_net *, struct afs_server *,
                                        struct afs_addr_cursor *, struct key *);
extern bool afs_fs_get_capabilities(struct afs_net *, struct afs_server *,
                                    struct afs_addr_cursor *, struct key *);
extern void afs_fs_inline_bulk_status(struct afs_operation *);

struct afs_acl {
        u32        size;
        u8        data[];
};

extern void afs_fs_fetch_acl(struct afs_operation *);
extern void afs_fs_store_acl(struct afs_operation *);

/*
 * fs_operation.c
 */
extern struct afs_operation *afs_alloc_operation(struct key *, struct afs_volume *);
extern int afs_put_operation(struct afs_operation *);
extern bool afs_begin_vnode_operation(struct afs_operation *);
extern void afs_wait_for_operation(struct afs_operation *);
extern int afs_do_sync_operation(struct afs_operation *);

static inline void afs_op_nomem(struct afs_operation *op)
{
        op->error = -ENOMEM;
}

static inline void afs_op_set_vnode(struct afs_operation *op, unsigned int n,
                                    struct afs_vnode *vnode)
{
        op->file[n].vnode = vnode;
        op->file[n].need_io_lock = true;
}

static inline void afs_op_set_fid(struct afs_operation *op, unsigned int n,
                                  const struct afs_fid *fid)
{
        op->file[n].fid = *fid;
}

/*
 * fs_probe.c
 */
extern void afs_fileserver_probe_result(struct afs_call *);
extern void afs_fs_probe_fileserver(struct afs_net *, struct afs_server *, struct key *, bool);
extern int afs_wait_for_fs_probes(struct afs_server_list *, unsigned long);
extern void afs_probe_fileserver(struct afs_net *, struct afs_server *);
extern void afs_fs_probe_dispatcher(struct work_struct *);
extern int afs_wait_for_one_fs_probe(struct afs_server *, bool);
extern void afs_fs_probe_cleanup(struct afs_net *);

/*
 * inode.c
 */
extern const struct afs_operation_ops afs_fetch_status_operation;

extern void afs_vnode_commit_status(struct afs_operation *, struct afs_vnode_param *);
extern int afs_fetch_status(struct afs_vnode *, struct key *, bool, afs_access_t *);
extern int afs_ilookup5_test_by_fid(struct inode *, void *);
extern struct inode *afs_iget_pseudo_dir(struct super_block *, bool);
extern struct inode *afs_iget(struct afs_operation *, struct afs_vnode_param *);
extern struct inode *afs_root_iget(struct super_block *, struct key *);
extern bool afs_check_validity(struct afs_vnode *);
extern int afs_validate(struct afs_vnode *, struct key *);
extern int afs_getattr(struct user_namespace *mnt_userns, const struct path *,
                       struct kstat *, u32, unsigned int);
extern int afs_setattr(struct user_namespace *mnt_userns, struct dentry *, struct iattr *);
extern void afs_evict_inode(struct inode *);
extern int afs_drop_inode(struct inode *);

/*
 * main.c
 */
extern struct workqueue_struct *afs_wq;
extern int afs_net_id;

static inline struct afs_net *afs_net(struct net *net)
{
        return net_generic(net, afs_net_id);
}

static inline struct afs_net *afs_sb2net(struct super_block *sb)
{
        return afs_net(AFS_FS_S(sb)->net_ns);
}

static inline struct afs_net *afs_d2net(struct dentry *dentry)
{
        return afs_sb2net(dentry->d_sb);
}

static inline struct afs_net *afs_i2net(struct inode *inode)
{
        return afs_sb2net(inode->i_sb);
}

static inline struct afs_net *afs_v2net(struct afs_vnode *vnode)
{
        return afs_i2net(&vnode->vfs_inode);
}

static inline struct afs_net *afs_sock2net(struct sock *sk)
{
        return net_generic(sock_net(sk), afs_net_id);
}

static inline void __afs_stat(atomic_t *s)
{
        atomic_inc(s);
}

#define afs_stat_v(vnode, n) __afs_stat(&afs_v2net(vnode)->n)

/*
 * misc.c
 */
extern int afs_abort_to_error(u32);
extern void afs_prioritise_error(struct afs_error *, int, u32);

/*
 * mntpt.c
 */
extern const struct inode_operations afs_mntpt_inode_operations;
extern const struct inode_operations afs_autocell_inode_operations;
extern const struct file_operations afs_mntpt_file_operations;

extern struct vfsmount *afs_d_automount(struct path *);
extern void afs_mntpt_kill_timer(void);

/*
 * proc.c
 */
#ifdef CONFIG_PROC_FS
extern int __net_init afs_proc_init(struct afs_net *);
extern void __net_exit afs_proc_cleanup(struct afs_net *);
extern int afs_proc_cell_setup(struct afs_cell *);
extern void afs_proc_cell_remove(struct afs_cell *);
extern void afs_put_sysnames(struct afs_sysnames *);
#else
static inline int afs_proc_init(struct afs_net *net) { return 0; }
static inline void afs_proc_cleanup(struct afs_net *net) {}
static inline int afs_proc_cell_setup(struct afs_cell *cell) { return 0; }
static inline void afs_proc_cell_remove(struct afs_cell *cell) {}
static inline void afs_put_sysnames(struct afs_sysnames *sysnames) {}
#endif

/*
 * rotate.c
 */
extern bool afs_select_fileserver(struct afs_operation *);
extern void afs_dump_edestaddrreq(const struct afs_operation *);

/*
 * rxrpc.c
 */
extern struct workqueue_struct *afs_async_calls;

extern int __net_init afs_open_socket(struct afs_net *);
extern void __net_exit afs_close_socket(struct afs_net *);
extern void afs_charge_preallocation(struct work_struct *);
extern void afs_put_call(struct afs_call *);
extern void afs_make_call(struct afs_addr_cursor *, struct afs_call *, gfp_t);
extern long afs_wait_for_call_to_complete(struct afs_call *, struct afs_addr_cursor *);
extern struct afs_call *afs_alloc_flat_call(struct afs_net *,
                                            const struct afs_call_type *,
                                            size_t, size_t);
extern void afs_flat_call_destructor(struct afs_call *);
extern void afs_send_empty_reply(struct afs_call *);
extern void afs_send_simple_reply(struct afs_call *, const void *, size_t);
extern int afs_extract_data(struct afs_call *, bool);
extern int afs_protocol_error(struct afs_call *, enum afs_eproto_cause);

static inline void afs_make_op_call(struct afs_operation *op, struct afs_call *call,
                                    gfp_t gfp)
{
        op->call = call;
        op->type = call->type;
        call->op = op;
        call->key = op->key;
        call->intr = !(op->flags & AFS_OPERATION_UNINTR);
        afs_make_call(&op->ac, call, gfp);
}

static inline void afs_extract_begin(struct afs_call *call, void *buf, size_t size)
{
        call->iov_len = size;
        call->kvec[0].iov_base = buf;
        call->kvec[0].iov_len = size;
        iov_iter_kvec(&call->def_iter, READ, call->kvec, 1, size);
}

static inline void afs_extract_to_tmp(struct afs_call *call)
{
        call->iov_len = sizeof(call->tmp);
        afs_extract_begin(call, &call->tmp, sizeof(call->tmp));
}

static inline void afs_extract_to_tmp64(struct afs_call *call)
{
        call->iov_len = sizeof(call->tmp64);
        afs_extract_begin(call, &call->tmp64, sizeof(call->tmp64));
}

static inline void afs_extract_discard(struct afs_call *call, size_t size)
{
        call->iov_len = size;
        iov_iter_discard(&call->def_iter, READ, size);
}

static inline void afs_extract_to_buf(struct afs_call *call, size_t size)
{
        call->iov_len = size;
        afs_extract_begin(call, call->buffer, size);
}

static inline int afs_transfer_reply(struct afs_call *call)
{
        return afs_extract_data(call, false);
}

static inline bool afs_check_call_state(struct afs_call *call,
                                        enum afs_call_state state)
{
        return READ_ONCE(call->state) == state;
}

static inline bool afs_set_call_state(struct afs_call *call,
                                      enum afs_call_state from,
                                      enum afs_call_state to)
{
        bool ok = false;

        spin_lock_bh(&call->state_lock);
        if (call->state == from) {
                call->state = to;
                trace_afs_call_state(call, from, to, 0, 0);
                ok = true;
        }
        spin_unlock_bh(&call->state_lock);
        return ok;
}

static inline void afs_set_call_complete(struct afs_call *call,
                                         int error, u32 remote_abort)
{
        enum afs_call_state state;
        bool ok = false;

        spin_lock_bh(&call->state_lock);
        state = call->state;
        if (state != AFS_CALL_COMPLETE) {
                call->abort_code = remote_abort;
                call->error = error;
                call->state = AFS_CALL_COMPLETE;
                trace_afs_call_state(call, state, AFS_CALL_COMPLETE,
                                     error, remote_abort);
                ok = true;
        }
        spin_unlock_bh(&call->state_lock);
        if (ok) {
                trace_afs_call_done(call);

                /* Asynchronous calls have two refs to release - one from the alloc and
                 * one queued with the work item - and we can't just deallocate the
                 * call because the work item may be queued again.
                 */
                if (call->drop_ref)
                        afs_put_call(call);
        }
}

/*
 * security.c
 */
extern void afs_put_permits(struct afs_permits *);
extern void afs_clear_permits(struct afs_vnode *);
extern void afs_cache_permit(struct afs_vnode *, struct key *, unsigned int,
                             struct afs_status_cb *);
extern void afs_zap_permits(struct rcu_head *);
extern struct key *afs_request_key(struct afs_cell *);
extern struct key *afs_request_key_rcu(struct afs_cell *);
extern int afs_check_permit(struct afs_vnode *, struct key *, afs_access_t *);
extern int afs_permission(struct user_namespace *, struct inode *, int);
extern void __exit afs_clean_up_permit_cache(void);

/*
 * server.c
 */
extern spinlock_t afs_server_peer_lock;

extern struct afs_server *afs_find_server(struct afs_net *,
                                          const struct sockaddr_rxrpc *);
extern struct afs_server *afs_find_server_by_uuid(struct afs_net *, const uuid_t *);
extern struct afs_server *afs_lookup_server(struct afs_cell *, struct key *, const uuid_t *, u32);
extern struct afs_server *afs_get_server(struct afs_server *, enum afs_server_trace);
extern struct afs_server *afs_use_server(struct afs_server *, enum afs_server_trace);
extern void afs_unuse_server(struct afs_net *, struct afs_server *, enum afs_server_trace);
extern void afs_unuse_server_notime(struct afs_net *, struct afs_server *, enum afs_server_trace);
extern void afs_put_server(struct afs_net *, struct afs_server *, enum afs_server_trace);
extern void afs_manage_servers(struct work_struct *);
extern void afs_servers_timer(struct timer_list *);
extern void afs_fs_probe_timer(struct timer_list *);
extern void __net_exit afs_purge_servers(struct afs_net *);
extern bool afs_check_server_record(struct afs_operation *, struct afs_server *);

static inline void afs_inc_servers_outstanding(struct afs_net *net)
{
        atomic_inc(&net->servers_outstanding);
}

static inline void afs_dec_servers_outstanding(struct afs_net *net)
{
        if (atomic_dec_and_test(&net->servers_outstanding))
                wake_up_var(&net->servers_outstanding);
}

static inline bool afs_is_probing_server(struct afs_server *server)
{
        return list_empty(&server->probe_link);
}

/*
 * server_list.c
 */
static inline struct afs_server_list *afs_get_serverlist(struct afs_server_list *slist)
{
        refcount_inc(&slist->usage);
        return slist;
}

extern void afs_put_serverlist(struct afs_net *, struct afs_server_list *);
extern struct afs_server_list *afs_alloc_server_list(struct afs_cell *, struct key *,
                                                     struct afs_vldb_entry *,
                                                     u8);
extern bool afs_annotate_server_list(struct afs_server_list *, struct afs_server_list *);

/*
 * super.c
 */
extern int __init afs_fs_init(void);
extern void afs_fs_exit(void);

/*
 * vlclient.c
 */
extern struct afs_vldb_entry *afs_vl_get_entry_by_name_u(struct afs_vl_cursor *,
                                                         const char *, int);
extern struct afs_addr_list *afs_vl_get_addrs_u(struct afs_vl_cursor *, const uuid_t *);
extern struct afs_call *afs_vl_get_capabilities(struct afs_net *, struct afs_addr_cursor *,
                                                struct key *, struct afs_vlserver *, unsigned int);
extern struct afs_addr_list *afs_yfsvl_get_endpoints(struct afs_vl_cursor *, const uuid_t *);
extern char *afs_yfsvl_get_cell_name(struct afs_vl_cursor *);

/*
 * vl_alias.c
 */
extern int afs_cell_detect_alias(struct afs_cell *, struct key *);

/*
 * vl_probe.c
 */
extern void afs_vlserver_probe_result(struct afs_call *);
extern int afs_send_vl_probes(struct afs_net *, struct key *, struct afs_vlserver_list *);
extern int afs_wait_for_vl_probes(struct afs_vlserver_list *, unsigned long);

/*
 * vl_rotate.c
 */
extern bool afs_begin_vlserver_operation(struct afs_vl_cursor *,
                                         struct afs_cell *, struct key *);
extern bool afs_select_vlserver(struct afs_vl_cursor *);
extern bool afs_select_current_vlserver(struct afs_vl_cursor *);
extern int afs_end_vlserver_operation(struct afs_vl_cursor *);

/*
 * vlserver_list.c
 */
static inline struct afs_vlserver *afs_get_vlserver(struct afs_vlserver *vlserver)
{
        refcount_inc(&vlserver->ref);
        return vlserver;
}

static inline struct afs_vlserver_list *afs_get_vlserverlist(struct afs_vlserver_list *vllist)
{
        if (vllist)
                refcount_inc(&vllist->ref);
        return vllist;
}

extern struct afs_vlserver *afs_alloc_vlserver(const char *, size_t, unsigned short);
extern void afs_put_vlserver(struct afs_net *, struct afs_vlserver *);
extern struct afs_vlserver_list *afs_alloc_vlserver_list(unsigned int);
extern void afs_put_vlserverlist(struct afs_net *, struct afs_vlserver_list *);
extern struct afs_vlserver_list *afs_extract_vlserver_list(struct afs_cell *,
                                                           const void *, size_t);

/*
 * volume.c
 */
extern struct afs_volume *afs_create_volume(struct afs_fs_context *);
extern void afs_activate_volume(struct afs_volume *);
extern void afs_deactivate_volume(struct afs_volume *);
bool afs_try_get_volume(struct afs_volume *volume, enum afs_volume_trace reason);
extern struct afs_volume *afs_get_volume(struct afs_volume *, enum afs_volume_trace);
extern void afs_put_volume(struct afs_net *, struct afs_volume *, enum afs_volume_trace);
extern int afs_check_volume_status(struct afs_volume *, struct afs_operation *);

/*
 * write.c
 */
extern int afs_set_page_dirty(struct page *);
extern int afs_write_begin(struct file *file, struct address_space *mapping,
                        loff_t pos, unsigned len, unsigned flags,
                        struct page **pagep, void **fsdata);
extern int afs_write_end(struct file *file, struct address_space *mapping,
                        loff_t pos, unsigned len, unsigned copied,
                        struct page *page, void *fsdata);
extern int afs_writepage(struct page *, struct writeback_control *);
extern int afs_writepages(struct address_space *, struct writeback_control *);
extern ssize_t afs_file_write(struct kiocb *, struct iov_iter *);
extern int afs_fsync(struct file *, loff_t, loff_t, int);
extern vm_fault_t afs_page_mkwrite(struct vm_fault *vmf);
extern void afs_prune_wb_keys(struct afs_vnode *);
extern int afs_launder_page(struct page *);

/*
 * xattr.c
 */
extern const struct xattr_handler *afs_xattr_handlers[];

/*
 * yfsclient.c
 */
extern void yfs_fs_fetch_data(struct afs_operation *);
extern void yfs_fs_create_file(struct afs_operation *);
extern void yfs_fs_make_dir(struct afs_operation *);
extern void yfs_fs_remove_file2(struct afs_operation *);
extern void yfs_fs_remove_file(struct afs_operation *);
extern void yfs_fs_remove_dir(struct afs_operation *);
extern void yfs_fs_link(struct afs_operation *);
extern void yfs_fs_symlink(struct afs_operation *);
extern void yfs_fs_rename(struct afs_operation *);
extern void yfs_fs_store_data(struct afs_operation *);
extern void yfs_fs_setattr(struct afs_operation *);
extern void yfs_fs_get_volume_status(struct afs_operation *);
extern void yfs_fs_set_lock(struct afs_operation *);
extern void yfs_fs_extend_lock(struct afs_operation *);
extern void yfs_fs_release_lock(struct afs_operation *);
extern void yfs_fs_fetch_status(struct afs_operation *);
extern void yfs_fs_inline_bulk_status(struct afs_operation *);

struct yfs_acl {
        struct afs_acl        *acl;                /* Dir/file/symlink ACL */
        struct afs_acl        *vol_acl;        /* Whole volume ACL */
        u32                inherit_flag;        /* True if ACL is inherited from parent dir */
        u32                num_cleaned;        /* Number of ACEs removed due to subject removal */
        unsigned int        flags;
#define YFS_ACL_WANT_ACL        0x01        /* Set if caller wants ->acl */
#define YFS_ACL_WANT_VOL_ACL        0x02        /* Set if caller wants ->vol_acl */
};

extern void yfs_free_opaque_acl(struct yfs_acl *);
extern void yfs_fs_fetch_opaque_acl(struct afs_operation *);
extern void yfs_fs_store_opaque_acl2(struct afs_operation *);

/*
 * Miscellaneous inline functions.
 */
static inline struct afs_vnode *AFS_FS_I(struct inode *inode)
{
        return container_of(inode, struct afs_vnode, vfs_inode);
}

static inline struct inode *AFS_VNODE_TO_I(struct afs_vnode *vnode)
{
        return &vnode->vfs_inode;
}

/*
 * Note that a dentry got changed.  We need to set d_fsdata to the data version
 * number derived from the result of the operation.  It doesn't matter if
 * d_fsdata goes backwards as we'll just revalidate.
 */
static inline void afs_update_dentry_version(struct afs_operation *op,
                                             struct afs_vnode_param *dir_vp,
                                             struct dentry *dentry)
{
        if (!op->error)
                dentry->d_fsdata =
                        (void *)(unsigned long)dir_vp->scb.status.data_version;
}

/*
 * Set the file size and block count.  Estimate the number of 512 bytes blocks
 * used, rounded up to nearest 1K for consistency with other AFS clients.
 */
static inline void afs_set_i_size(struct afs_vnode *vnode, u64 size)
{
        i_size_write(&vnode->vfs_inode, size);
        vnode->vfs_inode.i_blocks = ((size + 1023) >> 10) << 1;
}

/*
 * Check for a conflicting operation on a directory that we just unlinked from.
 * If someone managed to sneak a link or an unlink in on the file we just
 * unlinked, we won't be able to trust nlink on an AFS file (but not YFS).
 */
static inline void afs_check_dir_conflict(struct afs_operation *op,
                                          struct afs_vnode_param *dvp)
{
        if (dvp->dv_before + dvp->dv_delta != dvp->scb.status.data_version)
                op->flags |= AFS_OPERATION_DIR_CONFLICT;
}

static inline int afs_io_error(struct afs_call *call, enum afs_io_error where)
{
        trace_afs_io_error(call->debug_id, -EIO, where);
        return -EIO;
}

static inline int afs_bad(struct afs_vnode *vnode, enum afs_file_error where)
{
        trace_afs_file_error(vnode, -EIO, where);
        return -EIO;
}

/*****************************************************************************/
/*
 * debug tracing
 */
extern unsigned afs_debug;

#define dbgprintk(FMT,...) \
        printk("[%-6.6s] "FMT"\n", current->comm ,##__VA_ARGS__)

#define kenter(FMT,...)        dbgprintk("==> %s("FMT")",__func__ ,##__VA_ARGS__)
#define kleave(FMT,...)        dbgprintk("<== %s()"FMT"",__func__ ,##__VA_ARGS__)
#define kdebug(FMT,...)        dbgprintk("    "FMT ,##__VA_ARGS__)


#if defined(__KDEBUG)
#define _enter(FMT,...)        kenter(FMT,##__VA_ARGS__)
#define _leave(FMT,...)        kleave(FMT,##__VA_ARGS__)
#define _debug(FMT,...)        kdebug(FMT,##__VA_ARGS__)

#elif defined(CONFIG_AFS_DEBUG)
#define AFS_DEBUG_KENTER        0x01
#define AFS_DEBUG_KLEAVE        0x02
#define AFS_DEBUG_KDEBUG        0x04

#define _enter(FMT,...)                                        \
do {                                                        \
        if (unlikely(afs_debug & AFS_DEBUG_KENTER))        \
                kenter(FMT,##__VA_ARGS__);                \
} while (0)

#define _leave(FMT,...)                                        \
do {                                                        \
        if (unlikely(afs_debug & AFS_DEBUG_KLEAVE))        \
                kleave(FMT,##__VA_ARGS__);                \
} while (0)

#define _debug(FMT,...)                                        \
do {                                                        \
        if (unlikely(afs_debug & AFS_DEBUG_KDEBUG))        \
                kdebug(FMT,##__VA_ARGS__);                \
} while (0)

#else
#define _enter(FMT,...)        no_printk("==> %s("FMT")",__func__ ,##__VA_ARGS__)
#define _leave(FMT,...)        no_printk("<== %s()"FMT"",__func__ ,##__VA_ARGS__)
#define _debug(FMT,...)        no_printk("    "FMT ,##__VA_ARGS__)
#endif

/*
 * debug assertion checking
 */
#if 1 // defined(__KDEBUGALL)

#define ASSERT(X)                                                \
do {                                                                \
        if (unlikely(!(X))) {                                        \
                printk(KERN_ERR "\n");                                \
                printk(KERN_ERR "AFS: Assertion failed\n");        \
                BUG();                                                \
        }                                                        \
} while(0)

#define ASSERTCMP(X, OP, Y)                                                \
do {                                                                        \
        if (unlikely(!((X) OP (Y)))) {                                        \
                printk(KERN_ERR "\n");                                        \
                printk(KERN_ERR "AFS: Assertion failed\n");                \
                printk(KERN_ERR "%lu " #OP " %lu is false\n",                \
                       (unsigned long)(X), (unsigned long)(Y));                \
                printk(KERN_ERR "0x%lx " #OP " 0x%lx is false\n",        \
                       (unsigned long)(X), (unsigned long)(Y));                \
                BUG();                                                        \
        }                                                                \
} while(0)

#define ASSERTRANGE(L, OP1, N, OP2, H)                                        \
do {                                                                        \
        if (unlikely(!((L) OP1 (N)) || !((N) OP2 (H)))) {                \
                printk(KERN_ERR "\n");                                        \
                printk(KERN_ERR "AFS: Assertion failed\n");                \
                printk(KERN_ERR "%lu "#OP1" %lu "#OP2" %lu is false\n",        \
                       (unsigned long)(L), (unsigned long)(N),                \
                       (unsigned long)(H));                                \
                printk(KERN_ERR "0x%lx "#OP1" 0x%lx "#OP2" 0x%lx is false\n", \
                       (unsigned long)(L), (unsigned long)(N),                \
                       (unsigned long)(H));                                \
                BUG();                                                        \
        }                                                                \
} while(0)

#define ASSERTIF(C, X)                                                \
do {                                                                \
        if (unlikely((C) && !(X))) {                                \
                printk(KERN_ERR "\n");                                \
                printk(KERN_ERR "AFS: Assertion failed\n");        \
                BUG();                                                \
        }                                                        \
} while(0)

#define ASSERTIFCMP(C, X, OP, Y)                                        \
do {                                                                        \
        if (unlikely((C) && !((X) OP (Y)))) {                                \
                printk(KERN_ERR "\n");                                        \
                printk(KERN_ERR "AFS: Assertion failed\n");                \
                printk(KERN_ERR "%lu " #OP " %lu is false\n",                \
                       (unsigned long)(X), (unsigned long)(Y));                \
                printk(KERN_ERR "0x%lx " #OP " 0x%lx is false\n",        \
                       (unsigned long)(X), (unsigned long)(Y));                \
                BUG();                                                        \
        }                                                                \
} while(0)

#else

#define ASSERT(X)                                \
do {                                                \
} while(0)

#define ASSERTCMP(X, OP, Y)                        \
do {                                                \
} while(0)

#define ASSERTRANGE(L, OP1, N, OP2, H)                \
do {                                                \
} while(0)

#define ASSERTIF(C, X)                                \
do {                                                \
} while(0)

#define ASSERTIFCMP(C, X, OP, Y)                \
do {                                                \
} while(0)

#endif /* __KDEBUGALL */















































































































































































































































































   64 


























































































































































































































































































































































































































































































































































































































































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_WAIT_H
#define _LINUX_WAIT_H
/*
 * Linux wait queue related types and methods
 */
#include <linux/list.h>
#include <linux/stddef.h>
#include <linux/spinlock.h>

#include <asm/current.h>
#include <uapi/linux/wait.h>

typedef struct wait_queue_entry wait_queue_entry_t;

typedef int (*wait_queue_func_t)(struct wait_queue_entry *wq_entry, unsigned mode, int flags, void *key);
int default_wake_function(struct wait_queue_entry *wq_entry, unsigned mode, int flags, void *key);

/* wait_queue_entry::flags */
#define WQ_FLAG_EXCLUSIVE        0x01
#define WQ_FLAG_WOKEN                0x02
#define WQ_FLAG_BOOKMARK        0x04
#define WQ_FLAG_CUSTOM                0x08
#define WQ_FLAG_DONE                0x10
#define WQ_FLAG_PRIORITY        0x20

/*
 * A single wait-queue entry structure:
 */
struct wait_queue_entry {
        unsigned int                flags;
        void                        *private;
        wait_queue_func_t        func;
        struct list_head        entry;
};

struct wait_queue_head {
        spinlock_t                lock;
        struct list_head        head;
};
typedef struct wait_queue_head wait_queue_head_t;

struct task_struct;

/*
 * Macros for declaration and initialisaton of the datatypes
 */

#define __WAITQUEUE_INITIALIZER(name, tsk) {                                        \
        .private        = tsk,                                                        \
        .func                = default_wake_function,                                \
        .entry                = { NULL, NULL } }

#define DECLARE_WAITQUEUE(name, tsk)                                                \
        struct wait_queue_entry name = __WAITQUEUE_INITIALIZER(name, tsk)

#define __WAIT_QUEUE_HEAD_INITIALIZER(name) {                                        \
        .lock                = __SPIN_LOCK_UNLOCKED(name.lock),                        \
        .head                = LIST_HEAD_INIT(name.head) }

#define DECLARE_WAIT_QUEUE_HEAD(name) \
        struct wait_queue_head name = __WAIT_QUEUE_HEAD_INITIALIZER(name)

extern void __init_waitqueue_head(struct wait_queue_head *wq_head, const char *name, struct lock_class_key *);

#define init_waitqueue_head(wq_head)                                                \
        do {                                                                        \
                static struct lock_class_key __key;                                \
                                                                                \
                __init_waitqueue_head((wq_head), #wq_head, &__key);                \
        } while (0)

#ifdef CONFIG_LOCKDEP
# define __WAIT_QUEUE_HEAD_INIT_ONSTACK(name) \
        ({ init_waitqueue_head(&name); name; })
# define DECLARE_WAIT_QUEUE_HEAD_ONSTACK(name) \
        struct wait_queue_head name = __WAIT_QUEUE_HEAD_INIT_ONSTACK(name)
#else
# define DECLARE_WAIT_QUEUE_HEAD_ONSTACK(name) DECLARE_WAIT_QUEUE_HEAD(name)
#endif

static inline void init_waitqueue_entry(struct wait_queue_entry *wq_entry, struct task_struct *p)
{
        wq_entry->flags                = 0;
        wq_entry->private        = p;
        wq_entry->func                = default_wake_function;
}

static inline void
init_waitqueue_func_entry(struct wait_queue_entry *wq_entry, wait_queue_func_t func)
{
        wq_entry->flags                = 0;
        wq_entry->private        = NULL;
        wq_entry->func                = func;
}

/**
 * waitqueue_active -- locklessly test for waiters on the queue
 * @wq_head: the waitqueue to test for waiters
 *
 * returns true if the wait list is not empty
 *
 * NOTE: this function is lockless and requires care, incorrect usage _will_
 * lead to sporadic and non-obvious failure.
 *
 * Use either while holding wait_queue_head::lock or when used for wakeups
 * with an extra smp_mb() like::
 *
 *      CPU0 - waker                    CPU1 - waiter
 *
 *                                      for (;;) {
 *      @cond = true;                     prepare_to_wait(&wq_head, &wait, state);
 *      smp_mb();                         // smp_mb() from set_current_state()
 *      if (waitqueue_active(wq_head))         if (@cond)
 *        wake_up(wq_head);                      break;
 *                                        schedule();
 *                                      }
 *                                      finish_wait(&wq_head, &wait);
 *
 * Because without the explicit smp_mb() it's possible for the
 * waitqueue_active() load to get hoisted over the @cond store such that we'll
 * observe an empty wait list while the waiter might not observe @cond.
 *
 * Also note that this 'optimization' trades a spin_lock() for an smp_mb(),
 * which (when the lock is uncontended) are of roughly equal cost.
 */
static inline int waitqueue_active(struct wait_queue_head *wq_head)
{
        return !list_empty(&wq_head->head);
}

/**
 * wq_has_single_sleeper - check if there is only one sleeper
 * @wq_head: wait queue head
 *
 * Returns true of wq_head has only one sleeper on the list.
 *
 * Please refer to the comment for waitqueue_active.
 */
static inline bool wq_has_single_sleeper(struct wait_queue_head *wq_head)
{
        return list_is_singular(&wq_head->head);
}

/**
 * wq_has_sleeper - check if there are any waiting processes
 * @wq_head: wait queue head
 *
 * Returns true if wq_head has waiting processes
 *
 * Please refer to the comment for waitqueue_active.
 */
static inline bool wq_has_sleeper(struct wait_queue_head *wq_head)
{
        /*
         * We need to be sure we are in sync with the
         * add_wait_queue modifications to the wait queue.
         *
         * This memory barrier should be paired with one on the
         * waiting side.
         */
        smp_mb();
        return waitqueue_active(wq_head);
}

extern void add_wait_queue(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry);
extern void add_wait_queue_exclusive(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry);
extern void add_wait_queue_priority(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry);
extern void remove_wait_queue(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry);

static inline void __add_wait_queue(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry)
{
        struct list_head *head = &wq_head->head;
        struct wait_queue_entry *wq;

        list_for_each_entry(wq, &wq_head->head, entry) {
                if (!(wq->flags & WQ_FLAG_PRIORITY))
                        break;
                head = &wq->entry;
        }
        list_add(&wq_entry->entry, head);
}

/*
 * Used for wake-one threads:
 */
static inline void
__add_wait_queue_exclusive(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry)
{
        wq_entry->flags |= WQ_FLAG_EXCLUSIVE;
        __add_wait_queue(wq_head, wq_entry);
}

static inline void __add_wait_queue_entry_tail(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry)
{
        list_add_tail(&wq_entry->entry, &wq_head->head);
}

static inline void
__add_wait_queue_entry_tail_exclusive(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry)
{
        wq_entry->flags |= WQ_FLAG_EXCLUSIVE;
        __add_wait_queue_entry_tail(wq_head, wq_entry);
}

static inline void
__remove_wait_queue(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry)
{
        list_del(&wq_entry->entry);
}

void __wake_up(struct wait_queue_head *wq_head, unsigned int mode, int nr, void *key);
void __wake_up_locked_key(struct wait_queue_head *wq_head, unsigned int mode, void *key);
void __wake_up_locked_key_bookmark(struct wait_queue_head *wq_head,
                unsigned int mode, void *key, wait_queue_entry_t *bookmark);
void __wake_up_sync_key(struct wait_queue_head *wq_head, unsigned int mode, void *key);
void __wake_up_locked_sync_key(struct wait_queue_head *wq_head, unsigned int mode, void *key);
void __wake_up_locked(struct wait_queue_head *wq_head, unsigned int mode, int nr);
void __wake_up_sync(struct wait_queue_head *wq_head, unsigned int mode);
void __wake_up_pollfree(struct wait_queue_head *wq_head);

#define wake_up(x)                        __wake_up(x, TASK_NORMAL, 1, NULL)
#define wake_up_nr(x, nr)                __wake_up(x, TASK_NORMAL, nr, NULL)
#define wake_up_all(x)                        __wake_up(x, TASK_NORMAL, 0, NULL)
#define wake_up_locked(x)                __wake_up_locked((x), TASK_NORMAL, 1)
#define wake_up_all_locked(x)                __wake_up_locked((x), TASK_NORMAL, 0)
#define wake_up_sync(x)                        __wake_up_sync(x, TASK_NORMAL)

#define wake_up_interruptible(x)        __wake_up(x, TASK_INTERRUPTIBLE, 1, NULL)
#define wake_up_interruptible_nr(x, nr)        __wake_up(x, TASK_INTERRUPTIBLE, nr, NULL)
#define wake_up_interruptible_all(x)        __wake_up(x, TASK_INTERRUPTIBLE, 0, NULL)
#define wake_up_interruptible_sync(x)        __wake_up_sync((x), TASK_INTERRUPTIBLE)

/*
 * Wakeup macros to be used to report events to the targets.
 */
#define poll_to_key(m) ((void *)(__force uintptr_t)(__poll_t)(m))
#define key_to_poll(m) ((__force __poll_t)(uintptr_t)(void *)(m))
#define wake_up_poll(x, m)                                                        \
        __wake_up(x, TASK_NORMAL, 1, poll_to_key(m))
#define wake_up_locked_poll(x, m)                                                \
        __wake_up_locked_key((x), TASK_NORMAL, poll_to_key(m))
#define wake_up_interruptible_poll(x, m)                                        \
        __wake_up(x, TASK_INTERRUPTIBLE, 1, poll_to_key(m))
#define wake_up_interruptible_sync_poll(x, m)                                        \
        __wake_up_sync_key((x), TASK_INTERRUPTIBLE, poll_to_key(m))
#define wake_up_interruptible_sync_poll_locked(x, m)                                \
        __wake_up_locked_sync_key((x), TASK_INTERRUPTIBLE, poll_to_key(m))

/**
 * wake_up_pollfree - signal that a polled waitqueue is going away
 * @wq_head: the wait queue head
 *
 * In the very rare cases where a ->poll() implementation uses a waitqueue whose
 * lifetime is tied to a task rather than to the 'struct file' being polled,
 * this function must be called before the waitqueue is freed so that
 * non-blocking polls (e.g. epoll) are notified that the queue is going away.
 *
 * The caller must also RCU-delay the freeing of the wait_queue_head, e.g. via
 * an explicit synchronize_rcu() or call_rcu(), or via SLAB_TYPESAFE_BY_RCU.
 */
static inline void wake_up_pollfree(struct wait_queue_head *wq_head)
{
        /*
         * For performance reasons, we don't always take the queue lock here.
         * Therefore, we might race with someone removing the last entry from
         * the queue, and proceed while they still hold the queue lock.
         * However, rcu_read_lock() is required to be held in such cases, so we
         * can safely proceed with an RCU-delayed free.
         */
        if (waitqueue_active(wq_head))
                __wake_up_pollfree(wq_head);
}

#define ___wait_cond_timeout(condition)                                                \
({                                                                                \
        bool __cond = (condition);                                                \
        if (__cond && !__ret)                                                        \
                __ret = 1;                                                        \
        __cond || !__ret;                                                        \
})

#define ___wait_is_interruptible(state)                                                \
        (!__builtin_constant_p(state) ||                                        \
                state == TASK_INTERRUPTIBLE || state == TASK_KILLABLE)                \

extern void init_wait_entry(struct wait_queue_entry *wq_entry, int flags);

/*
 * The below macro ___wait_event() has an explicit shadow of the __ret
 * variable when used from the wait_event_*() macros.
 *
 * This is so that both can use the ___wait_cond_timeout() construct
 * to wrap the condition.
 *
 * The type inconsistency of the wait_event_*() __ret variable is also
 * on purpose; we use long where we can return timeout values and int
 * otherwise.
 */

#define ___wait_event(wq_head, condition, state, exclusive, ret, cmd)                \
({                                                                                \
        __label__ __out;                                                        \
        struct wait_queue_entry __wq_entry;                                        \
        long __ret = ret;        /* explicit shadow */                                \
                                                                                \
        init_wait_entry(&__wq_entry, exclusive ? WQ_FLAG_EXCLUSIVE : 0);        \
        for (;;) {                                                                \
                long __int = prepare_to_wait_event(&wq_head, &__wq_entry, state);\
                                                                                \
                if (condition)                                                        \
                        break;                                                        \
                                                                                \
                if (___wait_is_interruptible(state) && __int) {                        \
                        __ret = __int;                                                \
                        goto __out;                                                \
                }                                                                \
                                                                                \
                cmd;                                                                \
        }                                                                        \
        finish_wait(&wq_head, &__wq_entry);                                        \
__out:        __ret;                                                                        \
})

#define __wait_event(wq_head, condition)                                        \
        (void)___wait_event(wq_head, condition, TASK_UNINTERRUPTIBLE, 0, 0,        \
                            schedule())

/**
 * wait_event - sleep until a condition gets true
 * @wq_head: the waitqueue to wait on
 * @condition: a C expression for the event to wait for
 *
 * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
 * @condition evaluates to true. The @condition is checked each time
 * the waitqueue @wq_head is woken up.
 *
 * wake_up() has to be called after changing any variable that could
 * change the result of the wait condition.
 */
#define wait_event(wq_head, condition)                                                \
do {                                                                                \
        might_sleep();                                                                \
        if (condition)                                                                \
                break;                                                                \
        __wait_event(wq_head, condition);                                        \
} while (0)

#define __io_wait_event(wq_head, condition)                                        \
        (void)___wait_event(wq_head, condition, TASK_UNINTERRUPTIBLE, 0, 0,        \
                            io_schedule())

/*
 * io_wait_event() -- like wait_event() but with io_schedule()
 */
#define io_wait_event(wq_head, condition)                                        \
do {                                                                                \
        might_sleep();                                                                \
        if (condition)                                                                \
                break;                                                                \
        __io_wait_event(wq_head, condition);                                        \
} while (0)

#define __wait_event_freezable(wq_head, condition)                                \
        ___wait_event(wq_head, condition, TASK_INTERRUPTIBLE, 0, 0,                \
                            freezable_schedule())

/**
 * wait_event_freezable - sleep (or freeze) until a condition gets true
 * @wq_head: the waitqueue to wait on
 * @condition: a C expression for the event to wait for
 *
 * The process is put to sleep (TASK_INTERRUPTIBLE -- so as not to contribute
 * to system load) until the @condition evaluates to true. The
 * @condition is checked each time the waitqueue @wq_head is woken up.
 *
 * wake_up() has to be called after changing any variable that could
 * change the result of the wait condition.
 */
#define wait_event_freezable(wq_head, condition)                                \
({                                                                                \
        int __ret = 0;                                                                \
        might_sleep();                                                                \
        if (!(condition))                                                        \
                __ret = __wait_event_freezable(wq_head, condition);                \
        __ret;                                                                        \
})

#define __wait_event_timeout(wq_head, condition, timeout)                        \
        ___wait_event(wq_head, ___wait_cond_timeout(condition),                        \
                      TASK_UNINTERRUPTIBLE, 0, timeout,                                \
                      __ret = schedule_timeout(__ret))

/**
 * wait_event_timeout - sleep until a condition gets true or a timeout elapses
 * @wq_head: the waitqueue to wait on
 * @condition: a C expression for the event to wait for
 * @timeout: timeout, in jiffies
 *
 * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
 * @condition evaluates to true. The @condition is checked each time
 * the waitqueue @wq_head is woken up.
 *
 * wake_up() has to be called after changing any variable that could
 * change the result of the wait condition.
 *
 * Returns:
 * 0 if the @condition evaluated to %false after the @timeout elapsed,
 * 1 if the @condition evaluated to %true after the @timeout elapsed,
 * or the remaining jiffies (at least 1) if the @condition evaluated
 * to %true before the @timeout elapsed.
 */
#define wait_event_timeout(wq_head, condition, timeout)                                \
({                                                                                \
        long __ret = timeout;                                                        \
        might_sleep();                                                                \
        if (!___wait_cond_timeout(condition))                                        \
                __ret = __wait_event_timeout(wq_head, condition, timeout);        \
        __ret;                                                                        \
})

#define __wait_event_freezable_timeout(wq_head, condition, timeout)                \
        ___wait_event(wq_head, ___wait_cond_timeout(condition),                        \
                      TASK_INTERRUPTIBLE, 0, timeout,                                \
                      __ret = freezable_schedule_timeout(__ret))

/*
 * like wait_event_timeout() -- except it uses TASK_INTERRUPTIBLE to avoid
 * increasing load and is freezable.
 */
#define wait_event_freezable_timeout(wq_head, condition, timeout)                \
({                                                                                \
        long __ret = timeout;                                                        \
        might_sleep();                                                                \
        if (!___wait_cond_timeout(condition))                                        \
                __ret = __wait_event_freezable_timeout(wq_head, condition, timeout); \
        __ret;                                                                        \
})

#define __wait_event_exclusive_cmd(wq_head, condition, cmd1, cmd2)                \
        (void)___wait_event(wq_head, condition, TASK_UNINTERRUPTIBLE, 1, 0,        \
                            cmd1; schedule(); cmd2)
/*
 * Just like wait_event_cmd(), except it sets exclusive flag
 */
#define wait_event_exclusive_cmd(wq_head, condition, cmd1, cmd2)                \
do {                                                                                \
        if (condition)                                                                \
                break;                                                                \
        __wait_event_exclusive_cmd(wq_head, condition, cmd1, cmd2);                \
} while (0)

#define __wait_event_cmd(wq_head, condition, cmd1, cmd2)                        \
        (void)___wait_event(wq_head, condition, TASK_UNINTERRUPTIBLE, 0, 0,        \
                            cmd1; schedule(); cmd2)

/**
 * wait_event_cmd - sleep until a condition gets true
 * @wq_head: the waitqueue to wait on
 * @condition: a C expression for the event to wait for
 * @cmd1: the command will be executed before sleep
 * @cmd2: the command will be executed after sleep
 *
 * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
 * @condition evaluates to true. The @condition is checked each time
 * the waitqueue @wq_head is woken up.
 *
 * wake_up() has to be called after changing any variable that could
 * change the result of the wait condition.
 */
#define wait_event_cmd(wq_head, condition, cmd1, cmd2)                                \
do {                                                                                \
        if (condition)                                                                \
                break;                                                                \
        __wait_event_cmd(wq_head, condition, cmd1, cmd2);                        \
} while (0)

#define __wait_event_interruptible(wq_head, condition)                                \
        ___wait_event(wq_head, condition, TASK_INTERRUPTIBLE, 0, 0,                \
                      schedule())

/**
 * wait_event_interruptible - sleep until a condition gets true
 * @wq_head: the waitqueue to wait on
 * @condition: a C expression for the event to wait for
 *
 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
 * @condition evaluates to true or a signal is received.
 * The @condition is checked each time the waitqueue @wq_head is woken up.
 *
 * wake_up() has to be called after changing any variable that could
 * change the result of the wait condition.
 *
 * The function will return -ERESTARTSYS if it was interrupted by a
 * signal and 0 if @condition evaluated to true.
 */
#define wait_event_interruptible(wq_head, condition)                                \
({                                                                                \
        int __ret = 0;                                                                \
        might_sleep();                                                                \
        if (!(condition))                                                        \
                __ret = __wait_event_interruptible(wq_head, condition);                \
        __ret;                                                                        \
})

#define __wait_event_interruptible_timeout(wq_head, condition, timeout)                \
        ___wait_event(wq_head, ___wait_cond_timeout(condition),                        \
                      TASK_INTERRUPTIBLE, 0, timeout,                                \
                      __ret = schedule_timeout(__ret))

/**
 * wait_event_interruptible_timeout - sleep until a condition gets true or a timeout elapses
 * @wq_head: the waitqueue to wait on
 * @condition: a C expression for the event to wait for
 * @timeout: timeout, in jiffies
 *
 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
 * @condition evaluates to true or a signal is received.
 * The @condition is checked each time the waitqueue @wq_head is woken up.
 *
 * wake_up() has to be called after changing any variable that could
 * change the result of the wait condition.
 *
 * Returns:
 * 0 if the @condition evaluated to %false after the @timeout elapsed,
 * 1 if the @condition evaluated to %true after the @timeout elapsed,
 * the remaining jiffies (at least 1) if the @condition evaluated
 * to %true before the @timeout elapsed, or -%ERESTARTSYS if it was
 * interrupted by a signal.
 */
#define wait_event_interruptible_timeout(wq_head, condition, timeout)                \
({                                                                                \
        long __ret = timeout;                                                        \
        might_sleep();                                                                \
        if (!___wait_cond_timeout(condition))                                        \
                __ret = __wait_event_interruptible_timeout(wq_head,                \
                                                condition, timeout);                \
        __ret;                                                                        \
})

#define __wait_event_hrtimeout(wq_head, condition, timeout, state)                \
({                                                                                \
        int __ret = 0;                                                                \
        struct hrtimer_sleeper __t;                                                \
                                                                                \
        hrtimer_init_sleeper_on_stack(&__t, CLOCK_MONOTONIC,                        \
                                      HRTIMER_MODE_REL);                        \
        if ((timeout) != KTIME_MAX) {                                                \
                hrtimer_set_expires_range_ns(&__t.timer, timeout,                \
                                        current->timer_slack_ns);                \
                hrtimer_sleeper_start_expires(&__t, HRTIMER_MODE_REL);                \
        }                                                                        \
                                                                                \
        __ret = ___wait_event(wq_head, condition, state, 0, 0,                        \
                if (!__t.task) {                                                \
                        __ret = -ETIME;                                                \
                        break;                                                        \
                }                                                                \
                schedule());                                                        \
                                                                                \
        hrtimer_cancel(&__t.timer);                                                \
        destroy_hrtimer_on_stack(&__t.timer);                                        \
        __ret;                                                                        \
})

/**
 * wait_event_hrtimeout - sleep until a condition gets true or a timeout elapses
 * @wq_head: the waitqueue to wait on
 * @condition: a C expression for the event to wait for
 * @timeout: timeout, as a ktime_t
 *
 * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
 * @condition evaluates to true or a signal is received.
 * The @condition is checked each time the waitqueue @wq_head is woken up.
 *
 * wake_up() has to be called after changing any variable that could
 * change the result of the wait condition.
 *
 * The function returns 0 if @condition became true, or -ETIME if the timeout
 * elapsed.
 */
#define wait_event_hrtimeout(wq_head, condition, timeout)                        \
({                                                                                \
        int __ret = 0;                                                                \
        might_sleep();                                                                \
        if (!(condition))                                                        \
                __ret = __wait_event_hrtimeout(wq_head, condition, timeout,        \
                                               TASK_UNINTERRUPTIBLE);                \
        __ret;                                                                        \
})

/**
 * wait_event_interruptible_hrtimeout - sleep until a condition gets true or a timeout elapses
 * @wq: the waitqueue to wait on
 * @condition: a C expression for the event to wait for
 * @timeout: timeout, as a ktime_t
 *
 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
 * @condition evaluates to true or a signal is received.
 * The @condition is checked each time the waitqueue @wq is woken up.
 *
 * wake_up() has to be called after changing any variable that could
 * change the result of the wait condition.
 *
 * The function returns 0 if @condition became true, -ERESTARTSYS if it was
 * interrupted by a signal, or -ETIME if the timeout elapsed.
 */
#define wait_event_interruptible_hrtimeout(wq, condition, timeout)                \
({                                                                                \
        long __ret = 0;                                                                \
        might_sleep();                                                                \
        if (!(condition))                                                        \
                __ret = __wait_event_hrtimeout(wq, condition, timeout,                \
                                               TASK_INTERRUPTIBLE);                \
        __ret;                                                                        \
})

#define __wait_event_interruptible_exclusive(wq, condition)                        \
        ___wait_event(wq, condition, TASK_INTERRUPTIBLE, 1, 0,                        \
                      schedule())

#define wait_event_interruptible_exclusive(wq, condition)                        \
({                                                                                \
        int __ret = 0;                                                                \
        might_sleep();                                                                \
        if (!(condition))                                                        \
                __ret = __wait_event_interruptible_exclusive(wq, condition);        \
        __ret;                                                                        \
})

#define __wait_event_killable_exclusive(wq, condition)                                \
        ___wait_event(wq, condition, TASK_KILLABLE, 1, 0,                        \
                      schedule())

#define wait_event_killable_exclusive(wq, condition)                                \
({                                                                                \
        int __ret = 0;                                                                \
        might_sleep();                                                                \
        if (!(condition))                                                        \
                __ret = __wait_event_killable_exclusive(wq, condition);                \
        __ret;                                                                        \
})


#define __wait_event_freezable_exclusive(wq, condition)                                \
        ___wait_event(wq, condition, TASK_INTERRUPTIBLE, 1, 0,                        \
                        freezable_schedule())

#define wait_event_freezable_exclusive(wq, condition)                                \
({                                                                                \
        int __ret = 0;                                                                \
        might_sleep();                                                                \
        if (!(condition))                                                        \
                __ret = __wait_event_freezable_exclusive(wq, condition);        \
        __ret;                                                                        \
})

/**
 * wait_event_idle - wait for a condition without contributing to system load
 * @wq_head: the waitqueue to wait on
 * @condition: a C expression for the event to wait for
 *
 * The process is put to sleep (TASK_IDLE) until the
 * @condition evaluates to true.
 * The @condition is checked each time the waitqueue @wq_head is woken up.
 *
 * wake_up() has to be called after changing any variable that could
 * change the result of the wait condition.
 *
 */
#define wait_event_idle(wq_head, condition)                                        \
do {                                                                                \
        might_sleep();                                                                \
        if (!(condition))                                                        \
                ___wait_event(wq_head, condition, TASK_IDLE, 0, 0, schedule());        \
} while (0)

/**
 * wait_event_idle_exclusive - wait for a condition with contributing to system load
 * @wq_head: the waitqueue to wait on
 * @condition: a C expression for the event to wait for
 *
 * The process is put to sleep (TASK_IDLE) until the
 * @condition evaluates to true.
 * The @condition is checked each time the waitqueue @wq_head is woken up.
 *
 * The process is put on the wait queue with an WQ_FLAG_EXCLUSIVE flag
 * set thus if other processes wait on the same list, when this
 * process is woken further processes are not considered.
 *
 * wake_up() has to be called after changing any variable that could
 * change the result of the wait condition.
 *
 */
#define wait_event_idle_exclusive(wq_head, condition)                                \
do {                                                                                \
        might_sleep();                                                                \
        if (!(condition))                                                        \
                ___wait_event(wq_head, condition, TASK_IDLE, 1, 0, schedule());        \
} while (0)

#define __wait_event_idle_timeout(wq_head, condition, timeout)                        \
        ___wait_event(wq_head, ___wait_cond_timeout(condition),                        \
                      TASK_IDLE, 0, timeout,                                        \
                      __ret = schedule_timeout(__ret))

/**
 * wait_event_idle_timeout - sleep without load until a condition becomes true or a timeout elapses
 * @wq_head: the waitqueue to wait on
 * @condition: a C expression for the event to wait for
 * @timeout: timeout, in jiffies
 *
 * The process is put to sleep (TASK_IDLE) until the
 * @condition evaluates to true. The @condition is checked each time
 * the waitqueue @wq_head is woken up.
 *
 * wake_up() has to be called after changing any variable that could
 * change the result of the wait condition.
 *
 * Returns:
 * 0 if the @condition evaluated to %false after the @timeout elapsed,
 * 1 if the @condition evaluated to %true after the @timeout elapsed,
 * or the remaining jiffies (at least 1) if the @condition evaluated
 * to %true before the @timeout elapsed.
 */
#define wait_event_idle_timeout(wq_head, condition, timeout)                        \
({                                                                                \
        long __ret = timeout;                                                        \
        might_sleep();                                                                \
        if (!___wait_cond_timeout(condition))                                        \
                __ret = __wait_event_idle_timeout(wq_head, condition, timeout);        \
        __ret;                                                                        \
})

#define __wait_event_idle_exclusive_timeout(wq_head, condition, timeout)        \
        ___wait_event(wq_head, ___wait_cond_timeout(condition),                        \
                      TASK_IDLE, 1, timeout,                                        \
                      __ret = schedule_timeout(__ret))

/**
 * wait_event_idle_exclusive_timeout - sleep without load until a condition becomes true or a timeout elapses
 * @wq_head: the waitqueue to wait on
 * @condition: a C expression for the event to wait for
 * @timeout: timeout, in jiffies
 *
 * The process is put to sleep (TASK_IDLE) until the
 * @condition evaluates to true. The @condition is checked each time
 * the waitqueue @wq_head is woken up.
 *
 * The process is put on the wait queue with an WQ_FLAG_EXCLUSIVE flag
 * set thus if other processes wait on the same list, when this
 * process is woken further processes are not considered.
 *
 * wake_up() has to be called after changing any variable that could
 * change the result of the wait condition.
 *
 * Returns:
 * 0 if the @condition evaluated to %false after the @timeout elapsed,
 * 1 if the @condition evaluated to %true after the @timeout elapsed,
 * or the remaining jiffies (at least 1) if the @condition evaluated
 * to %true before the @timeout elapsed.
 */
#define wait_event_idle_exclusive_timeout(wq_head, condition, timeout)                \
({                                                                                \
        long __ret = timeout;                                                        \
        might_sleep();                                                                \
        if (!___wait_cond_timeout(condition))                                        \
                __ret = __wait_event_idle_exclusive_timeout(wq_head, condition, timeout);\
        __ret;                                                                        \
})

extern int do_wait_intr(wait_queue_head_t *, wait_queue_entry_t *);
extern int do_wait_intr_irq(wait_queue_head_t *, wait_queue_entry_t *);

#define __wait_event_interruptible_locked(wq, condition, exclusive, fn)                \
({                                                                                \
        int __ret;                                                                \
        DEFINE_WAIT(__wait);                                                        \
        if (exclusive)                                                                \
                __wait.flags |= WQ_FLAG_EXCLUSIVE;                                \
        do {                                                                        \
                __ret = fn(&(wq), &__wait);                                        \
                if (__ret)                                                        \
                        break;                                                        \
        } while (!(condition));                                                        \
        __remove_wait_queue(&(wq), &__wait);                                        \
        __set_current_state(TASK_RUNNING);                                        \
        __ret;                                                                        \
})


/**
 * wait_event_interruptible_locked - sleep until a condition gets true
 * @wq: the waitqueue to wait on
 * @condition: a C expression for the event to wait for
 *
 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
 * @condition evaluates to true or a signal is received.
 * The @condition is checked each time the waitqueue @wq is woken up.
 *
 * It must be called with wq.lock being held.  This spinlock is
 * unlocked while sleeping but @condition testing is done while lock
 * is held and when this macro exits the lock is held.
 *
 * The lock is locked/unlocked using spin_lock()/spin_unlock()
 * functions which must match the way they are locked/unlocked outside
 * of this macro.
 *
 * wake_up_locked() has to be called after changing any variable that could
 * change the result of the wait condition.
 *
 * The function will return -ERESTARTSYS if it was interrupted by a
 * signal and 0 if @condition evaluated to true.
 */
#define wait_event_interruptible_locked(wq, condition)                                \
        ((condition)                                                                \
         ? 0 : __wait_event_interruptible_locked(wq, condition, 0, do_wait_intr))

/**
 * wait_event_interruptible_locked_irq - sleep until a condition gets true
 * @wq: the waitqueue to wait on
 * @condition: a C expression for the event to wait for
 *
 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
 * @condition evaluates to true or a signal is received.
 * The @condition is checked each time the waitqueue @wq is woken up.
 *
 * It must be called with wq.lock being held.  This spinlock is
 * unlocked while sleeping but @condition testing is done while lock
 * is held and when this macro exits the lock is held.
 *
 * The lock is locked/unlocked using spin_lock_irq()/spin_unlock_irq()
 * functions which must match the way they are locked/unlocked outside
 * of this macro.
 *
 * wake_up_locked() has to be called after changing any variable that could
 * change the result of the wait condition.
 *
 * The function will return -ERESTARTSYS if it was interrupted by a
 * signal and 0 if @condition evaluated to true.
 */
#define wait_event_interruptible_locked_irq(wq, condition)                        \
        ((condition)                                                                \
         ? 0 : __wait_event_interruptible_locked(wq, condition, 0, do_wait_intr_irq))

/**
 * wait_event_interruptible_exclusive_locked - sleep exclusively until a condition gets true
 * @wq: the waitqueue to wait on
 * @condition: a C expression for the event to wait for
 *
 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
 * @condition evaluates to true or a signal is received.
 * The @condition is checked each time the waitqueue @wq is woken up.
 *
 * It must be called with wq.lock being held.  This spinlock is
 * unlocked while sleeping but @condition testing is done while lock
 * is held and when this macro exits the lock is held.
 *
 * The lock is locked/unlocked using spin_lock()/spin_unlock()
 * functions which must match the way they are locked/unlocked outside
 * of this macro.
 *
 * The process is put on the wait queue with an WQ_FLAG_EXCLUSIVE flag
 * set thus when other process waits process on the list if this
 * process is awaken further processes are not considered.
 *
 * wake_up_locked() has to be called after changing any variable that could
 * change the result of the wait condition.
 *
 * The function will return -ERESTARTSYS if it was interrupted by a
 * signal and 0 if @condition evaluated to true.
 */
#define wait_event_interruptible_exclusive_locked(wq, condition)                \
        ((condition)                                                                \
         ? 0 : __wait_event_interruptible_locked(wq, condition, 1, do_wait_intr))

/**
 * wait_event_interruptible_exclusive_locked_irq - sleep until a condition gets true
 * @wq: the waitqueue to wait on
 * @condition: a C expression for the event to wait for
 *
 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
 * @condition evaluates to true or a signal is received.
 * The @condition is checked each time the waitqueue @wq is woken up.
 *
 * It must be called with wq.lock being held.  This spinlock is
 * unlocked while sleeping but @condition testing is done while lock
 * is held and when this macro exits the lock is held.
 *
 * The lock is locked/unlocked using spin_lock_irq()/spin_unlock_irq()
 * functions which must match the way they are locked/unlocked outside
 * of this macro.
 *
 * The process is put on the wait queue with an WQ_FLAG_EXCLUSIVE flag
 * set thus when other process waits process on the list if this
 * process is awaken further processes are not considered.
 *
 * wake_up_locked() has to be called after changing any variable that could
 * change the result of the wait condition.
 *
 * The function will return -ERESTARTSYS if it was interrupted by a
 * signal and 0 if @condition evaluated to true.
 */
#define wait_event_interruptible_exclusive_locked_irq(wq, condition)                \
        ((condition)                                                                \
         ? 0 : __wait_event_interruptible_locked(wq, condition, 1, do_wait_intr_irq))


#define __wait_event_killable(wq, condition)                                        \
        ___wait_event(wq, condition, TASK_KILLABLE, 0, 0, schedule())

/**
 * wait_event_killable - sleep until a condition gets true
 * @wq_head: the waitqueue to wait on
 * @condition: a C expression for the event to wait for
 *
 * The process is put to sleep (TASK_KILLABLE) until the
 * @condition evaluates to true or a signal is received.
 * The @condition is checked each time the waitqueue @wq_head is woken up.
 *
 * wake_up() has to be called after changing any variable that could
 * change the result of the wait condition.
 *
 * The function will return -ERESTARTSYS if it was interrupted by a
 * signal and 0 if @condition evaluated to true.
 */
#define wait_event_killable(wq_head, condition)                                        \
({                                                                                \
        int __ret = 0;                                                                \
        might_sleep();                                                                \
        if (!(condition))                                                        \
                __ret = __wait_event_killable(wq_head, condition);                \
        __ret;                                                                        \
})

#define __wait_event_killable_timeout(wq_head, condition, timeout)                \
        ___wait_event(wq_head, ___wait_cond_timeout(condition),                        \
                      TASK_KILLABLE, 0, timeout,                                \
                      __ret = schedule_timeout(__ret))

/**
 * wait_event_killable_timeout - sleep until a condition gets true or a timeout elapses
 * @wq_head: the waitqueue to wait on
 * @condition: a C expression for the event to wait for
 * @timeout: timeout, in jiffies
 *
 * The process is put to sleep (TASK_KILLABLE) until the
 * @condition evaluates to true or a kill signal is received.
 * The @condition is checked each time the waitqueue @wq_head is woken up.
 *
 * wake_up() has to be called after changing any variable that could
 * change the result of the wait condition.
 *
 * Returns:
 * 0 if the @condition evaluated to %false after the @timeout elapsed,
 * 1 if the @condition evaluated to %true after the @timeout elapsed,
 * the remaining jiffies (at least 1) if the @condition evaluated
 * to %true before the @timeout elapsed, or -%ERESTARTSYS if it was
 * interrupted by a kill signal.
 *
 * Only kill signals interrupt this process.
 */
#define wait_event_killable_timeout(wq_head, condition, timeout)                \
({                                                                                \
        long __ret = timeout;                                                        \
        might_sleep();                                                                \
        if (!___wait_cond_timeout(condition))                                        \
                __ret = __wait_event_killable_timeout(wq_head,                        \
                                                condition, timeout);                \
        __ret;                                                                        \
})


#define __wait_event_lock_irq(wq_head, condition, lock, cmd)                        \
        (void)___wait_event(wq_head, condition, TASK_UNINTERRUPTIBLE, 0, 0,        \
                            spin_unlock_irq(&lock);                                \
                            cmd;                                                \
                            schedule();                                                \
                            spin_lock_irq(&lock))

/**
 * wait_event_lock_irq_cmd - sleep until a condition gets true. The
 *                             condition is checked under the lock. This
 *                             is expected to be called with the lock
 *                             taken.
 * @wq_head: the waitqueue to wait on
 * @condition: a C expression for the event to wait for
 * @lock: a locked spinlock_t, which will be released before cmd
 *          and schedule() and reacquired afterwards.
 * @cmd: a command which is invoked outside the critical section before
 *         sleep
 *
 * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
 * @condition evaluates to true. The @condition is checked each time
 * the waitqueue @wq_head is woken up.
 *
 * wake_up() has to be called after changing any variable that could
 * change the result of the wait condition.
 *
 * This is supposed to be called while holding the lock. The lock is
 * dropped before invoking the cmd and going to sleep and is reacquired
 * afterwards.
 */
#define wait_event_lock_irq_cmd(wq_head, condition, lock, cmd)                        \
do {                                                                                \
        if (condition)                                                                \
                break;                                                                \
        __wait_event_lock_irq(wq_head, condition, lock, cmd);                        \
} while (0)

/**
 * wait_event_lock_irq - sleep until a condition gets true. The
 *                         condition is checked under the lock. This
 *                         is expected to be called with the lock
 *                         taken.
 * @wq_head: the waitqueue to wait on
 * @condition: a C expression for the event to wait for
 * @lock: a locked spinlock_t, which will be released before schedule()
 *          and reacquired afterwards.
 *
 * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
 * @condition evaluates to true. The @condition is checked each time
 * the waitqueue @wq_head is woken up.
 *
 * wake_up() has to be called after changing any variable that could
 * change the result of the wait condition.
 *
 * This is supposed to be called while holding the lock. The lock is
 * dropped before going to sleep and is reacquired afterwards.
 */
#define wait_event_lock_irq(wq_head, condition, lock)                                \
do {                                                                                \
        if (condition)                                                                \
                break;                                                                \
        __wait_event_lock_irq(wq_head, condition, lock, );                        \
} while (0)


#define __wait_event_interruptible_lock_irq(wq_head, condition, lock, cmd)        \
        ___wait_event(wq_head, condition, TASK_INTERRUPTIBLE, 0, 0,                \
                      spin_unlock_irq(&lock);                                        \
                      cmd;                                                        \
                      schedule();                                                \
                      spin_lock_irq(&lock))

/**
 * wait_event_interruptible_lock_irq_cmd - sleep until a condition gets true.
 *                The condition is checked under the lock. This is expected to
 *                be called with the lock taken.
 * @wq_head: the waitqueue to wait on
 * @condition: a C expression for the event to wait for
 * @lock: a locked spinlock_t, which will be released before cmd and
 *          schedule() and reacquired afterwards.
 * @cmd: a command which is invoked outside the critical section before
 *         sleep
 *
 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
 * @condition evaluates to true or a signal is received. The @condition is
 * checked each time the waitqueue @wq_head is woken up.
 *
 * wake_up() has to be called after changing any variable that could
 * change the result of the wait condition.
 *
 * This is supposed to be called while holding the lock. The lock is
 * dropped before invoking the cmd and going to sleep and is reacquired
 * afterwards.
 *
 * The macro will return -ERESTARTSYS if it was interrupted by a signal
 * and 0 if @condition evaluated to true.
 */
#define wait_event_interruptible_lock_irq_cmd(wq_head, condition, lock, cmd)        \
({                                                                                \
        int __ret = 0;                                                                \
        if (!(condition))                                                        \
                __ret = __wait_event_interruptible_lock_irq(wq_head,                \
                                                condition, lock, cmd);                \
        __ret;                                                                        \
})

/**
 * wait_event_interruptible_lock_irq - sleep until a condition gets true.
 *                The condition is checked under the lock. This is expected
 *                to be called with the lock taken.
 * @wq_head: the waitqueue to wait on
 * @condition: a C expression for the event to wait for
 * @lock: a locked spinlock_t, which will be released before schedule()
 *          and reacquired afterwards.
 *
 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
 * @condition evaluates to true or signal is received. The @condition is
 * checked each time the waitqueue @wq_head is woken up.
 *
 * wake_up() has to be called after changing any variable that could
 * change the result of the wait condition.
 *
 * This is supposed to be called while holding the lock. The lock is
 * dropped before going to sleep and is reacquired afterwards.
 *
 * The macro will return -ERESTARTSYS if it was interrupted by a signal
 * and 0 if @condition evaluated to true.
 */
#define wait_event_interruptible_lock_irq(wq_head, condition, lock)                \
({                                                                                \
        int __ret = 0;                                                                \
        if (!(condition))                                                        \
                __ret = __wait_event_interruptible_lock_irq(wq_head,                \
                                                condition, lock,);                \
        __ret;                                                                        \
})

#define __wait_event_lock_irq_timeout(wq_head, condition, lock, timeout, state)        \
        ___wait_event(wq_head, ___wait_cond_timeout(condition),                        \
                      state, 0, timeout,                                        \
                      spin_unlock_irq(&lock);                                        \
                      __ret = schedule_timeout(__ret);                                \
                      spin_lock_irq(&lock));

/**
 * wait_event_interruptible_lock_irq_timeout - sleep until a condition gets
 *                true or a timeout elapses. The condition is checked under
 *                the lock. This is expected to be called with the lock taken.
 * @wq_head: the waitqueue to wait on
 * @condition: a C expression for the event to wait for
 * @lock: a locked spinlock_t, which will be released before schedule()
 *          and reacquired afterwards.
 * @timeout: timeout, in jiffies
 *
 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
 * @condition evaluates to true or signal is received. The @condition is
 * checked each time the waitqueue @wq_head is woken up.
 *
 * wake_up() has to be called after changing any variable that could
 * change the result of the wait condition.
 *
 * This is supposed to be called while holding the lock. The lock is
 * dropped before going to sleep and is reacquired afterwards.
 *
 * The function returns 0 if the @timeout elapsed, -ERESTARTSYS if it
 * was interrupted by a signal, and the remaining jiffies otherwise
 * if the condition evaluated to true before the timeout elapsed.
 */
#define wait_event_interruptible_lock_irq_timeout(wq_head, condition, lock,        \
                                                  timeout)                        \
({                                                                                \
        long __ret = timeout;                                                        \
        if (!___wait_cond_timeout(condition))                                        \
                __ret = __wait_event_lock_irq_timeout(                                \
                                        wq_head, condition, lock, timeout,        \
                                        TASK_INTERRUPTIBLE);                        \
        __ret;                                                                        \
})

#define wait_event_lock_irq_timeout(wq_head, condition, lock, timeout)                \
({                                                                                \
        long __ret = timeout;                                                        \
        if (!___wait_cond_timeout(condition))                                        \
                __ret = __wait_event_lock_irq_timeout(                                \
                                        wq_head, condition, lock, timeout,        \
                                        TASK_UNINTERRUPTIBLE);                        \
        __ret;                                                                        \
})

/*
 * Waitqueues which are removed from the waitqueue_head at wakeup time
 */
void prepare_to_wait(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state);
bool prepare_to_wait_exclusive(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state);
long prepare_to_wait_event(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state);
void finish_wait(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry);
long wait_woken(struct wait_queue_entry *wq_entry, unsigned mode, long timeout);
int woken_wake_function(struct wait_queue_entry *wq_entry, unsigned mode, int sync, void *key);
int autoremove_wake_function(struct wait_queue_entry *wq_entry, unsigned mode, int sync, void *key);

#define DEFINE_WAIT_FUNC(name, function)                                        \
        struct wait_queue_entry name = {                                        \
                .private        = current,                                        \
                .func                = function,                                        \
                .entry                = LIST_HEAD_INIT((name).entry),                        \
        }

#define DEFINE_WAIT(name) DEFINE_WAIT_FUNC(name, autoremove_wake_function)

#define init_wait(wait)                                                                \
        do {                                                                        \
                (wait)->private = current;                                        \
                (wait)->func = autoremove_wake_function;                        \
                INIT_LIST_HEAD(&(wait)->entry);                                        \
                (wait)->flags = 0;                                                \
        } while (0)

bool try_invoke_on_locked_down_task(struct task_struct *p, bool (*func)(struct task_struct *t, void *arg), void *arg);

#endif /* _LINUX_WAIT_H */














































































































































































































































































































































 3640 






 3640 


























 3639 




 3655 










































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
// SPDX-License-Identifier: GPL-2.0
/*
 * security/tomoyo/audit.c
 *
 * Copyright (C) 2005-2011  NTT DATA CORPORATION
 */

#include "common.h"
#include <linux/slab.h>

/**
 * tomoyo_print_bprm - Print "struct linux_binprm" for auditing.
 *
 * @bprm: Pointer to "struct linux_binprm".
 * @dump: Pointer to "struct tomoyo_page_dump".
 *
 * Returns the contents of @bprm on success, NULL otherwise.
 *
 * This function uses kzalloc(), so caller must kfree() if this function
 * didn't return NULL.
 */
static char *tomoyo_print_bprm(struct linux_binprm *bprm,
                               struct tomoyo_page_dump *dump)
{
        static const int tomoyo_buffer_len = 4096 * 2;
        char *buffer = kzalloc(tomoyo_buffer_len, GFP_NOFS);
        char *cp;
        char *last_start;
        int len;
        unsigned long pos = bprm->p;
        int offset = pos % PAGE_SIZE;
        int argv_count = bprm->argc;
        int envp_count = bprm->envc;
        bool truncated = false;

        if (!buffer)
                return NULL;
        len = snprintf(buffer, tomoyo_buffer_len - 1, "argv[]={ ");
        cp = buffer + len;
        if (!argv_count) {
                memmove(cp, "} envp[]={ ", 11);
                cp += 11;
        }
        last_start = cp;
        while (argv_count || envp_count) {
                if (!tomoyo_dump_page(bprm, pos, dump))
                        goto out;
                pos += PAGE_SIZE - offset;
                /* Read. */
                while (offset < PAGE_SIZE) {
                        const char *kaddr = dump->data;
                        const unsigned char c = kaddr[offset++];

                        if (cp == last_start)
                                *cp++ = '"';
                        if (cp >= buffer + tomoyo_buffer_len - 32) {
                                /* Reserve some room for "..." string. */
                                truncated = true;
                        } else if (c == '\\') {
                                *cp++ = '\\';
                                *cp++ = '\\';
                        } else if (c > ' ' && c < 127) {
                                *cp++ = c;
                        } else if (!c) {
                                *cp++ = '"';
                                *cp++ = ' ';
                                last_start = cp;
                        } else {
                                *cp++ = '\\';
                                *cp++ = (c >> 6) + '0';
                                *cp++ = ((c >> 3) & 7) + '0';
                                *cp++ = (c & 7) + '0';
                        }
                        if (c)
                                continue;
                        if (argv_count) {
                                if (--argv_count == 0) {
                                        if (truncated) {
                                                cp = last_start;
                                                memmove(cp, "... ", 4);
                                                cp += 4;
                                        }
                                        memmove(cp, "} envp[]={ ", 11);
                                        cp += 11;
                                        last_start = cp;
                                        truncated = false;
                                }
                        } else if (envp_count) {
                                if (--envp_count == 0) {
                                        if (truncated) {
                                                cp = last_start;
                                                memmove(cp, "... ", 4);
                                                cp += 4;
                                        }
                                }
                        }
                        if (!argv_count && !envp_count)
                                break;
                }
                offset = 0;
        }
        *cp++ = '}';
        *cp = '\0';
        return buffer;
out:
        snprintf(buffer, tomoyo_buffer_len - 1,
                 "argv[]={ ... } envp[]= { ... }");
        return buffer;
}

/**
 * tomoyo_filetype - Get string representation of file type.
 *
 * @mode: Mode value for stat().
 *
 * Returns file type string.
 */
static inline const char *tomoyo_filetype(const umode_t mode)
{
        switch (mode & S_IFMT) {
        case S_IFREG:
        case 0:
                return tomoyo_condition_keyword[TOMOYO_TYPE_IS_FILE];
        case S_IFDIR:
                return tomoyo_condition_keyword[TOMOYO_TYPE_IS_DIRECTORY];
        case S_IFLNK:
                return tomoyo_condition_keyword[TOMOYO_TYPE_IS_SYMLINK];
        case S_IFIFO:
                return tomoyo_condition_keyword[TOMOYO_TYPE_IS_FIFO];
        case S_IFSOCK:
                return tomoyo_condition_keyword[TOMOYO_TYPE_IS_SOCKET];
        case S_IFBLK:
                return tomoyo_condition_keyword[TOMOYO_TYPE_IS_BLOCK_DEV];
        case S_IFCHR:
                return tomoyo_condition_keyword[TOMOYO_TYPE_IS_CHAR_DEV];
        }
        return "unknown"; /* This should not happen. */
}

/**
 * tomoyo_print_header - Get header line of audit log.
 *
 * @r: Pointer to "struct tomoyo_request_info".
 *
 * Returns string representation.
 *
 * This function uses kmalloc(), so caller must kfree() if this function
 * didn't return NULL.
 */
static char *tomoyo_print_header(struct tomoyo_request_info *r)
{
        struct tomoyo_time stamp;
        const pid_t gpid = task_pid_nr(current);
        struct tomoyo_obj_info *obj = r->obj;
        static const int tomoyo_buffer_len = 4096;
        char *buffer = kmalloc(tomoyo_buffer_len, GFP_NOFS);
        int pos;
        u8 i;

        if (!buffer)
                return NULL;

        tomoyo_convert_time(ktime_get_real_seconds(), &stamp);

        pos = snprintf(buffer, tomoyo_buffer_len - 1,
                       "#%04u/%02u/%02u %02u:%02u:%02u# profile=%u mode=%s granted=%s (global-pid=%u) task={ pid=%u ppid=%u uid=%u gid=%u euid=%u egid=%u suid=%u sgid=%u fsuid=%u fsgid=%u }",
                       stamp.year, stamp.month, stamp.day, stamp.hour,
                       stamp.min, stamp.sec, r->profile, tomoyo_mode[r->mode],
                       tomoyo_yesno(r->granted), gpid, tomoyo_sys_getpid(),
                       tomoyo_sys_getppid(),
                       from_kuid(&init_user_ns, current_uid()),
                       from_kgid(&init_user_ns, current_gid()),
                       from_kuid(&init_user_ns, current_euid()),
                       from_kgid(&init_user_ns, current_egid()),
                       from_kuid(&init_user_ns, current_suid()),
                       from_kgid(&init_user_ns, current_sgid()),
                       from_kuid(&init_user_ns, current_fsuid()),
                       from_kgid(&init_user_ns, current_fsgid()));
        if (!obj)
                goto no_obj_info;
        if (!obj->validate_done) {
                tomoyo_get_attributes(obj);
                obj->validate_done = true;
        }
        for (i = 0; i < TOMOYO_MAX_PATH_STAT; i++) {
                struct tomoyo_mini_stat *stat;
                unsigned int dev;
                umode_t mode;

                if (!obj->stat_valid[i])
                        continue;
                stat = &obj->stat[i];
                dev = stat->dev;
                mode = stat->mode;
                if (i & 1) {
                        pos += snprintf(buffer + pos,
                                        tomoyo_buffer_len - 1 - pos,
                                        " path%u.parent={ uid=%u gid=%u ino=%lu perm=0%o }",
                                        (i >> 1) + 1,
                                        from_kuid(&init_user_ns, stat->uid),
                                        from_kgid(&init_user_ns, stat->gid),
                                        (unsigned long)stat->ino,
                                        stat->mode & S_IALLUGO);
                        continue;
                }
                pos += snprintf(buffer + pos, tomoyo_buffer_len - 1 - pos,
                                " path%u={ uid=%u gid=%u ino=%lu major=%u minor=%u perm=0%o type=%s",
                                (i >> 1) + 1,
                                from_kuid(&init_user_ns, stat->uid),
                                from_kgid(&init_user_ns, stat->gid),
                                (unsigned long)stat->ino,
                                MAJOR(dev), MINOR(dev),
                                mode & S_IALLUGO, tomoyo_filetype(mode));
                if (S_ISCHR(mode) || S_ISBLK(mode)) {
                        dev = stat->rdev;
                        pos += snprintf(buffer + pos,
                                        tomoyo_buffer_len - 1 - pos,
                                        " dev_major=%u dev_minor=%u",
                                        MAJOR(dev), MINOR(dev));
                }
                pos += snprintf(buffer + pos, tomoyo_buffer_len - 1 - pos,
                                " }");
        }
no_obj_info:
        if (pos < tomoyo_buffer_len - 1)
                return buffer;
        kfree(buffer);
        return NULL;
}

/**
 * tomoyo_init_log - Allocate buffer for audit logs.
 *
 * @r:    Pointer to "struct tomoyo_request_info".
 * @len:  Buffer size needed for @fmt and @args.
 * @fmt:  The printf()'s format string.
 * @args: va_list structure for @fmt.
 *
 * Returns pointer to allocated memory.
 *
 * This function uses kzalloc(), so caller must kfree() if this function
 * didn't return NULL.
 */
char *tomoyo_init_log(struct tomoyo_request_info *r, int len, const char *fmt,
                      va_list args)
{
        char *buf = NULL;
        char *bprm_info = NULL;
        const char *header = NULL;
        char *realpath = NULL;
        const char *symlink = NULL;
        int pos;
        const char *domainname = r->domain->domainname->name;

        header = tomoyo_print_header(r);
        if (!header)
                return NULL;
        /* +10 is for '\n' etc. and '\0'. */
        len += strlen(domainname) + strlen(header) + 10;
        if (r->ee) {
                struct file *file = r->ee->bprm->file;

                realpath = tomoyo_realpath_from_path(&file->f_path);
                bprm_info = tomoyo_print_bprm(r->ee->bprm, &r->ee->dump);
                if (!realpath || !bprm_info)
                        goto out;
                /* +80 is for " exec={ realpath=\"%s\" argc=%d envc=%d %s }" */
                len += strlen(realpath) + 80 + strlen(bprm_info);
        } else if (r->obj && r->obj->symlink_target) {
                symlink = r->obj->symlink_target->name;
                /* +18 is for " symlink.target=\"%s\"" */
                len += 18 + strlen(symlink);
        }
        len = tomoyo_round2(len);
        buf = kzalloc(len, GFP_NOFS);
        if (!buf)
                goto out;
        len--;
        pos = snprintf(buf, len, "%s", header);
        if (realpath) {
                struct linux_binprm *bprm = r->ee->bprm;

                pos += snprintf(buf + pos, len - pos,
                                " exec={ realpath=\"%s\" argc=%d envc=%d %s }",
                                realpath, bprm->argc, bprm->envc, bprm_info);
        } else if (symlink)
                pos += snprintf(buf + pos, len - pos, " symlink.target=\"%s\"",
                                symlink);
        pos += snprintf(buf + pos, len - pos, "\n%s\n", domainname);
        vsnprintf(buf + pos, len - pos, fmt, args);
out:
        kfree(realpath);
        kfree(bprm_info);
        kfree(header);
        return buf;
}

/* Wait queue for /sys/kernel/security/tomoyo/audit. */
static DECLARE_WAIT_QUEUE_HEAD(tomoyo_log_wait);

/* Structure for audit log. */
struct tomoyo_log {
        struct list_head list;
        char *log;
        int size;
};

/* The list for "struct tomoyo_log". */
static LIST_HEAD(tomoyo_log);

/* Lock for "struct list_head tomoyo_log". */
static DEFINE_SPINLOCK(tomoyo_log_lock);

/* Length of "struct list_head tomoyo_log". */
static unsigned int tomoyo_log_count;

/**
 * tomoyo_get_audit - Get audit mode.
 *
 * @ns:          Pointer to "struct tomoyo_policy_namespace".
 * @profile:     Profile number.
 * @index:       Index number of functionality.
 * @matched_acl: Pointer to "struct tomoyo_acl_info".
 * @is_granted:  True if granted log, false otherwise.
 *
 * Returns true if this request should be audited, false otherwise.
 */
static bool tomoyo_get_audit(const struct tomoyo_policy_namespace *ns,
                             const u8 profile, const u8 index,
                             const struct tomoyo_acl_info *matched_acl,
                             const bool is_granted)
{
        u8 mode;
        const u8 category = tomoyo_index2category[index] +
                TOMOYO_MAX_MAC_INDEX;
        struct tomoyo_profile *p;

        if (!tomoyo_policy_loaded)
                return false;
        p = tomoyo_profile(ns, profile);
        if (tomoyo_log_count >= p->pref[TOMOYO_PREF_MAX_AUDIT_LOG])
                return false;
        if (is_granted && matched_acl && matched_acl->cond &&
            matched_acl->cond->grant_log != TOMOYO_GRANTLOG_AUTO)
                return matched_acl->cond->grant_log == TOMOYO_GRANTLOG_YES;
        mode = p->config[index];
        if (mode == TOMOYO_CONFIG_USE_DEFAULT)
                mode = p->config[category];
        if (mode == TOMOYO_CONFIG_USE_DEFAULT)
                mode = p->default_config;
        if (is_granted)
                return mode & TOMOYO_CONFIG_WANT_GRANT_LOG;
        return mode & TOMOYO_CONFIG_WANT_REJECT_LOG;
}

/**
 * tomoyo_write_log2 - Write an audit log.
 *
 * @r:    Pointer to "struct tomoyo_request_info".
 * @len:  Buffer size needed for @fmt and @args.
 * @fmt:  The printf()'s format string.
 * @args: va_list structure for @fmt.
 *
 * Returns nothing.
 */
void tomoyo_write_log2(struct tomoyo_request_info *r, int len, const char *fmt,
                       va_list args)
{
        char *buf;
        struct tomoyo_log *entry;
        bool quota_exceeded = false;

        if (!tomoyo_get_audit(r->domain->ns, r->profile, r->type,
                              r->matched_acl, r->granted))
                goto out;
        buf = tomoyo_init_log(r, len, fmt, args);
        if (!buf)
                goto out;
        entry = kzalloc(sizeof(*entry), GFP_NOFS);
        if (!entry) {
                kfree(buf);
                goto out;
        }
        entry->log = buf;
        len = tomoyo_round2(strlen(buf) + 1);
        /*
         * The entry->size is used for memory quota checks.
         * Don't go beyond strlen(entry->log).
         */
        entry->size = len + tomoyo_round2(sizeof(*entry));
        spin_lock(&tomoyo_log_lock);
        if (tomoyo_memory_quota[TOMOYO_MEMORY_AUDIT] &&
            tomoyo_memory_used[TOMOYO_MEMORY_AUDIT] + entry->size >=
            tomoyo_memory_quota[TOMOYO_MEMORY_AUDIT]) {
                quota_exceeded = true;
        } else {
                tomoyo_memory_used[TOMOYO_MEMORY_AUDIT] += entry->size;
                list_add_tail(&entry->list, &tomoyo_log);
                tomoyo_log_count++;
        }
        spin_unlock(&tomoyo_log_lock);
        if (quota_exceeded) {
                kfree(buf);
                kfree(entry);
                goto out;
        }
        wake_up(&tomoyo_log_wait);
out:
        return;
}

/**
 * tomoyo_write_log - Write an audit log.
 *
 * @r:   Pointer to "struct tomoyo_request_info".
 * @fmt: The printf()'s format string, followed by parameters.
 *
 * Returns nothing.
 */
void tomoyo_write_log(struct tomoyo_request_info *r, const char *fmt, ...)
{
        va_list args;
        int len;

        va_start(args, fmt);
        len = vsnprintf((char *) &len, 1, fmt, args) + 1;
        va_end(args);
        va_start(args, fmt);
        tomoyo_write_log2(r, len, fmt, args);
        va_end(args);
}

/**
 * tomoyo_read_log - Read an audit log.
 *
 * @head: Pointer to "struct tomoyo_io_buffer".
 *
 * Returns nothing.
 */
void tomoyo_read_log(struct tomoyo_io_buffer *head)
{
        struct tomoyo_log *ptr = NULL;

        if (head->r.w_pos)
                return;
        kfree(head->read_buf);
        head->read_buf = NULL;
        spin_lock(&tomoyo_log_lock);
        if (!list_empty(&tomoyo_log)) {
                ptr = list_entry(tomoyo_log.next, typeof(*ptr), list);
                list_del(&ptr->list);
                tomoyo_log_count--;
                tomoyo_memory_used[TOMOYO_MEMORY_AUDIT] -= ptr->size;
        }
        spin_unlock(&tomoyo_log_lock);
        if (ptr) {
                head->read_buf = ptr->log;
                head->r.w[head->r.w_pos++] = head->read_buf;
                kfree(ptr);
        }
}

/**
 * tomoyo_poll_log - Wait for an audit log.
 *
 * @file: Pointer to "struct file".
 * @wait: Pointer to "poll_table". Maybe NULL.
 *
 * Returns EPOLLIN | EPOLLRDNORM when ready to read an audit log.
 */
__poll_t tomoyo_poll_log(struct file *file, poll_table *wait)
{
        if (tomoyo_log_count)
                return EPOLLIN | EPOLLRDNORM;
        poll_wait(file, &tomoyo_log_wait, wait);
        if (tomoyo_log_count)
                return EPOLLIN | EPOLLRDNORM;
        return 0;
}


































































































  725 

   11 

























































































































































































 1641 

































































 1639 



 1641 
 1637 







 1633 

 1639 




 1614 


 1641 

 1639 





























































 1638 







































 1639 








































 1641 
















 1638 














 1636 








 1636 
 1639 
 1640 








 1638 
 1639 


































   11 







  726 




 1637 


































 1640 
































































































































  726 





  725 




   11 
















  725 








































































  723 






































  726 
















  726 





















  725 

















  726 









  726 











   11 
























  726 
  726 





  725 









  725 






































  726 






















































































































































































































































































































































































































































 1640 

 1641 



























 1638 





 1637 










  726 











  726 






















 1616 











 1616 









 1613 





















 1636 




 1634 











 1641 





 1640 
 1618 
























 1641 









 1640 


















 1641 






















 1640 






























 1641 















































































































































 1636 















 1615 





























































































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright 2002 Andi Kleen, SuSE Labs.
 * Thanks to Ben LaHaise for precious feedback.
 */
#include <linux/highmem.h>
#include <linux/memblock.h>
#include <linux/sched.h>
#include <linux/mm.h>
#include <linux/interrupt.h>
#include <linux/seq_file.h>
#include <linux/debugfs.h>
#include <linux/pfn.h>
#include <linux/percpu.h>
#include <linux/gfp.h>
#include <linux/pci.h>
#include <linux/vmalloc.h>
#include <linux/libnvdimm.h>
#include <linux/vmstat.h>
#include <linux/kernel.h>

#include <asm/e820/api.h>
#include <asm/processor.h>
#include <asm/tlbflush.h>
#include <asm/sections.h>
#include <asm/setup.h>
#include <linux/uaccess.h>
#include <asm/pgalloc.h>
#include <asm/proto.h>
#include <asm/memtype.h>
#include <asm/set_memory.h>

#include "../mm_internal.h"

/*
 * The current flushing context - we pass it instead of 5 arguments:
 */
struct cpa_data {
        unsigned long        *vaddr;
        pgd_t                *pgd;
        pgprot_t        mask_set;
        pgprot_t        mask_clr;
        unsigned long        numpages;
        unsigned long        curpage;
        unsigned long        pfn;
        unsigned int        flags;
        unsigned int        force_split                : 1,
                        force_static_prot        : 1,
                        force_flush_all                : 1;
        struct page        **pages;
};

enum cpa_warn {
        CPA_CONFLICT,
        CPA_PROTECT,
        CPA_DETECT,
};

static const int cpa_warn_level = CPA_PROTECT;

/*
 * Serialize cpa() (for !DEBUG_PAGEALLOC which uses large identity mappings)
 * using cpa_lock. So that we don't allow any other cpu, with stale large tlb
 * entries change the page attribute in parallel to some other cpu
 * splitting a large page entry along with changing the attribute.
 */
static DEFINE_SPINLOCK(cpa_lock);

#define CPA_FLUSHTLB 1
#define CPA_ARRAY 2
#define CPA_PAGES_ARRAY 4
#define CPA_NO_CHECK_ALIAS 8 /* Do not search for aliases */

static inline pgprot_t cachemode2pgprot(enum page_cache_mode pcm)
{
        return __pgprot(cachemode2protval(pcm));
}

#ifdef CONFIG_PROC_FS
static unsigned long direct_pages_count[PG_LEVEL_NUM];

void update_page_count(int level, unsigned long pages)
{
        /* Protect against CPA */
        spin_lock(&pgd_lock);
        direct_pages_count[level] += pages;
        spin_unlock(&pgd_lock);
}

static void split_page_count(int level)
{
        if (direct_pages_count[level] == 0)
                return;

        direct_pages_count[level]--;
        if (system_state == SYSTEM_RUNNING) {
                if (level == PG_LEVEL_2M)
                        count_vm_event(DIRECT_MAP_LEVEL2_SPLIT);
                else if (level == PG_LEVEL_1G)
                        count_vm_event(DIRECT_MAP_LEVEL3_SPLIT);
        }
        direct_pages_count[level - 1] += PTRS_PER_PTE;
}

void arch_report_meminfo(struct seq_file *m)
{
        seq_printf(m, "DirectMap4k:    %8lu kB\n",
                        direct_pages_count[PG_LEVEL_4K] << 2);
#if defined(CONFIG_X86_64) || defined(CONFIG_X86_PAE)
        seq_printf(m, "DirectMap2M:    %8lu kB\n",
                        direct_pages_count[PG_LEVEL_2M] << 11);
#else
        seq_printf(m, "DirectMap4M:    %8lu kB\n",
                        direct_pages_count[PG_LEVEL_2M] << 12);
#endif
        if (direct_gbpages)
                seq_printf(m, "DirectMap1G:    %8lu kB\n",
                        direct_pages_count[PG_LEVEL_1G] << 20);
}
#else
static inline void split_page_count(int level) { }
#endif

#ifdef CONFIG_X86_CPA_STATISTICS

static unsigned long cpa_1g_checked;
static unsigned long cpa_1g_sameprot;
static unsigned long cpa_1g_preserved;
static unsigned long cpa_2m_checked;
static unsigned long cpa_2m_sameprot;
static unsigned long cpa_2m_preserved;
static unsigned long cpa_4k_install;

static inline void cpa_inc_1g_checked(void)
{
        cpa_1g_checked++;
}

static inline void cpa_inc_2m_checked(void)
{
        cpa_2m_checked++;
}

static inline void cpa_inc_4k_install(void)
{
        data_race(cpa_4k_install++);
}

static inline void cpa_inc_lp_sameprot(int level)
{
        if (level == PG_LEVEL_1G)
                cpa_1g_sameprot++;
        else
                cpa_2m_sameprot++;
}

static inline void cpa_inc_lp_preserved(int level)
{
        if (level == PG_LEVEL_1G)
                cpa_1g_preserved++;
        else
                cpa_2m_preserved++;
}

static int cpastats_show(struct seq_file *m, void *p)
{
        seq_printf(m, "1G pages checked:     %16lu\n", cpa_1g_checked);
        seq_printf(m, "1G pages sameprot:    %16lu\n", cpa_1g_sameprot);
        seq_printf(m, "1G pages preserved:   %16lu\n", cpa_1g_preserved);
        seq_printf(m, "2M pages checked:     %16lu\n", cpa_2m_checked);
        seq_printf(m, "2M pages sameprot:    %16lu\n", cpa_2m_sameprot);
        seq_printf(m, "2M pages preserved:   %16lu\n", cpa_2m_preserved);
        seq_printf(m, "4K pages set-checked: %16lu\n", cpa_4k_install);
        return 0;
}

static int cpastats_open(struct inode *inode, struct file *file)
{
        return single_open(file, cpastats_show, NULL);
}

static const struct file_operations cpastats_fops = {
        .open                = cpastats_open,
        .read                = seq_read,
        .llseek                = seq_lseek,
        .release        = single_release,
};

static int __init cpa_stats_init(void)
{
        debugfs_create_file("cpa_stats", S_IRUSR, arch_debugfs_dir, NULL,
                            &cpastats_fops);
        return 0;
}
late_initcall(cpa_stats_init);
#else
static inline void cpa_inc_1g_checked(void) { }
static inline void cpa_inc_2m_checked(void) { }
static inline void cpa_inc_4k_install(void) { }
static inline void cpa_inc_lp_sameprot(int level) { }
static inline void cpa_inc_lp_preserved(int level) { }
#endif


static inline int
within(unsigned long addr, unsigned long start, unsigned long end)
{
        return addr >= start && addr < end;
}

static inline int
within_inclusive(unsigned long addr, unsigned long start, unsigned long end)
{
        return addr >= start && addr <= end;
}

#ifdef CONFIG_X86_64

static inline unsigned long highmap_start_pfn(void)
{
        return __pa_symbol(_text) >> PAGE_SHIFT;
}

static inline unsigned long highmap_end_pfn(void)
{
        /* Do not reference physical address outside the kernel. */
        return __pa_symbol(roundup(_brk_end, PMD_SIZE) - 1) >> PAGE_SHIFT;
}

static bool __cpa_pfn_in_highmap(unsigned long pfn)
{
        /*
         * Kernel text has an alias mapping at a high address, known
         * here as "highmap".
         */
        return within_inclusive(pfn, highmap_start_pfn(), highmap_end_pfn());
}

#else

static bool __cpa_pfn_in_highmap(unsigned long pfn)
{
        /* There is no highmap on 32-bit */
        return false;
}

#endif

/*
 * See set_mce_nospec().
 *
 * Machine check recovery code needs to change cache mode of poisoned pages to
 * UC to avoid speculative access logging another error. But passing the
 * address of the 1:1 mapping to set_memory_uc() is a fine way to encourage a
 * speculative access. So we cheat and flip the top bit of the address. This
 * works fine for the code that updates the page tables. But at the end of the
 * process we need to flush the TLB and cache and the non-canonical address
 * causes a #GP fault when used by the INVLPG and CLFLUSH instructions.
 *
 * But in the common case we already have a canonical address. This code
 * will fix the top bit if needed and is a no-op otherwise.
 */
static inline unsigned long fix_addr(unsigned long addr)
{
#ifdef CONFIG_X86_64
        return (long)(addr << 1) >> 1;
#else
        return addr;
#endif
}

static unsigned long __cpa_addr(struct cpa_data *cpa, unsigned long idx)
{
        if (cpa->flags & CPA_PAGES_ARRAY) {
                struct page *page = cpa->pages[idx];

                if (unlikely(PageHighMem(page)))
                        return 0;

                return (unsigned long)page_address(page);
        }

        if (cpa->flags & CPA_ARRAY)
                return cpa->vaddr[idx];

        return *cpa->vaddr + idx * PAGE_SIZE;
}

/*
 * Flushing functions
 */

static void clflush_cache_range_opt(void *vaddr, unsigned int size)
{
        const unsigned long clflush_size = boot_cpu_data.x86_clflush_size;
        void *p = (void *)((unsigned long)vaddr & ~(clflush_size - 1));
        void *vend = vaddr + size;

        if (p >= vend)
                return;

        for (; p < vend; p += clflush_size)
                clflushopt(p);
}

/**
 * clflush_cache_range - flush a cache range with clflush
 * @vaddr:        virtual start address
 * @size:        number of bytes to flush
 *
 * CLFLUSHOPT is an unordered instruction which needs fencing with MFENCE or
 * SFENCE to avoid ordering issues.
 */
void clflush_cache_range(void *vaddr, unsigned int size)
{
        mb();
        clflush_cache_range_opt(vaddr, size);
        mb();
}
EXPORT_SYMBOL_GPL(clflush_cache_range);

#ifdef CONFIG_ARCH_HAS_PMEM_API
void arch_invalidate_pmem(void *addr, size_t size)
{
        clflush_cache_range(addr, size);
}
EXPORT_SYMBOL_GPL(arch_invalidate_pmem);
#endif

static void __cpa_flush_all(void *arg)
{
        unsigned long cache = (unsigned long)arg;

        /*
         * Flush all to work around Errata in early athlons regarding
         * large page flushing.
         */
        __flush_tlb_all();

        if (cache && boot_cpu_data.x86 >= 4)
                wbinvd();
}

static void cpa_flush_all(unsigned long cache)
{
        BUG_ON(irqs_disabled() && !early_boot_irqs_disabled);

        on_each_cpu(__cpa_flush_all, (void *) cache, 1);
}

static void __cpa_flush_tlb(void *data)
{
        struct cpa_data *cpa = data;
        unsigned int i;

        for (i = 0; i < cpa->numpages; i++)
                flush_tlb_one_kernel(fix_addr(__cpa_addr(cpa, i)));
}

static void cpa_flush(struct cpa_data *data, int cache)
{
        struct cpa_data *cpa = data;
        unsigned int i;

        BUG_ON(irqs_disabled() && !early_boot_irqs_disabled);

        if (cache && !static_cpu_has(X86_FEATURE_CLFLUSH)) {
                cpa_flush_all(cache);
                return;
        }

        if (cpa->force_flush_all || cpa->numpages > tlb_single_page_flush_ceiling)
                flush_tlb_all();
        else
                on_each_cpu(__cpa_flush_tlb, cpa, 1);

        if (!cache)
                return;

        mb();
        for (i = 0; i < cpa->numpages; i++) {
                unsigned long addr = __cpa_addr(cpa, i);
                unsigned int level;

                pte_t *pte = lookup_address(addr, &level);

                /*
                 * Only flush present addresses:
                 */
                if (pte && (pte_val(*pte) & _PAGE_PRESENT))
                        clflush_cache_range_opt((void *)fix_addr(addr), PAGE_SIZE);
        }
        mb();
}

static bool overlaps(unsigned long r1_start, unsigned long r1_end,
                     unsigned long r2_start, unsigned long r2_end)
{
        return (r1_start <= r2_end && r1_end >= r2_start) ||
                (r2_start <= r1_end && r2_end >= r1_start);
}

#ifdef CONFIG_PCI_BIOS
/*
 * The BIOS area between 640k and 1Mb needs to be executable for PCI BIOS
 * based config access (CONFIG_PCI_GOBIOS) support.
 */
#define BIOS_PFN        PFN_DOWN(BIOS_BEGIN)
#define BIOS_PFN_END        PFN_DOWN(BIOS_END - 1)

static pgprotval_t protect_pci_bios(unsigned long spfn, unsigned long epfn)
{
        if (pcibios_enabled && overlaps(spfn, epfn, BIOS_PFN, BIOS_PFN_END))
                return _PAGE_NX;
        return 0;
}
#else
static pgprotval_t protect_pci_bios(unsigned long spfn, unsigned long epfn)
{
        return 0;
}
#endif

/*
 * The .rodata section needs to be read-only. Using the pfn catches all
 * aliases.  This also includes __ro_after_init, so do not enforce until
 * kernel_set_to_readonly is true.
 */
static pgprotval_t protect_rodata(unsigned long spfn, unsigned long epfn)
{
        unsigned long epfn_ro, spfn_ro = PFN_DOWN(__pa_symbol(__start_rodata));

        /*
         * Note: __end_rodata is at page aligned and not inclusive, so
         * subtract 1 to get the last enforced PFN in the rodata area.
         */
        epfn_ro = PFN_DOWN(__pa_symbol(__end_rodata)) - 1;

        if (kernel_set_to_readonly && overlaps(spfn, epfn, spfn_ro, epfn_ro))
                return _PAGE_RW;
        return 0;
}

/*
 * Protect kernel text against becoming non executable by forbidding
 * _PAGE_NX.  This protects only the high kernel mapping (_text -> _etext)
 * out of which the kernel actually executes.  Do not protect the low
 * mapping.
 *
 * This does not cover __inittext since that is gone after boot.
 */
static pgprotval_t protect_kernel_text(unsigned long start, unsigned long end)
{
        unsigned long t_end = (unsigned long)_etext - 1;
        unsigned long t_start = (unsigned long)_text;

        if (overlaps(start, end, t_start, t_end))
                return _PAGE_NX;
        return 0;
}

#if defined(CONFIG_X86_64)
/*
 * Once the kernel maps the text as RO (kernel_set_to_readonly is set),
 * kernel text mappings for the large page aligned text, rodata sections
 * will be always read-only. For the kernel identity mappings covering the
 * holes caused by this alignment can be anything that user asks.
 *
 * This will preserve the large page mappings for kernel text/data at no
 * extra cost.
 */
static pgprotval_t protect_kernel_text_ro(unsigned long start,
                                          unsigned long end)
{
        unsigned long t_end = (unsigned long)__end_rodata_hpage_align - 1;
        unsigned long t_start = (unsigned long)_text;
        unsigned int level;

        if (!kernel_set_to_readonly || !overlaps(start, end, t_start, t_end))
                return 0;
        /*
         * Don't enforce the !RW mapping for the kernel text mapping, if
         * the current mapping is already using small page mapping.  No
         * need to work hard to preserve large page mappings in this case.
         *
         * This also fixes the Linux Xen paravirt guest boot failure caused
         * by unexpected read-only mappings for kernel identity
         * mappings. In this paravirt guest case, the kernel text mapping
         * and the kernel identity mapping share the same page-table pages,
         * so the protections for kernel text and identity mappings have to
         * be the same.
         */
        if (lookup_address(start, &level) && (level != PG_LEVEL_4K))
                return _PAGE_RW;
        return 0;
}
#else
static pgprotval_t protect_kernel_text_ro(unsigned long start,
                                          unsigned long end)
{
        return 0;
}
#endif

static inline bool conflicts(pgprot_t prot, pgprotval_t val)
{
        return (pgprot_val(prot) & ~val) != pgprot_val(prot);
}

static inline void check_conflict(int warnlvl, pgprot_t prot, pgprotval_t val,
                                  unsigned long start, unsigned long end,
                                  unsigned long pfn, const char *txt)
{
        static const char *lvltxt[] = {
                [CPA_CONFLICT]        = "conflict",
                [CPA_PROTECT]        = "protect",
                [CPA_DETECT]        = "detect",
        };

        if (warnlvl > cpa_warn_level || !conflicts(prot, val))
                return;

        pr_warn("CPA %8s %10s: 0x%016lx - 0x%016lx PFN %lx req %016llx prevent %016llx\n",
                lvltxt[warnlvl], txt, start, end, pfn, (unsigned long long)pgprot_val(prot),
                (unsigned long long)val);
}

/*
 * Certain areas of memory on x86 require very specific protection flags,
 * for example the BIOS area or kernel text. Callers don't always get this
 * right (again, ioremap() on BIOS memory is not uncommon) so this function
 * checks and fixes these known static required protection bits.
 */
static inline pgprot_t static_protections(pgprot_t prot, unsigned long start,
                                          unsigned long pfn, unsigned long npg,
                                          unsigned long lpsize, int warnlvl)
{
        pgprotval_t forbidden, res;
        unsigned long end;

        /*
         * There is no point in checking RW/NX conflicts when the requested
         * mapping is setting the page !PRESENT.
         */
        if (!(pgprot_val(prot) & _PAGE_PRESENT))
                return prot;

        /* Operate on the virtual address */
        end = start + npg * PAGE_SIZE - 1;

        res = protect_kernel_text(start, end);
        check_conflict(warnlvl, prot, res, start, end, pfn, "Text NX");
        forbidden = res;

        /*
         * Special case to preserve a large page. If the change spawns the
         * full large page mapping then there is no point to split it
         * up. Happens with ftrace and is going to be removed once ftrace
         * switched to text_poke().
         */
        if (lpsize != (npg * PAGE_SIZE) || (start & (lpsize - 1))) {
                res = protect_kernel_text_ro(start, end);
                check_conflict(warnlvl, prot, res, start, end, pfn, "Text RO");
                forbidden |= res;
        }

        /* Check the PFN directly */
        res = protect_pci_bios(pfn, pfn + npg - 1);
        check_conflict(warnlvl, prot, res, start, end, pfn, "PCIBIOS NX");
        forbidden |= res;

        res = protect_rodata(pfn, pfn + npg - 1);
        check_conflict(warnlvl, prot, res, start, end, pfn, "Rodata RO");
        forbidden |= res;

        return __pgprot(pgprot_val(prot) & ~forbidden);
}

/*
 * Lookup the page table entry for a virtual address in a specific pgd.
 * Return a pointer to the entry and the level of the mapping.
 */
pte_t *lookup_address_in_pgd(pgd_t *pgd, unsigned long address,
                             unsigned int *level)
{
        p4d_t *p4d;
        pud_t *pud;
        pmd_t *pmd;

        *level = PG_LEVEL_NONE;

        if (pgd_none(*pgd))
                return NULL;

        p4d = p4d_offset(pgd, address);
        if (p4d_none(*p4d))
                return NULL;

        *level = PG_LEVEL_512G;
        if (p4d_large(*p4d) || !p4d_present(*p4d))
                return (pte_t *)p4d;

        pud = pud_offset(p4d, address);
        if (pud_none(*pud))
                return NULL;

        *level = PG_LEVEL_1G;
        if (pud_large(*pud) || !pud_present(*pud))
                return (pte_t *)pud;

        pmd = pmd_offset(pud, address);
        if (pmd_none(*pmd))
                return NULL;

        *level = PG_LEVEL_2M;
        if (pmd_large(*pmd) || !pmd_present(*pmd))
                return (pte_t *)pmd;

        *level = PG_LEVEL_4K;

        return pte_offset_kernel(pmd, address);
}

/*
 * Lookup the page table entry for a virtual address. Return a pointer
 * to the entry and the level of the mapping.
 *
 * Note: We return pud and pmd either when the entry is marked large
 * or when the present bit is not set. Otherwise we would return a
 * pointer to a nonexisting mapping.
 */
pte_t *lookup_address(unsigned long address, unsigned int *level)
{
        return lookup_address_in_pgd(pgd_offset_k(address), address, level);
}
EXPORT_SYMBOL_GPL(lookup_address);

/*
 * Lookup the page table entry for a virtual address in a given mm. Return a
 * pointer to the entry and the level of the mapping.
 */
pte_t *lookup_address_in_mm(struct mm_struct *mm, unsigned long address,
                            unsigned int *level)
{
        return lookup_address_in_pgd(pgd_offset(mm, address), address, level);
}
EXPORT_SYMBOL_GPL(lookup_address_in_mm);

static pte_t *_lookup_address_cpa(struct cpa_data *cpa, unsigned long address,
                                  unsigned int *level)
{
        if (cpa->pgd)
                return lookup_address_in_pgd(cpa->pgd + pgd_index(address),
                                               address, level);

        return lookup_address(address, level);
}

/*
 * Lookup the PMD entry for a virtual address. Return a pointer to the entry
 * or NULL if not present.
 */
pmd_t *lookup_pmd_address(unsigned long address)
{
        pgd_t *pgd;
        p4d_t *p4d;
        pud_t *pud;

        pgd = pgd_offset_k(address);
        if (pgd_none(*pgd))
                return NULL;

        p4d = p4d_offset(pgd, address);
        if (p4d_none(*p4d) || p4d_large(*p4d) || !p4d_present(*p4d))
                return NULL;

        pud = pud_offset(p4d, address);
        if (pud_none(*pud) || pud_large(*pud) || !pud_present(*pud))
                return NULL;

        return pmd_offset(pud, address);
}

/*
 * This is necessary because __pa() does not work on some
 * kinds of memory, like vmalloc() or the alloc_remap()
 * areas on 32-bit NUMA systems.  The percpu areas can
 * end up in this kind of memory, for instance.
 *
 * This could be optimized, but it is only intended to be
 * used at initialization time, and keeping it
 * unoptimized should increase the testing coverage for
 * the more obscure platforms.
 */
phys_addr_t slow_virt_to_phys(void *__virt_addr)
{
        unsigned long virt_addr = (unsigned long)__virt_addr;
        phys_addr_t phys_addr;
        unsigned long offset;
        enum pg_level level;
        pte_t *pte;

        pte = lookup_address(virt_addr, &level);
        BUG_ON(!pte);

        /*
         * pXX_pfn() returns unsigned long, which must be cast to phys_addr_t
         * before being left-shifted PAGE_SHIFT bits -- this trick is to
         * make 32-PAE kernel work correctly.
         */
        switch (level) {
        case PG_LEVEL_1G:
                phys_addr = (phys_addr_t)pud_pfn(*(pud_t *)pte) << PAGE_SHIFT;
                offset = virt_addr & ~PUD_PAGE_MASK;
                break;
        case PG_LEVEL_2M:
                phys_addr = (phys_addr_t)pmd_pfn(*(pmd_t *)pte) << PAGE_SHIFT;
                offset = virt_addr & ~PMD_PAGE_MASK;
                break;
        default:
                phys_addr = (phys_addr_t)pte_pfn(*pte) << PAGE_SHIFT;
                offset = virt_addr & ~PAGE_MASK;
        }

        return (phys_addr_t)(phys_addr | offset);
}
EXPORT_SYMBOL_GPL(slow_virt_to_phys);

/*
 * Set the new pmd in all the pgds we know about:
 */
static void __set_pmd_pte(pte_t *kpte, unsigned long address, pte_t pte)
{
        /* change init_mm */
        set_pte_atomic(kpte, pte);
#ifdef CONFIG_X86_32
        if (!SHARED_KERNEL_PMD) {
                struct page *page;

                list_for_each_entry(page, &pgd_list, lru) {
                        pgd_t *pgd;
                        p4d_t *p4d;
                        pud_t *pud;
                        pmd_t *pmd;

                        pgd = (pgd_t *)page_address(page) + pgd_index(address);
                        p4d = p4d_offset(pgd, address);
                        pud = pud_offset(p4d, address);
                        pmd = pmd_offset(pud, address);
                        set_pte_atomic((pte_t *)pmd, pte);
                }
        }
#endif
}

static pgprot_t pgprot_clear_protnone_bits(pgprot_t prot)
{
        /*
         * _PAGE_GLOBAL means "global page" for present PTEs.
         * But, it is also used to indicate _PAGE_PROTNONE
         * for non-present PTEs.
         *
         * This ensures that a _PAGE_GLOBAL PTE going from
         * present to non-present is not confused as
         * _PAGE_PROTNONE.
         */
        if (!(pgprot_val(prot) & _PAGE_PRESENT))
                pgprot_val(prot) &= ~_PAGE_GLOBAL;

        return prot;
}

static int __should_split_large_page(pte_t *kpte, unsigned long address,
                                     struct cpa_data *cpa)
{
        unsigned long numpages, pmask, psize, lpaddr, pfn, old_pfn;
        pgprot_t old_prot, new_prot, req_prot, chk_prot;
        pte_t new_pte, *tmp;
        enum pg_level level;

        /*
         * Check for races, another CPU might have split this page
         * up already:
         */
        tmp = _lookup_address_cpa(cpa, address, &level);
        if (tmp != kpte)
                return 1;

        switch (level) {
        case PG_LEVEL_2M:
                old_prot = pmd_pgprot(*(pmd_t *)kpte);
                old_pfn = pmd_pfn(*(pmd_t *)kpte);
                cpa_inc_2m_checked();
                break;
        case PG_LEVEL_1G:
                old_prot = pud_pgprot(*(pud_t *)kpte);
                old_pfn = pud_pfn(*(pud_t *)kpte);
                cpa_inc_1g_checked();
                break;
        default:
                return -EINVAL;
        }

        psize = page_level_size(level);
        pmask = page_level_mask(level);

        /*
         * Calculate the number of pages, which fit into this large
         * page starting at address:
         */
        lpaddr = (address + psize) & pmask;
        numpages = (lpaddr - address) >> PAGE_SHIFT;
        if (numpages < cpa->numpages)
                cpa->numpages = numpages;

        /*
         * We are safe now. Check whether the new pgprot is the same:
         * Convert protection attributes to 4k-format, as cpa->mask* are set
         * up accordingly.
         */

        /* Clear PSE (aka _PAGE_PAT) and move PAT bit to correct position */
        req_prot = pgprot_large_2_4k(old_prot);

        pgprot_val(req_prot) &= ~pgprot_val(cpa->mask_clr);
        pgprot_val(req_prot) |= pgprot_val(cpa->mask_set);

        /*
         * req_prot is in format of 4k pages. It must be converted to large
         * page format: the caching mode includes the PAT bit located at
         * different bit positions in the two formats.
         */
        req_prot = pgprot_4k_2_large(req_prot);
        req_prot = pgprot_clear_protnone_bits(req_prot);
        if (pgprot_val(req_prot) & _PAGE_PRESENT)
                pgprot_val(req_prot) |= _PAGE_PSE;

        /*
         * old_pfn points to the large page base pfn. So we need to add the
         * offset of the virtual address:
         */
        pfn = old_pfn + ((address & (psize - 1)) >> PAGE_SHIFT);
        cpa->pfn = pfn;

        /*
         * Calculate the large page base address and the number of 4K pages
         * in the large page
         */
        lpaddr = address & pmask;
        numpages = psize >> PAGE_SHIFT;

        /*
         * Sanity check that the existing mapping is correct versus the static
         * protections. static_protections() guards against !PRESENT, so no
         * extra conditional required here.
         */
        chk_prot = static_protections(old_prot, lpaddr, old_pfn, numpages,
                                      psize, CPA_CONFLICT);

        if (WARN_ON_ONCE(pgprot_val(chk_prot) != pgprot_val(old_prot))) {
                /*
                 * Split the large page and tell the split code to
                 * enforce static protections.
                 */
                cpa->force_static_prot = 1;
                return 1;
        }

        /*
         * Optimization: If the requested pgprot is the same as the current
         * pgprot, then the large page can be preserved and no updates are
         * required independent of alignment and length of the requested
         * range. The above already established that the current pgprot is
         * correct, which in consequence makes the requested pgprot correct
         * as well if it is the same. The static protection scan below will
         * not come to a different conclusion.
         */
        if (pgprot_val(req_prot) == pgprot_val(old_prot)) {
                cpa_inc_lp_sameprot(level);
                return 0;
        }

        /*
         * If the requested range does not cover the full page, split it up
         */
        if (address != lpaddr || cpa->numpages != numpages)
                return 1;

        /*
         * Check whether the requested pgprot is conflicting with a static
         * protection requirement in the large page.
         */
        new_prot = static_protections(req_prot, lpaddr, old_pfn, numpages,
                                      psize, CPA_DETECT);

        /*
         * If there is a conflict, split the large page.
         *
         * There used to be a 4k wise evaluation trying really hard to
         * preserve the large pages, but experimentation has shown, that this
         * does not help at all. There might be corner cases which would
         * preserve one large page occasionally, but it's really not worth the
         * extra code and cycles for the common case.
         */
        if (pgprot_val(req_prot) != pgprot_val(new_prot))
                return 1;

        /* All checks passed. Update the large page mapping. */
        new_pte = pfn_pte(old_pfn, new_prot);
        __set_pmd_pte(kpte, address, new_pte);
        cpa->flags |= CPA_FLUSHTLB;
        cpa_inc_lp_preserved(level);
        return 0;
}

static int should_split_large_page(pte_t *kpte, unsigned long address,
                                   struct cpa_data *cpa)
{
        int do_split;

        if (cpa->force_split)
                return 1;

        spin_lock(&pgd_lock);
        do_split = __should_split_large_page(kpte, address, cpa);
        spin_unlock(&pgd_lock);

        return do_split;
}

static void split_set_pte(struct cpa_data *cpa, pte_t *pte, unsigned long pfn,
                          pgprot_t ref_prot, unsigned long address,
                          unsigned long size)
{
        unsigned int npg = PFN_DOWN(size);
        pgprot_t prot;

        /*
         * If should_split_large_page() discovered an inconsistent mapping,
         * remove the invalid protection in the split mapping.
         */
        if (!cpa->force_static_prot)
                goto set;

        /* Hand in lpsize = 0 to enforce the protection mechanism */
        prot = static_protections(ref_prot, address, pfn, npg, 0, CPA_PROTECT);

        if (pgprot_val(prot) == pgprot_val(ref_prot))
                goto set;

        /*
         * If this is splitting a PMD, fix it up. PUD splits cannot be
         * fixed trivially as that would require to rescan the newly
         * installed PMD mappings after returning from split_large_page()
         * so an eventual further split can allocate the necessary PTE
         * pages. Warn for now and revisit it in case this actually
         * happens.
         */
        if (size == PAGE_SIZE)
                ref_prot = prot;
        else
                pr_warn_once("CPA: Cannot fixup static protections for PUD split\n");
set:
        set_pte(pte, pfn_pte(pfn, ref_prot));
}

static int
__split_large_page(struct cpa_data *cpa, pte_t *kpte, unsigned long address,
                   struct page *base)
{
        unsigned long lpaddr, lpinc, ref_pfn, pfn, pfninc = 1;
        pte_t *pbase = (pte_t *)page_address(base);
        unsigned int i, level;
        pgprot_t ref_prot;
        pte_t *tmp;

        spin_lock(&pgd_lock);
        /*
         * Check for races, another CPU might have split this page
         * up for us already:
         */
        tmp = _lookup_address_cpa(cpa, address, &level);
        if (tmp != kpte) {
                spin_unlock(&pgd_lock);
                return 1;
        }

        paravirt_alloc_pte(&init_mm, page_to_pfn(base));

        switch (level) {
        case PG_LEVEL_2M:
                ref_prot = pmd_pgprot(*(pmd_t *)kpte);
                /*
                 * Clear PSE (aka _PAGE_PAT) and move
                 * PAT bit to correct position.
                 */
                ref_prot = pgprot_large_2_4k(ref_prot);
                ref_pfn = pmd_pfn(*(pmd_t *)kpte);
                lpaddr = address & PMD_MASK;
                lpinc = PAGE_SIZE;
                break;

        case PG_LEVEL_1G:
                ref_prot = pud_pgprot(*(pud_t *)kpte);
                ref_pfn = pud_pfn(*(pud_t *)kpte);
                pfninc = PMD_PAGE_SIZE >> PAGE_SHIFT;
                lpaddr = address & PUD_MASK;
                lpinc = PMD_SIZE;
                /*
                 * Clear the PSE flags if the PRESENT flag is not set
                 * otherwise pmd_present/pmd_huge will return true
                 * even on a non present pmd.
                 */
                if (!(pgprot_val(ref_prot) & _PAGE_PRESENT))
                        pgprot_val(ref_prot) &= ~_PAGE_PSE;
                break;

        default:
                spin_unlock(&pgd_lock);
                return 1;
        }

        ref_prot = pgprot_clear_protnone_bits(ref_prot);

        /*
         * Get the target pfn from the original entry:
         */
        pfn = ref_pfn;
        for (i = 0; i < PTRS_PER_PTE; i++, pfn += pfninc, lpaddr += lpinc)
                split_set_pte(cpa, pbase + i, pfn, ref_prot, lpaddr, lpinc);

        if (virt_addr_valid(address)) {
                unsigned long pfn = PFN_DOWN(__pa(address));

                if (pfn_range_is_mapped(pfn, pfn + 1))
                        split_page_count(level);
        }

        /*
         * Install the new, split up pagetable.
         *
         * We use the standard kernel pagetable protections for the new
         * pagetable protections, the actual ptes set above control the
         * primary protection behavior:
         */
        __set_pmd_pte(kpte, address, mk_pte(base, __pgprot(_KERNPG_TABLE)));

        /*
         * Do a global flush tlb after splitting the large page
         * and before we do the actual change page attribute in the PTE.
         *
         * Without this, we violate the TLB application note, that says:
         * "The TLBs may contain both ordinary and large-page
         *  translations for a 4-KByte range of linear addresses. This
         *  may occur if software modifies the paging structures so that
         *  the page size used for the address range changes. If the two
         *  translations differ with respect to page frame or attributes
         *  (e.g., permissions), processor behavior is undefined and may
         *  be implementation-specific."
         *
         * We do this global tlb flush inside the cpa_lock, so that we
         * don't allow any other cpu, with stale tlb entries change the
         * page attribute in parallel, that also falls into the
         * just split large page entry.
         */
        flush_tlb_all();
        spin_unlock(&pgd_lock);

        return 0;
}

static int split_large_page(struct cpa_data *cpa, pte_t *kpte,
                            unsigned long address)
{
        struct page *base;

        if (!debug_pagealloc_enabled())
                spin_unlock(&cpa_lock);
        base = alloc_pages(GFP_KERNEL, 0);
        if (!debug_pagealloc_enabled())
                spin_lock(&cpa_lock);
        if (!base)
                return -ENOMEM;

        if (__split_large_page(cpa, kpte, address, base))
                __free_page(base);

        return 0;
}

static bool try_to_free_pte_page(pte_t *pte)
{
        int i;

        for (i = 0; i < PTRS_PER_PTE; i++)
                if (!pte_none(pte[i]))
                        return false;

        free_page((unsigned long)pte);
        return true;
}

static bool try_to_free_pmd_page(pmd_t *pmd)
{
        int i;

        for (i = 0; i < PTRS_PER_PMD; i++)
                if (!pmd_none(pmd[i]))
                        return false;

        free_page((unsigned long)pmd);
        return true;
}

static bool unmap_pte_range(pmd_t *pmd, unsigned long start, unsigned long end)
{
        pte_t *pte = pte_offset_kernel(pmd, start);

        while (start < end) {
                set_pte(pte, __pte(0));

                start += PAGE_SIZE;
                pte++;
        }

        if (try_to_free_pte_page((pte_t *)pmd_page_vaddr(*pmd))) {
                pmd_clear(pmd);
                return true;
        }
        return false;
}

static void __unmap_pmd_range(pud_t *pud, pmd_t *pmd,
                              unsigned long start, unsigned long end)
{
        if (unmap_pte_range(pmd, start, end))
                if (try_to_free_pmd_page(pud_pgtable(*pud)))
                        pud_clear(pud);
}

static void unmap_pmd_range(pud_t *pud, unsigned long start, unsigned long end)
{
        pmd_t *pmd = pmd_offset(pud, start);

        /*
         * Not on a 2MB page boundary?
         */
        if (start & (PMD_SIZE - 1)) {
                unsigned long next_page = (start + PMD_SIZE) & PMD_MASK;
                unsigned long pre_end = min_t(unsigned long, end, next_page);

                __unmap_pmd_range(pud, pmd, start, pre_end);

                start = pre_end;
                pmd++;
        }

        /*
         * Try to unmap in 2M chunks.
         */
        while (end - start >= PMD_SIZE) {
                if (pmd_large(*pmd))
                        pmd_clear(pmd);
                else
                        __unmap_pmd_range(pud, pmd, start, start + PMD_SIZE);

                start += PMD_SIZE;
                pmd++;
        }

        /*
         * 4K leftovers?
         */
        if (start < end)
                return __unmap_pmd_range(pud, pmd, start, end);

        /*
         * Try again to free the PMD page if haven't succeeded above.
         */
        if (!pud_none(*pud))
                if (try_to_free_pmd_page(pud_pgtable(*pud)))
                        pud_clear(pud);
}

static void unmap_pud_range(p4d_t *p4d, unsigned long start, unsigned long end)
{
        pud_t *pud = pud_offset(p4d, start);

        /*
         * Not on a GB page boundary?
         */
        if (start & (PUD_SIZE - 1)) {
                unsigned long next_page = (start + PUD_SIZE) & PUD_MASK;
                unsigned long pre_end        = min_t(unsigned long, end, next_page);

                unmap_pmd_range(pud, start, pre_end);

                start = pre_end;
                pud++;
        }

        /*
         * Try to unmap in 1G chunks?
         */
        while (end - start >= PUD_SIZE) {

                if (pud_large(*pud))
                        pud_clear(pud);
                else
                        unmap_pmd_range(pud, start, start + PUD_SIZE);

                start += PUD_SIZE;
                pud++;
        }

        /*
         * 2M leftovers?
         */
        if (start < end)
                unmap_pmd_range(pud, start, end);

        /*
         * No need to try to free the PUD page because we'll free it in
         * populate_pgd's error path
         */
}

static int alloc_pte_page(pmd_t *pmd)
{
        pte_t *pte = (pte_t *)get_zeroed_page(GFP_KERNEL);
        if (!pte)
                return -1;

        set_pmd(pmd, __pmd(__pa(pte) | _KERNPG_TABLE));
        return 0;
}

static int alloc_pmd_page(pud_t *pud)
{
        pmd_t *pmd = (pmd_t *)get_zeroed_page(GFP_KERNEL);
        if (!pmd)
                return -1;

        set_pud(pud, __pud(__pa(pmd) | _KERNPG_TABLE));
        return 0;
}

static void populate_pte(struct cpa_data *cpa,
                         unsigned long start, unsigned long end,
                         unsigned num_pages, pmd_t *pmd, pgprot_t pgprot)
{
        pte_t *pte;

        pte = pte_offset_kernel(pmd, start);

        pgprot = pgprot_clear_protnone_bits(pgprot);

        while (num_pages-- && start < end) {
                set_pte(pte, pfn_pte(cpa->pfn, pgprot));

                start         += PAGE_SIZE;
                cpa->pfn++;
                pte++;
        }
}

static long populate_pmd(struct cpa_data *cpa,
                         unsigned long start, unsigned long end,
                         unsigned num_pages, pud_t *pud, pgprot_t pgprot)
{
        long cur_pages = 0;
        pmd_t *pmd;
        pgprot_t pmd_pgprot;

        /*
         * Not on a 2M boundary?
         */
        if (start & (PMD_SIZE - 1)) {
                unsigned long pre_end = start + (num_pages << PAGE_SHIFT);
                unsigned long next_page = (start + PMD_SIZE) & PMD_MASK;

                pre_end   = min_t(unsigned long, pre_end, next_page);
                cur_pages = (pre_end - start) >> PAGE_SHIFT;
                cur_pages = min_t(unsigned int, num_pages, cur_pages);

                /*
                 * Need a PTE page?
                 */
                pmd = pmd_offset(pud, start);
                if (pmd_none(*pmd))
                        if (alloc_pte_page(pmd))
                                return -1;

                populate_pte(cpa, start, pre_end, cur_pages, pmd, pgprot);

                start = pre_end;
        }

        /*
         * We mapped them all?
         */
        if (num_pages == cur_pages)
                return cur_pages;

        pmd_pgprot = pgprot_4k_2_large(pgprot);

        while (end - start >= PMD_SIZE) {

                /*
                 * We cannot use a 1G page so allocate a PMD page if needed.
                 */
                if (pud_none(*pud))
                        if (alloc_pmd_page(pud))
                                return -1;

                pmd = pmd_offset(pud, start);

                set_pmd(pmd, pmd_mkhuge(pfn_pmd(cpa->pfn,
                                        canon_pgprot(pmd_pgprot))));

                start          += PMD_SIZE;
                cpa->pfn  += PMD_SIZE >> PAGE_SHIFT;
                cur_pages += PMD_SIZE >> PAGE_SHIFT;
        }

        /*
         * Map trailing 4K pages.
         */
        if (start < end) {
                pmd = pmd_offset(pud, start);
                if (pmd_none(*pmd))
                        if (alloc_pte_page(pmd))
                                return -1;

                populate_pte(cpa, start, end, num_pages - cur_pages,
                             pmd, pgprot);
        }
        return num_pages;
}

static int populate_pud(struct cpa_data *cpa, unsigned long start, p4d_t *p4d,
                        pgprot_t pgprot)
{
        pud_t *pud;
        unsigned long end;
        long cur_pages = 0;
        pgprot_t pud_pgprot;

        end = start + (cpa->numpages << PAGE_SHIFT);

        /*
         * Not on a Gb page boundary? => map everything up to it with
         * smaller pages.
         */
        if (start & (PUD_SIZE - 1)) {
                unsigned long pre_end;
                unsigned long next_page = (start + PUD_SIZE) & PUD_MASK;

                pre_end   = min_t(unsigned long, end, next_page);
                cur_pages = (pre_end - start) >> PAGE_SHIFT;
                cur_pages = min_t(int, (int)cpa->numpages, cur_pages);

                pud = pud_offset(p4d, start);

                /*
                 * Need a PMD page?
                 */
                if (pud_none(*pud))
                        if (alloc_pmd_page(pud))
                                return -1;

                cur_pages = populate_pmd(cpa, start, pre_end, cur_pages,
                                         pud, pgprot);
                if (cur_pages < 0)
                        return cur_pages;

                start = pre_end;
        }

        /* We mapped them all? */
        if (cpa->numpages == cur_pages)
                return cur_pages;

        pud = pud_offset(p4d, start);
        pud_pgprot = pgprot_4k_2_large(pgprot);

        /*
         * Map everything starting from the Gb boundary, possibly with 1G pages
         */
        while (boot_cpu_has(X86_FEATURE_GBPAGES) && end - start >= PUD_SIZE) {
                set_pud(pud, pud_mkhuge(pfn_pud(cpa->pfn,
                                   canon_pgprot(pud_pgprot))));

                start          += PUD_SIZE;
                cpa->pfn  += PUD_SIZE >> PAGE_SHIFT;
                cur_pages += PUD_SIZE >> PAGE_SHIFT;
                pud++;
        }

        /* Map trailing leftover */
        if (start < end) {
                long tmp;

                pud = pud_offset(p4d, start);
                if (pud_none(*pud))
                        if (alloc_pmd_page(pud))
                                return -1;

                tmp = populate_pmd(cpa, start, end, cpa->numpages - cur_pages,
                                   pud, pgprot);
                if (tmp < 0)
                        return cur_pages;

                cur_pages += tmp;
        }
        return cur_pages;
}

/*
 * Restrictions for kernel page table do not necessarily apply when mapping in
 * an alternate PGD.
 */
static int populate_pgd(struct cpa_data *cpa, unsigned long addr)
{
        pgprot_t pgprot = __pgprot(_KERNPG_TABLE);
        pud_t *pud = NULL;        /* shut up gcc */
        p4d_t *p4d;
        pgd_t *pgd_entry;
        long ret;

        pgd_entry = cpa->pgd + pgd_index(addr);

        if (pgd_none(*pgd_entry)) {
                p4d = (p4d_t *)get_zeroed_page(GFP_KERNEL);
                if (!p4d)
                        return -1;

                set_pgd(pgd_entry, __pgd(__pa(p4d) | _KERNPG_TABLE));
        }

        /*
         * Allocate a PUD page and hand it down for mapping.
         */
        p4d = p4d_offset(pgd_entry, addr);
        if (p4d_none(*p4d)) {
                pud = (pud_t *)get_zeroed_page(GFP_KERNEL);
                if (!pud)
                        return -1;

                set_p4d(p4d, __p4d(__pa(pud) | _KERNPG_TABLE));
        }

        pgprot_val(pgprot) &= ~pgprot_val(cpa->mask_clr);
        pgprot_val(pgprot) |=  pgprot_val(cpa->mask_set);

        ret = populate_pud(cpa, addr, p4d, pgprot);
        if (ret < 0) {
                /*
                 * Leave the PUD page in place in case some other CPU or thread
                 * already found it, but remove any useless entries we just
                 * added to it.
                 */
                unmap_pud_range(p4d, addr,
                                addr + (cpa->numpages << PAGE_SHIFT));
                return ret;
        }

        cpa->numpages = ret;
        return 0;
}

static int __cpa_process_fault(struct cpa_data *cpa, unsigned long vaddr,
                               int primary)
{
        if (cpa->pgd) {
                /*
                 * Right now, we only execute this code path when mapping
                 * the EFI virtual memory map regions, no other users
                 * provide a ->pgd value. This may change in the future.
                 */
                return populate_pgd(cpa, vaddr);
        }

        /*
         * Ignore all non primary paths.
         */
        if (!primary) {
                cpa->numpages = 1;
                return 0;
        }

        /*
         * Ignore the NULL PTE for kernel identity mapping, as it is expected
         * to have holes.
         * Also set numpages to '1' indicating that we processed cpa req for
         * one virtual address page and its pfn. TBD: numpages can be set based
         * on the initial value and the level returned by lookup_address().
         */
        if (within(vaddr, PAGE_OFFSET,
                   PAGE_OFFSET + (max_pfn_mapped << PAGE_SHIFT))) {
                cpa->numpages = 1;
                cpa->pfn = __pa(vaddr) >> PAGE_SHIFT;
                return 0;

        } else if (__cpa_pfn_in_highmap(cpa->pfn)) {
                /* Faults in the highmap are OK, so do not warn: */
                return -EFAULT;
        } else {
                WARN(1, KERN_WARNING "CPA: called for zero pte. "
                        "vaddr = %lx cpa->vaddr = %lx\n", vaddr,
                        *cpa->vaddr);

                return -EFAULT;
        }
}

static int __change_page_attr(struct cpa_data *cpa, int primary)
{
        unsigned long address;
        int do_split, err;
        unsigned int level;
        pte_t *kpte, old_pte;

        address = __cpa_addr(cpa, cpa->curpage);
repeat:
        kpte = _lookup_address_cpa(cpa, address, &level);
        if (!kpte)
                return __cpa_process_fault(cpa, address, primary);

        old_pte = *kpte;
        if (pte_none(old_pte))
                return __cpa_process_fault(cpa, address, primary);

        if (level == PG_LEVEL_4K) {
                pte_t new_pte;
                pgprot_t new_prot = pte_pgprot(old_pte);
                unsigned long pfn = pte_pfn(old_pte);

                pgprot_val(new_prot) &= ~pgprot_val(cpa->mask_clr);
                pgprot_val(new_prot) |= pgprot_val(cpa->mask_set);

                cpa_inc_4k_install();
                /* Hand in lpsize = 0 to enforce the protection mechanism */
                new_prot = static_protections(new_prot, address, pfn, 1, 0,
                                              CPA_PROTECT);

                new_prot = pgprot_clear_protnone_bits(new_prot);

                /*
                 * We need to keep the pfn from the existing PTE,
                 * after all we're only going to change it's attributes
                 * not the memory it points to
                 */
                new_pte = pfn_pte(pfn, new_prot);
                cpa->pfn = pfn;
                /*
                 * Do we really change anything ?
                 */
                if (pte_val(old_pte) != pte_val(new_pte)) {
                        set_pte_atomic(kpte, new_pte);
                        cpa->flags |= CPA_FLUSHTLB;
                }
                cpa->numpages = 1;
                return 0;
        }

        /*
         * Check, whether we can keep the large page intact
         * and just change the pte:
         */
        do_split = should_split_large_page(kpte, address, cpa);
        /*
         * When the range fits into the existing large page,
         * return. cp->numpages and cpa->tlbflush have been updated in
         * try_large_page:
         */
        if (do_split <= 0)
                return do_split;

        /*
         * We have to split the large page:
         */
        err = split_large_page(cpa, kpte, address);
        if (!err)
                goto repeat;

        return err;
}

static int __change_page_attr_set_clr(struct cpa_data *cpa, int checkalias);

static int cpa_process_alias(struct cpa_data *cpa)
{
        struct cpa_data alias_cpa;
        unsigned long laddr = (unsigned long)__va(cpa->pfn << PAGE_SHIFT);
        unsigned long vaddr;
        int ret;

        if (!pfn_range_is_mapped(cpa->pfn, cpa->pfn + 1))
                return 0;

        /*
         * No need to redo, when the primary call touched the direct
         * mapping already:
         */
        vaddr = __cpa_addr(cpa, cpa->curpage);
        if (!(within(vaddr, PAGE_OFFSET,
                    PAGE_OFFSET + (max_pfn_mapped << PAGE_SHIFT)))) {

                alias_cpa = *cpa;
                alias_cpa.vaddr = &laddr;
                alias_cpa.flags &= ~(CPA_PAGES_ARRAY | CPA_ARRAY);
                alias_cpa.curpage = 0;

                cpa->force_flush_all = 1;

                ret = __change_page_attr_set_clr(&alias_cpa, 0);
                if (ret)
                        return ret;
        }

#ifdef CONFIG_X86_64
        /*
         * If the primary call didn't touch the high mapping already
         * and the physical address is inside the kernel map, we need
         * to touch the high mapped kernel as well:
         */
        if (!within(vaddr, (unsigned long)_text, _brk_end) &&
            __cpa_pfn_in_highmap(cpa->pfn)) {
                unsigned long temp_cpa_vaddr = (cpa->pfn << PAGE_SHIFT) +
                                               __START_KERNEL_map - phys_base;
                alias_cpa = *cpa;
                alias_cpa.vaddr = &temp_cpa_vaddr;
                alias_cpa.flags &= ~(CPA_PAGES_ARRAY | CPA_ARRAY);
                alias_cpa.curpage = 0;

                cpa->force_flush_all = 1;
                /*
                 * The high mapping range is imprecise, so ignore the
                 * return value.
                 */
                __change_page_attr_set_clr(&alias_cpa, 0);
        }
#endif

        return 0;
}

static int __change_page_attr_set_clr(struct cpa_data *cpa, int checkalias)
{
        unsigned long numpages = cpa->numpages;
        unsigned long rempages = numpages;
        int ret = 0;

        while (rempages) {
                /*
                 * Store the remaining nr of pages for the large page
                 * preservation check.
                 */
                cpa->numpages = rempages;
                /* for array changes, we can't use large page */
                if (cpa->flags & (CPA_ARRAY | CPA_PAGES_ARRAY))
                        cpa->numpages = 1;

                if (!debug_pagealloc_enabled())
                        spin_lock(&cpa_lock);
                ret = __change_page_attr(cpa, checkalias);
                if (!debug_pagealloc_enabled())
                        spin_unlock(&cpa_lock);
                if (ret)
                        goto out;

                if (checkalias) {
                        ret = cpa_process_alias(cpa);
                        if (ret)
                                goto out;
                }

                /*
                 * Adjust the number of pages with the result of the
                 * CPA operation. Either a large page has been
                 * preserved or a single page update happened.
                 */
                BUG_ON(cpa->numpages > rempages || !cpa->numpages);
                rempages -= cpa->numpages;
                cpa->curpage += cpa->numpages;
        }

out:
        /* Restore the original numpages */
        cpa->numpages = numpages;
        return ret;
}

static int change_page_attr_set_clr(unsigned long *addr, int numpages,
                                    pgprot_t mask_set, pgprot_t mask_clr,
                                    int force_split, int in_flag,
                                    struct page **pages)
{
        struct cpa_data cpa;
        int ret, cache, checkalias;

        memset(&cpa, 0, sizeof(cpa));

        /*
         * Check, if we are requested to set a not supported
         * feature.  Clearing non-supported features is OK.
         */
        mask_set = canon_pgprot(mask_set);

        if (!pgprot_val(mask_set) && !pgprot_val(mask_clr) && !force_split)
                return 0;

        /* Ensure we are PAGE_SIZE aligned */
        if (in_flag & CPA_ARRAY) {
                int i;
                for (i = 0; i < numpages; i++) {
                        if (addr[i] & ~PAGE_MASK) {
                                addr[i] &= PAGE_MASK;
                                WARN_ON_ONCE(1);
                        }
                }
        } else if (!(in_flag & CPA_PAGES_ARRAY)) {
                /*
                 * in_flag of CPA_PAGES_ARRAY implies it is aligned.
                 * No need to check in that case
                 */
                if (*addr & ~PAGE_MASK) {
                        *addr &= PAGE_MASK;
                        /*
                         * People should not be passing in unaligned addresses:
                         */
                        WARN_ON_ONCE(1);
                }
        }

        /* Must avoid aliasing mappings in the highmem code */
        kmap_flush_unused();

        vm_unmap_aliases();

        cpa.vaddr = addr;
        cpa.pages = pages;
        cpa.numpages = numpages;
        cpa.mask_set = mask_set;
        cpa.mask_clr = mask_clr;
        cpa.flags = 0;
        cpa.curpage = 0;
        cpa.force_split = force_split;

        if (in_flag & (CPA_ARRAY | CPA_PAGES_ARRAY))
                cpa.flags |= in_flag;

        /* No alias checking for _NX bit modifications */
        checkalias = (pgprot_val(mask_set) | pgprot_val(mask_clr)) != _PAGE_NX;
        /* Has caller explicitly disabled alias checking? */
        if (in_flag & CPA_NO_CHECK_ALIAS)
                checkalias = 0;

        ret = __change_page_attr_set_clr(&cpa, checkalias);

        /*
         * Check whether we really changed something:
         */
        if (!(cpa.flags & CPA_FLUSHTLB))
                goto out;

        /*
         * No need to flush, when we did not set any of the caching
         * attributes:
         */
        cache = !!pgprot2cachemode(mask_set);

        /*
         * On error; flush everything to be sure.
         */
        if (ret) {
                cpa_flush_all(cache);
                goto out;
        }

        cpa_flush(&cpa, cache);
out:
        return ret;
}

static inline int change_page_attr_set(unsigned long *addr, int numpages,
                                       pgprot_t mask, int array)
{
        return change_page_attr_set_clr(addr, numpages, mask, __pgprot(0), 0,
                (array ? CPA_ARRAY : 0), NULL);
}

static inline int change_page_attr_clear(unsigned long *addr, int numpages,
                                         pgprot_t mask, int array)
{
        return change_page_attr_set_clr(addr, numpages, __pgprot(0), mask, 0,
                (array ? CPA_ARRAY : 0), NULL);
}

static inline int cpa_set_pages_array(struct page **pages, int numpages,
                                       pgprot_t mask)
{
        return change_page_attr_set_clr(NULL, numpages, mask, __pgprot(0), 0,
                CPA_PAGES_ARRAY, pages);
}

static inline int cpa_clear_pages_array(struct page **pages, int numpages,
                                         pgprot_t mask)
{
        return change_page_attr_set_clr(NULL, numpages, __pgprot(0), mask, 0,
                CPA_PAGES_ARRAY, pages);
}

/*
 * _set_memory_prot is an internal helper for callers that have been passed
 * a pgprot_t value from upper layers and a reservation has already been taken.
 * If you want to set the pgprot to a specific page protocol, use the
 * set_memory_xx() functions.
 */
int __set_memory_prot(unsigned long addr, int numpages, pgprot_t prot)
{
        return change_page_attr_set_clr(&addr, numpages, prot,
                                        __pgprot(~pgprot_val(prot)), 0, 0,
                                        NULL);
}

int _set_memory_uc(unsigned long addr, int numpages)
{
        /*
         * for now UC MINUS. see comments in ioremap()
         * If you really need strong UC use ioremap_uc(), but note
         * that you cannot override IO areas with set_memory_*() as
         * these helpers cannot work with IO memory.
         */
        return change_page_attr_set(&addr, numpages,
                                    cachemode2pgprot(_PAGE_CACHE_MODE_UC_MINUS),
                                    0);
}

int set_memory_uc(unsigned long addr, int numpages)
{
        int ret;

        /*
         * for now UC MINUS. see comments in ioremap()
         */
        ret = memtype_reserve(__pa(addr), __pa(addr) + numpages * PAGE_SIZE,
                              _PAGE_CACHE_MODE_UC_MINUS, NULL);
        if (ret)
                goto out_err;

        ret = _set_memory_uc(addr, numpages);
        if (ret)
                goto out_free;

        return 0;

out_free:
        memtype_free(__pa(addr), __pa(addr) + numpages * PAGE_SIZE);
out_err:
        return ret;
}
EXPORT_SYMBOL(set_memory_uc);

int _set_memory_wc(unsigned long addr, int numpages)
{
        int ret;

        ret = change_page_attr_set(&addr, numpages,
                                   cachemode2pgprot(_PAGE_CACHE_MODE_UC_MINUS),
                                   0);
        if (!ret) {
                ret = change_page_attr_set_clr(&addr, numpages,
                                               cachemode2pgprot(_PAGE_CACHE_MODE_WC),
                                               __pgprot(_PAGE_CACHE_MASK),
                                               0, 0, NULL);
        }
        return ret;
}

int set_memory_wc(unsigned long addr, int numpages)
{
        int ret;

        ret = memtype_reserve(__pa(addr), __pa(addr) + numpages * PAGE_SIZE,
                _PAGE_CACHE_MODE_WC, NULL);
        if (ret)
                return ret;

        ret = _set_memory_wc(addr, numpages);
        if (ret)
                memtype_free(__pa(addr), __pa(addr) + numpages * PAGE_SIZE);

        return ret;
}
EXPORT_SYMBOL(set_memory_wc);

int _set_memory_wt(unsigned long addr, int numpages)
{
        return change_page_attr_set(&addr, numpages,
                                    cachemode2pgprot(_PAGE_CACHE_MODE_WT), 0);
}

int _set_memory_wb(unsigned long addr, int numpages)
{
        /* WB cache mode is hard wired to all cache attribute bits being 0 */
        return change_page_attr_clear(&addr, numpages,
                                      __pgprot(_PAGE_CACHE_MASK), 0);
}

int set_memory_wb(unsigned long addr, int numpages)
{
        int ret;

        ret = _set_memory_wb(addr, numpages);
        if (ret)
                return ret;

        memtype_free(__pa(addr), __pa(addr) + numpages * PAGE_SIZE);
        return 0;
}
EXPORT_SYMBOL(set_memory_wb);

int set_memory_x(unsigned long addr, int numpages)
{
        if (!(__supported_pte_mask & _PAGE_NX))
                return 0;

        return change_page_attr_clear(&addr, numpages, __pgprot(_PAGE_NX), 0);
}

int set_memory_nx(unsigned long addr, int numpages)
{
        if (!(__supported_pte_mask & _PAGE_NX))
                return 0;

        return change_page_attr_set(&addr, numpages, __pgprot(_PAGE_NX), 0);
}

int set_memory_ro(unsigned long addr, int numpages)
{
        return change_page_attr_clear(&addr, numpages, __pgprot(_PAGE_RW), 0);
}

int set_memory_rw(unsigned long addr, int numpages)
{
        return change_page_attr_set(&addr, numpages, __pgprot(_PAGE_RW), 0);
}

int set_memory_np(unsigned long addr, int numpages)
{
        return change_page_attr_clear(&addr, numpages, __pgprot(_PAGE_PRESENT), 0);
}

int set_memory_np_noalias(unsigned long addr, int numpages)
{
        int cpa_flags = CPA_NO_CHECK_ALIAS;

        return change_page_attr_set_clr(&addr, numpages, __pgprot(0),
                                        __pgprot(_PAGE_PRESENT), 0,
                                        cpa_flags, NULL);
}

int set_memory_4k(unsigned long addr, int numpages)
{
        return change_page_attr_set_clr(&addr, numpages, __pgprot(0),
                                        __pgprot(0), 1, 0, NULL);
}

int set_memory_nonglobal(unsigned long addr, int numpages)
{
        return change_page_attr_clear(&addr, numpages,
                                      __pgprot(_PAGE_GLOBAL), 0);
}

int set_memory_global(unsigned long addr, int numpages)
{
        return change_page_attr_set(&addr, numpages,
                                    __pgprot(_PAGE_GLOBAL), 0);
}

static int __set_memory_enc_dec(unsigned long addr, int numpages, bool enc)
{
        struct cpa_data cpa;
        int ret;

        /* Nothing to do if memory encryption is not active */
        if (!mem_encrypt_active())
                return 0;

        /* Should not be working on unaligned addresses */
        if (WARN_ONCE(addr & ~PAGE_MASK, "misaligned address: %#lx\n", addr))
                addr &= PAGE_MASK;

        memset(&cpa, 0, sizeof(cpa));
        cpa.vaddr = &addr;
        cpa.numpages = numpages;
        cpa.mask_set = enc ? __pgprot(_PAGE_ENC) : __pgprot(0);
        cpa.mask_clr = enc ? __pgprot(0) : __pgprot(_PAGE_ENC);
        cpa.pgd = init_mm.pgd;

        /* Must avoid aliasing mappings in the highmem code */
        kmap_flush_unused();
        vm_unmap_aliases();

        /*
         * Before changing the encryption attribute, we need to flush caches.
         */
        cpa_flush(&cpa, !this_cpu_has(X86_FEATURE_SME_COHERENT));

        ret = __change_page_attr_set_clr(&cpa, 1);

        /*
         * After changing the encryption attribute, we need to flush TLBs again
         * in case any speculative TLB caching occurred (but no need to flush
         * caches again).  We could just use cpa_flush_all(), but in case TLB
         * flushing gets optimized in the cpa_flush() path use the same logic
         * as above.
         */
        cpa_flush(&cpa, 0);

        return ret;
}

int set_memory_encrypted(unsigned long addr, int numpages)
{
        return __set_memory_enc_dec(addr, numpages, true);
}
EXPORT_SYMBOL_GPL(set_memory_encrypted);

int set_memory_decrypted(unsigned long addr, int numpages)
{
        return __set_memory_enc_dec(addr, numpages, false);
}
EXPORT_SYMBOL_GPL(set_memory_decrypted);

int set_pages_uc(struct page *page, int numpages)
{
        unsigned long addr = (unsigned long)page_address(page);

        return set_memory_uc(addr, numpages);
}
EXPORT_SYMBOL(set_pages_uc);

static int _set_pages_array(struct page **pages, int numpages,
                enum page_cache_mode new_type)
{
        unsigned long start;
        unsigned long end;
        enum page_cache_mode set_type;
        int i;
        int free_idx;
        int ret;

        for (i = 0; i < numpages; i++) {
                if (PageHighMem(pages[i]))
                        continue;
                start = page_to_pfn(pages[i]) << PAGE_SHIFT;
                end = start + PAGE_SIZE;
                if (memtype_reserve(start, end, new_type, NULL))
                        goto err_out;
        }

        /* If WC, set to UC- first and then WC */
        set_type = (new_type == _PAGE_CACHE_MODE_WC) ?
                                _PAGE_CACHE_MODE_UC_MINUS : new_type;

        ret = cpa_set_pages_array(pages, numpages,
                                  cachemode2pgprot(set_type));
        if (!ret && new_type == _PAGE_CACHE_MODE_WC)
                ret = change_page_attr_set_clr(NULL, numpages,
                                               cachemode2pgprot(
                                                _PAGE_CACHE_MODE_WC),
                                               __pgprot(_PAGE_CACHE_MASK),
                                               0, CPA_PAGES_ARRAY, pages);
        if (ret)
                goto err_out;
        return 0; /* Success */
err_out:
        free_idx = i;
        for (i = 0; i < free_idx; i++) {
                if (PageHighMem(pages[i]))
                        continue;
                start = page_to_pfn(pages[i]) << PAGE_SHIFT;
                end = start + PAGE_SIZE;
                memtype_free(start, end);
        }
        return -EINVAL;
}

int set_pages_array_uc(struct page **pages, int numpages)
{
        return _set_pages_array(pages, numpages, _PAGE_CACHE_MODE_UC_MINUS);
}
EXPORT_SYMBOL(set_pages_array_uc);

int set_pages_array_wc(struct page **pages, int numpages)
{
        return _set_pages_array(pages, numpages, _PAGE_CACHE_MODE_WC);
}
EXPORT_SYMBOL(set_pages_array_wc);

int set_pages_array_wt(struct page **pages, int numpages)
{
        return _set_pages_array(pages, numpages, _PAGE_CACHE_MODE_WT);
}
EXPORT_SYMBOL_GPL(set_pages_array_wt);

int set_pages_wb(struct page *page, int numpages)
{
        unsigned long addr = (unsigned long)page_address(page);

        return set_memory_wb(addr, numpages);
}
EXPORT_SYMBOL(set_pages_wb);

int set_pages_array_wb(struct page **pages, int numpages)
{
        int retval;
        unsigned long start;
        unsigned long end;
        int i;

        /* WB cache mode is hard wired to all cache attribute bits being 0 */
        retval = cpa_clear_pages_array(pages, numpages,
                        __pgprot(_PAGE_CACHE_MASK));
        if (retval)
                return retval;

        for (i = 0; i < numpages; i++) {
                if (PageHighMem(pages[i]))
                        continue;
                start = page_to_pfn(pages[i]) << PAGE_SHIFT;
                end = start + PAGE_SIZE;
                memtype_free(start, end);
        }

        return 0;
}
EXPORT_SYMBOL(set_pages_array_wb);

int set_pages_ro(struct page *page, int numpages)
{
        unsigned long addr = (unsigned long)page_address(page);

        return set_memory_ro(addr, numpages);
}

int set_pages_rw(struct page *page, int numpages)
{
        unsigned long addr = (unsigned long)page_address(page);

        return set_memory_rw(addr, numpages);
}

static int __set_pages_p(struct page *page, int numpages)
{
        unsigned long tempaddr = (unsigned long) page_address(page);
        struct cpa_data cpa = { .vaddr = &tempaddr,
                                .pgd = NULL,
                                .numpages = numpages,
                                .mask_set = __pgprot(_PAGE_PRESENT | _PAGE_RW),
                                .mask_clr = __pgprot(0),
                                .flags = 0};

        /*
         * No alias checking needed for setting present flag. otherwise,
         * we may need to break large pages for 64-bit kernel text
         * mappings (this adds to complexity if we want to do this from
         * atomic context especially). Let's keep it simple!
         */
        return __change_page_attr_set_clr(&cpa, 0);
}

static int __set_pages_np(struct page *page, int numpages)
{
        unsigned long tempaddr = (unsigned long) page_address(page);
        struct cpa_data cpa = { .vaddr = &tempaddr,
                                .pgd = NULL,
                                .numpages = numpages,
                                .mask_set = __pgprot(0),
                                .mask_clr = __pgprot(_PAGE_PRESENT | _PAGE_RW),
                                .flags = 0};

        /*
         * No alias checking needed for setting not present flag. otherwise,
         * we may need to break large pages for 64-bit kernel text
         * mappings (this adds to complexity if we want to do this from
         * atomic context especially). Let's keep it simple!
         */
        return __change_page_attr_set_clr(&cpa, 0);
}

int set_direct_map_invalid_noflush(struct page *page)
{
        return __set_pages_np(page, 1);
}

int set_direct_map_default_noflush(struct page *page)
{
        return __set_pages_p(page, 1);
}

#ifdef CONFIG_DEBUG_PAGEALLOC
void __kernel_map_pages(struct page *page, int numpages, int enable)
{
        if (PageHighMem(page))
                return;
        if (!enable) {
                debug_check_no_locks_freed(page_address(page),
                                           numpages * PAGE_SIZE);
        }

        /*
         * The return value is ignored as the calls cannot fail.
         * Large pages for identity mappings are not used at boot time
         * and hence no memory allocations during large page split.
         */
        if (enable)
                __set_pages_p(page, numpages);
        else
                __set_pages_np(page, numpages);

        /*
         * We should perform an IPI and flush all tlbs,
         * but that can deadlock->flush only current cpu.
         * Preemption needs to be disabled around __flush_tlb_all() due to
         * CR3 reload in __native_flush_tlb().
         */
        preempt_disable();
        __flush_tlb_all();
        preempt_enable();

        arch_flush_lazy_mmu_mode();
}
#endif /* CONFIG_DEBUG_PAGEALLOC */

bool kernel_page_present(struct page *page)
{
        unsigned int level;
        pte_t *pte;

        if (PageHighMem(page))
                return false;

        pte = lookup_address((unsigned long)page_address(page), &level);
        return (pte_val(*pte) & _PAGE_PRESENT);
}

int __init kernel_map_pages_in_pgd(pgd_t *pgd, u64 pfn, unsigned long address,
                                   unsigned numpages, unsigned long page_flags)
{
        int retval = -EINVAL;

        struct cpa_data cpa = {
                .vaddr = &address,
                .pfn = pfn,
                .pgd = pgd,
                .numpages = numpages,
                .mask_set = __pgprot(0),
                .mask_clr = __pgprot(~page_flags & (_PAGE_NX|_PAGE_RW)),
                .flags = 0,
        };

        WARN_ONCE(num_online_cpus() > 1, "Don't call after initializing SMP");

        if (!(__supported_pte_mask & _PAGE_NX))
                goto out;

        if (!(page_flags & _PAGE_ENC))
                cpa.mask_clr = pgprot_encrypted(cpa.mask_clr);

        cpa.mask_set = __pgprot(_PAGE_PRESENT | page_flags);

        retval = __change_page_attr_set_clr(&cpa, 0);
        __flush_tlb_all();

out:
        return retval;
}

/*
 * __flush_tlb_all() flushes mappings only on current CPU and hence this
 * function shouldn't be used in an SMP environment. Presently, it's used only
 * during boot (way before smp_init()) by EFI subsystem and hence is ok.
 */
int __init kernel_unmap_pages_in_pgd(pgd_t *pgd, unsigned long address,
                                     unsigned long numpages)
{
        int retval;

        /*
         * The typical sequence for unmapping is to find a pte through
         * lookup_address_in_pgd() (ideally, it should never return NULL because
         * the address is already mapped) and change it's protections. As pfn is
         * the *target* of a mapping, it's not useful while unmapping.
         */
        struct cpa_data cpa = {
                .vaddr                = &address,
                .pfn                = 0,
                .pgd                = pgd,
                .numpages        = numpages,
                .mask_set        = __pgprot(0),
                .mask_clr        = __pgprot(_PAGE_PRESENT | _PAGE_RW),
                .flags                = 0,
        };

        WARN_ONCE(num_online_cpus() > 1, "Don't call after initializing SMP");

        retval = __change_page_attr_set_clr(&cpa, 0);
        __flush_tlb_all();

        return retval;
}

/*
 * The testcases use internal knowledge of the implementation that shouldn't
 * be exposed to the rest of the kernel. Include these directly here.
 */
#ifdef CONFIG_CPA_DEBUG
#include "cpa-test.c"
#endif






































































































































































































































































































































































































































































































































































































































































































































































































































































































































    5 






























































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































    5 
































































    5 












    5 



























































































































































































































































































































    5 





































































































































































































































































































































































































































































































































    5 
    5 



    5 














    5 









































    5 
























































































































































































































































































   10 
















































































































































































































































    5 










1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965
2966
2967
2968
2969
2970
2971
2972
2973
2974
2975
2976
2977
2978
2979
2980
2981
2982
2983
2984
2985
2986
2987
2988
2989
2990
2991
2992
2993
2994
2995
2996
2997
2998
2999
3000
3001
3002
3003
3004
3005
3006
3007
3008
3009
3010
3011
3012
3013
3014
3015
3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029
3030
3031
3032
3033
3034
3035
3036
3037
3038
3039
3040
3041
3042
3043
3044
3045
3046
3047
3048
3049
3050
3051
3052
3053
3054
3055
3056
3057
3058
3059
3060
3061
3062
3063
3064
3065
3066
3067
3068
3069
3070
3071
3072
3073
3074
3075
3076
3077
3078
3079
3080
3081
3082
3083
3084
3085
3086
3087
3088
3089
3090
3091
3092
3093
3094
3095
3096
3097
3098
3099
3100
3101
3102
3103
3104
3105
3106
3107
3108
3109
3110
3111
3112
3113
3114
3115
3116
3117
3118
3119
3120
3121
3122
3123
3124
3125
3126
3127
3128
3129
3130
3131
3132
3133
3134
3135
3136
3137
3138
3139
3140
3141
3142
3143
3144
3145
3146
3147
3148
3149
3150
3151
3152
3153
3154
3155
3156
3157
3158
3159
3160
3161
3162
3163
3164
3165
3166
3167
3168
3169
3170
3171
3172
3173
3174
3175
3176
3177
3178
3179
3180
3181
3182
3183
3184
3185
3186
3187
3188
3189
3190
3191
3192
3193
3194
3195
3196
3197
3198
3199
3200
3201
3202
3203
3204
3205
3206
3207
3208
3209
3210
3211
3212
3213
3214
3215
3216
3217
3218
3219
3220
3221
3222
3223
3224
3225
3226
3227
3228
3229
3230
3231
3232
3233
3234
3235
3236
3237
3238
3239
3240
3241
3242
3243
3244
3245
3246
3247
3248
3249
3250
3251
3252
3253
3254
3255
3256
3257
3258
3259
3260
3261
3262
3263
3264
3265
3266
3267
3268
3269
3270
3271
3272
3273
3274
3275
3276
3277
3278
3279
3280
3281
3282
3283
3284
3285
3286
3287
3288
3289
3290
3291
3292
3293
3294
3295
3296
3297
3298
3299
3300
3301
3302
3303
3304
3305
3306
3307
3308
3309
3310
3311
3312
3313
3314
3315
3316
3317
3318
3319
3320
3321
3322
3323
3324
3325
3326
3327
3328
3329
3330
3331
3332
3333
3334
3335
3336
3337
3338
3339
3340
3341
3342
3343
3344
3345
3346
3347
3348
3349
3350
3351
3352
3353
3354
3355
3356
3357
3358
3359
3360
3361
3362
3363
3364
3365
3366
3367
3368
3369
3370
3371
3372
3373
3374
3375
3376
3377
3378
3379
3380
3381
3382
3383
3384
3385
3386
3387
3388
3389
3390
3391
3392
3393
3394
3395
3396
3397
3398
3399
3400
3401
3402
3403
3404
3405
3406
3407
3408
3409
3410
3411
3412
3413
3414
3415
3416
3417
3418
3419
3420
3421
3422
3423
3424
3425
3426
3427
3428
3429
3430
3431
3432
3433
3434
3435
3436
3437
3438
3439
3440
3441
3442
3443
3444
3445
3446
3447
3448
3449
3450
3451
3452
3453
3454
3455
3456
3457
3458
3459
3460
3461
3462
3463
3464
3465
3466
3467
3468
3469
3470
3471
3472
3473
3474
3475
3476
3477
3478
3479
3480
3481
3482
3483
3484
3485
3486
3487
3488
3489
3490
3491
3492
3493
3494
3495
3496
3497
3498
3499
3500
3501
3502
3503
3504
3505
3506
3507
3508
3509
3510
3511
3512
3513
3514
3515
3516
3517
3518
3519
3520
3521
3522
3523
3524
3525
3526
3527
3528
3529
3530
3531
3532
3533
3534
3535
3536
3537
3538
3539
3540
3541
3542
3543
3544
3545
3546
3547
3548
3549
3550
3551
3552
3553
3554
3555
3556
3557
3558
3559
3560
3561
3562
3563
3564
3565
3566
3567
3568
3569
3570
3571
3572
3573
3574
3575
3576
3577
3578
3579
3580
3581
3582
3583
3584
3585
3586
3587
3588
3589
3590
3591
3592
3593
3594
3595
3596
3597
3598
3599
3600
3601
3602
3603
3604
3605
3606
3607
3608
3609
3610
3611
3612
3613
3614
3615
3616
3617
3618
3619
3620
3621
3622
3623
3624
3625
3626
3627
3628
3629
3630
3631
3632
3633
3634
3635
3636
3637
3638
3639
3640
3641
3642
3643
3644
3645
3646
3647
3648
3649
3650
3651
3652
3653
3654
3655
3656
3657
3658
3659
3660
3661
3662
3663
3664
3665
3666
3667
3668
3669
3670
3671
3672
3673
3674
3675
3676
3677
3678
3679
3680
3681
3682
3683
3684
3685
3686
3687
3688
3689
3690
3691
3692
3693
3694
3695
3696
3697
3698
3699
3700
3701
3702
3703
3704
3705
3706
3707
3708
3709
3710
3711
3712
3713
3714
3715
3716
3717
3718
3719
3720
3721
3722
3723
3724
3725
3726
3727
3728
3729
3730
3731
3732
3733
3734
3735
3736
3737
3738
3739
3740
3741
3742
3743
3744
3745
3746
3747
3748
3749
3750
3751
3752
3753
3754
3755
3756
3757
3758
3759
3760
3761
3762
3763
3764
3765
3766
3767
3768
3769
3770
3771
3772
3773
3774
3775
3776
3777
3778
3779
3780
3781
3782
3783
3784
3785
3786
3787
3788
3789
3790
3791
3792
3793
3794
3795
3796
3797
3798
3799
3800
3801
3802
3803
3804
3805
3806
3807
3808
3809
3810
3811
3812
3813
3814
3815
3816
3817
3818
3819
3820
3821
3822
3823
3824
3825
3826
3827
3828
3829
3830
3831
3832
3833
3834
3835
3836
3837
3838
3839
3840
3841
3842
3843
3844
3845
3846
3847
3848
3849
3850
3851
3852
3853
3854
3855
3856
3857
3858
3859
3860
3861
3862
3863
3864
3865
3866
3867
3868
3869
3870
3871
3872
3873
3874
3875
3876
3877
3878
3879
// SPDX-License-Identifier: GPL-2.0
/*
 *  ext4.h
 *
 * Copyright (C) 1992, 1993, 1994, 1995
 * Remy Card (card@masi.ibp.fr)
 * Laboratoire MASI - Institut Blaise Pascal
 * Universite Pierre et Marie Curie (Paris VI)
 *
 *  from
 *
 *  linux/include/linux/minix_fs.h
 *
 *  Copyright (C) 1991, 1992  Linus Torvalds
 */

#ifndef _EXT4_H
#define _EXT4_H

#include <linux/types.h>
#include <linux/blkdev.h>
#include <linux/magic.h>
#include <linux/jbd2.h>
#include <linux/quota.h>
#include <linux/rwsem.h>
#include <linux/rbtree.h>
#include <linux/seqlock.h>
#include <linux/mutex.h>
#include <linux/timer.h>
#include <linux/wait.h>
#include <linux/sched/signal.h>
#include <linux/blockgroup_lock.h>
#include <linux/percpu_counter.h>
#include <linux/ratelimit.h>
#include <crypto/hash.h>
#include <linux/falloc.h>
#include <linux/percpu-rwsem.h>
#include <linux/fiemap.h>
#ifdef __KERNEL__
#include <linux/compat.h>
#endif

#include <linux/fscrypt.h>
#include <linux/fsverity.h>

#include <linux/compiler.h>

/*
 * The fourth extended filesystem constants/structures
 */

/*
 * with AGGRESSIVE_CHECK allocator runs consistency checks over
 * structures. these checks slow things down a lot
 */
#define AGGRESSIVE_CHECK__

/*
 * with DOUBLE_CHECK defined mballoc creates persistent in-core
 * bitmaps, maintains and uses them to check for double allocations
 */
#define DOUBLE_CHECK__

/*
 * Define EXT4FS_DEBUG to produce debug messages
 */
#undef EXT4FS_DEBUG

/*
 * Debug code
 */
#ifdef EXT4FS_DEBUG
#define ext4_debug(f, a...)                                                \
        do {                                                                \
                printk(KERN_DEBUG "EXT4-fs DEBUG (%s, %d): %s:",        \
                        __FILE__, __LINE__, __func__);                        \
                printk(KERN_DEBUG f, ## a);                                \
        } while (0)
#else
#define ext4_debug(fmt, ...)        no_printk(fmt, ##__VA_ARGS__)
#endif

 /*
  * Turn on EXT_DEBUG to enable ext4_ext_show_path/leaf/move in extents.c
  */
#define EXT_DEBUG__

/*
 * Dynamic printk for controlled extents debugging.
 */
#ifdef CONFIG_EXT4_DEBUG
#define ext_debug(ino, fmt, ...)                                        \
        pr_debug("[%s/%d] EXT4-fs (%s): ino %lu: (%s, %d): %s:" fmt,        \
                 current->comm, task_pid_nr(current),                        \
                 ino->i_sb->s_id, ino->i_ino, __FILE__, __LINE__,        \
                 __func__, ##__VA_ARGS__)
#else
#define ext_debug(ino, fmt, ...)        no_printk(fmt, ##__VA_ARGS__)
#endif

#define ASSERT(assert)                                                \
do {                                                                        \
        if (unlikely(!(assert))) {                                        \
                printk(KERN_EMERG                                        \
                       "Assertion failure in %s() at %s:%d: '%s'\n",        \
                       __func__, __FILE__, __LINE__, #assert);                \
                BUG();                                                        \
        }                                                                \
} while (0)

/* data type for block offset of block group */
typedef int ext4_grpblk_t;

/* data type for filesystem-wide blocks number */
typedef unsigned long long ext4_fsblk_t;

/* data type for file logical block number */
typedef __u32 ext4_lblk_t;

/* data type for block group number */
typedef unsigned int ext4_group_t;

enum SHIFT_DIRECTION {
        SHIFT_LEFT = 0,
        SHIFT_RIGHT,
};

/*
 * Flags used in mballoc's allocation_context flags field.
 *
 * Also used to show what's going on for debugging purposes when the
 * flag field is exported via the traceport interface
 */

/* prefer goal again. length */
#define EXT4_MB_HINT_MERGE                0x0001
/* blocks already reserved */
#define EXT4_MB_HINT_RESERVED                0x0002
/* metadata is being allocated */
#define EXT4_MB_HINT_METADATA                0x0004
/* first blocks in the file */
#define EXT4_MB_HINT_FIRST                0x0008
/* search for the best chunk */
#define EXT4_MB_HINT_BEST                0x0010
/* data is being allocated */
#define EXT4_MB_HINT_DATA                0x0020
/* don't preallocate (for tails) */
#define EXT4_MB_HINT_NOPREALLOC                0x0040
/* allocate for locality group */
#define EXT4_MB_HINT_GROUP_ALLOC        0x0080
/* allocate goal blocks or none */
#define EXT4_MB_HINT_GOAL_ONLY                0x0100
/* goal is meaningful */
#define EXT4_MB_HINT_TRY_GOAL                0x0200
/* blocks already pre-reserved by delayed allocation */
#define EXT4_MB_DELALLOC_RESERVED        0x0400
/* We are doing stream allocation */
#define EXT4_MB_STREAM_ALLOC                0x0800
/* Use reserved root blocks if needed */
#define EXT4_MB_USE_ROOT_BLOCKS                0x1000
/* Use blocks from reserved pool */
#define EXT4_MB_USE_RESERVED                0x2000
/* Do strict check for free blocks while retrying block allocation */
#define EXT4_MB_STRICT_CHECK                0x4000
/* Large fragment size list lookup succeeded at least once for cr = 0 */
#define EXT4_MB_CR0_OPTIMIZED                0x8000
/* Avg fragment size rb tree lookup succeeded at least once for cr = 1 */
#define EXT4_MB_CR1_OPTIMIZED                0x00010000
/* Perform linear traversal for one group */
#define EXT4_MB_SEARCH_NEXT_LINEAR        0x00020000
struct ext4_allocation_request {
        /* target inode for block we're allocating */
        struct inode *inode;
        /* how many blocks we want to allocate */
        unsigned int len;
        /* logical block in target inode */
        ext4_lblk_t logical;
        /* the closest logical allocated block to the left */
        ext4_lblk_t lleft;
        /* the closest logical allocated block to the right */
        ext4_lblk_t lright;
        /* phys. target (a hint) */
        ext4_fsblk_t goal;
        /* phys. block for the closest logical allocated block to the left */
        ext4_fsblk_t pleft;
        /* phys. block for the closest logical allocated block to the right */
        ext4_fsblk_t pright;
        /* flags. see above EXT4_MB_HINT_* */
        unsigned int flags;
};

/*
 * Logical to physical block mapping, used by ext4_map_blocks()
 *
 * This structure is used to pass requests into ext4_map_blocks() as
 * well as to store the information returned by ext4_map_blocks().  It
 * takes less room on the stack than a struct buffer_head.
 */
#define EXT4_MAP_NEW                BIT(BH_New)
#define EXT4_MAP_MAPPED                BIT(BH_Mapped)
#define EXT4_MAP_UNWRITTEN        BIT(BH_Unwritten)
#define EXT4_MAP_BOUNDARY        BIT(BH_Boundary)
#define EXT4_MAP_FLAGS                (EXT4_MAP_NEW | EXT4_MAP_MAPPED |\
                                 EXT4_MAP_UNWRITTEN | EXT4_MAP_BOUNDARY)

struct ext4_map_blocks {
        ext4_fsblk_t m_pblk;
        ext4_lblk_t m_lblk;
        unsigned int m_len;
        unsigned int m_flags;
};

/*
 * Block validity checking, system zone rbtree.
 */
struct ext4_system_blocks {
        struct rb_root root;
        struct rcu_head rcu;
};

/*
 * Flags for ext4_io_end->flags
 */
#define        EXT4_IO_END_UNWRITTEN        0x0001

struct ext4_io_end_vec {
        struct list_head list;                /* list of io_end_vec */
        loff_t offset;                        /* offset in the file */
        ssize_t size;                        /* size of the extent */
};

/*
 * For converting unwritten extents on a work queue. 'handle' is used for
 * buffered writeback.
 */
typedef struct ext4_io_end {
        struct list_head        list;                /* per-file finished IO list */
        handle_t                *handle;        /* handle reserved for extent
                                                 * conversion */
        struct inode                *inode;                /* file being written to */
        struct bio                *bio;                /* Linked list of completed
                                                 * bios covering the extent */
        unsigned int                flag;                /* unwritten or not */
        atomic_t                count;                /* reference counter */
        struct list_head        list_vec;        /* list of ext4_io_end_vec */
} ext4_io_end_t;

struct ext4_io_submit {
        struct writeback_control *io_wbc;
        struct bio                *io_bio;
        ext4_io_end_t                *io_end;
        sector_t                io_next_block;
};

/*
 * Special inodes numbers
 */
#define        EXT4_BAD_INO                 1        /* Bad blocks inode */
#define EXT4_ROOT_INO                 2        /* Root inode */
#define EXT4_USR_QUOTA_INO         3        /* User quota inode */
#define EXT4_GRP_QUOTA_INO         4        /* Group quota inode */
#define EXT4_BOOT_LOADER_INO         5        /* Boot loader inode */
#define EXT4_UNDEL_DIR_INO         6        /* Undelete directory inode */
#define EXT4_RESIZE_INO                 7        /* Reserved group descriptors inode */
#define EXT4_JOURNAL_INO         8        /* Journal inode */

/* First non-reserved inode for old ext4 filesystems */
#define EXT4_GOOD_OLD_FIRST_INO        11

/*
 * Maximal count of links to a file
 */
#define EXT4_LINK_MAX                65000

/*
 * Macro-instructions used to manage several block sizes
 */
#define EXT4_MIN_BLOCK_SIZE                1024
#define        EXT4_MAX_BLOCK_SIZE                65536
#define EXT4_MIN_BLOCK_LOG_SIZE                10
#define EXT4_MAX_BLOCK_LOG_SIZE                16
#define EXT4_MAX_CLUSTER_LOG_SIZE        30
#ifdef __KERNEL__
# define EXT4_BLOCK_SIZE(s)                ((s)->s_blocksize)
#else
# define EXT4_BLOCK_SIZE(s)                (EXT4_MIN_BLOCK_SIZE << (s)->s_log_block_size)
#endif
#define        EXT4_ADDR_PER_BLOCK(s)                (EXT4_BLOCK_SIZE(s) / sizeof(__u32))
#define EXT4_CLUSTER_SIZE(s)                (EXT4_BLOCK_SIZE(s) << \
                                         EXT4_SB(s)->s_cluster_bits)
#ifdef __KERNEL__
# define EXT4_BLOCK_SIZE_BITS(s)        ((s)->s_blocksize_bits)
# define EXT4_CLUSTER_BITS(s)                (EXT4_SB(s)->s_cluster_bits)
#else
# define EXT4_BLOCK_SIZE_BITS(s)        ((s)->s_log_block_size + 10)
#endif
#ifdef __KERNEL__
#define        EXT4_ADDR_PER_BLOCK_BITS(s)        (EXT4_SB(s)->s_addr_per_block_bits)
#define EXT4_INODE_SIZE(s)                (EXT4_SB(s)->s_inode_size)
#define EXT4_FIRST_INO(s)                (EXT4_SB(s)->s_first_ino)
#else
#define EXT4_INODE_SIZE(s)        (((s)->s_rev_level == EXT4_GOOD_OLD_REV) ? \
                                 EXT4_GOOD_OLD_INODE_SIZE : \
                                 (s)->s_inode_size)
#define EXT4_FIRST_INO(s)        (((s)->s_rev_level == EXT4_GOOD_OLD_REV) ? \
                                 EXT4_GOOD_OLD_FIRST_INO : \
                                 (s)->s_first_ino)
#endif
#define EXT4_BLOCK_ALIGN(size, blkbits)                ALIGN((size), (1 << (blkbits)))
#define EXT4_MAX_BLOCKS(size, offset, blkbits) \
        ((EXT4_BLOCK_ALIGN(size + offset, blkbits) >> blkbits) - (offset >> \
                                                                  blkbits))

/* Translate a block number to a cluster number */
#define EXT4_B2C(sbi, blk)        ((blk) >> (sbi)->s_cluster_bits)
/* Translate a cluster number to a block number */
#define EXT4_C2B(sbi, cluster)        ((cluster) << (sbi)->s_cluster_bits)
/* Translate # of blks to # of clusters */
#define EXT4_NUM_B2C(sbi, blks)        (((blks) + (sbi)->s_cluster_ratio - 1) >> \
                                 (sbi)->s_cluster_bits)
/* Mask out the low bits to get the starting block of the cluster */
#define EXT4_PBLK_CMASK(s, pblk) ((pblk) &                                \
                                  ~((ext4_fsblk_t) (s)->s_cluster_ratio - 1))
#define EXT4_LBLK_CMASK(s, lblk) ((lblk) &                                \
                                  ~((ext4_lblk_t) (s)->s_cluster_ratio - 1))
/* Fill in the low bits to get the last block of the cluster */
#define EXT4_LBLK_CFILL(sbi, lblk) ((lblk) |                                \
                                    ((ext4_lblk_t) (sbi)->s_cluster_ratio - 1))
/* Get the cluster offset */
#define EXT4_PBLK_COFF(s, pblk) ((pblk) &                                \
                                 ((ext4_fsblk_t) (s)->s_cluster_ratio - 1))
#define EXT4_LBLK_COFF(s, lblk) ((lblk) &                                \
                                 ((ext4_lblk_t) (s)->s_cluster_ratio - 1))

/*
 * Structure of a blocks group descriptor
 */
struct ext4_group_desc
{
        __le32        bg_block_bitmap_lo;        /* Blocks bitmap block */
        __le32        bg_inode_bitmap_lo;        /* Inodes bitmap block */
        __le32        bg_inode_table_lo;        /* Inodes table block */
        __le16        bg_free_blocks_count_lo;/* Free blocks count */
        __le16        bg_free_inodes_count_lo;/* Free inodes count */
        __le16        bg_used_dirs_count_lo;        /* Directories count */
        __le16        bg_flags;                /* EXT4_BG_flags (INODE_UNINIT, etc) */
        __le32  bg_exclude_bitmap_lo;   /* Exclude bitmap for snapshots */
        __le16  bg_block_bitmap_csum_lo;/* crc32c(s_uuid+grp_num+bbitmap) LE */
        __le16  bg_inode_bitmap_csum_lo;/* crc32c(s_uuid+grp_num+ibitmap) LE */
        __le16  bg_itable_unused_lo;        /* Unused inodes count */
        __le16  bg_checksum;                /* crc16(sb_uuid+group+desc) */
        __le32        bg_block_bitmap_hi;        /* Blocks bitmap block MSB */
        __le32        bg_inode_bitmap_hi;        /* Inodes bitmap block MSB */
        __le32        bg_inode_table_hi;        /* Inodes table block MSB */
        __le16        bg_free_blocks_count_hi;/* Free blocks count MSB */
        __le16        bg_free_inodes_count_hi;/* Free inodes count MSB */
        __le16        bg_used_dirs_count_hi;        /* Directories count MSB */
        __le16  bg_itable_unused_hi;    /* Unused inodes count MSB */
        __le32  bg_exclude_bitmap_hi;   /* Exclude bitmap block MSB */
        __le16  bg_block_bitmap_csum_hi;/* crc32c(s_uuid+grp_num+bbitmap) BE */
        __le16  bg_inode_bitmap_csum_hi;/* crc32c(s_uuid+grp_num+ibitmap) BE */
        __u32   bg_reserved;
};

#define EXT4_BG_INODE_BITMAP_CSUM_HI_END        \
        (offsetof(struct ext4_group_desc, bg_inode_bitmap_csum_hi) + \
         sizeof(__le16))
#define EXT4_BG_BLOCK_BITMAP_CSUM_HI_END        \
        (offsetof(struct ext4_group_desc, bg_block_bitmap_csum_hi) + \
         sizeof(__le16))

/*
 * Structure of a flex block group info
 */

struct flex_groups {
        atomic64_t        free_clusters;
        atomic_t        free_inodes;
        atomic_t        used_dirs;
};

#define EXT4_BG_INODE_UNINIT        0x0001 /* Inode table/bitmap not in use */
#define EXT4_BG_BLOCK_UNINIT        0x0002 /* Block bitmap not in use */
#define EXT4_BG_INODE_ZEROED        0x0004 /* On-disk itable initialized to zero */

/*
 * Macro-instructions used to manage group descriptors
 */
#define EXT4_MIN_DESC_SIZE                32
#define EXT4_MIN_DESC_SIZE_64BIT        64
#define        EXT4_MAX_DESC_SIZE                EXT4_MIN_BLOCK_SIZE
#define EXT4_DESC_SIZE(s)                (EXT4_SB(s)->s_desc_size)
#ifdef __KERNEL__
# define EXT4_BLOCKS_PER_GROUP(s)        (EXT4_SB(s)->s_blocks_per_group)
# define EXT4_CLUSTERS_PER_GROUP(s)        (EXT4_SB(s)->s_clusters_per_group)
# define EXT4_DESC_PER_BLOCK(s)                (EXT4_SB(s)->s_desc_per_block)
# define EXT4_INODES_PER_GROUP(s)        (EXT4_SB(s)->s_inodes_per_group)
# define EXT4_DESC_PER_BLOCK_BITS(s)        (EXT4_SB(s)->s_desc_per_block_bits)
#else
# define EXT4_BLOCKS_PER_GROUP(s)        ((s)->s_blocks_per_group)
# define EXT4_DESC_PER_BLOCK(s)                (EXT4_BLOCK_SIZE(s) / EXT4_DESC_SIZE(s))
# define EXT4_INODES_PER_GROUP(s)        ((s)->s_inodes_per_group)
#endif

/*
 * Constants relative to the data blocks
 */
#define        EXT4_NDIR_BLOCKS                12
#define        EXT4_IND_BLOCK                        EXT4_NDIR_BLOCKS
#define        EXT4_DIND_BLOCK                        (EXT4_IND_BLOCK + 1)
#define        EXT4_TIND_BLOCK                        (EXT4_DIND_BLOCK + 1)
#define        EXT4_N_BLOCKS                        (EXT4_TIND_BLOCK + 1)

/*
 * Inode flags
 */
#define        EXT4_SECRM_FL                        0x00000001 /* Secure deletion */
#define        EXT4_UNRM_FL                        0x00000002 /* Undelete */
#define        EXT4_COMPR_FL                        0x00000004 /* Compress file */
#define EXT4_SYNC_FL                        0x00000008 /* Synchronous updates */
#define EXT4_IMMUTABLE_FL                0x00000010 /* Immutable file */
#define EXT4_APPEND_FL                        0x00000020 /* writes to file may only append */
#define EXT4_NODUMP_FL                        0x00000040 /* do not dump file */
#define EXT4_NOATIME_FL                        0x00000080 /* do not update atime */
/* Reserved for compression usage... */
#define EXT4_DIRTY_FL                        0x00000100
#define EXT4_COMPRBLK_FL                0x00000200 /* One or more compressed clusters */
#define EXT4_NOCOMPR_FL                        0x00000400 /* Don't compress */
        /* nb: was previously EXT2_ECOMPR_FL */
#define EXT4_ENCRYPT_FL                        0x00000800 /* encrypted file */
/* End compression flags --- maybe not all used */
#define EXT4_INDEX_FL                        0x00001000 /* hash-indexed directory */
#define EXT4_IMAGIC_FL                        0x00002000 /* AFS directory */
#define EXT4_JOURNAL_DATA_FL                0x00004000 /* file data should be journaled */
#define EXT4_NOTAIL_FL                        0x00008000 /* file tail should not be merged */
#define EXT4_DIRSYNC_FL                        0x00010000 /* dirsync behaviour (directories only) */
#define EXT4_TOPDIR_FL                        0x00020000 /* Top of directory hierarchies*/
#define EXT4_HUGE_FILE_FL               0x00040000 /* Set to each huge file */
#define EXT4_EXTENTS_FL                        0x00080000 /* Inode uses extents */
#define EXT4_VERITY_FL                        0x00100000 /* Verity protected inode */
#define EXT4_EA_INODE_FL                0x00200000 /* Inode used for large EA */
/* 0x00400000 was formerly EXT4_EOFBLOCKS_FL */

#define EXT4_DAX_FL                        0x02000000 /* Inode is DAX */

#define EXT4_INLINE_DATA_FL                0x10000000 /* Inode has inline data. */
#define EXT4_PROJINHERIT_FL                0x20000000 /* Create with parents projid */
#define EXT4_CASEFOLD_FL                0x40000000 /* Casefolded directory */
#define EXT4_RESERVED_FL                0x80000000 /* reserved for ext4 lib */

/* User modifiable flags */
#define EXT4_FL_USER_MODIFIABLE                (EXT4_SECRM_FL | \
                                         EXT4_UNRM_FL | \
                                         EXT4_COMPR_FL | \
                                         EXT4_SYNC_FL | \
                                         EXT4_IMMUTABLE_FL | \
                                         EXT4_APPEND_FL | \
                                         EXT4_NODUMP_FL | \
                                         EXT4_NOATIME_FL | \
                                         EXT4_JOURNAL_DATA_FL | \
                                         EXT4_NOTAIL_FL | \
                                         EXT4_DIRSYNC_FL | \
                                         EXT4_TOPDIR_FL | \
                                         EXT4_EXTENTS_FL | \
                                         0x00400000 /* EXT4_EOFBLOCKS_FL */ | \
                                         EXT4_DAX_FL | \
                                         EXT4_PROJINHERIT_FL | \
                                         EXT4_CASEFOLD_FL)

/* User visible flags */
#define EXT4_FL_USER_VISIBLE                (EXT4_FL_USER_MODIFIABLE | \
                                         EXT4_DIRTY_FL | \
                                         EXT4_COMPRBLK_FL | \
                                         EXT4_NOCOMPR_FL | \
                                         EXT4_ENCRYPT_FL | \
                                         EXT4_INDEX_FL | \
                                         EXT4_VERITY_FL | \
                                         EXT4_INLINE_DATA_FL)

/* Flags that should be inherited by new inodes from their parent. */
#define EXT4_FL_INHERITED (EXT4_SECRM_FL | EXT4_UNRM_FL | EXT4_COMPR_FL |\
                           EXT4_SYNC_FL | EXT4_NODUMP_FL | EXT4_NOATIME_FL |\
                           EXT4_NOCOMPR_FL | EXT4_JOURNAL_DATA_FL |\
                           EXT4_NOTAIL_FL | EXT4_DIRSYNC_FL |\
                           EXT4_PROJINHERIT_FL | EXT4_CASEFOLD_FL |\
                           EXT4_DAX_FL)

/* Flags that are appropriate for regular files (all but dir-specific ones). */
#define EXT4_REG_FLMASK (~(EXT4_DIRSYNC_FL | EXT4_TOPDIR_FL | EXT4_CASEFOLD_FL |\
                           EXT4_PROJINHERIT_FL))

/* Flags that are appropriate for non-directories/regular files. */
#define EXT4_OTHER_FLMASK (EXT4_NODUMP_FL | EXT4_NOATIME_FL)

/* The only flags that should be swapped */
#define EXT4_FL_SHOULD_SWAP (EXT4_HUGE_FILE_FL | EXT4_EXTENTS_FL)

/* Flags which are mutually exclusive to DAX */
#define EXT4_DAX_MUT_EXCL (EXT4_VERITY_FL | EXT4_ENCRYPT_FL |\
                           EXT4_JOURNAL_DATA_FL | EXT4_INLINE_DATA_FL)

/* Mask out flags that are inappropriate for the given type of inode. */
static inline __u32 ext4_mask_flags(umode_t mode, __u32 flags)
{
        if (S_ISDIR(mode))
                return flags;
        else if (S_ISREG(mode))
                return flags & EXT4_REG_FLMASK;
        else
                return flags & EXT4_OTHER_FLMASK;
}

/*
 * Inode flags used for atomic set/get
 */
enum {
        EXT4_INODE_SECRM        = 0,        /* Secure deletion */
        EXT4_INODE_UNRM                = 1,        /* Undelete */
        EXT4_INODE_COMPR        = 2,        /* Compress file */
        EXT4_INODE_SYNC                = 3,        /* Synchronous updates */
        EXT4_INODE_IMMUTABLE        = 4,        /* Immutable file */
        EXT4_INODE_APPEND        = 5,        /* writes to file may only append */
        EXT4_INODE_NODUMP        = 6,        /* do not dump file */
        EXT4_INODE_NOATIME        = 7,        /* do not update atime */
/* Reserved for compression usage... */
        EXT4_INODE_DIRTY        = 8,
        EXT4_INODE_COMPRBLK        = 9,        /* One or more compressed clusters */
        EXT4_INODE_NOCOMPR        = 10,        /* Don't compress */
        EXT4_INODE_ENCRYPT        = 11,        /* Encrypted file */
/* End compression flags --- maybe not all used */
        EXT4_INODE_INDEX        = 12,        /* hash-indexed directory */
        EXT4_INODE_IMAGIC        = 13,        /* AFS directory */
        EXT4_INODE_JOURNAL_DATA        = 14,        /* file data should be journaled */
        EXT4_INODE_NOTAIL        = 15,        /* file tail should not be merged */
        EXT4_INODE_DIRSYNC        = 16,        /* dirsync behaviour (directories only) */
        EXT4_INODE_TOPDIR        = 17,        /* Top of directory hierarchies*/
        EXT4_INODE_HUGE_FILE        = 18,        /* Set to each huge file */
        EXT4_INODE_EXTENTS        = 19,        /* Inode uses extents */
        EXT4_INODE_VERITY        = 20,        /* Verity protected inode */
        EXT4_INODE_EA_INODE        = 21,        /* Inode used for large EA */
/* 22 was formerly EXT4_INODE_EOFBLOCKS */
        EXT4_INODE_DAX                = 25,        /* Inode is DAX */
        EXT4_INODE_INLINE_DATA        = 28,        /* Data in inode. */
        EXT4_INODE_PROJINHERIT        = 29,        /* Create with parents projid */
        EXT4_INODE_CASEFOLD        = 30,        /* Casefolded directory */
        EXT4_INODE_RESERVED        = 31,        /* reserved for ext4 lib */
};

/*
 * Since it's pretty easy to mix up bit numbers and hex values, we use a
 * build-time check to make sure that EXT4_XXX_FL is consistent with respect to
 * EXT4_INODE_XXX. If all is well, the macros will be dropped, so, it won't cost
 * any extra space in the compiled kernel image, otherwise, the build will fail.
 * It's important that these values are the same, since we are using
 * EXT4_INODE_XXX to test for flag values, but EXT4_XXX_FL must be consistent
 * with the values of FS_XXX_FL defined in include/linux/fs.h and the on-disk
 * values found in ext2, ext3 and ext4 filesystems, and of course the values
 * defined in e2fsprogs.
 *
 * It's not paranoia if the Murphy's Law really *is* out to get you.  :-)
 */
#define TEST_FLAG_VALUE(FLAG) (EXT4_##FLAG##_FL == (1U << EXT4_INODE_##FLAG))
#define CHECK_FLAG_VALUE(FLAG) BUILD_BUG_ON(!TEST_FLAG_VALUE(FLAG))

static inline void ext4_check_flag_values(void)
{
        CHECK_FLAG_VALUE(SECRM);
        CHECK_FLAG_VALUE(UNRM);
        CHECK_FLAG_VALUE(COMPR);
        CHECK_FLAG_VALUE(SYNC);
        CHECK_FLAG_VALUE(IMMUTABLE);
        CHECK_FLAG_VALUE(APPEND);
        CHECK_FLAG_VALUE(NODUMP);
        CHECK_FLAG_VALUE(NOATIME);
        CHECK_FLAG_VALUE(DIRTY);
        CHECK_FLAG_VALUE(COMPRBLK);
        CHECK_FLAG_VALUE(NOCOMPR);
        CHECK_FLAG_VALUE(ENCRYPT);
        CHECK_FLAG_VALUE(INDEX);
        CHECK_FLAG_VALUE(IMAGIC);
        CHECK_FLAG_VALUE(JOURNAL_DATA);
        CHECK_FLAG_VALUE(NOTAIL);
        CHECK_FLAG_VALUE(DIRSYNC);
        CHECK_FLAG_VALUE(TOPDIR);
        CHECK_FLAG_VALUE(HUGE_FILE);
        CHECK_FLAG_VALUE(EXTENTS);
        CHECK_FLAG_VALUE(VERITY);
        CHECK_FLAG_VALUE(EA_INODE);
        CHECK_FLAG_VALUE(INLINE_DATA);
        CHECK_FLAG_VALUE(PROJINHERIT);
        CHECK_FLAG_VALUE(CASEFOLD);
        CHECK_FLAG_VALUE(RESERVED);
}

/* Used to pass group descriptor data when online resize is done */
struct ext4_new_group_input {
        __u32 group;                /* Group number for this data */
        __u64 block_bitmap;        /* Absolute block number of block bitmap */
        __u64 inode_bitmap;        /* Absolute block number of inode bitmap */
        __u64 inode_table;        /* Absolute block number of inode table start */
        __u32 blocks_count;        /* Total number of blocks in this group */
        __u16 reserved_blocks;        /* Number of reserved blocks in this group */
        __u16 unused;
};

#if defined(__KERNEL__) && defined(CONFIG_COMPAT)
struct compat_ext4_new_group_input {
        u32 group;
        compat_u64 block_bitmap;
        compat_u64 inode_bitmap;
        compat_u64 inode_table;
        u32 blocks_count;
        u16 reserved_blocks;
        u16 unused;
};
#endif

/* The struct ext4_new_group_input in kernel space, with free_blocks_count */
struct ext4_new_group_data {
        __u32 group;
        __u64 block_bitmap;
        __u64 inode_bitmap;
        __u64 inode_table;
        __u32 blocks_count;
        __u16 reserved_blocks;
        __u16 mdata_blocks;
        __u32 free_clusters_count;
};

/* Indexes used to index group tables in ext4_new_group_data */
enum {
        BLOCK_BITMAP = 0,        /* block bitmap */
        INODE_BITMAP,                /* inode bitmap */
        INODE_TABLE,                /* inode tables */
        GROUP_TABLE_COUNT,
};

/*
 * Flags used by ext4_map_blocks()
 */
        /* Allocate any needed blocks and/or convert an unwritten
           extent to be an initialized ext4 */
#define EXT4_GET_BLOCKS_CREATE                        0x0001
        /* Request the creation of an unwritten extent */
#define EXT4_GET_BLOCKS_UNWRIT_EXT                0x0002
#define EXT4_GET_BLOCKS_CREATE_UNWRIT_EXT        (EXT4_GET_BLOCKS_UNWRIT_EXT|\
                                                 EXT4_GET_BLOCKS_CREATE)
        /* Caller is from the delayed allocation writeout path
         * finally doing the actual allocation of delayed blocks */
#define EXT4_GET_BLOCKS_DELALLOC_RESERVE        0x0004
        /* caller is from the direct IO path, request to creation of an
        unwritten extents if not allocated, split the unwritten
        extent if blocks has been preallocated already*/
#define EXT4_GET_BLOCKS_PRE_IO                        0x0008
#define EXT4_GET_BLOCKS_CONVERT                        0x0010
#define EXT4_GET_BLOCKS_IO_CREATE_EXT                (EXT4_GET_BLOCKS_PRE_IO|\
                                         EXT4_GET_BLOCKS_CREATE_UNWRIT_EXT)
        /* Convert extent to initialized after IO complete */
#define EXT4_GET_BLOCKS_IO_CONVERT_EXT                (EXT4_GET_BLOCKS_CONVERT|\
                                         EXT4_GET_BLOCKS_CREATE_UNWRIT_EXT)
        /* Eventual metadata allocation (due to growing extent tree)
         * should not fail, so try to use reserved blocks for that.*/
#define EXT4_GET_BLOCKS_METADATA_NOFAIL                0x0020
        /* Don't normalize allocation size (used for fallocate) */
#define EXT4_GET_BLOCKS_NO_NORMALIZE                0x0040
        /* Convert written extents to unwritten */
#define EXT4_GET_BLOCKS_CONVERT_UNWRITTEN        0x0100
        /* Write zeros to newly created written extents */
#define EXT4_GET_BLOCKS_ZERO                        0x0200
#define EXT4_GET_BLOCKS_CREATE_ZERO                (EXT4_GET_BLOCKS_CREATE |\
                                        EXT4_GET_BLOCKS_ZERO)
        /* Caller will submit data before dropping transaction handle. This
         * allows jbd2 to avoid submitting data before commit. */
#define EXT4_GET_BLOCKS_IO_SUBMIT                0x0400

/*
 * The bit position of these flags must not overlap with any of the
 * EXT4_GET_BLOCKS_*.  They are used by ext4_find_extent(),
 * read_extent_tree_block(), ext4_split_extent_at(),
 * ext4_ext_insert_extent(), and ext4_ext_create_new_leaf().
 * EXT4_EX_NOCACHE is used to indicate that the we shouldn't be
 * caching the extents when reading from the extent tree while a
 * truncate or punch hole operation is in progress.
 */
#define EXT4_EX_NOCACHE                                0x40000000
#define EXT4_EX_FORCE_CACHE                        0x20000000
#define EXT4_EX_NOFAIL                                0x10000000

/*
 * Flags used by ext4_free_blocks
 */
#define EXT4_FREE_BLOCKS_METADATA                0x0001
#define EXT4_FREE_BLOCKS_FORGET                        0x0002
#define EXT4_FREE_BLOCKS_VALIDATED                0x0004
#define EXT4_FREE_BLOCKS_NO_QUOT_UPDATE                0x0008
#define EXT4_FREE_BLOCKS_NOFREE_FIRST_CLUSTER        0x0010
#define EXT4_FREE_BLOCKS_NOFREE_LAST_CLUSTER        0x0020
#define EXT4_FREE_BLOCKS_RERESERVE_CLUSTER      0x0040

/*
 * ioctl commands
 */
#define        EXT4_IOC_GETVERSION                _IOR('f', 3, long)
#define        EXT4_IOC_SETVERSION                _IOW('f', 4, long)
#define        EXT4_IOC_GETVERSION_OLD                FS_IOC_GETVERSION
#define        EXT4_IOC_SETVERSION_OLD                FS_IOC_SETVERSION
#define EXT4_IOC_GETRSVSZ                _IOR('f', 5, long)
#define EXT4_IOC_SETRSVSZ                _IOW('f', 6, long)
#define EXT4_IOC_GROUP_EXTEND                _IOW('f', 7, unsigned long)
#define EXT4_IOC_GROUP_ADD                _IOW('f', 8, struct ext4_new_group_input)
#define EXT4_IOC_MIGRATE                _IO('f', 9)
 /* note ioctl 10 reserved for an early version of the FIEMAP ioctl */
 /* note ioctl 11 reserved for filesystem-independent FIEMAP ioctl */
#define EXT4_IOC_ALLOC_DA_BLKS                _IO('f', 12)
#define EXT4_IOC_MOVE_EXT                _IOWR('f', 15, struct move_extent)
#define EXT4_IOC_RESIZE_FS                _IOW('f', 16, __u64)
#define EXT4_IOC_SWAP_BOOT                _IO('f', 17)
#define EXT4_IOC_PRECACHE_EXTENTS        _IO('f', 18)
/* ioctl codes 19--39 are reserved for fscrypt */
#define EXT4_IOC_CLEAR_ES_CACHE                _IO('f', 40)
#define EXT4_IOC_GETSTATE                _IOW('f', 41, __u32)
#define EXT4_IOC_GET_ES_CACHE                _IOWR('f', 42, struct fiemap)
#define EXT4_IOC_CHECKPOINT                _IOW('f', 43, __u32)

#define EXT4_IOC_SHUTDOWN _IOR ('X', 125, __u32)

/*
 * Flags for going down operation
 */
#define EXT4_GOING_FLAGS_DEFAULT                0x0        /* going down */
#define EXT4_GOING_FLAGS_LOGFLUSH                0x1        /* flush log but not data */
#define EXT4_GOING_FLAGS_NOLOGFLUSH                0x2        /* don't flush log nor data */

/*
 * Flags returned by EXT4_IOC_GETSTATE
 *
 * We only expose to userspace a subset of the state flags in
 * i_state_flags
 */
#define EXT4_STATE_FLAG_EXT_PRECACHED        0x00000001
#define EXT4_STATE_FLAG_NEW                0x00000002
#define EXT4_STATE_FLAG_NEWENTRY        0x00000004
#define EXT4_STATE_FLAG_DA_ALLOC_CLOSE        0x00000008

/* flags for ioctl EXT4_IOC_CHECKPOINT */
#define EXT4_IOC_CHECKPOINT_FLAG_DISCARD        0x1
#define EXT4_IOC_CHECKPOINT_FLAG_ZEROOUT        0x2
#define EXT4_IOC_CHECKPOINT_FLAG_DRY_RUN        0x4
#define EXT4_IOC_CHECKPOINT_FLAG_VALID                (EXT4_IOC_CHECKPOINT_FLAG_DISCARD | \
                                                EXT4_IOC_CHECKPOINT_FLAG_ZEROOUT | \
                                                EXT4_IOC_CHECKPOINT_FLAG_DRY_RUN)

#if defined(__KERNEL__) && defined(CONFIG_COMPAT)
/*
 * ioctl commands in 32 bit emulation
 */
#define EXT4_IOC32_GETVERSION                _IOR('f', 3, int)
#define EXT4_IOC32_SETVERSION                _IOW('f', 4, int)
#define EXT4_IOC32_GETRSVSZ                _IOR('f', 5, int)
#define EXT4_IOC32_SETRSVSZ                _IOW('f', 6, int)
#define EXT4_IOC32_GROUP_EXTEND                _IOW('f', 7, unsigned int)
#define EXT4_IOC32_GROUP_ADD                _IOW('f', 8, struct compat_ext4_new_group_input)
#define EXT4_IOC32_GETVERSION_OLD        FS_IOC32_GETVERSION
#define EXT4_IOC32_SETVERSION_OLD        FS_IOC32_SETVERSION
#endif

/*
 * Returned by EXT4_IOC_GET_ES_CACHE as an additional possible flag.
 * It indicates that the entry in extent status cache is for a hole.
 */
#define EXT4_FIEMAP_EXTENT_HOLE                0x08000000

/* Max physical block we can address w/o extents */
#define EXT4_MAX_BLOCK_FILE_PHYS        0xFFFFFFFF

/* Max logical block we can support */
#define EXT4_MAX_LOGICAL_BLOCK                0xFFFFFFFE

/*
 * Structure of an inode on the disk
 */
struct ext4_inode {
        __le16        i_mode;                /* File mode */
        __le16        i_uid;                /* Low 16 bits of Owner Uid */
        __le32        i_size_lo;        /* Size in bytes */
        __le32        i_atime;        /* Access time */
        __le32        i_ctime;        /* Inode Change time */
        __le32        i_mtime;        /* Modification time */
        __le32        i_dtime;        /* Deletion Time */
        __le16        i_gid;                /* Low 16 bits of Group Id */
        __le16        i_links_count;        /* Links count */
        __le32        i_blocks_lo;        /* Blocks count */
        __le32        i_flags;        /* File flags */
        union {
                struct {
                        __le32  l_i_version;
                } linux1;
                struct {
                        __u32  h_i_translator;
                } hurd1;
                struct {
                        __u32  m_i_reserved1;
                } masix1;
        } osd1;                                /* OS dependent 1 */
        __le32        i_block[EXT4_N_BLOCKS];/* Pointers to blocks */
        __le32        i_generation;        /* File version (for NFS) */
        __le32        i_file_acl_lo;        /* File ACL */
        __le32        i_size_high;
        __le32        i_obso_faddr;        /* Obsoleted fragment address */
        union {
                struct {
                        __le16        l_i_blocks_high; /* were l_i_reserved1 */
                        __le16        l_i_file_acl_high;
                        __le16        l_i_uid_high;        /* these 2 fields */
                        __le16        l_i_gid_high;        /* were reserved2[0] */
                        __le16        l_i_checksum_lo;/* crc32c(uuid+inum+inode) LE */
                        __le16        l_i_reserved;
                } linux2;
                struct {
                        __le16        h_i_reserved1;        /* Obsoleted fragment number/size which are removed in ext4 */
                        __u16        h_i_mode_high;
                        __u16        h_i_uid_high;
                        __u16        h_i_gid_high;
                        __u32        h_i_author;
                } hurd2;
                struct {
                        __le16        h_i_reserved1;        /* Obsoleted fragment number/size which are removed in ext4 */
                        __le16        m_i_file_acl_high;
                        __u32        m_i_reserved2[2];
                } masix2;
        } osd2;                                /* OS dependent 2 */
        __le16        i_extra_isize;
        __le16        i_checksum_hi;        /* crc32c(uuid+inum+inode) BE */
        __le32  i_ctime_extra;  /* extra Change time      (nsec << 2 | epoch) */
        __le32  i_mtime_extra;  /* extra Modification time(nsec << 2 | epoch) */
        __le32  i_atime_extra;  /* extra Access time      (nsec << 2 | epoch) */
        __le32  i_crtime;       /* File Creation time */
        __le32  i_crtime_extra; /* extra FileCreationtime (nsec << 2 | epoch) */
        __le32  i_version_hi;        /* high 32 bits for 64-bit version */
        __le32        i_projid;        /* Project ID */
};

struct move_extent {
        __u32 reserved;                /* should be zero */
        __u32 donor_fd;                /* donor file descriptor */
        __u64 orig_start;        /* logical start offset in block for orig */
        __u64 donor_start;        /* logical start offset in block for donor */
        __u64 len;                /* block length to be moved */
        __u64 moved_len;        /* moved block length */
};

#define EXT4_EPOCH_BITS 2
#define EXT4_EPOCH_MASK ((1 << EXT4_EPOCH_BITS) - 1)
#define EXT4_NSEC_MASK  (~0UL << EXT4_EPOCH_BITS)

/*
 * Extended fields will fit into an inode if the filesystem was formatted
 * with large inodes (-I 256 or larger) and there are not currently any EAs
 * consuming all of the available space. For new inodes we always reserve
 * enough space for the kernel's known extended fields, but for inodes
 * created with an old kernel this might not have been the case. None of
 * the extended inode fields is critical for correct filesystem operation.
 * This macro checks if a certain field fits in the inode. Note that
 * inode-size = GOOD_OLD_INODE_SIZE + i_extra_isize
 */
#define EXT4_FITS_IN_INODE(ext4_inode, einode, field)        \
        ((offsetof(typeof(*ext4_inode), field) +        \
          sizeof((ext4_inode)->field))                        \
        <= (EXT4_GOOD_OLD_INODE_SIZE +                        \
            (einode)->i_extra_isize))                        \

/*
 * We use an encoding that preserves the times for extra epoch "00":
 *
 * extra  msb of                         adjust for signed
 * epoch  32-bit                         32-bit tv_sec to
 * bits   time    decoded 64-bit tv_sec  64-bit tv_sec      valid time range
 * 0 0    1    -0x80000000..-0x00000001  0x000000000 1901-12-13..1969-12-31
 * 0 0    0    0x000000000..0x07fffffff  0x000000000 1970-01-01..2038-01-19
 * 0 1    1    0x080000000..0x0ffffffff  0x100000000 2038-01-19..2106-02-07
 * 0 1    0    0x100000000..0x17fffffff  0x100000000 2106-02-07..2174-02-25
 * 1 0    1    0x180000000..0x1ffffffff  0x200000000 2174-02-25..2242-03-16
 * 1 0    0    0x200000000..0x27fffffff  0x200000000 2242-03-16..2310-04-04
 * 1 1    1    0x280000000..0x2ffffffff  0x300000000 2310-04-04..2378-04-22
 * 1 1    0    0x300000000..0x37fffffff  0x300000000 2378-04-22..2446-05-10
 *
 * Note that previous versions of the kernel on 64-bit systems would
 * incorrectly use extra epoch bits 1,1 for dates between 1901 and
 * 1970.  e2fsck will correct this, assuming that it is run on the
 * affected filesystem before 2242.
 */

static inline __le32 ext4_encode_extra_time(struct timespec64 *time)
{
        u32 extra =((time->tv_sec - (s32)time->tv_sec) >> 32) & EXT4_EPOCH_MASK;
        return cpu_to_le32(extra | (time->tv_nsec << EXT4_EPOCH_BITS));
}

static inline void ext4_decode_extra_time(struct timespec64 *time,
                                          __le32 extra)
{
        if (unlikely(extra & cpu_to_le32(EXT4_EPOCH_MASK)))
                time->tv_sec += (u64)(le32_to_cpu(extra) & EXT4_EPOCH_MASK) << 32;
        time->tv_nsec = (le32_to_cpu(extra) & EXT4_NSEC_MASK) >> EXT4_EPOCH_BITS;
}

#define EXT4_INODE_SET_XTIME(xtime, inode, raw_inode)                                \
do {                                                                                \
        if (EXT4_FITS_IN_INODE(raw_inode, EXT4_I(inode), xtime ## _extra))     {\
                (raw_inode)->xtime = cpu_to_le32((inode)->xtime.tv_sec);        \
                (raw_inode)->xtime ## _extra =                                        \
                                ext4_encode_extra_time(&(inode)->xtime);        \
                }                                                                \
        else        \
                (raw_inode)->xtime = cpu_to_le32(clamp_t(int32_t, (inode)->xtime.tv_sec, S32_MIN, S32_MAX));        \
} while (0)

#define EXT4_EINODE_SET_XTIME(xtime, einode, raw_inode)                               \
do {                                                                               \
        if (EXT4_FITS_IN_INODE(raw_inode, einode, xtime))                       \
                (raw_inode)->xtime = cpu_to_le32((einode)->xtime.tv_sec);      \
        if (EXT4_FITS_IN_INODE(raw_inode, einode, xtime ## _extra))               \
                (raw_inode)->xtime ## _extra =                                       \
                                ext4_encode_extra_time(&(einode)->xtime);      \
} while (0)

#define EXT4_INODE_GET_XTIME(xtime, inode, raw_inode)                                \
do {                                                                                \
        (inode)->xtime.tv_sec = (signed)le32_to_cpu((raw_inode)->xtime);        \
        if (EXT4_FITS_IN_INODE(raw_inode, EXT4_I(inode), xtime ## _extra)) {        \
                ext4_decode_extra_time(&(inode)->xtime,                                \
                                       raw_inode->xtime ## _extra);                \
                }                                                                \
        else                                                                        \
                (inode)->xtime.tv_nsec = 0;                                        \
} while (0)


#define EXT4_EINODE_GET_XTIME(xtime, einode, raw_inode)                               \
do {                                                                               \
        if (EXT4_FITS_IN_INODE(raw_inode, einode, xtime))                       \
                (einode)->xtime.tv_sec =                                        \
                        (signed)le32_to_cpu((raw_inode)->xtime);               \
        else                                                                       \
                (einode)->xtime.tv_sec = 0;                                       \
        if (EXT4_FITS_IN_INODE(raw_inode, einode, xtime ## _extra))               \
                ext4_decode_extra_time(&(einode)->xtime,                       \
                                       raw_inode->xtime ## _extra);               \
        else                                                                       \
                (einode)->xtime.tv_nsec = 0;                                       \
} while (0)

#define i_disk_version osd1.linux1.l_i_version

#if defined(__KERNEL__) || defined(__linux__)
#define i_reserved1        osd1.linux1.l_i_reserved1
#define i_file_acl_high        osd2.linux2.l_i_file_acl_high
#define i_blocks_high        osd2.linux2.l_i_blocks_high
#define i_uid_low        i_uid
#define i_gid_low        i_gid
#define i_uid_high        osd2.linux2.l_i_uid_high
#define i_gid_high        osd2.linux2.l_i_gid_high
#define i_checksum_lo        osd2.linux2.l_i_checksum_lo

#elif defined(__GNU__)

#define i_translator        osd1.hurd1.h_i_translator
#define i_uid_high        osd2.hurd2.h_i_uid_high
#define i_gid_high        osd2.hurd2.h_i_gid_high
#define i_author        osd2.hurd2.h_i_author

#elif defined(__masix__)

#define i_reserved1        osd1.masix1.m_i_reserved1
#define i_file_acl_high        osd2.masix2.m_i_file_acl_high
#define i_reserved2        osd2.masix2.m_i_reserved2

#endif /* defined(__KERNEL__) || defined(__linux__) */

#include "extents_status.h"
#include "fast_commit.h"

/*
 * Lock subclasses for i_data_sem in the ext4_inode_info structure.
 *
 * These are needed to avoid lockdep false positives when we need to
 * allocate blocks to the quota inode during ext4_map_blocks(), while
 * holding i_data_sem for a normal (non-quota) inode.  Since we don't
 * do quota tracking for the quota inode, this avoids deadlock (as
 * well as infinite recursion, since it isn't turtles all the way
 * down...)
 *
 *  I_DATA_SEM_NORMAL - Used for most inodes
 *  I_DATA_SEM_OTHER  - Used by move_inode.c for the second normal inode
 *                          where the second inode has larger inode number
 *                          than the first
 *  I_DATA_SEM_QUOTA  - Used for quota inodes only
 *  I_DATA_SEM_EA     - Used for ea_inodes only
 */
enum {
        I_DATA_SEM_NORMAL = 0,
        I_DATA_SEM_OTHER,
        I_DATA_SEM_QUOTA,
        I_DATA_SEM_EA
};


/*
 * fourth extended file system inode data in memory
 */
struct ext4_inode_info {
        __le32        i_data[15];        /* unconverted */
        __u32        i_dtime;
        ext4_fsblk_t        i_file_acl;

        /*
         * i_block_group is the number of the block group which contains
         * this file's inode.  Constant across the lifetime of the inode,
         * it is used for making block allocation decisions - we try to
         * place a file's data blocks near its inode block, and new inodes
         * near to their parent directory's inode.
         */
        ext4_group_t        i_block_group;
        ext4_lblk_t        i_dir_start_lookup;
#if (BITS_PER_LONG < 64)
        unsigned long        i_state_flags;                /* Dynamic state flags */
#endif
        unsigned long        i_flags;

        /*
         * Extended attributes can be read independently of the main file
         * data. Taking i_mutex even when reading would cause contention
         * between readers of EAs and writers of regular file data, so
         * instead we synchronize on xattr_sem when reading or changing
         * EAs.
         */
        struct rw_semaphore xattr_sem;

        /*
         * Inodes with EXT4_STATE_ORPHAN_FILE use i_orphan_idx. Otherwise
         * i_orphan is used.
         */
        union {
                struct list_head i_orphan;        /* unlinked but open inodes */
                unsigned int i_orphan_idx;        /* Index in orphan file */
        };

        /* Fast commit related info */

        struct list_head i_fc_list;        /*
                                         * inodes that need fast commit
                                         * protected by sbi->s_fc_lock.
                                         */

        /* Start of lblk range that needs to be committed in this fast commit */
        ext4_lblk_t i_fc_lblk_start;

        /* End of lblk range that needs to be committed in this fast commit */
        ext4_lblk_t i_fc_lblk_len;

        /* Number of ongoing updates on this inode */
        atomic_t  i_fc_updates;

        /* Fast commit wait queue for this inode */
        wait_queue_head_t i_fc_wait;

        /* Protect concurrent accesses on i_fc_lblk_start, i_fc_lblk_len */
        struct mutex i_fc_lock;

        /*
         * i_disksize keeps track of what the inode size is ON DISK, not
         * in memory.  During truncate, i_size is set to the new size by
         * the VFS prior to calling ext4_truncate(), but the filesystem won't
         * set i_disksize to 0 until the truncate is actually under way.
         *
         * The intent is that i_disksize always represents the blocks which
         * are used by this file.  This allows recovery to restart truncate
         * on orphans if we crash during truncate.  We actually write i_disksize
         * into the on-disk inode when writing inodes out, instead of i_size.
         *
         * The only time when i_disksize and i_size may be different is when
         * a truncate is in progress.  The only things which change i_disksize
         * are ext4_get_block (growth) and ext4_truncate (shrinkth).
         */
        loff_t        i_disksize;

        /*
         * i_data_sem is for serialising ext4_truncate() against
         * ext4_getblock().  In the 2.4 ext2 design, great chunks of inode's
         * data tree are chopped off during truncate. We can't do that in
         * ext4 because whenever we perform intermediate commits during
         * truncate, the inode and all the metadata blocks *must* be in a
         * consistent state which allows truncation of the orphans to restart
         * during recovery.  Hence we must fix the get_block-vs-truncate race
         * by other means, so we have i_data_sem.
         */
        struct rw_semaphore i_data_sem;
        struct inode vfs_inode;
        struct jbd2_inode *jinode;

        spinlock_t i_raw_lock;        /* protects updates to the raw inode */

        /*
         * File creation time. Its function is same as that of
         * struct timespec64 i_{a,c,m}time in the generic inode.
         */
        struct timespec64 i_crtime;

        /* mballoc */
        atomic_t i_prealloc_active;
        struct list_head i_prealloc_list;
        spinlock_t i_prealloc_lock;

        /* extents status tree */
        struct ext4_es_tree i_es_tree;
        rwlock_t i_es_lock;
        struct list_head i_es_list;
        unsigned int i_es_all_nr;        /* protected by i_es_lock */
        unsigned int i_es_shk_nr;        /* protected by i_es_lock */
        ext4_lblk_t i_es_shrink_lblk;        /* Offset where we start searching for
                                           extents to shrink. Protected by
                                           i_es_lock  */

        /* ialloc */
        ext4_group_t        i_last_alloc_group;

        /* allocation reservation info for delalloc */
        /* In case of bigalloc, this refer to clusters rather than blocks */
        unsigned int i_reserved_data_blocks;

        /* pending cluster reservations for bigalloc file systems */
        struct ext4_pending_tree i_pending_tree;

        /* on-disk additional length */
        __u16 i_extra_isize;

        /* Indicate the inline data space. */
        u16 i_inline_off;
        u16 i_inline_size;

#ifdef CONFIG_QUOTA
        /* quota space reservation, managed internally by quota code */
        qsize_t i_reserved_quota;
#endif

        /* Lock protecting lists below */
        spinlock_t i_completed_io_lock;
        /*
         * Completed IOs that need unwritten extents handling and have
         * transaction reserved
         */
        struct list_head i_rsv_conversion_list;
        struct work_struct i_rsv_conversion_work;
        atomic_t i_unwritten; /* Nr. of inflight conversions pending */

        spinlock_t i_block_reservation_lock;

        /*
         * Transactions that contain inode's metadata needed to complete
         * fsync and fdatasync, respectively.
         */
        tid_t i_sync_tid;
        tid_t i_datasync_tid;

#ifdef CONFIG_QUOTA
        struct dquot *i_dquot[MAXQUOTAS];
#endif

        /* Precomputed uuid+inum+igen checksum for seeding inode checksums */
        __u32 i_csum_seed;

        kprojid_t i_projid;
};

/*
 * File system states
 */
#define        EXT4_VALID_FS                        0x0001        /* Unmounted cleanly */
#define        EXT4_ERROR_FS                        0x0002        /* Errors detected */
#define        EXT4_ORPHAN_FS                        0x0004        /* Orphans being recovered */
#define EXT4_FC_REPLAY                        0x0020        /* Fast commit replay ongoing */

/*
 * Misc. filesystem flags
 */
#define EXT2_FLAGS_SIGNED_HASH                0x0001  /* Signed dirhash in use */
#define EXT2_FLAGS_UNSIGNED_HASH        0x0002  /* Unsigned dirhash in use */
#define EXT2_FLAGS_TEST_FILESYS                0x0004        /* to test development code */

/*
 * Mount flags set via mount options or defaults
 */
#define EXT4_MOUNT_NO_MBCACHE                0x00001 /* Do not use mbcache */
#define EXT4_MOUNT_GRPID                0x00004        /* Create files with directory's group */
#define EXT4_MOUNT_DEBUG                0x00008        /* Some debugging messages */
#define EXT4_MOUNT_ERRORS_CONT                0x00010        /* Continue on errors */
#define EXT4_MOUNT_ERRORS_RO                0x00020        /* Remount fs ro on errors */
#define EXT4_MOUNT_ERRORS_PANIC                0x00040        /* Panic on errors */
#define EXT4_MOUNT_ERRORS_MASK                0x00070
#define EXT4_MOUNT_MINIX_DF                0x00080        /* Mimics the Minix statfs */
#define EXT4_MOUNT_NOLOAD                0x00100        /* Don't use existing journal*/
#ifdef CONFIG_FS_DAX
#define EXT4_MOUNT_DAX_ALWAYS                0x00200        /* Direct Access */
#else
#define EXT4_MOUNT_DAX_ALWAYS                0
#endif
#define EXT4_MOUNT_DATA_FLAGS                0x00C00        /* Mode for data writes: */
#define EXT4_MOUNT_JOURNAL_DATA                0x00400        /* Write data to journal */
#define EXT4_MOUNT_ORDERED_DATA                0x00800        /* Flush data before commit */
#define EXT4_MOUNT_WRITEBACK_DATA        0x00C00        /* No data ordering */
#define EXT4_MOUNT_UPDATE_JOURNAL        0x01000        /* Update the journal format */
#define EXT4_MOUNT_NO_UID32                0x02000  /* Disable 32-bit UIDs */
#define EXT4_MOUNT_XATTR_USER                0x04000        /* Extended user attributes */
#define EXT4_MOUNT_POSIX_ACL                0x08000        /* POSIX Access Control Lists */
#define EXT4_MOUNT_NO_AUTO_DA_ALLOC        0x10000        /* No auto delalloc mapping */
#define EXT4_MOUNT_BARRIER                0x20000 /* Use block barriers */
#define EXT4_MOUNT_QUOTA                0x40000 /* Some quota option set */
#define EXT4_MOUNT_USRQUOTA                0x80000 /* "old" user quota,
                                                 * enable enforcement for hidden
                                                 * quota files */
#define EXT4_MOUNT_GRPQUOTA                0x100000 /* "old" group quota, enable
                                                  * enforcement for hidden quota
                                                  * files */
#define EXT4_MOUNT_PRJQUOTA                0x200000 /* Enable project quota
                                                  * enforcement */
#define EXT4_MOUNT_DIOREAD_NOLOCK        0x400000 /* Enable support for dio read nolocking */
#define EXT4_MOUNT_JOURNAL_CHECKSUM        0x800000 /* Journal checksums */
#define EXT4_MOUNT_JOURNAL_ASYNC_COMMIT        0x1000000 /* Journal Async Commit */
#define EXT4_MOUNT_WARN_ON_ERROR        0x2000000 /* Trigger WARN_ON on error */
#define EXT4_MOUNT_NO_PREFETCH_BLOCK_BITMAPS 0x4000000
#define EXT4_MOUNT_DELALLOC                0x8000000 /* Delalloc support */
#define EXT4_MOUNT_DATA_ERR_ABORT        0x10000000 /* Abort on file data write */
#define EXT4_MOUNT_BLOCK_VALIDITY        0x20000000 /* Block validity checking */
#define EXT4_MOUNT_DISCARD                0x40000000 /* Issue DISCARD requests */
#define EXT4_MOUNT_INIT_INODE_TABLE        0x80000000 /* Initialize uninitialized itables */

/*
 * Mount flags set either automatically (could not be set by mount option)
 * based on per file system feature or property or in special cases such as
 * distinguishing between explicit mount option definition and default.
 */
#define EXT4_MOUNT2_EXPLICIT_DELALLOC        0x00000001 /* User explicitly
                                                      specified delalloc */
#define EXT4_MOUNT2_STD_GROUP_SIZE        0x00000002 /* We have standard group
                                                      size of blocksize * 8
                                                      blocks */
#define EXT4_MOUNT2_HURD_COMPAT                0x00000004 /* Support HURD-castrated
                                                      file systems */
#define EXT4_MOUNT2_EXPLICIT_JOURNAL_CHECKSUM        0x00000008 /* User explicitly
                                                specified journal checksum */

#define EXT4_MOUNT2_JOURNAL_FAST_COMMIT        0x00000010 /* Journal fast commit */
#define EXT4_MOUNT2_DAX_NEVER                0x00000020 /* Do not allow Direct Access */
#define EXT4_MOUNT2_DAX_INODE                0x00000040 /* For printing options only */
#define EXT4_MOUNT2_MB_OPTIMIZE_SCAN        0x00000080 /* Optimize group
                                                    * scanning in mballoc
                                                    */

#define clear_opt(sb, opt)                EXT4_SB(sb)->s_mount_opt &= \
                                                ~EXT4_MOUNT_##opt
#define set_opt(sb, opt)                EXT4_SB(sb)->s_mount_opt |= \
                                                EXT4_MOUNT_##opt
#define test_opt(sb, opt)                (EXT4_SB(sb)->s_mount_opt & \
                                         EXT4_MOUNT_##opt)

#define clear_opt2(sb, opt)                EXT4_SB(sb)->s_mount_opt2 &= \
                                                ~EXT4_MOUNT2_##opt
#define set_opt2(sb, opt)                EXT4_SB(sb)->s_mount_opt2 |= \
                                                EXT4_MOUNT2_##opt
#define test_opt2(sb, opt)                (EXT4_SB(sb)->s_mount_opt2 & \
                                         EXT4_MOUNT2_##opt)

#define ext4_test_and_set_bit                __test_and_set_bit_le
#define ext4_set_bit                        __set_bit_le
#define ext4_set_bit_atomic                ext2_set_bit_atomic
#define ext4_test_and_clear_bit                __test_and_clear_bit_le
#define ext4_clear_bit                        __clear_bit_le
#define ext4_clear_bit_atomic                ext2_clear_bit_atomic
#define ext4_test_bit                        test_bit_le
#define ext4_find_next_zero_bit                find_next_zero_bit_le
#define ext4_find_next_bit                find_next_bit_le

extern void ext4_set_bits(void *bm, int cur, int len);

/*
 * Maximal mount counts between two filesystem checks
 */
#define EXT4_DFL_MAX_MNT_COUNT                20        /* Allow 20 mounts */
#define EXT4_DFL_CHECKINTERVAL                0        /* Don't use interval check */

/*
 * Behaviour when detecting errors
 */
#define EXT4_ERRORS_CONTINUE                1        /* Continue execution */
#define EXT4_ERRORS_RO                        2        /* Remount fs read-only */
#define EXT4_ERRORS_PANIC                3        /* Panic */
#define EXT4_ERRORS_DEFAULT                EXT4_ERRORS_CONTINUE

/* Metadata checksum algorithm codes */
#define EXT4_CRC32C_CHKSUM                1

/*
 * Structure of the super block
 */
struct ext4_super_block {
/*00*/        __le32        s_inodes_count;                /* Inodes count */
        __le32        s_blocks_count_lo;        /* Blocks count */
        __le32        s_r_blocks_count_lo;        /* Reserved blocks count */
        __le32        s_free_blocks_count_lo;        /* Free blocks count */
/*10*/        __le32        s_free_inodes_count;        /* Free inodes count */
        __le32        s_first_data_block;        /* First Data Block */
        __le32        s_log_block_size;        /* Block size */
        __le32        s_log_cluster_size;        /* Allocation cluster size */
/*20*/        __le32        s_blocks_per_group;        /* # Blocks per group */
        __le32        s_clusters_per_group;        /* # Clusters per group */
        __le32        s_inodes_per_group;        /* # Inodes per group */
        __le32        s_mtime;                /* Mount time */
/*30*/        __le32        s_wtime;                /* Write time */
        __le16        s_mnt_count;                /* Mount count */
        __le16        s_max_mnt_count;        /* Maximal mount count */
        __le16        s_magic;                /* Magic signature */
        __le16        s_state;                /* File system state */
        __le16        s_errors;                /* Behaviour when detecting errors */
        __le16        s_minor_rev_level;        /* minor revision level */
/*40*/        __le32        s_lastcheck;                /* time of last check */
        __le32        s_checkinterval;        /* max. time between checks */
        __le32        s_creator_os;                /* OS */
        __le32        s_rev_level;                /* Revision level */
/*50*/        __le16        s_def_resuid;                /* Default uid for reserved blocks */
        __le16        s_def_resgid;                /* Default gid for reserved blocks */
        /*
         * These fields are for EXT4_DYNAMIC_REV superblocks only.
         *
         * Note: the difference between the compatible feature set and
         * the incompatible feature set is that if there is a bit set
         * in the incompatible feature set that the kernel doesn't
         * know about, it should refuse to mount the filesystem.
         *
         * e2fsck's requirements are more strict; if it doesn't know
         * about a feature in either the compatible or incompatible
         * feature set, it must abort and not try to meddle with
         * things it doesn't understand...
         */
        __le32        s_first_ino;                /* First non-reserved inode */
        __le16  s_inode_size;                /* size of inode structure */
        __le16        s_block_group_nr;        /* block group # of this superblock */
        __le32        s_feature_compat;        /* compatible feature set */
/*60*/        __le32        s_feature_incompat;        /* incompatible feature set */
        __le32        s_feature_ro_compat;        /* readonly-compatible feature set */
/*68*/        __u8        s_uuid[16];                /* 128-bit uuid for volume */
/*78*/        char        s_volume_name[16];        /* volume name */
/*88*/        char        s_last_mounted[64] __nonstring;        /* directory where last mounted */
/*C8*/        __le32        s_algorithm_usage_bitmap; /* For compression */
        /*
         * Performance hints.  Directory preallocation should only
         * happen if the EXT4_FEATURE_COMPAT_DIR_PREALLOC flag is on.
         */
        __u8        s_prealloc_blocks;        /* Nr of blocks to try to preallocate*/
        __u8        s_prealloc_dir_blocks;        /* Nr to preallocate for dirs */
        __le16        s_reserved_gdt_blocks;        /* Per group desc for online growth */
        /*
         * Journaling support valid if EXT4_FEATURE_COMPAT_HAS_JOURNAL set.
         */
/*D0*/        __u8        s_journal_uuid[16];        /* uuid of journal superblock */
/*E0*/        __le32        s_journal_inum;                /* inode number of journal file */
        __le32        s_journal_dev;                /* device number of journal file */
        __le32        s_last_orphan;                /* start of list of inodes to delete */
        __le32        s_hash_seed[4];                /* HTREE hash seed */
        __u8        s_def_hash_version;        /* Default hash version to use */
        __u8        s_jnl_backup_type;
        __le16  s_desc_size;                /* size of group descriptor */
/*100*/        __le32        s_default_mount_opts;
        __le32        s_first_meta_bg;        /* First metablock block group */
        __le32        s_mkfs_time;                /* When the filesystem was created */
        __le32        s_jnl_blocks[17];        /* Backup of the journal inode */
        /* 64bit support valid if EXT4_FEATURE_COMPAT_64BIT */
/*150*/        __le32        s_blocks_count_hi;        /* Blocks count */
        __le32        s_r_blocks_count_hi;        /* Reserved blocks count */
        __le32        s_free_blocks_count_hi;        /* Free blocks count */
        __le16        s_min_extra_isize;        /* All inodes have at least # bytes */
        __le16        s_want_extra_isize;         /* New inodes should reserve # bytes */
        __le32        s_flags;                /* Miscellaneous flags */
        __le16  s_raid_stride;                /* RAID stride */
        __le16  s_mmp_update_interval;  /* # seconds to wait in MMP checking */
        __le64  s_mmp_block;            /* Block for multi-mount protection */
        __le32  s_raid_stripe_width;    /* blocks on all data disks (N*stride)*/
        __u8        s_log_groups_per_flex;  /* FLEX_BG group size */
        __u8        s_checksum_type;        /* metadata checksum algorithm used */
        __u8        s_encryption_level;        /* versioning level for encryption */
        __u8        s_reserved_pad;                /* Padding to next 32bits */
        __le64        s_kbytes_written;        /* nr of lifetime kilobytes written */
        __le32        s_snapshot_inum;        /* Inode number of active snapshot */
        __le32        s_snapshot_id;                /* sequential ID of active snapshot */
        __le64        s_snapshot_r_blocks_count; /* reserved blocks for active
                                              snapshot's future use */
        __le32        s_snapshot_list;        /* inode number of the head of the
                                           on-disk snapshot list */
#define EXT4_S_ERR_START offsetof(struct ext4_super_block, s_error_count)
        __le32        s_error_count;                /* number of fs errors */
        __le32        s_first_error_time;        /* first time an error happened */
        __le32        s_first_error_ino;        /* inode involved in first error */
        __le64        s_first_error_block;        /* block involved of first error */
        __u8        s_first_error_func[32] __nonstring;        /* function where the error happened */
        __le32        s_first_error_line;        /* line number where error happened */
        __le32        s_last_error_time;        /* most recent time of an error */
        __le32        s_last_error_ino;        /* inode involved in last error */
        __le32        s_last_error_line;        /* line number where error happened */
        __le64        s_last_error_block;        /* block involved of last error */
        __u8        s_last_error_func[32] __nonstring;        /* function where the error happened */
#define EXT4_S_ERR_END offsetof(struct ext4_super_block, s_mount_opts)
        __u8        s_mount_opts[64];
        __le32        s_usr_quota_inum;        /* inode for tracking user quota */
        __le32        s_grp_quota_inum;        /* inode for tracking group quota */
        __le32        s_overhead_clusters;        /* overhead blocks/clusters in fs */
        __le32        s_backup_bgs[2];        /* groups with sparse_super2 SBs */
        __u8        s_encrypt_algos[4];        /* Encryption algorithms in use  */
        __u8        s_encrypt_pw_salt[16];        /* Salt used for string2key algorithm */
        __le32        s_lpf_ino;                /* Location of the lost+found inode */
        __le32        s_prj_quota_inum;        /* inode for tracking project quota */
        __le32        s_checksum_seed;        /* crc32c(uuid) if csum_seed set */
        __u8        s_wtime_hi;
        __u8        s_mtime_hi;
        __u8        s_mkfs_time_hi;
        __u8        s_lastcheck_hi;
        __u8        s_first_error_time_hi;
        __u8        s_last_error_time_hi;
        __u8        s_first_error_errcode;
        __u8    s_last_error_errcode;
        __le16  s_encoding;                /* Filename charset encoding */
        __le16  s_encoding_flags;        /* Filename charset encoding flags */
        __le32  s_orphan_file_inum;        /* Inode for tracking orphan inodes */
        __le32        s_reserved[94];                /* Padding to the end of the block */
        __le32        s_checksum;                /* crc32c(superblock) */
};

#define EXT4_S_ERR_LEN (EXT4_S_ERR_END - EXT4_S_ERR_START)

#ifdef __KERNEL__

/* Number of quota types we support */
#define EXT4_MAXQUOTAS 3

#define EXT4_ENC_UTF8_12_1        1

/* Types of ext4 journal triggers */
enum ext4_journal_trigger_type {
        EXT4_JTR_ORPHAN_FILE,
        EXT4_JTR_NONE        /* This must be the last entry for indexing to work! */
};

#define EXT4_JOURNAL_TRIGGER_COUNT EXT4_JTR_NONE

struct ext4_journal_trigger {
        struct jbd2_buffer_trigger_type tr_triggers;
        struct super_block *sb;
};

static inline struct ext4_journal_trigger *EXT4_TRIGGER(
                                struct jbd2_buffer_trigger_type *trigger)
{
        return container_of(trigger, struct ext4_journal_trigger, tr_triggers);
}

#define EXT4_ORPHAN_BLOCK_MAGIC 0x0b10ca04

/* Structure at the tail of orphan block */
struct ext4_orphan_block_tail {
        __le32 ob_magic;
        __le32 ob_checksum;
};

static inline int ext4_inodes_per_orphan_block(struct super_block *sb)
{
        return (sb->s_blocksize - sizeof(struct ext4_orphan_block_tail)) /
                        sizeof(u32);
}

struct ext4_orphan_block {
        atomic_t ob_free_entries;        /* Number of free orphan entries in block */
        struct buffer_head *ob_bh;        /* Buffer for orphan block */
};

/*
 * Info about orphan file.
 */
struct ext4_orphan_info {
        int of_blocks;                        /* Number of orphan blocks in a file */
        __u32 of_csum_seed;                /* Checksum seed for orphan file */
        struct ext4_orphan_block *of_binfo;        /* Array with info about orphan
                                                 * file blocks */
};

/*
 * fourth extended-fs super-block data in memory
 */
struct ext4_sb_info {
        unsigned long s_desc_size;        /* Size of a group descriptor in bytes */
        unsigned long s_inodes_per_block;/* Number of inodes per block */
        unsigned long s_blocks_per_group;/* Number of blocks in a group */
        unsigned long s_clusters_per_group; /* Number of clusters in a group */
        unsigned long s_inodes_per_group;/* Number of inodes in a group */
        unsigned long s_itb_per_group;        /* Number of inode table blocks per group */
        unsigned long s_gdb_count;        /* Number of group descriptor blocks */
        unsigned long s_desc_per_block;        /* Number of group descriptors per block */
        ext4_group_t s_groups_count;        /* Number of groups in the fs */
        ext4_group_t s_blockfile_groups;/* Groups acceptable for non-extent files */
        unsigned long s_overhead;  /* # of fs overhead clusters */
        unsigned int s_cluster_ratio;        /* Number of blocks per cluster */
        unsigned int s_cluster_bits;        /* log2 of s_cluster_ratio */
        loff_t s_bitmap_maxbytes;        /* max bytes for bitmap files */
        struct buffer_head * s_sbh;        /* Buffer containing the super block */
        struct ext4_super_block *s_es;        /* Pointer to the super block in the buffer */
        struct buffer_head * __rcu *s_group_desc;
        unsigned int s_mount_opt;
        unsigned int s_mount_opt2;
        unsigned long s_mount_flags;
        unsigned int s_def_mount_opt;
        ext4_fsblk_t s_sb_block;
        atomic64_t s_resv_clusters;
        kuid_t s_resuid;
        kgid_t s_resgid;
        unsigned short s_mount_state;
        unsigned short s_pad;
        int s_addr_per_block_bits;
        int s_desc_per_block_bits;
        int s_inode_size;
        int s_first_ino;
        unsigned int s_inode_readahead_blks;
        unsigned int s_inode_goal;
        u32 s_hash_seed[4];
        int s_def_hash_version;
        int s_hash_unsigned;        /* 3 if hash should be unsigned, 0 if not */
        struct percpu_counter s_freeclusters_counter;
        struct percpu_counter s_freeinodes_counter;
        struct percpu_counter s_dirs_counter;
        struct percpu_counter s_dirtyclusters_counter;
        struct percpu_counter s_sra_exceeded_retry_limit;
        struct blockgroup_lock *s_blockgroup_lock;
        struct proc_dir_entry *s_proc;
        struct kobject s_kobj;
        struct completion s_kobj_unregister;
        struct super_block *s_sb;
        struct buffer_head *s_mmp_bh;

        /* Journaling */
        struct journal_s *s_journal;
        unsigned long s_ext4_flags;                /* Ext4 superblock flags */
        struct mutex s_orphan_lock;        /* Protects on disk list changes */
        struct list_head s_orphan;        /* List of orphaned inodes in on disk
                                           list */
        struct ext4_orphan_info s_orphan_info;
        unsigned long s_commit_interval;
        u32 s_max_batch_time;
        u32 s_min_batch_time;
        struct block_device *s_journal_bdev;
#ifdef CONFIG_QUOTA
        /* Names of quota files with journalled quota */
        char __rcu *s_qf_names[EXT4_MAXQUOTAS];
        int s_jquota_fmt;                        /* Format of quota to use */
#endif
        unsigned int s_want_extra_isize; /* New inodes should reserve # bytes */
        struct ext4_system_blocks __rcu *s_system_blks;

#ifdef EXTENTS_STATS
        /* ext4 extents stats */
        unsigned long s_ext_min;
        unsigned long s_ext_max;
        unsigned long s_depth_max;
        spinlock_t s_ext_stats_lock;
        unsigned long s_ext_blocks;
        unsigned long s_ext_extents;
#endif

        /* for buddy allocator */
        struct ext4_group_info ** __rcu *s_group_info;
        struct inode *s_buddy_cache;
        spinlock_t s_md_lock;
        unsigned short *s_mb_offsets;
        unsigned int *s_mb_maxs;
        unsigned int s_group_info_size;
        unsigned int s_mb_free_pending;
        struct list_head s_freed_data_list;        /* List of blocks to be freed
                                                   after commit completed */
        struct list_head s_discard_list;
        struct work_struct s_discard_work;
        atomic_t s_retry_alloc_pending;
        struct rb_root s_mb_avg_fragment_size_root;
        rwlock_t s_mb_rb_lock;
        struct list_head *s_mb_largest_free_orders;
        rwlock_t *s_mb_largest_free_orders_locks;

        /* tunables */
        unsigned long s_stripe;
        unsigned int s_mb_max_linear_groups;
        unsigned int s_mb_stream_request;
        unsigned int s_mb_max_to_scan;
        unsigned int s_mb_min_to_scan;
        unsigned int s_mb_stats;
        unsigned int s_mb_order2_reqs;
        unsigned int s_mb_group_prealloc;
        unsigned int s_mb_max_inode_prealloc;
        unsigned int s_max_dir_size_kb;
        /* where last allocation was done - for stream allocation */
        unsigned long s_mb_last_group;
        unsigned long s_mb_last_start;
        unsigned int s_mb_prefetch;
        unsigned int s_mb_prefetch_limit;

        /* stats for buddy allocator */
        atomic_t s_bal_reqs;        /* number of reqs with len > 1 */
        atomic_t s_bal_success;        /* we found long enough chunks */
        atomic_t s_bal_allocated;        /* in blocks */
        atomic_t s_bal_ex_scanned;        /* total extents scanned */
        atomic_t s_bal_groups_scanned;        /* number of groups scanned */
        atomic_t s_bal_goals;        /* goal hits */
        atomic_t s_bal_breaks;        /* too long searches */
        atomic_t s_bal_2orders;        /* 2^order hits */
        atomic_t s_bal_cr0_bad_suggestions;
        atomic_t s_bal_cr1_bad_suggestions;
        atomic64_t s_bal_cX_groups_considered[4];
        atomic64_t s_bal_cX_hits[4];
        atomic64_t s_bal_cX_failed[4];                /* cX loop didn't find blocks */
        atomic_t s_mb_buddies_generated;        /* number of buddies generated */
        atomic64_t s_mb_generation_time;
        atomic_t s_mb_lost_chunks;
        atomic_t s_mb_preallocated;
        atomic_t s_mb_discarded;
        atomic_t s_lock_busy;

        /* locality groups */
        struct ext4_locality_group __percpu *s_locality_groups;

        /* for write statistics */
        unsigned long s_sectors_written_start;
        u64 s_kbytes_written;

        /* the size of zero-out chunk */
        unsigned int s_extent_max_zeroout_kb;

        unsigned int s_log_groups_per_flex;
        struct flex_groups * __rcu *s_flex_groups;
        ext4_group_t s_flex_groups_allocated;

        /* workqueue for reserved extent conversions (buffered io) */
        struct workqueue_struct *rsv_conversion_wq;

        /* timer for periodic error stats printing */
        struct timer_list s_err_report;

        /* Lazy inode table initialization info */
        struct ext4_li_request *s_li_request;
        /* Wait multiplier for lazy initialization thread */
        unsigned int s_li_wait_mult;

        /* Kernel thread for multiple mount protection */
        struct task_struct *s_mmp_tsk;

        /* record the last minlen when FITRIM is called. */
        unsigned long s_last_trim_minblks;

        /* Reference to checksum algorithm driver via cryptoapi */
        struct crypto_shash *s_chksum_driver;

        /* Precomputed FS UUID checksum for seeding other checksums */
        __u32 s_csum_seed;

        /* Reclaim extents from extent status tree */
        struct shrinker s_es_shrinker;
        struct list_head s_es_list;        /* List of inodes with reclaimable extents */
        long s_es_nr_inode;
        struct ext4_es_stats s_es_stats;
        struct mb_cache *s_ea_block_cache;
        struct mb_cache *s_ea_inode_cache;
        spinlock_t s_es_lock ____cacheline_aligned_in_smp;

        /* Journal triggers for checksum computation */
        struct ext4_journal_trigger s_journal_triggers[EXT4_JOURNAL_TRIGGER_COUNT];

        /* Ratelimit ext4 messages. */
        struct ratelimit_state s_err_ratelimit_state;
        struct ratelimit_state s_warning_ratelimit_state;
        struct ratelimit_state s_msg_ratelimit_state;
        atomic_t s_warning_count;
        atomic_t s_msg_count;

        /* Encryption policy for '-o test_dummy_encryption' */
        struct fscrypt_dummy_policy s_dummy_enc_policy;

        /*
         * Barrier between writepages ops and changing any inode's JOURNAL_DATA
         * or EXTENTS flag.
         */
        struct percpu_rw_semaphore s_writepages_rwsem;
        struct dax_device *s_daxdev;
#ifdef CONFIG_EXT4_DEBUG
        unsigned long s_simulate_fail;
#endif
        /* Record the errseq of the backing block device */
        errseq_t s_bdev_wb_err;
        spinlock_t s_bdev_wb_lock;

        /* Information about errors that happened during this mount */
        spinlock_t s_error_lock;
        int s_add_error_count;
        int s_first_error_code;
        __u32 s_first_error_line;
        __u32 s_first_error_ino;
        __u64 s_first_error_block;
        const char *s_first_error_func;
        time64_t s_first_error_time;
        int s_last_error_code;
        __u32 s_last_error_line;
        __u32 s_last_error_ino;
        __u64 s_last_error_block;
        const char *s_last_error_func;
        time64_t s_last_error_time;
        /*
         * If we are in a context where we cannot update error information in
         * the on-disk superblock, we queue this work to do it.
         */
        struct work_struct s_error_work;

        /* Ext4 fast commit sub transaction ID */
        atomic_t s_fc_subtid;

        /*
         * After commit starts, the main queue gets locked, and the further
         * updates get added in the staging queue.
         */
#define FC_Q_MAIN        0
#define FC_Q_STAGING        1
        struct list_head s_fc_q[2];        /* Inodes staged for fast commit
                                         * that have data changes in them.
                                         */
        struct list_head s_fc_dentry_q[2];        /* directory entry updates */
        unsigned int s_fc_bytes;
        /*
         * Main fast commit lock. This lock protects accesses to the
         * following fields:
         * ei->i_fc_list, s_fc_dentry_q, s_fc_q, s_fc_bytes, s_fc_bh.
         */
        spinlock_t s_fc_lock;
        struct buffer_head *s_fc_bh;
        struct ext4_fc_stats s_fc_stats;
        tid_t s_fc_ineligible_tid;
#ifdef CONFIG_EXT4_DEBUG
        int s_fc_debug_max_replay;
#endif
        struct ext4_fc_replay_state s_fc_replay_state;
};

static inline struct ext4_sb_info *EXT4_SB(struct super_block *sb)
{
        return sb->s_fs_info;
}
static inline struct ext4_inode_info *EXT4_I(struct inode *inode)
{
        return container_of(inode, struct ext4_inode_info, vfs_inode);
}

static inline int ext4_valid_inum(struct super_block *sb, unsigned long ino)
{
        return ino == EXT4_ROOT_INO ||
                (ino >= EXT4_FIRST_INO(sb) &&
                 ino <= le32_to_cpu(EXT4_SB(sb)->s_es->s_inodes_count));
}

/*
 * Returns: sbi->field[index]
 * Used to access an array element from the following sbi fields which require
 * rcu protection to avoid dereferencing an invalid pointer due to reassignment
 * - s_group_desc
 * - s_group_info
 * - s_flex_group
 */
#define sbi_array_rcu_deref(sbi, field, index)                                   \
({                                                                           \
        typeof(*((sbi)->field)) _v;                                           \
        rcu_read_lock();                                                   \
        _v = ((typeof(_v)*)rcu_dereference((sbi)->field))[index];           \
        rcu_read_unlock();                                                   \
        _v;                                                                   \
})

/*
 * run-time mount flags
 */
enum {
        EXT4_MF_MNTDIR_SAMPLED,
        EXT4_MF_FS_ABORTED,        /* Fatal error detected */
        EXT4_MF_FC_INELIGIBLE        /* Fast commit ineligible */
};

static inline void ext4_set_mount_flag(struct super_block *sb, int bit)
{
        set_bit(bit, &EXT4_SB(sb)->s_mount_flags);
}

static inline void ext4_clear_mount_flag(struct super_block *sb, int bit)
{
        clear_bit(bit, &EXT4_SB(sb)->s_mount_flags);
}

static inline int ext4_test_mount_flag(struct super_block *sb, int bit)
{
        return test_bit(bit, &EXT4_SB(sb)->s_mount_flags);
}


/*
 * Simulate_fail codes
 */
#define EXT4_SIM_BBITMAP_EIO        1
#define EXT4_SIM_BBITMAP_CRC        2
#define EXT4_SIM_IBITMAP_EIO        3
#define EXT4_SIM_IBITMAP_CRC        4
#define EXT4_SIM_INODE_EIO        5
#define EXT4_SIM_INODE_CRC        6
#define EXT4_SIM_DIRBLOCK_EIO        7
#define EXT4_SIM_DIRBLOCK_CRC        8

static inline bool ext4_simulate_fail(struct super_block *sb,
                                     unsigned long code)
{
#ifdef CONFIG_EXT4_DEBUG
        struct ext4_sb_info *sbi = EXT4_SB(sb);

        if (unlikely(sbi->s_simulate_fail == code)) {
                sbi->s_simulate_fail = 0;
                return true;
        }
#endif
        return false;
}

static inline void ext4_simulate_fail_bh(struct super_block *sb,
                                         struct buffer_head *bh,
                                         unsigned long code)
{
        if (!IS_ERR(bh) && ext4_simulate_fail(sb, code))
                clear_buffer_uptodate(bh);
}

/*
 * Error number codes for s_{first,last}_error_errno
 *
 * Linux errno numbers are architecture specific, so we need to translate
 * them into something which is architecture independent.   We don't define
 * codes for all errno's; just the ones which are most likely to be the cause
 * of an ext4_error() call.
 */
#define EXT4_ERR_UNKNOWN         1
#define EXT4_ERR_EIO                 2
#define EXT4_ERR_ENOMEM                 3
#define EXT4_ERR_EFSBADCRC         4
#define EXT4_ERR_EFSCORRUPTED         5
#define EXT4_ERR_ENOSPC                 6
#define EXT4_ERR_ENOKEY                 7
#defi